diff options
Diffstat (limited to 'drivers')
1052 files changed, 39652 insertions, 16168 deletions
diff --git a/drivers/acpi/acpi_dbg.c b/drivers/acpi/acpi_dbg.c index 6041974c7627..fb7290338593 100644 --- a/drivers/acpi/acpi_dbg.c +++ b/drivers/acpi/acpi_dbg.c @@ -749,6 +749,9 @@ static int __init acpi_aml_init(void) { int ret; + if (acpi_disabled) + return -ENODEV; + /* Initialize AML IO interface */ mutex_init(&acpi_aml_io.lock); init_waitqueue_head(&acpi_aml_io.wait); diff --git a/drivers/acpi/acpi_memhotplug.c b/drivers/acpi/acpi_memhotplug.c index ad6e90fbc813..b02fd51e5589 100644 --- a/drivers/acpi/acpi_memhotplug.c +++ b/drivers/acpi/acpi_memhotplug.c @@ -194,7 +194,8 @@ static int acpi_memory_enable_device(struct acpi_memory_device *mem_device) if (node < 0) node = memory_add_physaddr_to_nid(info->start_addr); - result = __add_memory(node, info->start_addr, info->length); + result = __add_memory(node, info->start_addr, info->length, + MHP_NONE); /* * If the memory block has been used by the kernel, add_memory() diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c index 412a9725cc1e..2ee5e05a0d69 100644 --- a/drivers/acpi/acpi_processor.c +++ b/drivers/acpi/acpi_processor.c @@ -264,7 +264,6 @@ static int acpi_processor_get_info(struct acpi_device *device) } else { /* * Declared with "Device" statement; match _UID. - * Note that we don't handle string _UIDs yet. */ status = acpi_evaluate_integer(pr->handle, METHOD_NAME__UID, NULL, &value); diff --git a/drivers/acpi/acpica/hwgpe.c b/drivers/acpi/acpica/hwgpe.c index 37bb67ef3232..b13a4ed5bc63 100644 --- a/drivers/acpi/acpica/hwgpe.c +++ b/drivers/acpi/acpica/hwgpe.c @@ -47,7 +47,7 @@ acpi_status acpi_hw_gpe_read(u64 *value, struct acpi_gpe_address *reg) if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) { #ifdef ACPI_GPE_USE_LOGICAL_ADDRESSES - *value = (u64)ACPI_GET8(reg->address); + *value = (u64)ACPI_GET8((unsigned long)reg->address); return_ACPI_STATUS(AE_OK); #else return acpi_os_read_memory((acpi_physical_address)reg->address, @@ -82,7 +82,7 @@ acpi_status acpi_hw_gpe_write(u64 value, struct acpi_gpe_address *reg) { if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) { #ifdef ACPI_GPE_USE_LOGICAL_ADDRESSES - ACPI_SET8(reg->address, value); + ACPI_SET8((unsigned long)reg->address, value); return_ACPI_STATUS(AE_OK); #else return acpi_os_write_memory((acpi_physical_address)reg->address, diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 81bf71b10d44..fce7ade2aba9 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -79,6 +79,12 @@ ((struct acpi_hest_generic_status *) \ ((struct ghes_estatus_node *)(estatus_node) + 1)) +#define GHES_VENDOR_ENTRY_LEN(gdata_len) \ + (sizeof(struct ghes_vendor_record_entry) + (gdata_len)) +#define GHES_GDATA_FROM_VENDOR_ENTRY(vendor_entry) \ + ((struct acpi_hest_generic_data *) \ + ((struct ghes_vendor_record_entry *)(vendor_entry) + 1)) + /* * NMI-like notifications vary by architecture, before the compiler can prune * unused static functions it needs a value for these enums. @@ -123,6 +129,12 @@ static DEFINE_MUTEX(ghes_list_mutex); */ static DEFINE_SPINLOCK(ghes_notify_lock_irq); +struct ghes_vendor_record_entry { + struct work_struct work; + int error_severity; + char vendor_record[]; +}; + static struct gen_pool *ghes_estatus_pool; static unsigned long ghes_estatus_pool_size_request; @@ -511,6 +523,56 @@ static void ghes_handle_aer(struct acpi_hest_generic_data *gdata) #endif } +static BLOCKING_NOTIFIER_HEAD(vendor_record_notify_list); + +int ghes_register_vendor_record_notifier(struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&vendor_record_notify_list, nb); +} +EXPORT_SYMBOL_GPL(ghes_register_vendor_record_notifier); + +void ghes_unregister_vendor_record_notifier(struct notifier_block *nb) +{ + blocking_notifier_chain_unregister(&vendor_record_notify_list, nb); +} +EXPORT_SYMBOL_GPL(ghes_unregister_vendor_record_notifier); + +static void ghes_vendor_record_work_func(struct work_struct *work) +{ + struct ghes_vendor_record_entry *entry; + struct acpi_hest_generic_data *gdata; + u32 len; + + entry = container_of(work, struct ghes_vendor_record_entry, work); + gdata = GHES_GDATA_FROM_VENDOR_ENTRY(entry); + + blocking_notifier_call_chain(&vendor_record_notify_list, + entry->error_severity, gdata); + + len = GHES_VENDOR_ENTRY_LEN(acpi_hest_get_record_size(gdata)); + gen_pool_free(ghes_estatus_pool, (unsigned long)entry, len); +} + +static void ghes_defer_non_standard_event(struct acpi_hest_generic_data *gdata, + int sev) +{ + struct acpi_hest_generic_data *copied_gdata; + struct ghes_vendor_record_entry *entry; + u32 len; + + len = GHES_VENDOR_ENTRY_LEN(acpi_hest_get_record_size(gdata)); + entry = (void *)gen_pool_alloc(ghes_estatus_pool, len); + if (!entry) + return; + + copied_gdata = GHES_GDATA_FROM_VENDOR_ENTRY(entry); + memcpy(copied_gdata, gdata, acpi_hest_get_record_size(gdata)); + entry->error_severity = sev; + + INIT_WORK(&entry->work, ghes_vendor_record_work_func); + schedule_work(&entry->work); +} + static bool ghes_do_proc(struct ghes *ghes, const struct acpi_hest_generic_status *estatus) { @@ -549,6 +611,7 @@ static bool ghes_do_proc(struct ghes *ghes, } else { void *err = acpi_hest_get_payload(gdata); + ghes_defer_non_standard_event(gdata, sev); log_non_standard_event(sec_type, fru_id, fru_text, sec_sev, err, gdata->error_data_length); @@ -879,7 +942,7 @@ static void ghes_proc_in_irq(struct irq_work *irq_work) estatus_node->task_work.func = ghes_kick_task_work; estatus_node->task_work_cpu = smp_processor_id(); ret = task_work_add(current, &estatus_node->task_work, - true); + TWA_RESUME); if (ret) estatus_node->task_work.func = NULL; } diff --git a/drivers/acpi/dptf/Kconfig b/drivers/acpi/dptf/Kconfig index 51f06f36cafa..1e8c7ce89bf1 100644 --- a/drivers/acpi/dptf/Kconfig +++ b/drivers/acpi/dptf/Kconfig @@ -1,8 +1,25 @@ # SPDX-License-Identifier: GPL-2.0 -config DPTF_POWER - tristate "DPTF Platform Power Participant" + +menuconfig ACPI_DPTF + bool "Intel DPTF (Dynamic Platform and Thermal Framework) Support" depends on X86 help + Intel Dynamic Platform and Thermal Framework (DPTF) is a platform + level hardware/software solution for power and thermal management. + + As a container for multiple power/thermal technologies, DPTF provides + a coordinated approach for different policies to effect the hardware + state of a system. + + For more information see: + <https://01.org/intel%C2%AE-dynamic-platform-and-thermal-framework-dptf-chromium-os/overview> + +if ACPI_DPTF + +config DPTF_POWER + tristate "Platform Power DPTF Participant" + default m + help This driver adds support for Dynamic Platform and Thermal Framework (DPTF) Platform Power Participant device (INT3407) support. This participant is responsible for exposing platform telemetry: @@ -16,15 +33,17 @@ config DPTF_POWER the module will be called dptf_power. config DPTF_PCH_FIVR - tristate "DPTF PCH FIVR Participant" - depends on X86 + tristate "PCH FIVR DPTF Participant" + default m help This driver adds support for Dynamic Platform and Thermal Framework (DPTF) PCH FIVR Participant device support. This driver allows to - switch PCH FIVR (Fully Integrated Voltage Regulator) frequency. + switch the PCH FIVR (Fully Integrated Voltage Regulator) frequency. This participant is responsible for exposing: freq_mhz_low_clock freq_mhz_high_clock To compile this driver as a module, choose M here: the module will be called dptf_pch_fivr. + +endif diff --git a/drivers/acpi/dptf/dptf_pch_fivr.c b/drivers/acpi/dptf/dptf_pch_fivr.c index 4ab288827747..4c1992fce150 100644 --- a/drivers/acpi/dptf/dptf_pch_fivr.c +++ b/drivers/acpi/dptf/dptf_pch_fivr.c @@ -114,7 +114,7 @@ static struct platform_driver pch_fivr_driver = { .probe = pch_fivr_add, .remove = pch_fivr_remove, .driver = { - .name = "DPTF PCH FIVR", + .name = "dptf_pch_fivr", .acpi_match_table = pch_fivr_device_ids, }, }; diff --git a/drivers/acpi/dptf/dptf_power.c b/drivers/acpi/dptf/dptf_power.c index 92b996a564d0..06741305fc77 100644 --- a/drivers/acpi/dptf/dptf_power.c +++ b/drivers/acpi/dptf/dptf_power.c @@ -237,7 +237,7 @@ static struct platform_driver dptf_power_driver = { .probe = dptf_power_add, .remove = dptf_power_remove, .driver = { - .name = "DPTF Platform Power", + .name = "dptf_power", .acpi_match_table = int3407_device_ids, }, }; diff --git a/drivers/acpi/pci_mcfg.c b/drivers/acpi/pci_mcfg.c index 54b36b7ad47d..7ddd57abadd1 100644 --- a/drivers/acpi/pci_mcfg.c +++ b/drivers/acpi/pci_mcfg.c @@ -142,6 +142,26 @@ static struct mcfg_fixup mcfg_quirks[] = { XGENE_V2_ECAM_MCFG(4, 0), XGENE_V2_ECAM_MCFG(4, 1), XGENE_V2_ECAM_MCFG(4, 2), + +#define ALTRA_ECAM_QUIRK(rev, seg) \ + { "Ampere", "Altra ", rev, seg, MCFG_BUS_ANY, &pci_32b_read_ops } + + ALTRA_ECAM_QUIRK(1, 0), + ALTRA_ECAM_QUIRK(1, 1), + ALTRA_ECAM_QUIRK(1, 2), + ALTRA_ECAM_QUIRK(1, 3), + ALTRA_ECAM_QUIRK(1, 4), + ALTRA_ECAM_QUIRK(1, 5), + ALTRA_ECAM_QUIRK(1, 6), + ALTRA_ECAM_QUIRK(1, 7), + ALTRA_ECAM_QUIRK(1, 8), + ALTRA_ECAM_QUIRK(1, 9), + ALTRA_ECAM_QUIRK(1, 10), + ALTRA_ECAM_QUIRK(1, 11), + ALTRA_ECAM_QUIRK(1, 12), + ALTRA_ECAM_QUIRK(1, 13), + ALTRA_ECAM_QUIRK(1, 14), + ALTRA_ECAM_QUIRK(1, 15), }; static char mcfg_oem_id[ACPI_OEM_ID_SIZE]; @@ -280,5 +300,5 @@ void __init pci_mmcfg_late_init(void) { int err = acpi_table_parse(ACPI_SIG_MCFG, pci_mcfg_parse); if (err) - pr_err("Failed to parse MCFG (%d)\n", err); + pr_debug("Failed to parse MCFG (%d)\n", err); } diff --git a/drivers/acpi/reboot.c b/drivers/acpi/reboot.c index ca707f5b521d..2a61f884e222 100644 --- a/drivers/acpi/reboot.c +++ b/drivers/acpi/reboot.c @@ -3,6 +3,7 @@ #include <linux/pci.h> #include <linux/acpi.h> #include <acpi/reboot.h> +#include <linux/delay.h> #ifdef CONFIG_PCI static void acpi_pci_reboot(struct acpi_generic_address *rr, u8 reset_value) @@ -66,4 +67,14 @@ void acpi_reboot(void) acpi_reset(); break; } + + /* + * Some platforms do not shut down immediately after writing to the + * ACPI reset register, and this results in racing with the + * subsequent reboot mechanism. + * + * The 15ms delay has been found to be long enough for the system + * to reboot on the affected platforms. + */ + mdelay(15); } diff --git a/drivers/acpi/utils.c b/drivers/acpi/utils.c index 838b719ec7ce..d5411a166685 100644 --- a/drivers/acpi/utils.c +++ b/drivers/acpi/utils.c @@ -104,7 +104,6 @@ acpi_extract_package(union acpi_object *package, " [%c]\n", i, format_string[i]); return AE_BAD_DATA; - break; } break; @@ -129,7 +128,6 @@ acpi_extract_package(union acpi_object *package, " expecting [%c]\n", i, format_string[i]); return AE_BAD_DATA; - break; } break; case ACPI_TYPE_LOCAL_REFERENCE: @@ -144,7 +142,6 @@ acpi_extract_package(union acpi_object *package, " expecting [%c]\n", i, format_string[i]); return AE_BAD_DATA; - break; } break; @@ -155,7 +152,6 @@ acpi_extract_package(union acpi_object *package, i)); /* TBD: handle nested packages... */ return AE_SUPPORT; - break; } } diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 4b9476521da6..b5117576792b 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -2229,7 +2229,7 @@ static void binder_deferred_fd_close(int fd) __close_fd_get_file(fd, &twcb->file); if (twcb->file) { filp_close(twcb->file, current->files); - task_work_add(current, &twcb->twork, true); + task_work_add(current, &twcb->twork, TWA_RESUME); } else { kfree(twcb); } diff --git a/drivers/base/core.c b/drivers/base/core.c index b919e6d01d9a..c852f16c111b 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -4212,13 +4212,16 @@ define_dev_printk_level(_dev_info, KERN_INFO); * -EPROBE_DEFER and propagate error upwards. * In case of -EPROBE_DEFER it sets also defer probe reason, which can be * checked later by reading devices_deferred debugfs attribute. - * It replaces code sequence: + * It replaces code sequence:: + * * if (err != -EPROBE_DEFER) * dev_err(dev, ...); * else * dev_dbg(dev, ...); * return err; - * with + * + * with:: + * * return dev_err_probe(dev, err, ...); * * Returns @err. diff --git a/drivers/base/memory.c b/drivers/base/memory.c index adf828dfccf0..eef4ffb6122c 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -432,7 +432,8 @@ static ssize_t probe_store(struct device *dev, struct device_attribute *attr, nid = memory_add_physaddr_to_nid(phys_addr); ret = __add_memory(nid, phys_addr, - MIN_MEMORY_BLOCK_SIZE * sections_per_block); + MIN_MEMORY_BLOCK_SIZE * sections_per_block, + MHP_NONE); if (ret) goto out; diff --git a/drivers/base/node.c b/drivers/base/node.c index 43d21f9e88b1..6ffa470e2984 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -772,8 +772,8 @@ static int __ref get_nid_for_pfn(unsigned long pfn) return pfn_to_nid(pfn); } -static int do_register_memory_block_under_node(int nid, - struct memory_block *mem_blk) +static void do_register_memory_block_under_node(int nid, + struct memory_block *mem_blk) { int ret; @@ -786,12 +786,19 @@ static int do_register_memory_block_under_node(int nid, ret = sysfs_create_link_nowarn(&node_devices[nid]->dev.kobj, &mem_blk->dev.kobj, kobject_name(&mem_blk->dev.kobj)); - if (ret) - return ret; + if (ret && ret != -EEXIST) + dev_err_ratelimited(&node_devices[nid]->dev, + "can't create link to %s in sysfs (%d)\n", + kobject_name(&mem_blk->dev.kobj), ret); - return sysfs_create_link_nowarn(&mem_blk->dev.kobj, + ret = sysfs_create_link_nowarn(&mem_blk->dev.kobj, &node_devices[nid]->dev.kobj, kobject_name(&node_devices[nid]->dev.kobj)); + if (ret && ret != -EEXIST) + dev_err_ratelimited(&mem_blk->dev, + "can't create link to %s in sysfs (%d)\n", + kobject_name(&node_devices[nid]->dev.kobj), + ret); } /* register memory section under specified node if it spans that node */ @@ -827,7 +834,8 @@ static int register_mem_block_under_node_early(struct memory_block *mem_blk, if (page_nid != nid) continue; - return do_register_memory_block_under_node(nid, mem_blk); + do_register_memory_block_under_node(nid, mem_blk); + return 0; } /* mem section does not span the specified node */ return 0; @@ -842,7 +850,8 @@ static int register_mem_block_under_node_hotplug(struct memory_block *mem_blk, { int nid = *(int *)arg; - return do_register_memory_block_under_node(nid, mem_blk); + do_register_memory_block_under_node(nid, mem_blk); + return 0; } /* @@ -860,8 +869,8 @@ void unregister_memory_block_under_nodes(struct memory_block *mem_blk) kobject_name(&node_devices[mem_blk->nid]->dev.kobj)); } -int link_mem_sections(int nid, unsigned long start_pfn, unsigned long end_pfn, - enum meminit_context context) +void link_mem_sections(int nid, unsigned long start_pfn, unsigned long end_pfn, + enum meminit_context context) { walk_memory_blocks_func_t func; @@ -870,9 +879,9 @@ int link_mem_sections(int nid, unsigned long start_pfn, unsigned long end_pfn, else func = register_mem_block_under_node_early; - return walk_memory_blocks(PFN_PHYS(start_pfn), - PFN_PHYS(end_pfn - start_pfn), (void *)&nid, - func); + walk_memory_blocks(PFN_PHYS(start_pfn), PFN_PHYS(end_pfn - start_pfn), + (void *)&nid, func); + return; } #ifdef CONFIG_HUGETLBFS diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index 05bb4d4401b2..743268996336 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -415,26 +415,45 @@ static int _genpd_power_on(struct generic_pm_domain *genpd, bool timed) s64 elapsed_ns; int ret; + /* Notify consumers that we are about to power on. */ + ret = raw_notifier_call_chain_robust(&genpd->power_notifiers, + GENPD_NOTIFY_PRE_ON, + GENPD_NOTIFY_OFF, NULL); + ret = notifier_to_errno(ret); + if (ret) + return ret; + if (!genpd->power_on) - return 0; + goto out; - if (!timed) - return genpd->power_on(genpd); + if (!timed) { + ret = genpd->power_on(genpd); + if (ret) + goto err; + + goto out; + } time_start = ktime_get(); ret = genpd->power_on(genpd); if (ret) - return ret; + goto err; elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start)); if (elapsed_ns <= genpd->states[state_idx].power_on_latency_ns) - return ret; + goto out; genpd->states[state_idx].power_on_latency_ns = elapsed_ns; genpd->max_off_time_changed = true; pr_debug("%s: Power-%s latency exceeded, new value %lld ns\n", genpd->name, "on", elapsed_ns); +out: + raw_notifier_call_chain(&genpd->power_notifiers, GENPD_NOTIFY_ON, NULL); + return 0; +err: + raw_notifier_call_chain(&genpd->power_notifiers, GENPD_NOTIFY_OFF, + NULL); return ret; } @@ -445,27 +464,46 @@ static int _genpd_power_off(struct generic_pm_domain *genpd, bool timed) s64 elapsed_ns; int ret; + /* Notify consumers that we are about to power off. */ + ret = raw_notifier_call_chain_robust(&genpd->power_notifiers, + GENPD_NOTIFY_PRE_OFF, + GENPD_NOTIFY_ON, NULL); + ret = notifier_to_errno(ret); + if (ret) + return ret; + if (!genpd->power_off) - return 0; + goto out; + + if (!timed) { + ret = genpd->power_off(genpd); + if (ret) + goto busy; - if (!timed) - return genpd->power_off(genpd); + goto out; + } time_start = ktime_get(); ret = genpd->power_off(genpd); if (ret) - return ret; + goto busy; elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start)); if (elapsed_ns <= genpd->states[state_idx].power_off_latency_ns) - return 0; + goto out; genpd->states[state_idx].power_off_latency_ns = elapsed_ns; genpd->max_off_time_changed = true; pr_debug("%s: Power-%s latency exceeded, new value %lld ns\n", genpd->name, "off", elapsed_ns); +out: + raw_notifier_call_chain(&genpd->power_notifiers, GENPD_NOTIFY_OFF, + NULL); return 0; +busy: + raw_notifier_call_chain(&genpd->power_notifiers, GENPD_NOTIFY_ON, NULL); + return ret; } /** @@ -550,11 +588,14 @@ static int genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, return -EBUSY; ret = _genpd_power_off(genpd, true); - if (ret) + if (ret) { + genpd->states[genpd->state_idx].rejected++; return ret; + } genpd->status = GENPD_STATE_OFF; genpd_update_accounting(genpd); + genpd->states[genpd->state_idx].usage++; list_for_each_entry(link, &genpd->child_links, child_node) { genpd_sd_counter_dec(link->parent); @@ -1270,13 +1311,14 @@ static int genpd_restore_noirq(struct device *dev) * first time for the given domain in the present cycle. */ genpd_lock(genpd); - if (genpd->suspended_count++ == 0) + if (genpd->suspended_count++ == 0) { /* * The boot kernel might put the domain into arbitrary state, * so make it appear as powered off to genpd_sync_power_on(), * so that it tries to power it on in case it was really off. */ genpd->status = GENPD_STATE_OFF; + } genpd_sync_power_on(genpd, true, 0); genpd_unlock(genpd); @@ -1592,6 +1634,101 @@ int pm_genpd_remove_device(struct device *dev) } EXPORT_SYMBOL_GPL(pm_genpd_remove_device); +/** + * dev_pm_genpd_add_notifier - Add a genpd power on/off notifier for @dev + * + * @dev: Device that should be associated with the notifier + * @nb: The notifier block to register + * + * Users may call this function to add a genpd power on/off notifier for an + * attached @dev. Only one notifier per device is allowed. The notifier is + * sent when genpd is powering on/off the PM domain. + * + * It is assumed that the user guarantee that the genpd wouldn't be detached + * while this routine is getting called. + * + * Returns 0 on success and negative error values on failures. + */ +int dev_pm_genpd_add_notifier(struct device *dev, struct notifier_block *nb) +{ + struct generic_pm_domain *genpd; + struct generic_pm_domain_data *gpd_data; + int ret; + + genpd = dev_to_genpd_safe(dev); + if (!genpd) + return -ENODEV; + + if (WARN_ON(!dev->power.subsys_data || + !dev->power.subsys_data->domain_data)) + return -EINVAL; + + gpd_data = to_gpd_data(dev->power.subsys_data->domain_data); + if (gpd_data->power_nb) + return -EEXIST; + + genpd_lock(genpd); + ret = raw_notifier_chain_register(&genpd->power_notifiers, nb); + genpd_unlock(genpd); + + if (ret) { + dev_warn(dev, "failed to add notifier for PM domain %s\n", + genpd->name); + return ret; + } + + gpd_data->power_nb = nb; + return 0; +} +EXPORT_SYMBOL_GPL(dev_pm_genpd_add_notifier); + +/** + * dev_pm_genpd_remove_notifier - Remove a genpd power on/off notifier for @dev + * + * @dev: Device that is associated with the notifier + * + * Users may call this function to remove a genpd power on/off notifier for an + * attached @dev. + * + * It is assumed that the user guarantee that the genpd wouldn't be detached + * while this routine is getting called. + * + * Returns 0 on success and negative error values on failures. + */ +int dev_pm_genpd_remove_notifier(struct device *dev) +{ + struct generic_pm_domain *genpd; + struct generic_pm_domain_data *gpd_data; + int ret; + + genpd = dev_to_genpd_safe(dev); + if (!genpd) + return -ENODEV; + + if (WARN_ON(!dev->power.subsys_data || + !dev->power.subsys_data->domain_data)) + return -EINVAL; + + gpd_data = to_gpd_data(dev->power.subsys_data->domain_data); + if (!gpd_data->power_nb) + return -ENODEV; + + genpd_lock(genpd); + ret = raw_notifier_chain_unregister(&genpd->power_notifiers, + gpd_data->power_nb); + genpd_unlock(genpd); + + if (ret) { + dev_warn(dev, "failed to remove notifier for PM domain %s\n", + genpd->name); + return ret; + } + + gpd_data->power_nb = NULL; + return 0; +} +EXPORT_SYMBOL_GPL(dev_pm_genpd_remove_notifier); + static int genpd_add_subdomain(struct generic_pm_domain *genpd, struct generic_pm_domain *subdomain) { @@ -1762,6 +1899,7 @@ int pm_genpd_init(struct generic_pm_domain *genpd, INIT_LIST_HEAD(&genpd->parent_links); INIT_LIST_HEAD(&genpd->child_links); INIT_LIST_HEAD(&genpd->dev_list); + RAW_INIT_NOTIFIER_HEAD(&genpd->power_notifiers); genpd_lock_init(genpd); genpd->gov = gov; INIT_WORK(&genpd->power_off_work, genpd_power_off_work_fn); @@ -2923,7 +3061,7 @@ static int idle_states_show(struct seq_file *s, void *data) if (ret) return -ERESTARTSYS; - seq_puts(s, "State Time Spent(ms)\n"); + seq_puts(s, "State Time Spent(ms) Usage Rejected\n"); for (i = 0; i < genpd->state_count; i++) { ktime_t delta = 0; @@ -2935,7 +3073,8 @@ static int idle_states_show(struct seq_file *s, void *data) msecs = ktime_to_ms( ktime_add(genpd->states[i].idle_time, delta)); - seq_printf(s, "S%-13i %lld\n", i, msecs); + seq_printf(s, "S%-13i %-14lld %-14llu %llu\n", i, msecs, + genpd->states[i].usage, genpd->states[i].rejected); } genpd_unlock(genpd); diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 205a06752ca9..c7ac49042cee 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -363,7 +363,6 @@ static pm_callback_t pm_op(const struct dev_pm_ops *ops, pm_message_t state) case PM_EVENT_THAW: case PM_EVENT_RECOVER: return ops->thaw; - break; case PM_EVENT_RESTORE: return ops->restore; #endif /* CONFIG_HIBERNATE_CALLBACKS */ diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index 3e89b5d48ee6..f84128abade3 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -4010,10 +4010,10 @@ static int rbd_try_lock(struct rbd_device *rbd_dev) rbd_warn(rbd_dev, "breaking header lock owned by %s%llu", ENTITY_NAME(lockers[0].id.name)); - ret = ceph_monc_blacklist_add(&client->monc, + ret = ceph_monc_blocklist_add(&client->monc, &lockers[0].info.addr); if (ret) { - rbd_warn(rbd_dev, "blacklist of %s%llu failed: %d", + rbd_warn(rbd_dev, "blocklist of %s%llu failed: %d", ENTITY_NAME(lockers[0].id.name), ret); goto out; } @@ -4077,7 +4077,7 @@ static int rbd_try_acquire_lock(struct rbd_device *rbd_dev) ret = rbd_try_lock(rbd_dev); if (ret < 0) { rbd_warn(rbd_dev, "failed to lock header: %d", ret); - if (ret == -EBLACKLISTED) + if (ret == -EBLOCKLISTED) goto out; ret = 1; /* request lock anyway */ @@ -4613,7 +4613,7 @@ static void rbd_reregister_watch(struct work_struct *work) ret = __rbd_register_watch(rbd_dev); if (ret) { rbd_warn(rbd_dev, "failed to reregister watch: %d", ret); - if (ret != -EBLACKLISTED && ret != -ENOENT) { + if (ret != -EBLOCKLISTED && ret != -ENOENT) { queue_delayed_work(rbd_dev->task_wq, &rbd_dev->watch_dwork, RBD_RETRY_DELAY); diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c index adfc9352351d..501e9dacfff9 100644 --- a/drivers/block/xen-blkback/blkback.c +++ b/drivers/block/xen-blkback/blkback.c @@ -201,7 +201,7 @@ static inline void shrink_free_pagepool(struct xen_blkif_ring *ring, int num) #define vaddr(page) ((unsigned long)pfn_to_kaddr(page_to_pfn(page))) -static int do_block_io_op(struct xen_blkif_ring *ring); +static int do_block_io_op(struct xen_blkif_ring *ring, unsigned int *eoi_flags); static int dispatch_rw_block_io(struct xen_blkif_ring *ring, struct blkif_request *req, struct pending_req *pending_req); @@ -612,6 +612,8 @@ int xen_blkif_schedule(void *arg) struct xen_vbd *vbd = &blkif->vbd; unsigned long timeout; int ret; + bool do_eoi; + unsigned int eoi_flags = XEN_EOI_FLAG_SPURIOUS; set_freezable(); while (!kthread_should_stop()) { @@ -636,16 +638,23 @@ int xen_blkif_schedule(void *arg) if (timeout == 0) goto purge_gnt_list; + do_eoi = ring->waiting_reqs; + ring->waiting_reqs = 0; smp_mb(); /* clear flag *before* checking for work */ - ret = do_block_io_op(ring); + ret = do_block_io_op(ring, &eoi_flags); if (ret > 0) ring->waiting_reqs = 1; if (ret == -EACCES) wait_event_interruptible(ring->shutdown_wq, kthread_should_stop()); + if (do_eoi && !ring->waiting_reqs) { + xen_irq_lateeoi(ring->irq, eoi_flags); + eoi_flags |= XEN_EOI_FLAG_SPURIOUS; + } + purge_gnt_list: if (blkif->vbd.feature_gnt_persistent && time_after(jiffies, ring->next_lru)) { @@ -1121,7 +1130,7 @@ static void end_block_io_op(struct bio *bio) * and transmute it to the block API to hand it over to the proper block disk. */ static int -__do_block_io_op(struct xen_blkif_ring *ring) +__do_block_io_op(struct xen_blkif_ring *ring, unsigned int *eoi_flags) { union blkif_back_rings *blk_rings = &ring->blk_rings; struct blkif_request req; @@ -1144,6 +1153,9 @@ __do_block_io_op(struct xen_blkif_ring *ring) if (RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, rc)) break; + /* We've seen a request, so clear spurious eoi flag. */ + *eoi_flags &= ~XEN_EOI_FLAG_SPURIOUS; + if (kthread_should_stop()) { more_to_do = 1; break; @@ -1202,13 +1214,13 @@ done: } static int -do_block_io_op(struct xen_blkif_ring *ring) +do_block_io_op(struct xen_blkif_ring *ring, unsigned int *eoi_flags) { union blkif_back_rings *blk_rings = &ring->blk_rings; int more_to_do; do { - more_to_do = __do_block_io_op(ring); + more_to_do = __do_block_io_op(ring, eoi_flags); if (more_to_do) break; diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c index b9aa5d1ac10b..5e7c36d73dc6 100644 --- a/drivers/block/xen-blkback/xenbus.c +++ b/drivers/block/xen-blkback/xenbus.c @@ -246,9 +246,8 @@ static int xen_blkif_map(struct xen_blkif_ring *ring, grant_ref_t *gref, if (req_prod - rsp_prod > size) goto fail; - err = bind_interdomain_evtchn_to_irqhandler(blkif->domid, evtchn, - xen_blkif_be_int, 0, - "blkif-backend", ring); + err = bind_interdomain_evtchn_to_irqhandler_lateeoi(blkif->domid, + evtchn, xen_blkif_be_int, 0, "blkif-backend", ring); if (err < 0) goto fail; ring->irq = err; diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index bff3d4021c18..029403c18ca3 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1270,7 +1270,7 @@ static int __zram_bvec_read(struct zram *zram, struct page *page, u32 index, zram_slot_unlock(zram, index); /* Should NEVER happen. Return bio error if it does. */ - if (unlikely(ret)) + if (WARN_ON(ret)) pr_err("Decompression failed! err=%d, page=%u\n", ret, index); return ret; diff --git a/drivers/bus/brcmstb_gisb.c b/drivers/bus/brcmstb_gisb.c index ec1004c858b8..7355fa2cb439 100644 --- a/drivers/bus/brcmstb_gisb.c +++ b/drivers/bus/brcmstb_gisb.c @@ -30,8 +30,22 @@ #define ARB_ERR_CAP_STATUS_WRITE (1 << 1) #define ARB_ERR_CAP_STATUS_VALID (1 << 0) +#define ARB_BP_CAP_CLEAR (1 << 0) +#define ARB_BP_CAP_STATUS_PROT_SHIFT 14 +#define ARB_BP_CAP_STATUS_TYPE (1 << 13) +#define ARB_BP_CAP_STATUS_RSP_SHIFT 10 +#define ARB_BP_CAP_STATUS_MASK GENMASK(1, 0) +#define ARB_BP_CAP_STATUS_BS_SHIFT 2 +#define ARB_BP_CAP_STATUS_WRITE (1 << 1) +#define ARB_BP_CAP_STATUS_VALID (1 << 0) + enum { ARB_TIMER, + ARB_BP_CAP_CLR, + ARB_BP_CAP_HI_ADDR, + ARB_BP_CAP_ADDR, + ARB_BP_CAP_STATUS, + ARB_BP_CAP_MASTER, ARB_ERR_CAP_CLR, ARB_ERR_CAP_HI_ADDR, ARB_ERR_CAP_ADDR, @@ -41,6 +55,11 @@ enum { static const int gisb_offsets_bcm7038[] = { [ARB_TIMER] = 0x00c, + [ARB_BP_CAP_CLR] = 0x014, + [ARB_BP_CAP_HI_ADDR] = -1, + [ARB_BP_CAP_ADDR] = 0x0b8, + [ARB_BP_CAP_STATUS] = 0x0c0, + [ARB_BP_CAP_MASTER] = -1, [ARB_ERR_CAP_CLR] = 0x0c4, [ARB_ERR_CAP_HI_ADDR] = -1, [ARB_ERR_CAP_ADDR] = 0x0c8, @@ -50,6 +69,11 @@ static const int gisb_offsets_bcm7038[] = { static const int gisb_offsets_bcm7278[] = { [ARB_TIMER] = 0x008, + [ARB_BP_CAP_CLR] = 0x01c, + [ARB_BP_CAP_HI_ADDR] = -1, + [ARB_BP_CAP_ADDR] = 0x220, + [ARB_BP_CAP_STATUS] = 0x230, + [ARB_BP_CAP_MASTER] = 0x234, [ARB_ERR_CAP_CLR] = 0x7f8, [ARB_ERR_CAP_HI_ADDR] = -1, [ARB_ERR_CAP_ADDR] = 0x7e0, @@ -59,6 +83,11 @@ static const int gisb_offsets_bcm7278[] = { static const int gisb_offsets_bcm7400[] = { [ARB_TIMER] = 0x00c, + [ARB_BP_CAP_CLR] = 0x014, + [ARB_BP_CAP_HI_ADDR] = -1, + [ARB_BP_CAP_ADDR] = 0x0b8, + [ARB_BP_CAP_STATUS] = 0x0c0, + [ARB_BP_CAP_MASTER] = 0x0c4, [ARB_ERR_CAP_CLR] = 0x0c8, [ARB_ERR_CAP_HI_ADDR] = -1, [ARB_ERR_CAP_ADDR] = 0x0cc, @@ -68,6 +97,11 @@ static const int gisb_offsets_bcm7400[] = { static const int gisb_offsets_bcm7435[] = { [ARB_TIMER] = 0x00c, + [ARB_BP_CAP_CLR] = 0x014, + [ARB_BP_CAP_HI_ADDR] = -1, + [ARB_BP_CAP_ADDR] = 0x158, + [ARB_BP_CAP_STATUS] = 0x160, + [ARB_BP_CAP_MASTER] = 0x164, [ARB_ERR_CAP_CLR] = 0x168, [ARB_ERR_CAP_HI_ADDR] = -1, [ARB_ERR_CAP_ADDR] = 0x16c, @@ -77,6 +111,11 @@ static const int gisb_offsets_bcm7435[] = { static const int gisb_offsets_bcm7445[] = { [ARB_TIMER] = 0x008, + [ARB_BP_CAP_CLR] = 0x010, + [ARB_BP_CAP_HI_ADDR] = -1, + [ARB_BP_CAP_ADDR] = 0x1d8, + [ARB_BP_CAP_STATUS] = 0x1e0, + [ARB_BP_CAP_MASTER] = 0x1e4, [ARB_ERR_CAP_CLR] = 0x7e4, [ARB_ERR_CAP_HI_ADDR] = 0x7e8, [ARB_ERR_CAP_ADDR] = 0x7ec, @@ -125,6 +164,16 @@ static u64 gisb_read_address(struct brcmstb_gisb_arb_device *gdev) return value; } +static u64 gisb_read_bp_address(struct brcmstb_gisb_arb_device *gdev) +{ + u64 value; + + value = gisb_read(gdev, ARB_BP_CAP_ADDR); + value |= (u64)gisb_read(gdev, ARB_BP_CAP_HI_ADDR) << 32; + + return value; +} + static void gisb_write(struct brcmstb_gisb_arb_device *gdev, u32 val, int reg) { int offset = gdev->gisb_offsets[reg]; @@ -210,8 +259,8 @@ static int brcmstb_gisb_arb_decode_addr(struct brcmstb_gisb_arb_device *gdev, m_name = m_fmt; } - pr_crit("%s: %s at 0x%llx [%c %s], core: %s\n", - __func__, reason, arb_addr, + pr_crit("GISB: %s at 0x%llx [%c %s], core: %s\n", + reason, arb_addr, cap_status & ARB_ERR_CAP_STATUS_WRITE ? 'W' : 'R', cap_status & ARB_ERR_CAP_STATUS_TIMEOUT ? "timeout" : "", m_name); @@ -259,6 +308,41 @@ static irqreturn_t brcmstb_gisb_tea_handler(int irq, void *dev_id) return IRQ_HANDLED; } +static irqreturn_t brcmstb_gisb_bp_handler(int irq, void *dev_id) +{ + struct brcmstb_gisb_arb_device *gdev = dev_id; + const char *m_name; + u32 bp_status; + u64 arb_addr; + u32 master; + char m_fmt[11]; + + bp_status = gisb_read(gdev, ARB_BP_CAP_STATUS); + + /* Invalid captured address, bail out */ + if (!(bp_status & ARB_BP_CAP_STATUS_VALID)) + return IRQ_HANDLED; + + /* Read the address and master */ + arb_addr = gisb_read_bp_address(gdev); + master = gisb_read(gdev, ARB_BP_CAP_MASTER); + + m_name = brcmstb_gisb_master_to_str(gdev, master); + if (!m_name) { + snprintf(m_fmt, sizeof(m_fmt), "0x%08x", master); + m_name = m_fmt; + } + + pr_crit("GISB: breakpoint at 0x%llx [%c], core: %s\n", + arb_addr, bp_status & ARB_BP_CAP_STATUS_WRITE ? 'W' : 'R', + m_name); + + /* clear the GISB error */ + gisb_write(gdev, ARB_ERR_CAP_CLEAR, ARB_ERR_CAP_CLR); + + return IRQ_HANDLED; +} + /* * Dump out gisb errors on die or panic. */ @@ -317,13 +401,14 @@ static int __init brcmstb_gisb_arb_probe(struct platform_device *pdev) struct brcmstb_gisb_arb_device *gdev; const struct of_device_id *of_id; struct resource *r; - int err, timeout_irq, tea_irq; + int err, timeout_irq, tea_irq, bp_irq; unsigned int num_masters, j = 0; int i, first, last; r = platform_get_resource(pdev, IORESOURCE_MEM, 0); timeout_irq = platform_get_irq(pdev, 0); tea_irq = platform_get_irq(pdev, 1); + bp_irq = platform_get_irq(pdev, 2); gdev = devm_kzalloc(&pdev->dev, sizeof(*gdev), GFP_KERNEL); if (!gdev) @@ -356,6 +441,15 @@ static int __init brcmstb_gisb_arb_probe(struct platform_device *pdev) if (err < 0) return err; + /* Interrupt is optional */ + if (bp_irq > 0) { + err = devm_request_irq(&pdev->dev, bp_irq, + brcmstb_gisb_bp_handler, 0, pdev->name, + gdev); + if (err < 0) + return err; + } + /* If we do not have a valid mask, assume all masters are enabled */ if (of_property_read_u32(dn, "brcm,gisb-arb-master-mask", &gdev->valid_mask)) diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index 4026fac9fac3..c715d4681a0b 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -49,7 +49,7 @@ source "drivers/clk/versatile/Kconfig" config CLK_HSDK bool "PLL Driver for HSDK platform" - depends on OF || COMPILE_TEST + depends on ARC_SOC_HSDK || COMPILE_TEST depends on HAS_IOMEM help This driver supports the HSDK core, system, ddr, tunnel and hdmi PLLs @@ -373,6 +373,7 @@ source "drivers/clk/meson/Kconfig" source "drivers/clk/mvebu/Kconfig" source "drivers/clk/qcom/Kconfig" source "drivers/clk/renesas/Kconfig" +source "drivers/clk/rockchip/Kconfig" source "drivers/clk/samsung/Kconfig" source "drivers/clk/sifive/Kconfig" source "drivers/clk/sprd/Kconfig" diff --git a/drivers/clk/at91/at91sam9g45.c b/drivers/clk/at91/at91sam9g45.c index c88ee20bee31..cb4a406ed15d 100644 --- a/drivers/clk/at91/at91sam9g45.c +++ b/drivers/clk/at91/at91sam9g45.c @@ -46,13 +46,6 @@ static const struct { { .n = "pck1", .p = "prog1", .id = 9 }, }; -static const struct clk_pcr_layout at91sam9g45_pcr_layout = { - .offset = 0x10c, - .cmd = BIT(12), - .pid_mask = GENMASK(5, 0), - .div_mask = GENMASK(17, 16), -}; - struct pck { char *n; u8 id; diff --git a/drivers/clk/at91/clk-main.c b/drivers/clk/at91/clk-main.c index 5c83e899084f..cfae2f59df66 100644 --- a/drivers/clk/at91/clk-main.c +++ b/drivers/clk/at91/clk-main.c @@ -437,12 +437,17 @@ static int clk_sam9x5_main_set_parent(struct clk_hw *hw, u8 index) return -EINVAL; regmap_read(regmap, AT91_CKGR_MOR, &tmp); - tmp &= ~MOR_KEY_MASK; if (index && !(tmp & AT91_PMC_MOSCSEL)) - regmap_write(regmap, AT91_CKGR_MOR, tmp | AT91_PMC_MOSCSEL); + tmp = AT91_PMC_MOSCSEL; else if (!index && (tmp & AT91_PMC_MOSCSEL)) - regmap_write(regmap, AT91_CKGR_MOR, tmp & ~AT91_PMC_MOSCSEL); + tmp = 0; + else + return 0; + + regmap_update_bits(regmap, AT91_CKGR_MOR, + AT91_PMC_MOSCSEL | MOR_KEY_MASK, + tmp | AT91_PMC_KEY); while (!clk_sam9x5_main_ready(regmap)) cpu_relax(); diff --git a/drivers/clk/at91/clk-peripheral.c b/drivers/clk/at91/clk-peripheral.c index 7867eaf0447f..7a27ba8e0577 100644 --- a/drivers/clk/at91/clk-peripheral.c +++ b/drivers/clk/at91/clk-peripheral.c @@ -112,8 +112,8 @@ at91_clk_register_peripheral(struct regmap *regmap, const char *name, init.name = name; init.ops = &peripheral_ops; - init.parent_names = (parent_name ? &parent_name : NULL); - init.num_parents = (parent_name ? 1 : 0); + init.parent_names = &parent_name; + init.num_parents = 1; init.flags = 0; periph->id = id; diff --git a/drivers/clk/at91/clk-sam9x60-pll.c b/drivers/clk/at91/clk-sam9x60-pll.c index b473298ef7e6..78f458a7b2ef 100644 --- a/drivers/clk/at91/clk-sam9x60-pll.c +++ b/drivers/clk/at91/clk-sam9x60-pll.c @@ -331,7 +331,7 @@ static long sam9x60_div_pll_compute_div(struct sam9x60_pll_core *core, struct clk_hw *parent = clk_hw_get_parent(&core->hw); unsigned long tmp_rate, tmp_parent_rate, tmp_diff; long best_diff = -1, best_rate = -EINVAL; - u32 divid, best_div; + u32 divid; if (!rate) return 0; @@ -352,7 +352,6 @@ static long sam9x60_div_pll_compute_div(struct sam9x60_pll_core *core, *parent_rate = tmp_parent_rate; best_rate = tmp_rate; best_diff = tmp_diff; - best_div = divid; } if (!best_diff) diff --git a/drivers/clk/at91/sam9x60.c b/drivers/clk/at91/sam9x60.c index ab6318c0589e..3c4c95603595 100644 --- a/drivers/clk/at91/sam9x60.c +++ b/drivers/clk/at91/sam9x60.c @@ -279,7 +279,7 @@ static void __init sam9x60_pmc_setup(struct device_node *np) parent_names[3] = "masterck"; parent_names[4] = "pllack_divck"; parent_names[5] = "upllck_divck"; - for (i = 0; i < 8; i++) { + for (i = 0; i < 2; i++) { char name[6]; snprintf(name, sizeof(name), "prog%d", i); diff --git a/drivers/clk/baikal-t1/clk-ccu-pll.c b/drivers/clk/baikal-t1/clk-ccu-pll.c index 1eec8c0b8f50..2445d4b12baf 100644 --- a/drivers/clk/baikal-t1/clk-ccu-pll.c +++ b/drivers/clk/baikal-t1/clk-ccu-pll.c @@ -51,11 +51,13 @@ struct ccu_pll_info { }; /* - * Mark as critical all PLLs except Ethernet one. CPU and DDR PLLs are sources - * of CPU cores and DDR controller reference clocks, due to which they - * obviously shouldn't be ever gated. SATA and PCIe PLLs are the parents of - * APB-bus and DDR controller AXI-bus clocks. If they are gated the system will - * be unusable. + * Alas we have to mark all PLLs as critical. CPU and DDR PLLs are sources of + * CPU cores and DDR controller reference clocks, due to which they obviously + * shouldn't be ever gated. SATA and PCIe PLLs are the parents of APB-bus and + * DDR controller AXI-bus clocks. If they are gated the system will be + * unusable. Moreover disabling SATA and Ethernet PLLs causes automatic reset + * of the corresponding subsystems. So until we aren't ready to re-initialize + * all the devices consuming those PLLs, they will be marked as critical too. */ static const struct ccu_pll_info pll_info[] = { CCU_PLL_INFO(CCU_CPU_PLL, "cpu_pll", "ref_clk", CCU_CPU_PLL_BASE, @@ -67,7 +69,7 @@ static const struct ccu_pll_info pll_info[] = { CCU_PLL_INFO(CCU_PCIE_PLL, "pcie_pll", "ref_clk", CCU_PCIE_PLL_BASE, CLK_IS_CRITICAL), CCU_PLL_INFO(CCU_ETH_PLL, "eth_pll", "ref_clk", CCU_ETH_PLL_BASE, - CLK_SET_RATE_GATE) + CLK_IS_CRITICAL | CLK_SET_RATE_GATE) }; struct ccu_pll_data { diff --git a/drivers/clk/bcm/clk-bcm2835.c b/drivers/clk/bcm/clk-bcm2835.c index 3439bc65bb4e..1ac803e14fa3 100644 --- a/drivers/clk/bcm/clk-bcm2835.c +++ b/drivers/clk/bcm/clk-bcm2835.c @@ -1338,8 +1338,10 @@ static struct clk_hw *bcm2835_register_pll(struct bcm2835_cprman *cprman, pll->hw.init = &init; ret = devm_clk_hw_register(cprman->dev, &pll->hw); - if (ret) + if (ret) { + kfree(pll); return NULL; + } return &pll->hw; } diff --git a/drivers/clk/bcm/clk-raspberrypi.c b/drivers/clk/bcm/clk-raspberrypi.c index 5cc82954e1ce..f89b9cfc4309 100644 --- a/drivers/clk/bcm/clk-raspberrypi.c +++ b/drivers/clk/bcm/clk-raspberrypi.c @@ -271,6 +271,7 @@ static int raspberrypi_discover_clocks(struct raspberrypi_clk *rpi, case RPI_FIRMWARE_CORE_CLK_ID: case RPI_FIRMWARE_M2MC_CLK_ID: case RPI_FIRMWARE_V3D_CLK_ID: + case RPI_FIRMWARE_PIXEL_BVB_CLK_ID: hw = raspberrypi_clk_register(rpi, clks->parent, clks->id); if (IS_ERR(hw)) diff --git a/drivers/clk/clk-axi-clkgen.c b/drivers/clk/clk-axi-clkgen.c index 96f351785b41..14d803e6af62 100644 --- a/drivers/clk/clk-axi-clkgen.c +++ b/drivers/clk/clk-axi-clkgen.c @@ -27,19 +27,23 @@ #define AXI_CLKGEN_V2_DRP_STATUS_BUSY BIT(16) +#define MMCM_REG_CLKOUT5_2 0x07 #define MMCM_REG_CLKOUT0_1 0x08 #define MMCM_REG_CLKOUT0_2 0x09 +#define MMCM_REG_CLKOUT6_2 0x13 #define MMCM_REG_CLK_FB1 0x14 #define MMCM_REG_CLK_FB2 0x15 #define MMCM_REG_CLK_DIV 0x16 #define MMCM_REG_LOCK1 0x18 #define MMCM_REG_LOCK2 0x19 #define MMCM_REG_LOCK3 0x1a +#define MMCM_REG_POWER 0x28 #define MMCM_REG_FILTER1 0x4e #define MMCM_REG_FILTER2 0x4f #define MMCM_CLKOUT_NOCOUNT BIT(6) +#define MMCM_CLK_DIV_DIVIDE BIT(11) #define MMCM_CLK_DIV_NOCOUNT BIT(12) struct axi_clkgen { @@ -107,6 +111,8 @@ static void axi_clkgen_calc_params(unsigned long fin, unsigned long fout, unsigned long d, d_min, d_max, _d_min, _d_max; unsigned long m, m_min, m_max; unsigned long f, dout, best_f, fvco; + unsigned long fract_shift = 0; + unsigned long fvco_min_fract, fvco_max_fract; fin /= 1000; fout /= 1000; @@ -119,42 +125,89 @@ static void axi_clkgen_calc_params(unsigned long fin, unsigned long fout, d_min = max_t(unsigned long, DIV_ROUND_UP(fin, fpfd_max), 1); d_max = min_t(unsigned long, fin / fpfd_min, 80); - m_min = max_t(unsigned long, DIV_ROUND_UP(fvco_min, fin) * d_min, 1); - m_max = min_t(unsigned long, fvco_max * d_max / fin, 64); +again: + fvco_min_fract = fvco_min << fract_shift; + fvco_max_fract = fvco_max << fract_shift; + + m_min = max_t(unsigned long, DIV_ROUND_UP(fvco_min_fract, fin) * d_min, 1); + m_max = min_t(unsigned long, fvco_max_fract * d_max / fin, 64 << fract_shift); for (m = m_min; m <= m_max; m++) { - _d_min = max(d_min, DIV_ROUND_UP(fin * m, fvco_max)); - _d_max = min(d_max, fin * m / fvco_min); + _d_min = max(d_min, DIV_ROUND_UP(fin * m, fvco_max_fract)); + _d_max = min(d_max, fin * m / fvco_min_fract); for (d = _d_min; d <= _d_max; d++) { fvco = fin * m / d; dout = DIV_ROUND_CLOSEST(fvco, fout); - dout = clamp_t(unsigned long, dout, 1, 128); + dout = clamp_t(unsigned long, dout, 1, 128 << fract_shift); f = fvco / dout; if (abs(f - fout) < abs(best_f - fout)) { best_f = f; *best_d = d; - *best_m = m; - *best_dout = dout; + *best_m = m << (3 - fract_shift); + *best_dout = dout << (3 - fract_shift); if (best_f == fout) return; } } } + + /* Lets see if we find a better setting in fractional mode */ + if (fract_shift == 0) { + fract_shift = 3; + goto again; + } } -static void axi_clkgen_calc_clk_params(unsigned int divider, unsigned int *low, - unsigned int *high, unsigned int *edge, unsigned int *nocount) +struct axi_clkgen_div_params { + unsigned int low; + unsigned int high; + unsigned int edge; + unsigned int nocount; + unsigned int frac_en; + unsigned int frac; + unsigned int frac_wf_f; + unsigned int frac_wf_r; + unsigned int frac_phase; +}; + +static void axi_clkgen_calc_clk_params(unsigned int divider, + unsigned int frac_divider, struct axi_clkgen_div_params *params) { - if (divider == 1) - *nocount = 1; - else - *nocount = 0; - *high = divider / 2; - *edge = divider % 2; - *low = divider - *high; + memset(params, 0x0, sizeof(*params)); + + if (divider == 1) { + params->nocount = 1; + return; + } + + if (frac_divider == 0) { + params->high = divider / 2; + params->edge = divider % 2; + params->low = divider - params->high; + } else { + params->frac_en = 1; + params->frac = frac_divider; + + params->high = divider / 2; + params->edge = divider % 2; + params->low = params->high; + + if (params->edge == 0) { + params->high--; + params->frac_wf_r = 1; + } + + if (params->edge == 0 || frac_divider == 1) + params->low--; + if (((params->edge == 0) ^ (frac_divider == 1)) || + (divider == 2 && frac_divider == 1)) + params->frac_wf_f = 1; + + params->frac_phase = params->edge * 4 + frac_divider / 2; + } } static void axi_clkgen_write(struct axi_clkgen *axi_clkgen, @@ -246,15 +299,29 @@ static struct axi_clkgen *clk_hw_to_axi_clkgen(struct clk_hw *clk_hw) return container_of(clk_hw, struct axi_clkgen, clk_hw); } +static void axi_clkgen_set_div(struct axi_clkgen *axi_clkgen, + unsigned int reg1, unsigned int reg2, unsigned int reg3, + struct axi_clkgen_div_params *params) +{ + axi_clkgen_mmcm_write(axi_clkgen, reg1, + (params->high << 6) | params->low, 0xefff); + axi_clkgen_mmcm_write(axi_clkgen, reg2, + (params->frac << 12) | (params->frac_en << 11) | + (params->frac_wf_r << 10) | (params->edge << 7) | + (params->nocount << 6), 0x7fff); + if (reg3 != 0) { + axi_clkgen_mmcm_write(axi_clkgen, reg3, + (params->frac_phase << 11) | (params->frac_wf_f << 10), 0x3c00); + } +} + static int axi_clkgen_set_rate(struct clk_hw *clk_hw, unsigned long rate, unsigned long parent_rate) { struct axi_clkgen *axi_clkgen = clk_hw_to_axi_clkgen(clk_hw); unsigned int d, m, dout; - unsigned int nocount; - unsigned int high; - unsigned int edge; - unsigned int low; + struct axi_clkgen_div_params params; + uint32_t power = 0; uint32_t filter; uint32_t lock; @@ -266,24 +333,26 @@ static int axi_clkgen_set_rate(struct clk_hw *clk_hw, if (d == 0 || dout == 0 || m == 0) return -EINVAL; + if ((dout & 0x7) != 0 || (m & 0x7) != 0) + power |= 0x9800; + + axi_clkgen_mmcm_write(axi_clkgen, MMCM_REG_POWER, power, 0x9800); + filter = axi_clkgen_lookup_filter(m - 1); lock = axi_clkgen_lookup_lock(m - 1); - axi_clkgen_calc_clk_params(dout, &low, &high, &edge, &nocount); - axi_clkgen_mmcm_write(axi_clkgen, MMCM_REG_CLKOUT0_1, - (high << 6) | low, 0xefff); - axi_clkgen_mmcm_write(axi_clkgen, MMCM_REG_CLKOUT0_2, - (edge << 7) | (nocount << 6), 0x03ff); + axi_clkgen_calc_clk_params(dout >> 3, dout & 0x7, ¶ms); + axi_clkgen_set_div(axi_clkgen, MMCM_REG_CLKOUT0_1, MMCM_REG_CLKOUT0_2, + MMCM_REG_CLKOUT5_2, ¶ms); - axi_clkgen_calc_clk_params(d, &low, &high, &edge, &nocount); + axi_clkgen_calc_clk_params(d, 0, ¶ms); axi_clkgen_mmcm_write(axi_clkgen, MMCM_REG_CLK_DIV, - (edge << 13) | (nocount << 12) | (high << 6) | low, 0x3fff); + (params.edge << 13) | (params.nocount << 12) | + (params.high << 6) | params.low, 0x3fff); - axi_clkgen_calc_clk_params(m, &low, &high, &edge, &nocount); - axi_clkgen_mmcm_write(axi_clkgen, MMCM_REG_CLK_FB1, - (high << 6) | low, 0xefff); - axi_clkgen_mmcm_write(axi_clkgen, MMCM_REG_CLK_FB2, - (edge << 7) | (nocount << 6), 0x03ff); + axi_clkgen_calc_clk_params(m >> 3, m & 0x7, ¶ms); + axi_clkgen_set_div(axi_clkgen, MMCM_REG_CLK_FB1, MMCM_REG_CLK_FB2, + MMCM_REG_CLKOUT6_2, ¶ms); axi_clkgen_mmcm_write(axi_clkgen, MMCM_REG_LOCK1, lock & 0x3ff, 0x3ff); axi_clkgen_mmcm_write(axi_clkgen, MMCM_REG_LOCK2, @@ -313,35 +382,51 @@ static long axi_clkgen_round_rate(struct clk_hw *hw, unsigned long rate, return min_t(unsigned long long, tmp, LONG_MAX); } +static unsigned int axi_clkgen_get_div(struct axi_clkgen *axi_clkgen, + unsigned int reg1, unsigned int reg2) +{ + unsigned int val1, val2; + unsigned int div; + + axi_clkgen_mmcm_read(axi_clkgen, reg2, &val2); + if (val2 & MMCM_CLKOUT_NOCOUNT) + return 8; + + axi_clkgen_mmcm_read(axi_clkgen, reg1, &val1); + + div = (val1 & 0x3f) + ((val1 >> 6) & 0x3f); + div <<= 3; + + if (val2 & MMCM_CLK_DIV_DIVIDE) { + if ((val2 & BIT(7)) && (val2 & 0x7000) != 0x1000) + div += 8; + else + div += 16; + + div += (val2 >> 12) & 0x7; + } + + return div; +} + static unsigned long axi_clkgen_recalc_rate(struct clk_hw *clk_hw, unsigned long parent_rate) { struct axi_clkgen *axi_clkgen = clk_hw_to_axi_clkgen(clk_hw); unsigned int d, m, dout; - unsigned int reg; unsigned long long tmp; + unsigned int val; - axi_clkgen_mmcm_read(axi_clkgen, MMCM_REG_CLKOUT0_2, ®); - if (reg & MMCM_CLKOUT_NOCOUNT) { - dout = 1; - } else { - axi_clkgen_mmcm_read(axi_clkgen, MMCM_REG_CLKOUT0_1, ®); - dout = (reg & 0x3f) + ((reg >> 6) & 0x3f); - } + dout = axi_clkgen_get_div(axi_clkgen, MMCM_REG_CLKOUT0_1, + MMCM_REG_CLKOUT0_2); + m = axi_clkgen_get_div(axi_clkgen, MMCM_REG_CLK_FB1, + MMCM_REG_CLK_FB2); - axi_clkgen_mmcm_read(axi_clkgen, MMCM_REG_CLK_DIV, ®); - if (reg & MMCM_CLK_DIV_NOCOUNT) + axi_clkgen_mmcm_read(axi_clkgen, MMCM_REG_CLK_DIV, &val); + if (val & MMCM_CLK_DIV_NOCOUNT) d = 1; else - d = (reg & 0x3f) + ((reg >> 6) & 0x3f); - - axi_clkgen_mmcm_read(axi_clkgen, MMCM_REG_CLK_FB2, ®); - if (reg & MMCM_CLKOUT_NOCOUNT) { - m = 1; - } else { - axi_clkgen_mmcm_read(axi_clkgen, MMCM_REG_CLK_FB1, ®); - m = (reg & 0x3f) + ((reg >> 6) & 0x3f); - } + d = (val & 0x3f) + ((val >> 6) & 0x3f); if (d == 0 || dout == 0) return 0; diff --git a/drivers/clk/clk-composite.c b/drivers/clk/clk-composite.c index 7376f573bfdb..2ddb54f7d3ab 100644 --- a/drivers/clk/clk-composite.c +++ b/drivers/clk/clk-composite.c @@ -328,6 +328,7 @@ struct clk_hw *clk_hw_register_composite(struct device *dev, const char *name, rate_hw, rate_ops, gate_hw, gate_ops, flags); } +EXPORT_SYMBOL_GPL(clk_hw_register_composite); struct clk_hw *clk_hw_register_composite_pdata(struct device *dev, const char *name, diff --git a/drivers/clk/clk-fixed-factor.c b/drivers/clk/clk-fixed-factor.c index 8b343e59dc61..910e6e74ae90 100644 --- a/drivers/clk/clk-fixed-factor.c +++ b/drivers/clk/clk-fixed-factor.c @@ -206,6 +206,7 @@ static struct clk_hw *_of_fixed_factor_clk_setup(struct device_node *node) /** * of_fixed_factor_clk_setup() - Setup function for simple fixed factor clock + * @node: device node for the clock */ void __init of_fixed_factor_clk_setup(struct device_node *node) { diff --git a/drivers/clk/clk-fixed-rate.c b/drivers/clk/clk-fixed-rate.c index 77499a27c8fb..45501637705c 100644 --- a/drivers/clk/clk-fixed-rate.c +++ b/drivers/clk/clk-fixed-rate.c @@ -168,6 +168,7 @@ static struct clk_hw *_of_fixed_clk_setup(struct device_node *node) /** * of_fixed_clk_setup() - Setup function for simple fixed rate clock + * @node: device node for the clock */ void __init of_fixed_clk_setup(struct device_node *node) { diff --git a/drivers/clk/clk-qoriq.c b/drivers/clk/clk-qoriq.c index 5942e9874bc0..46101c6a20f2 100644 --- a/drivers/clk/clk-qoriq.c +++ b/drivers/clk/clk-qoriq.c @@ -31,7 +31,7 @@ #define CGA_PLL4 4 /* only on clockgen-1.0, which lacks CGB */ #define CGB_PLL1 4 #define CGB_PLL2 5 -#define MAX_PLL_DIV 16 +#define MAX_PLL_DIV 32 struct clockgen_pll_div { struct clk *clk; diff --git a/drivers/clk/clk-s2mps11.c b/drivers/clk/clk-s2mps11.c index 2ce370c804aa..aa21371f9104 100644 --- a/drivers/clk/clk-s2mps11.c +++ b/drivers/clk/clk-s2mps11.c @@ -267,18 +267,7 @@ static struct platform_driver s2mps11_clk_driver = { .remove = s2mps11_clk_remove, .id_table = s2mps11_clk_id, }; - -static int __init s2mps11_clk_init(void) -{ - return platform_driver_register(&s2mps11_clk_driver); -} -subsys_initcall(s2mps11_clk_init); - -static void __exit s2mps11_clk_cleanup(void) -{ - platform_driver_unregister(&s2mps11_clk_driver); -} -module_exit(s2mps11_clk_cleanup); +module_platform_driver(s2mps11_clk_driver); MODULE_DESCRIPTION("S2MPS11 Clock Driver"); MODULE_AUTHOR("Yadwinder Singh Brar <yadi.brar@samsung.com>"); diff --git a/drivers/clk/clk-si5341.c b/drivers/clk/clk-si5341.c index 3d7acab9d280..e0446e66fa64 100644 --- a/drivers/clk/clk-si5341.c +++ b/drivers/clk/clk-si5341.c @@ -883,11 +883,9 @@ static int si5341_output_set_parent(struct clk_hw *hw, u8 index) static u8 si5341_output_get_parent(struct clk_hw *hw) { struct clk_si5341_output *output = to_clk_si5341_output(hw); - int err; u32 val; - err = regmap_read(output->data->regmap, - SI5341_OUT_MUX_SEL(output), &val); + regmap_read(output->data->regmap, SI5341_OUT_MUX_SEL(output), &val); return val & 0x7; } diff --git a/drivers/clk/davinci/da8xx-cfgchip.c b/drivers/clk/davinci/da8xx-cfgchip.c index bdc52364b421..77d18276bfe8 100644 --- a/drivers/clk/davinci/da8xx-cfgchip.c +++ b/drivers/clk/davinci/da8xx-cfgchip.c @@ -571,6 +571,7 @@ static const struct clk_ops da8xx_usb1_clk48_ops = { /** * da8xx_cfgchip_register_usb1_clk48 - Register a new USB 1.1 PHY clock + * @dev: The device * @regmap: The CFGCHIP regmap */ static struct da8xx_usb1_clk48 * diff --git a/drivers/clk/imx/Kconfig b/drivers/clk/imx/Kconfig index db0253fa3d64..3b393cb07295 100644 --- a/drivers/clk/imx/Kconfig +++ b/drivers/clk/imx/Kconfig @@ -1,40 +1,102 @@ # SPDX-License-Identifier: GPL-2.0 # common clock support for NXP i.MX SoC family. config MXC_CLK - bool - def_bool ARCH_MXC + tristate "IMX clock" + depends on ARCH_MXC || COMPILE_TEST config MXC_CLK_SCU - bool - depends on IMX_SCU + tristate "IMX SCU clock" + depends on ARCH_MXC || COMPILE_TEST + depends on IMX_SCU && HAVE_ARM_SMCCC + +config CLK_IMX1 + def_bool SOC_IMX1 + select MXC_CLK + +config CLK_IMX25 + def_bool SOC_IMX25 + select MXC_CLK + +config CLK_IMX27 + def_bool SOC_IMX27 + select MXC_CLK + +config CLK_IMX31 + def_bool SOC_IMX31 + select MXC_CLK + +config CLK_IMX35 + def_bool SOC_IMX35 + select MXC_CLK + +config CLK_IMX5 + def_bool SOC_IMX5 + select MXC_CLK + +config CLK_IMX6Q + def_bool SOC_IMX6Q + select MXC_CLK + +config CLK_IMX6SL + def_bool SOC_IMX6SL + select MXC_CLK + +config CLK_IMX6SLL + def_bool SOC_IMX6SLL + select MXC_CLK + +config CLK_IMX6SX + def_bool SOC_IMX6SX + select MXC_CLK + +config CLK_IMX6UL + def_bool SOC_IMX6UL + select MXC_CLK + +config CLK_IMX7D + def_bool SOC_IMX7D + select MXC_CLK + +config CLK_IMX7ULP + def_bool SOC_IMX7ULP + select MXC_CLK + +config CLK_VF610 + def_bool SOC_VF610 + select MXC_CLK config CLK_IMX8MM - bool "IMX8MM CCM Clock Driver" - depends on ARCH_MXC + tristate "IMX8MM CCM Clock Driver" + depends on ARCH_MXC || COMPILE_TEST + select MXC_CLK help Build the driver for i.MX8MM CCM Clock Driver config CLK_IMX8MN - bool "IMX8MN CCM Clock Driver" - depends on ARCH_MXC + tristate "IMX8MN CCM Clock Driver" + depends on ARCH_MXC || COMPILE_TEST + select MXC_CLK help Build the driver for i.MX8MN CCM Clock Driver config CLK_IMX8MP - bool "IMX8MP CCM Clock Driver" - depends on ARCH_MXC + tristate "IMX8MP CCM Clock Driver" + depends on ARCH_MXC || COMPILE_TEST + select MXC_CLK help Build the driver for i.MX8MP CCM Clock Driver config CLK_IMX8MQ - bool "IMX8MQ CCM Clock Driver" - depends on ARCH_MXC + tristate "IMX8MQ CCM Clock Driver" + depends on ARCH_MXC || COMPILE_TEST + select MXC_CLK help Build the driver for i.MX8MQ CCM Clock Driver config CLK_IMX8QXP - bool "IMX8QXP SCU Clock" - depends on ARCH_MXC && IMX_SCU && ARM64 + tristate "IMX8QXP SCU Clock" + depends on (ARCH_MXC && ARM64) || COMPILE_TEST + depends on IMX_SCU && HAVE_ARM_SMCCC select MXC_CLK_SCU help Build the driver for IMX8QXP SCU based clocks. diff --git a/drivers/clk/imx/Makefile b/drivers/clk/imx/Makefile index 928f874c73d2..dd6a737d060b 100644 --- a/drivers/clk/imx/Makefile +++ b/drivers/clk/imx/Makefile @@ -1,48 +1,46 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_MXC_CLK) += \ - clk.o \ - clk-busy.o \ - clk-composite-8m.o \ - clk-cpu.o \ - clk-composite-7ulp.o \ - clk-divider-gate.o \ - clk-fixup-div.o \ - clk-fixup-mux.o \ - clk-frac-pll.o \ - clk-gate-exclusive.o \ - clk-gate2.o \ - clk-pfd.o \ - clk-pfdv2.o \ - clk-pllv1.o \ - clk-pllv2.o \ - clk-pllv3.o \ - clk-pllv4.o \ - clk-sscg-pll.o \ - clk-pll14xx.o - -obj-$(CONFIG_MXC_CLK_SCU) += \ - clk-scu.o \ - clk-lpcg-scu.o +mxc-clk-objs += clk.o +mxc-clk-objs += clk-busy.o +mxc-clk-objs += clk-composite-7ulp.o +mxc-clk-objs += clk-composite-8m.o +mxc-clk-objs += clk-cpu.o +mxc-clk-objs += clk-divider-gate.o +mxc-clk-objs += clk-fixup-div.o +mxc-clk-objs += clk-fixup-mux.o +mxc-clk-objs += clk-frac-pll.o +mxc-clk-objs += clk-gate2.o +mxc-clk-objs += clk-gate-exclusive.o +mxc-clk-objs += clk-pfd.o +mxc-clk-objs += clk-pfdv2.o +mxc-clk-objs += clk-pllv1.o +mxc-clk-objs += clk-pllv2.o +mxc-clk-objs += clk-pllv3.o +mxc-clk-objs += clk-pllv4.o +mxc-clk-objs += clk-pll14xx.o +mxc-clk-objs += clk-sscg-pll.o +obj-$(CONFIG_MXC_CLK) += mxc-clk.o obj-$(CONFIG_CLK_IMX8MM) += clk-imx8mm.o obj-$(CONFIG_CLK_IMX8MN) += clk-imx8mn.o obj-$(CONFIG_CLK_IMX8MP) += clk-imx8mp.o obj-$(CONFIG_CLK_IMX8MQ) += clk-imx8mq.o -obj-$(CONFIG_CLK_IMX8QXP) += clk-imx8qxp.o clk-imx8qxp-lpcg.o -obj-$(CONFIG_SOC_IMX1) += clk-imx1.o -obj-$(CONFIG_SOC_IMX21) += clk-imx21.o -obj-$(CONFIG_SOC_IMX25) += clk-imx25.o -obj-$(CONFIG_SOC_IMX27) += clk-imx27.o -obj-$(CONFIG_SOC_IMX31) += clk-imx31.o -obj-$(CONFIG_SOC_IMX35) += clk-imx35.o -obj-$(CONFIG_SOC_IMX5) += clk-imx5.o -obj-$(CONFIG_SOC_IMX6Q) += clk-imx6q.o -obj-$(CONFIG_SOC_IMX6SL) += clk-imx6sl.o -obj-$(CONFIG_SOC_IMX6SLL) += clk-imx6sll.o -obj-$(CONFIG_SOC_IMX6SX) += clk-imx6sx.o -obj-$(CONFIG_SOC_IMX6UL) += clk-imx6ul.o -obj-$(CONFIG_SOC_IMX7D) += clk-imx7d.o -obj-$(CONFIG_SOC_IMX7ULP) += clk-imx7ulp.o -obj-$(CONFIG_SOC_VF610) += clk-vf610.o +obj-$(CONFIG_MXC_CLK_SCU) += clk-imx-scu.o clk-imx-lpcg-scu.o +clk-imx-scu-$(CONFIG_CLK_IMX8QXP) += clk-scu.o clk-imx8qxp.o +clk-imx-lpcg-scu-$(CONFIG_CLK_IMX8QXP) += clk-lpcg-scu.o clk-imx8qxp-lpcg.o + +obj-$(CONFIG_CLK_IMX1) += clk-imx1.o +obj-$(CONFIG_CLK_IMX25) += clk-imx25.o +obj-$(CONFIG_CLK_IMX27) += clk-imx27.o +obj-$(CONFIG_CLK_IMX31) += clk-imx31.o +obj-$(CONFIG_CLK_IMX35) += clk-imx35.o +obj-$(CONFIG_CLK_IMX5) += clk-imx5.o +obj-$(CONFIG_CLK_IMX6Q) += clk-imx6q.o +obj-$(CONFIG_CLK_IMX6SL) += clk-imx6sl.o +obj-$(CONFIG_CLK_IMX6SLL) += clk-imx6sll.o +obj-$(CONFIG_CLK_IMX6SX) += clk-imx6sx.o +obj-$(CONFIG_CLK_IMX6UL) += clk-imx6ul.o +obj-$(CONFIG_CLK_IMX7D) += clk-imx7d.o +obj-$(CONFIG_CLK_IMX7ULP) += clk-imx7ulp.o +obj-$(CONFIG_CLK_VF610) += clk-vf610.o diff --git a/drivers/clk/imx/clk-busy.c b/drivers/clk/imx/clk-busy.c index 25c863da32c7..6f17311647f3 100644 --- a/drivers/clk/imx/clk-busy.c +++ b/drivers/clk/imx/clk-busy.c @@ -4,6 +4,7 @@ * Copyright 2012 Linaro Ltd. */ +#include <linux/bits.h> #include <linux/clk.h> #include <linux/clk-provider.h> #include <linux/io.h> diff --git a/drivers/clk/imx/clk-composite-7ulp.c b/drivers/clk/imx/clk-composite-7ulp.c index b9efcc8a855d..7c4f31b31eb0 100644 --- a/drivers/clk/imx/clk-composite-7ulp.c +++ b/drivers/clk/imx/clk-composite-7ulp.c @@ -5,6 +5,7 @@ * */ +#include <linux/bits.h> #include <linux/clk-provider.h> #include <linux/err.h> #include <linux/slab.h> diff --git a/drivers/clk/imx/clk-composite-8m.c b/drivers/clk/imx/clk-composite-8m.c index d2b5af826f2c..2c309e3dc8e3 100644 --- a/drivers/clk/imx/clk-composite-8m.c +++ b/drivers/clk/imx/clk-composite-8m.c @@ -5,6 +5,7 @@ #include <linux/clk-provider.h> #include <linux/errno.h> +#include <linux/export.h> #include <linux/io.h> #include <linux/slab.h> @@ -215,6 +216,7 @@ struct clk_hw *imx8m_clk_hw_composite_flags(const char *name, div->width = PCG_PREDIV_WIDTH; divider_ops = &imx8m_clk_composite_divider_ops; mux_ops = &clk_mux_ops; + flags |= CLK_SET_PARENT_GATE; } div->lock = &imx_ccm_lock; @@ -243,3 +245,4 @@ fail: kfree(mux); return ERR_CAST(hw); } +EXPORT_SYMBOL_GPL(imx8m_clk_hw_composite_flags); diff --git a/drivers/clk/imx/clk-cpu.c b/drivers/clk/imx/clk-cpu.c index cb182bec79ba..cb6ca4cf0535 100644 --- a/drivers/clk/imx/clk-cpu.c +++ b/drivers/clk/imx/clk-cpu.c @@ -5,6 +5,7 @@ #include <linux/clk.h> #include <linux/clk-provider.h> +#include <linux/export.h> #include <linux/slab.h> #include "clk.h" @@ -104,3 +105,4 @@ struct clk_hw *imx_clk_hw_cpu(const char *name, const char *parent_name, return hw; } +EXPORT_SYMBOL_GPL(imx_clk_hw_cpu); diff --git a/drivers/clk/imx/clk-fixup-mux.c b/drivers/clk/imx/clk-fixup-mux.c index 58a67630bb6a..c82401570c84 100644 --- a/drivers/clk/imx/clk-fixup-mux.c +++ b/drivers/clk/imx/clk-fixup-mux.c @@ -3,6 +3,7 @@ * Copyright (C) 2013 Freescale Semiconductor, Inc. */ +#include <linux/bits.h> #include <linux/clk-provider.h> #include <linux/err.h> #include <linux/io.h> diff --git a/drivers/clk/imx/clk-frac-pll.c b/drivers/clk/imx/clk-frac-pll.c index 101e0a300376..c703056fae85 100644 --- a/drivers/clk/imx/clk-frac-pll.c +++ b/drivers/clk/imx/clk-frac-pll.c @@ -10,6 +10,7 @@ #include <linux/clk-provider.h> #include <linux/err.h> +#include <linux/export.h> #include <linux/io.h> #include <linux/iopoll.h> #include <linux/slab.h> @@ -233,3 +234,4 @@ struct clk_hw *imx_clk_hw_frac_pll(const char *name, return hw; } +EXPORT_SYMBOL_GPL(imx_clk_hw_frac_pll); diff --git a/drivers/clk/imx/clk-gate2.c b/drivers/clk/imx/clk-gate2.c index b87ab3c3ba1e..7eed7083f46e 100644 --- a/drivers/clk/imx/clk-gate2.c +++ b/drivers/clk/imx/clk-gate2.c @@ -7,6 +7,7 @@ */ #include <linux/clk-provider.h> +#include <linux/export.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/io.h> @@ -15,7 +16,7 @@ #include "clk.h" /** - * DOC: basic gatable clock which can gate and ungate it's ouput + * DOC: basic gateable clock which can gate and ungate its output * * Traits of this clock: * prepare - clk_(un)prepare only ensures parent is (un)prepared @@ -177,3 +178,4 @@ struct clk_hw *clk_hw_register_gate2(struct device *dev, const char *name, return hw; } +EXPORT_SYMBOL_GPL(clk_hw_register_gate2); diff --git a/drivers/clk/imx/clk-imx21.c b/drivers/clk/imx/clk-imx21.c deleted file mode 100644 index 077b4a7123ce..000000000000 --- a/drivers/clk/imx/clk-imx21.c +++ /dev/null @@ -1,171 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright 2004-2007 Freescale Semiconductor, Inc. All Rights Reserved. - * Copyright 2008 Juergen Beisert, kernel@pengutronix.de - * Copyright 2008 Martin Fuzzey, mfuzzey@gmail.com - */ - -#include <linux/clk-provider.h> -#include <linux/clkdev.h> -#include <linux/io.h> -#include <linux/of.h> -#include <linux/of_address.h> -#include <dt-bindings/clock/imx21-clock.h> -#include <soc/imx/timer.h> -#include <asm/irq.h> - -#include "clk.h" - -#define MX21_CCM_BASE_ADDR 0x10027000 -#define MX21_GPT1_BASE_ADDR 0x10003000 -#define MX21_INT_GPT1 (NR_IRQS_LEGACY + 26) - -static void __iomem *ccm __initdata; - -/* Register offsets */ -#define CCM_CSCR (ccm + 0x00) -#define CCM_MPCTL0 (ccm + 0x04) -#define CCM_SPCTL0 (ccm + 0x0c) -#define CCM_PCDR0 (ccm + 0x18) -#define CCM_PCDR1 (ccm + 0x1c) -#define CCM_PCCR0 (ccm + 0x20) -#define CCM_PCCR1 (ccm + 0x24) - -static const char *mpll_osc_sel_clks[] = { "ckih_gate", "ckih_div1p5", }; -static const char *mpll_sel_clks[] = { "fpm_gate", "mpll_osc_sel", }; -static const char *spll_sel_clks[] = { "fpm_gate", "mpll_osc_sel", }; -static const char *ssi_sel_clks[] = { "spll_gate", "mpll_gate", }; - -static struct clk *clk[IMX21_CLK_MAX]; -static struct clk_onecell_data clk_data; - -static void __init _mx21_clocks_init(unsigned long lref, unsigned long href) -{ - BUG_ON(!ccm); - - clk[IMX21_CLK_DUMMY] = imx_clk_fixed("dummy", 0); - clk[IMX21_CLK_CKIL] = imx_obtain_fixed_clock("ckil", lref); - clk[IMX21_CLK_CKIH] = imx_obtain_fixed_clock("ckih", href); - clk[IMX21_CLK_FPM] = imx_clk_fixed_factor("fpm", "ckil", 512, 1); - clk[IMX21_CLK_CKIH_DIV1P5] = imx_clk_fixed_factor("ckih_div1p5", "ckih_gate", 2, 3); - - clk[IMX21_CLK_MPLL_GATE] = imx_clk_gate("mpll_gate", "mpll", CCM_CSCR, 0); - clk[IMX21_CLK_SPLL_GATE] = imx_clk_gate("spll_gate", "spll", CCM_CSCR, 1); - clk[IMX21_CLK_FPM_GATE] = imx_clk_gate("fpm_gate", "fpm", CCM_CSCR, 2); - clk[IMX21_CLK_CKIH_GATE] = imx_clk_gate_dis("ckih_gate", "ckih", CCM_CSCR, 3); - clk[IMX21_CLK_MPLL_OSC_SEL] = imx_clk_mux("mpll_osc_sel", CCM_CSCR, 4, 1, mpll_osc_sel_clks, ARRAY_SIZE(mpll_osc_sel_clks)); - clk[IMX21_CLK_IPG] = imx_clk_divider("ipg", "hclk", CCM_CSCR, 9, 1); - clk[IMX21_CLK_HCLK] = imx_clk_divider("hclk", "fclk", CCM_CSCR, 10, 4); - clk[IMX21_CLK_MPLL_SEL] = imx_clk_mux("mpll_sel", CCM_CSCR, 16, 1, mpll_sel_clks, ARRAY_SIZE(mpll_sel_clks)); - clk[IMX21_CLK_SPLL_SEL] = imx_clk_mux("spll_sel", CCM_CSCR, 17, 1, spll_sel_clks, ARRAY_SIZE(spll_sel_clks)); - clk[IMX21_CLK_SSI1_SEL] = imx_clk_mux("ssi1_sel", CCM_CSCR, 19, 1, ssi_sel_clks, ARRAY_SIZE(ssi_sel_clks)); - clk[IMX21_CLK_SSI2_SEL] = imx_clk_mux("ssi2_sel", CCM_CSCR, 20, 1, ssi_sel_clks, ARRAY_SIZE(ssi_sel_clks)); - clk[IMX21_CLK_USB_DIV] = imx_clk_divider("usb_div", "spll_gate", CCM_CSCR, 26, 3); - clk[IMX21_CLK_FCLK] = imx_clk_divider("fclk", "mpll_gate", CCM_CSCR, 29, 3); - - clk[IMX21_CLK_MPLL] = imx_clk_pllv1(IMX_PLLV1_IMX21, "mpll", "mpll_sel", CCM_MPCTL0); - - clk[IMX21_CLK_SPLL] = imx_clk_pllv1(IMX_PLLV1_IMX21, "spll", "spll_sel", CCM_SPCTL0); - - clk[IMX21_CLK_NFC_DIV] = imx_clk_divider("nfc_div", "fclk", CCM_PCDR0, 12, 4); - clk[IMX21_CLK_SSI1_DIV] = imx_clk_divider("ssi1_div", "ssi1_sel", CCM_PCDR0, 16, 6); - clk[IMX21_CLK_SSI2_DIV] = imx_clk_divider("ssi2_div", "ssi2_sel", CCM_PCDR0, 26, 6); - - clk[IMX21_CLK_PER1] = imx_clk_divider("per1", "mpll_gate", CCM_PCDR1, 0, 6); - clk[IMX21_CLK_PER2] = imx_clk_divider("per2", "mpll_gate", CCM_PCDR1, 8, 6); - clk[IMX21_CLK_PER3] = imx_clk_divider("per3", "mpll_gate", CCM_PCDR1, 16, 6); - clk[IMX21_CLK_PER4] = imx_clk_divider("per4", "mpll_gate", CCM_PCDR1, 24, 6); - - clk[IMX21_CLK_UART1_IPG_GATE] = imx_clk_gate("uart1_ipg_gate", "ipg", CCM_PCCR0, 0); - clk[IMX21_CLK_UART2_IPG_GATE] = imx_clk_gate("uart2_ipg_gate", "ipg", CCM_PCCR0, 1); - clk[IMX21_CLK_UART3_IPG_GATE] = imx_clk_gate("uart3_ipg_gate", "ipg", CCM_PCCR0, 2); - clk[IMX21_CLK_UART4_IPG_GATE] = imx_clk_gate("uart4_ipg_gate", "ipg", CCM_PCCR0, 3); - clk[IMX21_CLK_CSPI1_IPG_GATE] = imx_clk_gate("cspi1_ipg_gate", "ipg", CCM_PCCR0, 4); - clk[IMX21_CLK_CSPI2_IPG_GATE] = imx_clk_gate("cspi2_ipg_gate", "ipg", CCM_PCCR0, 5); - clk[IMX21_CLK_SSI1_GATE] = imx_clk_gate("ssi1_gate", "ipg", CCM_PCCR0, 6); - clk[IMX21_CLK_SSI2_GATE] = imx_clk_gate("ssi2_gate", "ipg", CCM_PCCR0, 7); - clk[IMX21_CLK_SDHC1_IPG_GATE] = imx_clk_gate("sdhc1_ipg_gate", "ipg", CCM_PCCR0, 9); - clk[IMX21_CLK_SDHC2_IPG_GATE] = imx_clk_gate("sdhc2_ipg_gate", "ipg", CCM_PCCR0, 10); - clk[IMX21_CLK_GPIO_GATE] = imx_clk_gate("gpio_gate", "ipg", CCM_PCCR0, 11); - clk[IMX21_CLK_I2C_GATE] = imx_clk_gate("i2c_gate", "ipg", CCM_PCCR0, 12); - clk[IMX21_CLK_DMA_GATE] = imx_clk_gate("dma_gate", "ipg", CCM_PCCR0, 13); - clk[IMX21_CLK_USB_GATE] = imx_clk_gate("usb_gate", "usb_div", CCM_PCCR0, 14); - clk[IMX21_CLK_EMMA_GATE] = imx_clk_gate("emma_gate", "ipg", CCM_PCCR0, 15); - clk[IMX21_CLK_SSI2_BAUD_GATE] = imx_clk_gate("ssi2_baud_gate", "ipg", CCM_PCCR0, 16); - clk[IMX21_CLK_SSI1_BAUD_GATE] = imx_clk_gate("ssi1_baud_gate", "ipg", CCM_PCCR0, 17); - clk[IMX21_CLK_LCDC_IPG_GATE] = imx_clk_gate("lcdc_ipg_gate", "ipg", CCM_PCCR0, 18); - clk[IMX21_CLK_NFC_GATE] = imx_clk_gate("nfc_gate", "nfc_div", CCM_PCCR0, 19); - clk[IMX21_CLK_SLCDC_HCLK_GATE] = imx_clk_gate("slcdc_hclk_gate", "hclk", CCM_PCCR0, 21); - clk[IMX21_CLK_PER4_GATE] = imx_clk_gate("per4_gate", "per4", CCM_PCCR0, 22); - clk[IMX21_CLK_BMI_GATE] = imx_clk_gate("bmi_gate", "hclk", CCM_PCCR0, 23); - clk[IMX21_CLK_USB_HCLK_GATE] = imx_clk_gate("usb_hclk_gate", "hclk", CCM_PCCR0, 24); - clk[IMX21_CLK_SLCDC_GATE] = imx_clk_gate("slcdc_gate", "hclk", CCM_PCCR0, 25); - clk[IMX21_CLK_LCDC_HCLK_GATE] = imx_clk_gate("lcdc_hclk_gate", "hclk", CCM_PCCR0, 26); - clk[IMX21_CLK_EMMA_HCLK_GATE] = imx_clk_gate("emma_hclk_gate", "hclk", CCM_PCCR0, 27); - clk[IMX21_CLK_BROM_GATE] = imx_clk_gate("brom_gate", "hclk", CCM_PCCR0, 28); - clk[IMX21_CLK_DMA_HCLK_GATE] = imx_clk_gate("dma_hclk_gate", "hclk", CCM_PCCR0, 30); - clk[IMX21_CLK_CSI_HCLK_GATE] = imx_clk_gate("csi_hclk_gate", "hclk", CCM_PCCR0, 31); - - clk[IMX21_CLK_CSPI3_IPG_GATE] = imx_clk_gate("cspi3_ipg_gate", "ipg", CCM_PCCR1, 23); - clk[IMX21_CLK_WDOG_GATE] = imx_clk_gate("wdog_gate", "ipg", CCM_PCCR1, 24); - clk[IMX21_CLK_GPT1_IPG_GATE] = imx_clk_gate("gpt1_ipg_gate", "ipg", CCM_PCCR1, 25); - clk[IMX21_CLK_GPT2_IPG_GATE] = imx_clk_gate("gpt2_ipg_gate", "ipg", CCM_PCCR1, 26); - clk[IMX21_CLK_GPT3_IPG_GATE] = imx_clk_gate("gpt3_ipg_gate", "ipg", CCM_PCCR1, 27); - clk[IMX21_CLK_PWM_IPG_GATE] = imx_clk_gate("pwm_ipg_gate", "ipg", CCM_PCCR1, 28); - clk[IMX21_CLK_RTC_GATE] = imx_clk_gate("rtc_gate", "ipg", CCM_PCCR1, 29); - clk[IMX21_CLK_KPP_GATE] = imx_clk_gate("kpp_gate", "ipg", CCM_PCCR1, 30); - clk[IMX21_CLK_OWIRE_GATE] = imx_clk_gate("owire_gate", "ipg", CCM_PCCR1, 31); - - imx_check_clocks(clk, ARRAY_SIZE(clk)); -} - -int __init mx21_clocks_init(unsigned long lref, unsigned long href) -{ - ccm = ioremap(MX21_CCM_BASE_ADDR, SZ_2K); - - _mx21_clocks_init(lref, href); - - clk_register_clkdev(clk[IMX21_CLK_PER1], "per", "imx21-uart.0"); - clk_register_clkdev(clk[IMX21_CLK_UART1_IPG_GATE], "ipg", "imx21-uart.0"); - clk_register_clkdev(clk[IMX21_CLK_PER1], "per", "imx21-uart.1"); - clk_register_clkdev(clk[IMX21_CLK_UART2_IPG_GATE], "ipg", "imx21-uart.1"); - clk_register_clkdev(clk[IMX21_CLK_PER1], "per", "imx21-uart.2"); - clk_register_clkdev(clk[IMX21_CLK_UART3_IPG_GATE], "ipg", "imx21-uart.2"); - clk_register_clkdev(clk[IMX21_CLK_PER1], "per", "imx21-uart.3"); - clk_register_clkdev(clk[IMX21_CLK_UART4_IPG_GATE], "ipg", "imx21-uart.3"); - clk_register_clkdev(clk[IMX21_CLK_GPT1_IPG_GATE], "ipg", "imx-gpt.0"); - clk_register_clkdev(clk[IMX21_CLK_PER1], "per", "imx-gpt.0"); - clk_register_clkdev(clk[IMX21_CLK_PER2], "per", "imx21-cspi.0"); - clk_register_clkdev(clk[IMX21_CLK_CSPI1_IPG_GATE], "ipg", "imx21-cspi.0"); - clk_register_clkdev(clk[IMX21_CLK_PER2], "per", "imx21-cspi.1"); - clk_register_clkdev(clk[IMX21_CLK_CSPI2_IPG_GATE], "ipg", "imx21-cspi.1"); - clk_register_clkdev(clk[IMX21_CLK_PER2], "per", "imx21-cspi.2"); - clk_register_clkdev(clk[IMX21_CLK_CSPI3_IPG_GATE], "ipg", "imx21-cspi.2"); - clk_register_clkdev(clk[IMX21_CLK_PER3], "per", "imx21-fb.0"); - clk_register_clkdev(clk[IMX21_CLK_LCDC_IPG_GATE], "ipg", "imx21-fb.0"); - clk_register_clkdev(clk[IMX21_CLK_LCDC_HCLK_GATE], "ahb", "imx21-fb.0"); - clk_register_clkdev(clk[IMX21_CLK_USB_GATE], "per", "imx21-hcd.0"); - clk_register_clkdev(clk[IMX21_CLK_USB_HCLK_GATE], "ahb", "imx21-hcd.0"); - clk_register_clkdev(clk[IMX21_CLK_NFC_GATE], NULL, "imx21-nand.0"); - clk_register_clkdev(clk[IMX21_CLK_DMA_HCLK_GATE], "ahb", "imx21-dma"); - clk_register_clkdev(clk[IMX21_CLK_DMA_GATE], "ipg", "imx21-dma"); - clk_register_clkdev(clk[IMX21_CLK_WDOG_GATE], NULL, "imx2-wdt.0"); - clk_register_clkdev(clk[IMX21_CLK_I2C_GATE], NULL, "imx21-i2c.0"); - clk_register_clkdev(clk[IMX21_CLK_OWIRE_GATE], NULL, "mxc_w1.0"); - - mxc_timer_init(MX21_GPT1_BASE_ADDR, MX21_INT_GPT1, GPT_TYPE_IMX21); - - return 0; -} - -static void __init mx21_clocks_init_dt(struct device_node *np) -{ - ccm = of_iomap(np, 0); - - _mx21_clocks_init(32768, 26000000); - - clk_data.clks = clk; - clk_data.clk_num = ARRAY_SIZE(clk); - of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); -} -CLK_OF_DECLARE(imx27_ccm, "fsl,imx21-ccm", mx21_clocks_init_dt); diff --git a/drivers/clk/imx/clk-imx27.c b/drivers/clk/imx/clk-imx27.c index a3753067fc12..5585ded8b8c6 100644 --- a/drivers/clk/imx/clk-imx27.c +++ b/drivers/clk/imx/clk-imx27.c @@ -181,79 +181,6 @@ static void __init _mx27_clocks_init(unsigned long fref) imx_print_silicon_rev("i.MX27", mx27_revision()); } -int __init mx27_clocks_init(unsigned long fref) -{ - ccm = ioremap(MX27_CCM_BASE_ADDR, SZ_4K); - - _mx27_clocks_init(fref); - - clk_register_clkdev(clk[IMX27_CLK_UART1_IPG_GATE], "ipg", "imx21-uart.0"); - clk_register_clkdev(clk[IMX27_CLK_PER1_GATE], "per", "imx21-uart.0"); - clk_register_clkdev(clk[IMX27_CLK_UART2_IPG_GATE], "ipg", "imx21-uart.1"); - clk_register_clkdev(clk[IMX27_CLK_PER1_GATE], "per", "imx21-uart.1"); - clk_register_clkdev(clk[IMX27_CLK_UART3_IPG_GATE], "ipg", "imx21-uart.2"); - clk_register_clkdev(clk[IMX27_CLK_PER1_GATE], "per", "imx21-uart.2"); - clk_register_clkdev(clk[IMX27_CLK_UART4_IPG_GATE], "ipg", "imx21-uart.3"); - clk_register_clkdev(clk[IMX27_CLK_PER1_GATE], "per", "imx21-uart.3"); - clk_register_clkdev(clk[IMX27_CLK_UART5_IPG_GATE], "ipg", "imx21-uart.4"); - clk_register_clkdev(clk[IMX27_CLK_PER1_GATE], "per", "imx21-uart.4"); - clk_register_clkdev(clk[IMX27_CLK_UART6_IPG_GATE], "ipg", "imx21-uart.5"); - clk_register_clkdev(clk[IMX27_CLK_PER1_GATE], "per", "imx21-uart.5"); - clk_register_clkdev(clk[IMX27_CLK_GPT1_IPG_GATE], "ipg", "imx-gpt.0"); - clk_register_clkdev(clk[IMX27_CLK_PER1_GATE], "per", "imx-gpt.0"); - clk_register_clkdev(clk[IMX27_CLK_PER2_GATE], "per", "imx21-mmc.0"); - clk_register_clkdev(clk[IMX27_CLK_SDHC1_IPG_GATE], "ipg", "imx21-mmc.0"); - clk_register_clkdev(clk[IMX27_CLK_PER2_GATE], "per", "imx21-mmc.1"); - clk_register_clkdev(clk[IMX27_CLK_SDHC2_IPG_GATE], "ipg", "imx21-mmc.1"); - clk_register_clkdev(clk[IMX27_CLK_PER2_GATE], "per", "imx21-mmc.2"); - clk_register_clkdev(clk[IMX27_CLK_SDHC2_IPG_GATE], "ipg", "imx21-mmc.2"); - clk_register_clkdev(clk[IMX27_CLK_PER2_GATE], "per", "imx27-cspi.0"); - clk_register_clkdev(clk[IMX27_CLK_CSPI1_IPG_GATE], "ipg", "imx27-cspi.0"); - clk_register_clkdev(clk[IMX27_CLK_PER2_GATE], "per", "imx27-cspi.1"); - clk_register_clkdev(clk[IMX27_CLK_CSPI2_IPG_GATE], "ipg", "imx27-cspi.1"); - clk_register_clkdev(clk[IMX27_CLK_PER2_GATE], "per", "imx27-cspi.2"); - clk_register_clkdev(clk[IMX27_CLK_CSPI3_IPG_GATE], "ipg", "imx27-cspi.2"); - clk_register_clkdev(clk[IMX27_CLK_PER3_GATE], "per", "imx21-fb.0"); - clk_register_clkdev(clk[IMX27_CLK_LCDC_IPG_GATE], "ipg", "imx21-fb.0"); - clk_register_clkdev(clk[IMX27_CLK_LCDC_AHB_GATE], "ahb", "imx21-fb.0"); - clk_register_clkdev(clk[IMX27_CLK_CSI_AHB_GATE], "ahb", "imx27-camera.0"); - clk_register_clkdev(clk[IMX27_CLK_PER4_GATE], "per", "imx27-camera.0"); - clk_register_clkdev(clk[IMX27_CLK_USB_DIV], "per", "imx-udc-mx27"); - clk_register_clkdev(clk[IMX27_CLK_USB_IPG_GATE], "ipg", "imx-udc-mx27"); - clk_register_clkdev(clk[IMX27_CLK_USB_AHB_GATE], "ahb", "imx-udc-mx27"); - clk_register_clkdev(clk[IMX27_CLK_USB_DIV], "per", "mxc-ehci.0"); - clk_register_clkdev(clk[IMX27_CLK_USB_IPG_GATE], "ipg", "mxc-ehci.0"); - clk_register_clkdev(clk[IMX27_CLK_USB_AHB_GATE], "ahb", "mxc-ehci.0"); - clk_register_clkdev(clk[IMX27_CLK_USB_DIV], "per", "mxc-ehci.1"); - clk_register_clkdev(clk[IMX27_CLK_USB_IPG_GATE], "ipg", "mxc-ehci.1"); - clk_register_clkdev(clk[IMX27_CLK_USB_AHB_GATE], "ahb", "mxc-ehci.1"); - clk_register_clkdev(clk[IMX27_CLK_USB_DIV], "per", "mxc-ehci.2"); - clk_register_clkdev(clk[IMX27_CLK_USB_IPG_GATE], "ipg", "mxc-ehci.2"); - clk_register_clkdev(clk[IMX27_CLK_USB_AHB_GATE], "ahb", "mxc-ehci.2"); - clk_register_clkdev(clk[IMX27_CLK_SSI1_IPG_GATE], NULL, "imx-ssi.0"); - clk_register_clkdev(clk[IMX27_CLK_SSI2_IPG_GATE], NULL, "imx-ssi.1"); - clk_register_clkdev(clk[IMX27_CLK_NFC_BAUD_GATE], NULL, "imx27-nand.0"); - clk_register_clkdev(clk[IMX27_CLK_VPU_BAUD_GATE], "per", "coda-imx27.0"); - clk_register_clkdev(clk[IMX27_CLK_VPU_AHB_GATE], "ahb", "coda-imx27.0"); - clk_register_clkdev(clk[IMX27_CLK_DMA_AHB_GATE], "ahb", "imx27-dma"); - clk_register_clkdev(clk[IMX27_CLK_DMA_IPG_GATE], "ipg", "imx27-dma"); - clk_register_clkdev(clk[IMX27_CLK_FEC_IPG_GATE], "ipg", "imx27-fec.0"); - clk_register_clkdev(clk[IMX27_CLK_FEC_AHB_GATE], "ahb", "imx27-fec.0"); - clk_register_clkdev(clk[IMX27_CLK_WDOG_IPG_GATE], NULL, "imx2-wdt.0"); - clk_register_clkdev(clk[IMX27_CLK_I2C1_IPG_GATE], NULL, "imx21-i2c.0"); - clk_register_clkdev(clk[IMX27_CLK_I2C2_IPG_GATE], NULL, "imx21-i2c.1"); - clk_register_clkdev(clk[IMX27_CLK_OWIRE_IPG_GATE], NULL, "mxc_w1.0"); - clk_register_clkdev(clk[IMX27_CLK_KPP_IPG_GATE], NULL, "imx-keypad"); - clk_register_clkdev(clk[IMX27_CLK_EMMA_AHB_GATE], "emma-ahb", "imx27-camera.0"); - clk_register_clkdev(clk[IMX27_CLK_EMMA_IPG_GATE], "emma-ipg", "imx27-camera.0"); - clk_register_clkdev(clk[IMX27_CLK_EMMA_AHB_GATE], "ahb", "m2m-emmaprp.0"); - clk_register_clkdev(clk[IMX27_CLK_EMMA_IPG_GATE], "ipg", "m2m-emmaprp.0"); - - mxc_timer_init(MX27_GPT1_BASE_ADDR, MX27_INT_GPT1, GPT_TYPE_IMX21); - - return 0; -} - static void __init mx27_clocks_init_dt(struct device_node *np) { struct device_node *refnp; diff --git a/drivers/clk/imx/clk-imx31.c b/drivers/clk/imx/clk-imx31.c index 4bb05e440cdd..7b13fb57d842 100644 --- a/drivers/clk/imx/clk-imx31.c +++ b/drivers/clk/imx/clk-imx31.c @@ -132,77 +132,6 @@ static void __init _mx31_clocks_init(void __iomem *base, unsigned long fref) clk_disable_unprepare(clk[iim_gate]); } -int __init mx31_clocks_init(unsigned long fref) -{ - void __iomem *base; - - base = ioremap(MX31_CCM_BASE_ADDR, SZ_4K); - if (!base) - panic("%s: failed to map registers\n", __func__); - - _mx31_clocks_init(base, fref); - - clk_register_clkdev(clk[gpt_gate], "per", "imx-gpt.0"); - clk_register_clkdev(clk[ipg], "ipg", "imx-gpt.0"); - clk_register_clkdev(clk[cspi1_gate], NULL, "imx31-cspi.0"); - clk_register_clkdev(clk[cspi2_gate], NULL, "imx31-cspi.1"); - clk_register_clkdev(clk[cspi3_gate], NULL, "imx31-cspi.2"); - clk_register_clkdev(clk[pwm_gate], "pwm", NULL); - clk_register_clkdev(clk[wdog_gate], NULL, "imx2-wdt.0"); - clk_register_clkdev(clk[ckil], "ref", "imx21-rtc"); - clk_register_clkdev(clk[rtc_gate], "ipg", "imx21-rtc"); - clk_register_clkdev(clk[epit1_gate], "epit", NULL); - clk_register_clkdev(clk[epit2_gate], "epit", NULL); - clk_register_clkdev(clk[nfc], NULL, "imx27-nand.0"); - clk_register_clkdev(clk[ipu_gate], NULL, "ipu-core"); - clk_register_clkdev(clk[ipu_gate], NULL, "mx3_sdc_fb"); - clk_register_clkdev(clk[kpp_gate], NULL, "imx-keypad"); - clk_register_clkdev(clk[usb_div_post], "per", "mxc-ehci.0"); - clk_register_clkdev(clk[usb_gate], "ahb", "mxc-ehci.0"); - clk_register_clkdev(clk[ipg], "ipg", "mxc-ehci.0"); - clk_register_clkdev(clk[usb_div_post], "per", "mxc-ehci.1"); - clk_register_clkdev(clk[usb_gate], "ahb", "mxc-ehci.1"); - clk_register_clkdev(clk[ipg], "ipg", "mxc-ehci.1"); - clk_register_clkdev(clk[usb_div_post], "per", "mxc-ehci.2"); - clk_register_clkdev(clk[usb_gate], "ahb", "mxc-ehci.2"); - clk_register_clkdev(clk[ipg], "ipg", "mxc-ehci.2"); - clk_register_clkdev(clk[usb_div_post], "per", "imx-udc-mx27"); - clk_register_clkdev(clk[usb_gate], "ahb", "imx-udc-mx27"); - clk_register_clkdev(clk[ipg], "ipg", "imx-udc-mx27"); - clk_register_clkdev(clk[csi_gate], NULL, "mx3-camera.0"); - /* i.mx31 has the i.mx21 type uart */ - clk_register_clkdev(clk[uart1_gate], "per", "imx21-uart.0"); - clk_register_clkdev(clk[ipg], "ipg", "imx21-uart.0"); - clk_register_clkdev(clk[uart2_gate], "per", "imx21-uart.1"); - clk_register_clkdev(clk[ipg], "ipg", "imx21-uart.1"); - clk_register_clkdev(clk[uart3_gate], "per", "imx21-uart.2"); - clk_register_clkdev(clk[ipg], "ipg", "imx21-uart.2"); - clk_register_clkdev(clk[uart4_gate], "per", "imx21-uart.3"); - clk_register_clkdev(clk[ipg], "ipg", "imx21-uart.3"); - clk_register_clkdev(clk[uart5_gate], "per", "imx21-uart.4"); - clk_register_clkdev(clk[ipg], "ipg", "imx21-uart.4"); - clk_register_clkdev(clk[i2c1_gate], NULL, "imx21-i2c.0"); - clk_register_clkdev(clk[i2c2_gate], NULL, "imx21-i2c.1"); - clk_register_clkdev(clk[i2c3_gate], NULL, "imx21-i2c.2"); - clk_register_clkdev(clk[owire_gate], NULL, "mxc_w1.0"); - clk_register_clkdev(clk[sdhc1_gate], NULL, "imx31-mmc.0"); - clk_register_clkdev(clk[sdhc2_gate], NULL, "imx31-mmc.1"); - clk_register_clkdev(clk[ssi1_gate], NULL, "imx-ssi.0"); - clk_register_clkdev(clk[ssi2_gate], NULL, "imx-ssi.1"); - clk_register_clkdev(clk[firi_gate], "firi", NULL); - clk_register_clkdev(clk[ata_gate], NULL, "pata_imx"); - clk_register_clkdev(clk[rtic_gate], "rtic", NULL); - clk_register_clkdev(clk[rng_gate], NULL, "mxc_rnga"); - clk_register_clkdev(clk[sdma_gate], NULL, "imx31-sdma"); - clk_register_clkdev(clk[iim_gate], "iim", NULL); - - - imx_register_uart_clocks(uart_clks); - mxc_timer_init(MX31_GPT1_BASE_ADDR, MX31_INT_GPT, GPT_TYPE_IMX31); - - return 0; -} - static void __init mx31_clocks_init_dt(struct device_node *np) { struct device_node *osc_np; diff --git a/drivers/clk/imx/clk-imx35.c b/drivers/clk/imx/clk-imx35.c index e595f559907f..c1df03665c09 100644 --- a/drivers/clk/imx/clk-imx35.c +++ b/drivers/clk/imx/clk-imx35.c @@ -248,74 +248,6 @@ static void __init _mx35_clocks_init(void) imx_print_silicon_rev("i.MX35", mx35_revision()); } -int __init mx35_clocks_init(void) -{ - _mx35_clocks_init(); - - clk_register_clkdev(clk[pata_gate], NULL, "pata_imx"); - clk_register_clkdev(clk[can1_gate], NULL, "flexcan.0"); - clk_register_clkdev(clk[can2_gate], NULL, "flexcan.1"); - clk_register_clkdev(clk[cspi1_gate], "per", "imx35-cspi.0"); - clk_register_clkdev(clk[cspi1_gate], "ipg", "imx35-cspi.0"); - clk_register_clkdev(clk[cspi2_gate], "per", "imx35-cspi.1"); - clk_register_clkdev(clk[cspi2_gate], "ipg", "imx35-cspi.1"); - clk_register_clkdev(clk[epit1_gate], NULL, "imx-epit.0"); - clk_register_clkdev(clk[epit2_gate], NULL, "imx-epit.1"); - clk_register_clkdev(clk[esdhc1_gate], "per", "sdhci-esdhc-imx35.0"); - clk_register_clkdev(clk[ipg], "ipg", "sdhci-esdhc-imx35.0"); - clk_register_clkdev(clk[ahb], "ahb", "sdhci-esdhc-imx35.0"); - clk_register_clkdev(clk[esdhc2_gate], "per", "sdhci-esdhc-imx35.1"); - clk_register_clkdev(clk[ipg], "ipg", "sdhci-esdhc-imx35.1"); - clk_register_clkdev(clk[ahb], "ahb", "sdhci-esdhc-imx35.1"); - clk_register_clkdev(clk[esdhc3_gate], "per", "sdhci-esdhc-imx35.2"); - clk_register_clkdev(clk[ipg], "ipg", "sdhci-esdhc-imx35.2"); - clk_register_clkdev(clk[ahb], "ahb", "sdhci-esdhc-imx35.2"); - /* i.mx35 has the i.mx27 type fec */ - clk_register_clkdev(clk[fec_gate], NULL, "imx27-fec.0"); - clk_register_clkdev(clk[gpt_gate], "per", "imx-gpt.0"); - clk_register_clkdev(clk[ipg], "ipg", "imx-gpt.0"); - clk_register_clkdev(clk[i2c1_gate], NULL, "imx21-i2c.0"); - clk_register_clkdev(clk[i2c2_gate], NULL, "imx21-i2c.1"); - clk_register_clkdev(clk[i2c3_gate], NULL, "imx21-i2c.2"); - clk_register_clkdev(clk[ipu_gate], NULL, "ipu-core"); - clk_register_clkdev(clk[ipu_gate], NULL, "mx3_sdc_fb"); - clk_register_clkdev(clk[kpp_gate], NULL, "imx-keypad"); - clk_register_clkdev(clk[owire_gate], NULL, "mxc_w1"); - clk_register_clkdev(clk[sdma_gate], NULL, "imx35-sdma"); - clk_register_clkdev(clk[ssi1_gate], NULL, "imx-ssi.0"); - clk_register_clkdev(clk[ssi2_gate], NULL, "imx-ssi.1"); - /* i.mx35 has the i.mx21 type uart */ - clk_register_clkdev(clk[uart1_gate], "per", "imx21-uart.0"); - clk_register_clkdev(clk[ipg], "ipg", "imx21-uart.0"); - clk_register_clkdev(clk[uart2_gate], "per", "imx21-uart.1"); - clk_register_clkdev(clk[ipg], "ipg", "imx21-uart.1"); - clk_register_clkdev(clk[uart3_gate], "per", "imx21-uart.2"); - clk_register_clkdev(clk[ipg], "ipg", "imx21-uart.2"); - /* i.mx35 has the i.mx21 type rtc */ - clk_register_clkdev(clk[ckil], "ref", "imx21-rtc"); - clk_register_clkdev(clk[rtc_gate], "ipg", "imx21-rtc"); - clk_register_clkdev(clk[usb_div], "per", "mxc-ehci.0"); - clk_register_clkdev(clk[ipg], "ipg", "mxc-ehci.0"); - clk_register_clkdev(clk[usbotg_gate], "ahb", "mxc-ehci.0"); - clk_register_clkdev(clk[usb_div], "per", "mxc-ehci.1"); - clk_register_clkdev(clk[ipg], "ipg", "mxc-ehci.1"); - clk_register_clkdev(clk[usbotg_gate], "ahb", "mxc-ehci.1"); - clk_register_clkdev(clk[usb_div], "per", "mxc-ehci.2"); - clk_register_clkdev(clk[ipg], "ipg", "mxc-ehci.2"); - clk_register_clkdev(clk[usbotg_gate], "ahb", "mxc-ehci.2"); - clk_register_clkdev(clk[usb_div], "per", "imx-udc-mx27"); - clk_register_clkdev(clk[ipg], "ipg", "imx-udc-mx27"); - clk_register_clkdev(clk[usbotg_gate], "ahb", "imx-udc-mx27"); - clk_register_clkdev(clk[wdog_gate], NULL, "imx2-wdt.0"); - clk_register_clkdev(clk[nfc_div], NULL, "imx25-nand.0"); - clk_register_clkdev(clk[csi_gate], NULL, "mx3-camera.0"); - clk_register_clkdev(clk[admux_gate], "audmux", NULL); - - mxc_timer_init(MX35_GPT1_BASE_ADDR, MX35_INT_GPT, GPT_TYPE_IMX31); - - return 0; -} - static void __init mx35_clocks_init_dt(struct device_node *ccm_node) { _mx35_clocks_init(); diff --git a/drivers/clk/imx/clk-imx6q.c b/drivers/clk/imx/clk-imx6q.c index ba33c79158de..b2ff187cedab 100644 --- a/drivers/clk/imx/clk-imx6q.c +++ b/drivers/clk/imx/clk-imx6q.c @@ -6,6 +6,7 @@ #include <linux/init.h> #include <linux/types.h> +#include <linux/bits.h> #include <linux/clk.h> #include <linux/clkdev.h> #include <linux/clk-provider.h> diff --git a/drivers/clk/imx/clk-imx6sl.c b/drivers/clk/imx/clk-imx6sl.c index 0f647d148abf..2f9361946a0e 100644 --- a/drivers/clk/imx/clk-imx6sl.c +++ b/drivers/clk/imx/clk-imx6sl.c @@ -3,6 +3,7 @@ * Copyright 2013-2014 Freescale Semiconductor, Inc. */ +#include <linux/bits.h> #include <linux/clk.h> #include <linux/clkdev.h> #include <linux/err.h> @@ -14,19 +15,19 @@ #include "clk.h" #define CCSR 0xc -#define BM_CCSR_PLL1_SW_CLK_SEL (1 << 2) +#define BM_CCSR_PLL1_SW_CLK_SEL BIT(2) #define CACRR 0x10 #define CDHIPR 0x48 -#define BM_CDHIPR_ARM_PODF_BUSY (1 << 16) +#define BM_CDHIPR_ARM_PODF_BUSY BIT(16) #define ARM_WAIT_DIV_396M 2 #define ARM_WAIT_DIV_792M 4 #define ARM_WAIT_DIV_996M 6 #define PLL_ARM 0x0 -#define BM_PLL_ARM_DIV_SELECT (0x7f << 0) -#define BM_PLL_ARM_POWERDOWN (1 << 12) -#define BM_PLL_ARM_ENABLE (1 << 13) -#define BM_PLL_ARM_LOCK (1 << 31) +#define BM_PLL_ARM_DIV_SELECT 0x7f +#define BM_PLL_ARM_POWERDOWN BIT(12) +#define BM_PLL_ARM_ENABLE BIT(13) +#define BM_PLL_ARM_LOCK BIT(31) #define PLL_ARM_DIV_792M 66 static const char *step_sels[] = { "osc", "pll2_pfd2", }; @@ -145,7 +146,7 @@ static void imx6sl_enable_pll_arm(bool enable) val |= BM_PLL_ARM_ENABLE; val &= ~BM_PLL_ARM_POWERDOWN; writel_relaxed(val, anatop_base + PLL_ARM); - while (!(__raw_readl(anatop_base + PLL_ARM) & BM_PLL_ARM_LOCK)) + while (!(readl_relaxed(anatop_base + PLL_ARM) & BM_PLL_ARM_LOCK)) ; } else { writel_relaxed(saved_pll_arm, anatop_base + PLL_ARM); diff --git a/drivers/clk/imx/clk-imx6sx.c b/drivers/clk/imx/clk-imx6sx.c index 89ba71271e5c..20dcce526d07 100644 --- a/drivers/clk/imx/clk-imx6sx.c +++ b/drivers/clk/imx/clk-imx6sx.c @@ -4,6 +4,7 @@ */ #include <dt-bindings/clock/imx6sx-clock.h> +#include <linux/bits.h> #include <linux/clk.h> #include <linux/clkdev.h> #include <linux/clk-provider.h> diff --git a/drivers/clk/imx/clk-imx7d.c b/drivers/clk/imx/clk-imx7d.c index b2057bd42e25..22d24a6a05e7 100644 --- a/drivers/clk/imx/clk-imx7d.c +++ b/drivers/clk/imx/clk-imx7d.c @@ -4,6 +4,7 @@ */ #include <dt-bindings/clock/imx7d-clock.h> +#include <linux/bits.h> #include <linux/clk.h> #include <linux/clkdev.h> #include <linux/clk-provider.h> @@ -505,72 +506,73 @@ static void __init imx7d_clocks_init(struct device_node *ccm_node) hws[IMX7D_ARM_M4_ROOT_SRC] = imx_clk_hw_mux2("arm_m4_src", base + 0x8080, 24, 3, arm_m4_sel, ARRAY_SIZE(arm_m4_sel)); hws[IMX7D_MAIN_AXI_ROOT_SRC] = imx_clk_hw_mux2("axi_src", base + 0x8800, 24, 3, axi_sel, ARRAY_SIZE(axi_sel)); hws[IMX7D_DISP_AXI_ROOT_SRC] = imx_clk_hw_mux2("disp_axi_src", base + 0x8880, 24, 3, disp_axi_sel, ARRAY_SIZE(disp_axi_sel)); - hws[IMX7D_ENET_AXI_ROOT_SRC] = imx_clk_hw_mux2("enet_axi_src", base + 0x8900, 24, 3, enet_axi_sel, ARRAY_SIZE(enet_axi_sel)); - hws[IMX7D_NAND_USDHC_BUS_ROOT_SRC] = imx_clk_hw_mux2("nand_usdhc_src", base + 0x8980, 24, 3, nand_usdhc_bus_sel, ARRAY_SIZE(nand_usdhc_bus_sel)); hws[IMX7D_AHB_CHANNEL_ROOT_SRC] = imx_clk_hw_mux2("ahb_src", base + 0x9000, 24, 3, ahb_channel_sel, ARRAY_SIZE(ahb_channel_sel)); - hws[IMX7D_DRAM_PHYM_ROOT_SRC] = imx_clk_hw_mux2("dram_phym_src", base + 0x9800, 24, 1, dram_phym_sel, ARRAY_SIZE(dram_phym_sel)); - hws[IMX7D_DRAM_ROOT_SRC] = imx_clk_hw_mux2("dram_src", base + 0x9880, 24, 1, dram_sel, ARRAY_SIZE(dram_sel)); - hws[IMX7D_DRAM_PHYM_ALT_ROOT_SRC] = imx_clk_hw_mux2("dram_phym_alt_src", base + 0xa000, 24, 3, dram_phym_alt_sel, ARRAY_SIZE(dram_phym_alt_sel)); - hws[IMX7D_DRAM_ALT_ROOT_SRC] = imx_clk_hw_mux2("dram_alt_src", base + 0xa080, 24, 3, dram_alt_sel, ARRAY_SIZE(dram_alt_sel)); - hws[IMX7D_USB_HSIC_ROOT_SRC] = imx_clk_hw_mux2("usb_hsic_src", base + 0xa100, 24, 3, usb_hsic_sel, ARRAY_SIZE(usb_hsic_sel)); - hws[IMX7D_PCIE_CTRL_ROOT_SRC] = imx_clk_hw_mux2("pcie_ctrl_src", base + 0xa180, 24, 3, pcie_ctrl_sel, ARRAY_SIZE(pcie_ctrl_sel)); - hws[IMX7D_PCIE_PHY_ROOT_SRC] = imx_clk_hw_mux2("pcie_phy_src", base + 0xa200, 24, 3, pcie_phy_sel, ARRAY_SIZE(pcie_phy_sel)); - hws[IMX7D_EPDC_PIXEL_ROOT_SRC] = imx_clk_hw_mux2("epdc_pixel_src", base + 0xa280, 24, 3, epdc_pixel_sel, ARRAY_SIZE(epdc_pixel_sel)); - hws[IMX7D_LCDIF_PIXEL_ROOT_SRC] = imx_clk_hw_mux2("lcdif_pixel_src", base + 0xa300, 24, 3, lcdif_pixel_sel, ARRAY_SIZE(lcdif_pixel_sel)); - hws[IMX7D_MIPI_DSI_ROOT_SRC] = imx_clk_hw_mux2("mipi_dsi_src", base + 0xa380, 24, 3, mipi_dsi_sel, ARRAY_SIZE(mipi_dsi_sel)); - hws[IMX7D_MIPI_CSI_ROOT_SRC] = imx_clk_hw_mux2("mipi_csi_src", base + 0xa400, 24, 3, mipi_csi_sel, ARRAY_SIZE(mipi_csi_sel)); - hws[IMX7D_MIPI_DPHY_ROOT_SRC] = imx_clk_hw_mux2("mipi_dphy_src", base + 0xa480, 24, 3, mipi_dphy_sel, ARRAY_SIZE(mipi_dphy_sel)); - hws[IMX7D_SAI1_ROOT_SRC] = imx_clk_hw_mux2("sai1_src", base + 0xa500, 24, 3, sai1_sel, ARRAY_SIZE(sai1_sel)); - hws[IMX7D_SAI2_ROOT_SRC] = imx_clk_hw_mux2("sai2_src", base + 0xa580, 24, 3, sai2_sel, ARRAY_SIZE(sai2_sel)); - hws[IMX7D_SAI3_ROOT_SRC] = imx_clk_hw_mux2("sai3_src", base + 0xa600, 24, 3, sai3_sel, ARRAY_SIZE(sai3_sel)); - hws[IMX7D_SPDIF_ROOT_SRC] = imx_clk_hw_mux2("spdif_src", base + 0xa680, 24, 3, spdif_sel, ARRAY_SIZE(spdif_sel)); - hws[IMX7D_ENET1_REF_ROOT_SRC] = imx_clk_hw_mux2("enet1_ref_src", base + 0xa700, 24, 3, enet1_ref_sel, ARRAY_SIZE(enet1_ref_sel)); - hws[IMX7D_ENET1_TIME_ROOT_SRC] = imx_clk_hw_mux2("enet1_time_src", base + 0xa780, 24, 3, enet1_time_sel, ARRAY_SIZE(enet1_time_sel)); - hws[IMX7D_ENET2_REF_ROOT_SRC] = imx_clk_hw_mux2("enet2_ref_src", base + 0xa800, 24, 3, enet2_ref_sel, ARRAY_SIZE(enet2_ref_sel)); - hws[IMX7D_ENET2_TIME_ROOT_SRC] = imx_clk_hw_mux2("enet2_time_src", base + 0xa880, 24, 3, enet2_time_sel, ARRAY_SIZE(enet2_time_sel)); - hws[IMX7D_ENET_PHY_REF_ROOT_SRC] = imx_clk_hw_mux2("enet_phy_ref_src", base + 0xa900, 24, 3, enet_phy_ref_sel, ARRAY_SIZE(enet_phy_ref_sel)); - hws[IMX7D_EIM_ROOT_SRC] = imx_clk_hw_mux2("eim_src", base + 0xa980, 24, 3, eim_sel, ARRAY_SIZE(eim_sel)); - hws[IMX7D_NAND_ROOT_SRC] = imx_clk_hw_mux2("nand_src", base + 0xaa00, 24, 3, nand_sel, ARRAY_SIZE(nand_sel)); - hws[IMX7D_QSPI_ROOT_SRC] = imx_clk_hw_mux2("qspi_src", base + 0xaa80, 24, 3, qspi_sel, ARRAY_SIZE(qspi_sel)); - hws[IMX7D_USDHC1_ROOT_SRC] = imx_clk_hw_mux2("usdhc1_src", base + 0xab00, 24, 3, usdhc1_sel, ARRAY_SIZE(usdhc1_sel)); - hws[IMX7D_USDHC2_ROOT_SRC] = imx_clk_hw_mux2("usdhc2_src", base + 0xab80, 24, 3, usdhc2_sel, ARRAY_SIZE(usdhc2_sel)); - hws[IMX7D_USDHC3_ROOT_SRC] = imx_clk_hw_mux2("usdhc3_src", base + 0xac00, 24, 3, usdhc3_sel, ARRAY_SIZE(usdhc3_sel)); - hws[IMX7D_CAN1_ROOT_SRC] = imx_clk_hw_mux2("can1_src", base + 0xac80, 24, 3, can1_sel, ARRAY_SIZE(can1_sel)); - hws[IMX7D_CAN2_ROOT_SRC] = imx_clk_hw_mux2("can2_src", base + 0xad00, 24, 3, can2_sel, ARRAY_SIZE(can2_sel)); - hws[IMX7D_I2C1_ROOT_SRC] = imx_clk_hw_mux2("i2c1_src", base + 0xad80, 24, 3, i2c1_sel, ARRAY_SIZE(i2c1_sel)); - hws[IMX7D_I2C2_ROOT_SRC] = imx_clk_hw_mux2("i2c2_src", base + 0xae00, 24, 3, i2c2_sel, ARRAY_SIZE(i2c2_sel)); - hws[IMX7D_I2C3_ROOT_SRC] = imx_clk_hw_mux2("i2c3_src", base + 0xae80, 24, 3, i2c3_sel, ARRAY_SIZE(i2c3_sel)); - hws[IMX7D_I2C4_ROOT_SRC] = imx_clk_hw_mux2("i2c4_src", base + 0xaf00, 24, 3, i2c4_sel, ARRAY_SIZE(i2c4_sel)); - hws[IMX7D_UART1_ROOT_SRC] = imx_clk_hw_mux2("uart1_src", base + 0xaf80, 24, 3, uart1_sel, ARRAY_SIZE(uart1_sel)); - hws[IMX7D_UART2_ROOT_SRC] = imx_clk_hw_mux2("uart2_src", base + 0xb000, 24, 3, uart2_sel, ARRAY_SIZE(uart2_sel)); - hws[IMX7D_UART3_ROOT_SRC] = imx_clk_hw_mux2("uart3_src", base + 0xb080, 24, 3, uart3_sel, ARRAY_SIZE(uart3_sel)); - hws[IMX7D_UART4_ROOT_SRC] = imx_clk_hw_mux2("uart4_src", base + 0xb100, 24, 3, uart4_sel, ARRAY_SIZE(uart4_sel)); - hws[IMX7D_UART5_ROOT_SRC] = imx_clk_hw_mux2("uart5_src", base + 0xb180, 24, 3, uart5_sel, ARRAY_SIZE(uart5_sel)); - hws[IMX7D_UART6_ROOT_SRC] = imx_clk_hw_mux2("uart6_src", base + 0xb200, 24, 3, uart6_sel, ARRAY_SIZE(uart6_sel)); - hws[IMX7D_UART7_ROOT_SRC] = imx_clk_hw_mux2("uart7_src", base + 0xb280, 24, 3, uart7_sel, ARRAY_SIZE(uart7_sel)); - hws[IMX7D_ECSPI1_ROOT_SRC] = imx_clk_hw_mux2("ecspi1_src", base + 0xb300, 24, 3, ecspi1_sel, ARRAY_SIZE(ecspi1_sel)); - hws[IMX7D_ECSPI2_ROOT_SRC] = imx_clk_hw_mux2("ecspi2_src", base + 0xb380, 24, 3, ecspi2_sel, ARRAY_SIZE(ecspi2_sel)); - hws[IMX7D_ECSPI3_ROOT_SRC] = imx_clk_hw_mux2("ecspi3_src", base + 0xb400, 24, 3, ecspi3_sel, ARRAY_SIZE(ecspi3_sel)); - hws[IMX7D_ECSPI4_ROOT_SRC] = imx_clk_hw_mux2("ecspi4_src", base + 0xb480, 24, 3, ecspi4_sel, ARRAY_SIZE(ecspi4_sel)); - hws[IMX7D_PWM1_ROOT_SRC] = imx_clk_hw_mux2("pwm1_src", base + 0xb500, 24, 3, pwm1_sel, ARRAY_SIZE(pwm1_sel)); - hws[IMX7D_PWM2_ROOT_SRC] = imx_clk_hw_mux2("pwm2_src", base + 0xb580, 24, 3, pwm2_sel, ARRAY_SIZE(pwm2_sel)); - hws[IMX7D_PWM3_ROOT_SRC] = imx_clk_hw_mux2("pwm3_src", base + 0xb600, 24, 3, pwm3_sel, ARRAY_SIZE(pwm3_sel)); - hws[IMX7D_PWM4_ROOT_SRC] = imx_clk_hw_mux2("pwm4_src", base + 0xb680, 24, 3, pwm4_sel, ARRAY_SIZE(pwm4_sel)); - hws[IMX7D_FLEXTIMER1_ROOT_SRC] = imx_clk_hw_mux2("flextimer1_src", base + 0xb700, 24, 3, flextimer1_sel, ARRAY_SIZE(flextimer1_sel)); - hws[IMX7D_FLEXTIMER2_ROOT_SRC] = imx_clk_hw_mux2("flextimer2_src", base + 0xb780, 24, 3, flextimer2_sel, ARRAY_SIZE(flextimer2_sel)); - hws[IMX7D_SIM1_ROOT_SRC] = imx_clk_hw_mux2("sim1_src", base + 0xb800, 24, 3, sim1_sel, ARRAY_SIZE(sim1_sel)); - hws[IMX7D_SIM2_ROOT_SRC] = imx_clk_hw_mux2("sim2_src", base + 0xb880, 24, 3, sim2_sel, ARRAY_SIZE(sim2_sel)); - hws[IMX7D_GPT1_ROOT_SRC] = imx_clk_hw_mux2("gpt1_src", base + 0xb900, 24, 3, gpt1_sel, ARRAY_SIZE(gpt1_sel)); - hws[IMX7D_GPT2_ROOT_SRC] = imx_clk_hw_mux2("gpt2_src", base + 0xb980, 24, 3, gpt2_sel, ARRAY_SIZE(gpt2_sel)); - hws[IMX7D_GPT3_ROOT_SRC] = imx_clk_hw_mux2("gpt3_src", base + 0xba00, 24, 3, gpt3_sel, ARRAY_SIZE(gpt3_sel)); - hws[IMX7D_GPT4_ROOT_SRC] = imx_clk_hw_mux2("gpt4_src", base + 0xba80, 24, 3, gpt4_sel, ARRAY_SIZE(gpt4_sel)); - hws[IMX7D_TRACE_ROOT_SRC] = imx_clk_hw_mux2("trace_src", base + 0xbb00, 24, 3, trace_sel, ARRAY_SIZE(trace_sel)); - hws[IMX7D_WDOG_ROOT_SRC] = imx_clk_hw_mux2("wdog_src", base + 0xbb80, 24, 3, wdog_sel, ARRAY_SIZE(wdog_sel)); - hws[IMX7D_CSI_MCLK_ROOT_SRC] = imx_clk_hw_mux2("csi_mclk_src", base + 0xbc00, 24, 3, csi_mclk_sel, ARRAY_SIZE(csi_mclk_sel)); - hws[IMX7D_AUDIO_MCLK_ROOT_SRC] = imx_clk_hw_mux2("audio_mclk_src", base + 0xbc80, 24, 3, audio_mclk_sel, ARRAY_SIZE(audio_mclk_sel)); - hws[IMX7D_WRCLK_ROOT_SRC] = imx_clk_hw_mux2("wrclk_src", base + 0xbd00, 24, 3, wrclk_sel, ARRAY_SIZE(wrclk_sel)); - hws[IMX7D_CLKO1_ROOT_SRC] = imx_clk_hw_mux2("clko1_src", base + 0xbd80, 24, 3, clko1_sel, ARRAY_SIZE(clko1_sel)); - hws[IMX7D_CLKO2_ROOT_SRC] = imx_clk_hw_mux2("clko2_src", base + 0xbe00, 24, 3, clko2_sel, ARRAY_SIZE(clko2_sel)); + + hws[IMX7D_ENET_AXI_ROOT_SRC] = imx_clk_hw_mux2_flags("enet_axi_src", base + 0x8900, 24, 3, enet_axi_sel, ARRAY_SIZE(enet_axi_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_NAND_USDHC_BUS_ROOT_SRC] = imx_clk_hw_mux2_flags("nand_usdhc_src", base + 0x8980, 24, 3, nand_usdhc_bus_sel, ARRAY_SIZE(nand_usdhc_bus_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_DRAM_PHYM_ROOT_SRC] = imx_clk_hw_mux2_flags("dram_phym_src", base + 0x9800, 24, 1, dram_phym_sel, ARRAY_SIZE(dram_phym_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_DRAM_ROOT_SRC] = imx_clk_hw_mux2_flags("dram_src", base + 0x9880, 24, 1, dram_sel, ARRAY_SIZE(dram_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_DRAM_PHYM_ALT_ROOT_SRC] = imx_clk_hw_mux2_flags("dram_phym_alt_src", base + 0xa000, 24, 3, dram_phym_alt_sel, ARRAY_SIZE(dram_phym_alt_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_DRAM_ALT_ROOT_SRC] = imx_clk_hw_mux2_flags("dram_alt_src", base + 0xa080, 24, 3, dram_alt_sel, ARRAY_SIZE(dram_alt_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_USB_HSIC_ROOT_SRC] = imx_clk_hw_mux2_flags("usb_hsic_src", base + 0xa100, 24, 3, usb_hsic_sel, ARRAY_SIZE(usb_hsic_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_PCIE_CTRL_ROOT_SRC] = imx_clk_hw_mux2_flags("pcie_ctrl_src", base + 0xa180, 24, 3, pcie_ctrl_sel, ARRAY_SIZE(pcie_ctrl_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_PCIE_PHY_ROOT_SRC] = imx_clk_hw_mux2_flags("pcie_phy_src", base + 0xa200, 24, 3, pcie_phy_sel, ARRAY_SIZE(pcie_phy_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_EPDC_PIXEL_ROOT_SRC] = imx_clk_hw_mux2_flags("epdc_pixel_src", base + 0xa280, 24, 3, epdc_pixel_sel, ARRAY_SIZE(epdc_pixel_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_LCDIF_PIXEL_ROOT_SRC] = imx_clk_hw_mux2_flags("lcdif_pixel_src", base + 0xa300, 24, 3, lcdif_pixel_sel, ARRAY_SIZE(lcdif_pixel_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_MIPI_DSI_ROOT_SRC] = imx_clk_hw_mux2_flags("mipi_dsi_src", base + 0xa380, 24, 3, mipi_dsi_sel, ARRAY_SIZE(mipi_dsi_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_MIPI_CSI_ROOT_SRC] = imx_clk_hw_mux2_flags("mipi_csi_src", base + 0xa400, 24, 3, mipi_csi_sel, ARRAY_SIZE(mipi_csi_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_MIPI_DPHY_ROOT_SRC] = imx_clk_hw_mux2_flags("mipi_dphy_src", base + 0xa480, 24, 3, mipi_dphy_sel, ARRAY_SIZE(mipi_dphy_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_SAI1_ROOT_SRC] = imx_clk_hw_mux2_flags("sai1_src", base + 0xa500, 24, 3, sai1_sel, ARRAY_SIZE(sai1_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_SAI2_ROOT_SRC] = imx_clk_hw_mux2_flags("sai2_src", base + 0xa580, 24, 3, sai2_sel, ARRAY_SIZE(sai2_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_SAI3_ROOT_SRC] = imx_clk_hw_mux2_flags("sai3_src", base + 0xa600, 24, 3, sai3_sel, ARRAY_SIZE(sai3_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_SPDIF_ROOT_SRC] = imx_clk_hw_mux2_flags("spdif_src", base + 0xa680, 24, 3, spdif_sel, ARRAY_SIZE(spdif_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_ENET1_REF_ROOT_SRC] = imx_clk_hw_mux2_flags("enet1_ref_src", base + 0xa700, 24, 3, enet1_ref_sel, ARRAY_SIZE(enet1_ref_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_ENET1_TIME_ROOT_SRC] = imx_clk_hw_mux2_flags("enet1_time_src", base + 0xa780, 24, 3, enet1_time_sel, ARRAY_SIZE(enet1_time_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_ENET2_REF_ROOT_SRC] = imx_clk_hw_mux2_flags("enet2_ref_src", base + 0xa800, 24, 3, enet2_ref_sel, ARRAY_SIZE(enet2_ref_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_ENET2_TIME_ROOT_SRC] = imx_clk_hw_mux2_flags("enet2_time_src", base + 0xa880, 24, 3, enet2_time_sel, ARRAY_SIZE(enet2_time_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_ENET_PHY_REF_ROOT_SRC] = imx_clk_hw_mux2_flags("enet_phy_ref_src", base + 0xa900, 24, 3, enet_phy_ref_sel, ARRAY_SIZE(enet_phy_ref_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_EIM_ROOT_SRC] = imx_clk_hw_mux2_flags("eim_src", base + 0xa980, 24, 3, eim_sel, ARRAY_SIZE(eim_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_NAND_ROOT_SRC] = imx_clk_hw_mux2_flags("nand_src", base + 0xaa00, 24, 3, nand_sel, ARRAY_SIZE(nand_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_QSPI_ROOT_SRC] = imx_clk_hw_mux2_flags("qspi_src", base + 0xaa80, 24, 3, qspi_sel, ARRAY_SIZE(qspi_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_USDHC1_ROOT_SRC] = imx_clk_hw_mux2_flags("usdhc1_src", base + 0xab00, 24, 3, usdhc1_sel, ARRAY_SIZE(usdhc1_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_USDHC2_ROOT_SRC] = imx_clk_hw_mux2_flags("usdhc2_src", base + 0xab80, 24, 3, usdhc2_sel, ARRAY_SIZE(usdhc2_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_USDHC3_ROOT_SRC] = imx_clk_hw_mux2_flags("usdhc3_src", base + 0xac00, 24, 3, usdhc3_sel, ARRAY_SIZE(usdhc3_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_CAN1_ROOT_SRC] = imx_clk_hw_mux2_flags("can1_src", base + 0xac80, 24, 3, can1_sel, ARRAY_SIZE(can1_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_CAN2_ROOT_SRC] = imx_clk_hw_mux2_flags("can2_src", base + 0xad00, 24, 3, can2_sel, ARRAY_SIZE(can2_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_I2C1_ROOT_SRC] = imx_clk_hw_mux2_flags("i2c1_src", base + 0xad80, 24, 3, i2c1_sel, ARRAY_SIZE(i2c1_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_I2C2_ROOT_SRC] = imx_clk_hw_mux2_flags("i2c2_src", base + 0xae00, 24, 3, i2c2_sel, ARRAY_SIZE(i2c2_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_I2C3_ROOT_SRC] = imx_clk_hw_mux2_flags("i2c3_src", base + 0xae80, 24, 3, i2c3_sel, ARRAY_SIZE(i2c3_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_I2C4_ROOT_SRC] = imx_clk_hw_mux2_flags("i2c4_src", base + 0xaf00, 24, 3, i2c4_sel, ARRAY_SIZE(i2c4_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_UART1_ROOT_SRC] = imx_clk_hw_mux2_flags("uart1_src", base + 0xaf80, 24, 3, uart1_sel, ARRAY_SIZE(uart1_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_UART2_ROOT_SRC] = imx_clk_hw_mux2_flags("uart2_src", base + 0xb000, 24, 3, uart2_sel, ARRAY_SIZE(uart2_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_UART3_ROOT_SRC] = imx_clk_hw_mux2_flags("uart3_src", base + 0xb080, 24, 3, uart3_sel, ARRAY_SIZE(uart3_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_UART4_ROOT_SRC] = imx_clk_hw_mux2_flags("uart4_src", base + 0xb100, 24, 3, uart4_sel, ARRAY_SIZE(uart4_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_UART5_ROOT_SRC] = imx_clk_hw_mux2_flags("uart5_src", base + 0xb180, 24, 3, uart5_sel, ARRAY_SIZE(uart5_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_UART6_ROOT_SRC] = imx_clk_hw_mux2_flags("uart6_src", base + 0xb200, 24, 3, uart6_sel, ARRAY_SIZE(uart6_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_UART7_ROOT_SRC] = imx_clk_hw_mux2_flags("uart7_src", base + 0xb280, 24, 3, uart7_sel, ARRAY_SIZE(uart7_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_ECSPI1_ROOT_SRC] = imx_clk_hw_mux2_flags("ecspi1_src", base + 0xb300, 24, 3, ecspi1_sel, ARRAY_SIZE(ecspi1_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_ECSPI2_ROOT_SRC] = imx_clk_hw_mux2_flags("ecspi2_src", base + 0xb380, 24, 3, ecspi2_sel, ARRAY_SIZE(ecspi2_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_ECSPI3_ROOT_SRC] = imx_clk_hw_mux2_flags("ecspi3_src", base + 0xb400, 24, 3, ecspi3_sel, ARRAY_SIZE(ecspi3_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_ECSPI4_ROOT_SRC] = imx_clk_hw_mux2_flags("ecspi4_src", base + 0xb480, 24, 3, ecspi4_sel, ARRAY_SIZE(ecspi4_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_PWM1_ROOT_SRC] = imx_clk_hw_mux2_flags("pwm1_src", base + 0xb500, 24, 3, pwm1_sel, ARRAY_SIZE(pwm1_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_PWM2_ROOT_SRC] = imx_clk_hw_mux2_flags("pwm2_src", base + 0xb580, 24, 3, pwm2_sel, ARRAY_SIZE(pwm2_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_PWM3_ROOT_SRC] = imx_clk_hw_mux2_flags("pwm3_src", base + 0xb600, 24, 3, pwm3_sel, ARRAY_SIZE(pwm3_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_PWM4_ROOT_SRC] = imx_clk_hw_mux2_flags("pwm4_src", base + 0xb680, 24, 3, pwm4_sel, ARRAY_SIZE(pwm4_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_FLEXTIMER1_ROOT_SRC] = imx_clk_hw_mux2_flags("flextimer1_src", base + 0xb700, 24, 3, flextimer1_sel, ARRAY_SIZE(flextimer1_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_FLEXTIMER2_ROOT_SRC] = imx_clk_hw_mux2_flags("flextimer2_src", base + 0xb780, 24, 3, flextimer2_sel, ARRAY_SIZE(flextimer2_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_SIM1_ROOT_SRC] = imx_clk_hw_mux2_flags("sim1_src", base + 0xb800, 24, 3, sim1_sel, ARRAY_SIZE(sim1_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_SIM2_ROOT_SRC] = imx_clk_hw_mux2_flags("sim2_src", base + 0xb880, 24, 3, sim2_sel, ARRAY_SIZE(sim2_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_GPT1_ROOT_SRC] = imx_clk_hw_mux2_flags("gpt1_src", base + 0xb900, 24, 3, gpt1_sel, ARRAY_SIZE(gpt1_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_GPT2_ROOT_SRC] = imx_clk_hw_mux2_flags("gpt2_src", base + 0xb980, 24, 3, gpt2_sel, ARRAY_SIZE(gpt2_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_GPT3_ROOT_SRC] = imx_clk_hw_mux2_flags("gpt3_src", base + 0xba00, 24, 3, gpt3_sel, ARRAY_SIZE(gpt3_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_GPT4_ROOT_SRC] = imx_clk_hw_mux2_flags("gpt4_src", base + 0xba80, 24, 3, gpt4_sel, ARRAY_SIZE(gpt4_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_TRACE_ROOT_SRC] = imx_clk_hw_mux2_flags("trace_src", base + 0xbb00, 24, 3, trace_sel, ARRAY_SIZE(trace_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_WDOG_ROOT_SRC] = imx_clk_hw_mux2_flags("wdog_src", base + 0xbb80, 24, 3, wdog_sel, ARRAY_SIZE(wdog_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_CSI_MCLK_ROOT_SRC] = imx_clk_hw_mux2_flags("csi_mclk_src", base + 0xbc00, 24, 3, csi_mclk_sel, ARRAY_SIZE(csi_mclk_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_AUDIO_MCLK_ROOT_SRC] = imx_clk_hw_mux2_flags("audio_mclk_src", base + 0xbc80, 24, 3, audio_mclk_sel, ARRAY_SIZE(audio_mclk_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_WRCLK_ROOT_SRC] = imx_clk_hw_mux2_flags("wrclk_src", base + 0xbd00, 24, 3, wrclk_sel, ARRAY_SIZE(wrclk_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_CLKO1_ROOT_SRC] = imx_clk_hw_mux2_flags("clko1_src", base + 0xbd80, 24, 3, clko1_sel, ARRAY_SIZE(clko1_sel), CLK_SET_PARENT_GATE); + hws[IMX7D_CLKO2_ROOT_SRC] = imx_clk_hw_mux2_flags("clko2_src", base + 0xbe00, 24, 3, clko2_sel, ARRAY_SIZE(clko2_sel), CLK_SET_PARENT_GATE); hws[IMX7D_ARM_A7_ROOT_CG] = imx_clk_hw_gate3("arm_a7_cg", "arm_a7_src", base + 0x8000, 28); hws[IMX7D_ARM_M4_ROOT_CG] = imx_clk_hw_gate3("arm_m4_cg", "arm_m4_src", base + 0x8080, 28); diff --git a/drivers/clk/imx/clk-imx8mm.c b/drivers/clk/imx/clk-imx8mm.c index b793264c21c6..0de0be0cf548 100644 --- a/drivers/clk/imx/clk-imx8mm.c +++ b/drivers/clk/imx/clk-imx8mm.c @@ -657,3 +657,7 @@ static struct platform_driver imx8mm_clk_driver = { }, }; module_platform_driver(imx8mm_clk_driver); + +MODULE_AUTHOR("Bai Ping <ping.bai@nxp.com>"); +MODULE_DESCRIPTION("NXP i.MX8MM clock driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/imx/clk-imx8mn.c b/drivers/clk/imx/clk-imx8mn.c index 213cc37b3173..e984de543f0b 100644 --- a/drivers/clk/imx/clk-imx8mn.c +++ b/drivers/clk/imx/clk-imx8mn.c @@ -608,3 +608,7 @@ static struct platform_driver imx8mn_clk_driver = { }, }; module_platform_driver(imx8mn_clk_driver); + +MODULE_AUTHOR("Anson Huang <Anson.Huang@nxp.com>"); +MODULE_DESCRIPTION("NXP i.MX8MN clock driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/imx/clk-imx8mp.c b/drivers/clk/imx/clk-imx8mp.c index ca747712400f..12ce4770f702 100644 --- a/drivers/clk/imx/clk-imx8mp.c +++ b/drivers/clk/imx/clk-imx8mp.c @@ -152,10 +152,6 @@ static const char * const imx8mp_can2_sels[] = {"osc_24m", "sys_pll2_200m", "sys "sys_pll1_160m", "sys_pll1_800m", "sys_pll3_out", "sys_pll2_250m", "audio_pll2_out", }; -static const char * const imx8mp_memrepair_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m", - "sys_pll3_out", "audio_pll1_out", "video_pll1_out", - "audio_pll2_out", "sys_pll1_133m", }; - static const char * const imx8mp_pcie_phy_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll2_500m", "clk_ext1", "clk_ext2", "clk_ext3", "clk_ext4", "sys_pll1_400m", }; @@ -375,15 +371,14 @@ static const char * const imx8mp_media_cam2_pix_sels[] = {"osc_24m", "sys_pll1_2 "sys_pll3_out", "audio_pll2_out", "video_pll1_out", }; -static const char * const imx8mp_media_mipi_phy2_ref_sels[] = {"osc_24m", "sys_pll2_333m", "sys_pll2_100m", - "sys_pll1_800m", "sys_pll2_1000m", - "clk_ext2", "audio_pll2_out", - "video_pll1_out", }; +static const char * const imx8mp_media_ldb_sels[] = {"osc_24m", "sys_pll2_333m", "sys_pll2_100m", + "sys_pll1_800m", "sys_pll2_1000m", + "clk_ext2", "audio_pll2_out", + "video_pll1_out", }; -static const char * const imx8mp_media_mipi_csi2_esc_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_80m", - "sys_pll1_800m", "sys_pll2_1000m", - "sys_pll3_out", "clk_ext3", - "audio_pll2_out", }; +static const char * const imx8mp_memrepair_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_80m", + "sys_pll1_800m", "sys_pll2_1000m", "sys_pll3_out", + "clk_ext3", "audio_pll2_out", }; static const char * const imx8mp_pcie2_ctrl_sels[] = {"osc_24m", "sys_pll2_250m", "sys_pll2_200m", "sys_pll1_266m", "sys_pll1_800m", "sys_pll2_500m", @@ -590,7 +585,6 @@ static int imx8mp_clocks_probe(struct platform_device *pdev) hws[IMX8MP_CLK_VPU_G2] = imx8m_clk_hw_composite("vpu_g2", imx8mp_vpu_g2_sels, ccm_base + 0xa180); hws[IMX8MP_CLK_CAN1] = imx8m_clk_hw_composite("can1", imx8mp_can1_sels, ccm_base + 0xa200); hws[IMX8MP_CLK_CAN2] = imx8m_clk_hw_composite("can2", imx8mp_can2_sels, ccm_base + 0xa280); - hws[IMX8MP_CLK_MEMREPAIR] = imx8m_clk_hw_composite("memrepair", imx8mp_memrepair_sels, ccm_base + 0xa300); hws[IMX8MP_CLK_PCIE_PHY] = imx8m_clk_hw_composite("pcie_phy", imx8mp_pcie_phy_sels, ccm_base + 0xa380); hws[IMX8MP_CLK_PCIE_AUX] = imx8m_clk_hw_composite("pcie_aux", imx8mp_pcie_aux_sels, ccm_base + 0xa400); hws[IMX8MP_CLK_I2C5] = imx8m_clk_hw_composite("i2c5", imx8mp_i2c5_sels, ccm_base + 0xa480); @@ -647,8 +641,8 @@ static int imx8mp_clocks_probe(struct platform_device *pdev) hws[IMX8MP_CLK_MEDIA_MIPI_PHY1_REF] = imx8m_clk_hw_composite("media_mipi_phy1_ref", imx8mp_media_mipi_phy1_ref_sels, ccm_base + 0xbd80); hws[IMX8MP_CLK_MEDIA_DISP1_PIX] = imx8m_clk_hw_composite("media_disp1_pix", imx8mp_media_disp1_pix_sels, ccm_base + 0xbe00); hws[IMX8MP_CLK_MEDIA_CAM2_PIX] = imx8m_clk_hw_composite("media_cam2_pix", imx8mp_media_cam2_pix_sels, ccm_base + 0xbe80); - hws[IMX8MP_CLK_MEDIA_MIPI_PHY2_REF] = imx8m_clk_hw_composite("media_mipi_phy2_ref", imx8mp_media_mipi_phy2_ref_sels, ccm_base + 0xbf00); - hws[IMX8MP_CLK_MEDIA_MIPI_CSI2_ESC] = imx8m_clk_hw_composite("media_mipi_csi2_esc", imx8mp_media_mipi_csi2_esc_sels, ccm_base + 0xbf80); + hws[IMX8MP_CLK_MEDIA_LDB] = imx8m_clk_hw_composite("media_ldb", imx8mp_media_ldb_sels, ccm_base + 0xbf00); + hws[IMX8MP_CLK_MEMREPAIR] = imx8m_clk_hw_composite_critical("mem_repair", imx8mp_memrepair_sels, ccm_base + 0xbf80); hws[IMX8MP_CLK_PCIE2_CTRL] = imx8m_clk_hw_composite("pcie2_ctrl", imx8mp_pcie2_ctrl_sels, ccm_base + 0xc000); hws[IMX8MP_CLK_PCIE2_PHY] = imx8m_clk_hw_composite("pcie2_phy", imx8mp_pcie2_phy_sels, ccm_base + 0xc080); hws[IMX8MP_CLK_MEDIA_MIPI_TEST_BYTE] = imx8m_clk_hw_composite("media_mipi_test_byte", imx8mp_media_mipi_test_byte_sels, ccm_base + 0xc100); @@ -773,3 +767,7 @@ static struct platform_driver imx8mp_clk_driver = { }, }; module_platform_driver(imx8mp_clk_driver); + +MODULE_AUTHOR("Anson Huang <Anson.Huang@nxp.com>"); +MODULE_DESCRIPTION("NXP i.MX8MP clock driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/imx/clk-imx8mq.c b/drivers/clk/imx/clk-imx8mq.c index a64aace213c2..8265d1d48af4 100644 --- a/drivers/clk/imx/clk-imx8mq.c +++ b/drivers/clk/imx/clk-imx8mq.c @@ -157,10 +157,10 @@ static const char * const imx8mq_qspi_sels[] = {"osc_25m", "sys1_pll_400m", "sys "audio_pll2_out", "sys1_pll_266m", "sys3_pll_out", "sys1_pll_100m", }; static const char * const imx8mq_usdhc1_sels[] = {"osc_25m", "sys1_pll_400m", "sys1_pll_800m", "sys2_pll_500m", - "audio_pll2_out", "sys1_pll_266m", "sys3_pll_out", "sys1_pll_100m", }; + "sys3_pll_out", "sys1_pll_266m", "audio_pll2_out", "sys1_pll_100m", }; static const char * const imx8mq_usdhc2_sels[] = {"osc_25m", "sys1_pll_400m", "sys1_pll_800m", "sys2_pll_500m", - "audio_pll2_out", "sys1_pll_266m", "sys3_pll_out", "sys1_pll_100m", }; + "sys3_pll_out", "sys1_pll_266m", "audio_pll2_out", "sys1_pll_100m", }; static const char * const imx8mq_i2c1_sels[] = {"osc_25m", "sys1_pll_160m", "sys2_pll_50m", "sys3_pll_out", "audio_pll1_out", "video_pll1_out", "audio_pll2_out", "sys1_pll_133m", }; @@ -643,3 +643,7 @@ static struct platform_driver imx8mq_clk_driver = { }, }; module_platform_driver(imx8mq_clk_driver); + +MODULE_AUTHOR("Abel Vesa <abel.vesa@nxp.com>"); +MODULE_DESCRIPTION("NXP i.MX8MQ clock driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/imx/clk-imx8qxp-lpcg.c b/drivers/clk/imx/clk-imx8qxp-lpcg.c index 04c8ee35e14c..e947a70054ac 100644 --- a/drivers/clk/imx/clk-imx8qxp-lpcg.c +++ b/drivers/clk/imx/clk-imx8qxp-lpcg.c @@ -232,3 +232,7 @@ static struct platform_driver imx8qxp_lpcg_clk_driver = { }; builtin_platform_driver(imx8qxp_lpcg_clk_driver); + +MODULE_AUTHOR("Aisheng Dong <aisheng.dong@nxp.com>"); +MODULE_DESCRIPTION("NXP i.MX8QXP LPCG clock driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/imx/clk-imx8qxp.c b/drivers/clk/imx/clk-imx8qxp.c index 5e2903efc488..d650ca33cdc8 100644 --- a/drivers/clk/imx/clk-imx8qxp.c +++ b/drivers/clk/imx/clk-imx8qxp.c @@ -152,3 +152,7 @@ static struct platform_driver imx8qxp_clk_driver = { .probe = imx8qxp_clk_probe, }; builtin_platform_driver(imx8qxp_clk_driver); + +MODULE_AUTHOR("Aisheng Dong <aisheng.dong@nxp.com>"); +MODULE_DESCRIPTION("NXP i.MX8QXP clock driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/imx/clk-lpcg-scu.c b/drivers/clk/imx/clk-lpcg-scu.c index a73a799fb777..1f0e44f921ae 100644 --- a/drivers/clk/imx/clk-lpcg-scu.c +++ b/drivers/clk/imx/clk-lpcg-scu.c @@ -4,6 +4,7 @@ * Dong Aisheng <aisheng.dong@nxp.com> */ +#include <linux/bits.h> #include <linux/clk-provider.h> #include <linux/err.h> #include <linux/io.h> diff --git a/drivers/clk/imx/clk-pfd.c b/drivers/clk/imx/clk-pfd.c index 50b7c30296f7..5d2a9a3be95e 100644 --- a/drivers/clk/imx/clk-pfd.c +++ b/drivers/clk/imx/clk-pfd.c @@ -12,7 +12,7 @@ /** * struct clk_pfd - IMX PFD clock - * @clk_hw: clock source + * @hw: clock source * @reg: PFD register address * @idx: the index of PFD encoded in the register * diff --git a/drivers/clk/imx/clk-pfdv2.c b/drivers/clk/imx/clk-pfdv2.c index 78e1f7641aaa..6b744c84278e 100644 --- a/drivers/clk/imx/clk-pfdv2.c +++ b/drivers/clk/imx/clk-pfdv2.c @@ -17,7 +17,7 @@ /** * struct clk_pfdv2 - IMX PFD clock - * @clk_hw: clock source + * @hw: clock source * @reg: PFD register address * @gate_bit: Gate bit offset * @vld_bit: Valid bit offset diff --git a/drivers/clk/imx/clk-pll14xx.c b/drivers/clk/imx/clk-pll14xx.c index f9eb189b93c0..aba36e4217d2 100644 --- a/drivers/clk/imx/clk-pll14xx.c +++ b/drivers/clk/imx/clk-pll14xx.c @@ -3,9 +3,10 @@ * Copyright 2017-2018 NXP. */ -#include <linux/bitops.h> +#include <linux/bits.h> #include <linux/clk-provider.h> #include <linux/err.h> +#include <linux/export.h> #include <linux/io.h> #include <linux/iopoll.h> #include <linux/slab.h> @@ -68,6 +69,7 @@ struct imx_pll14xx_clk imx_1443x_pll = { .rate_table = imx_pll1443x_tbl, .rate_count = ARRAY_SIZE(imx_pll1443x_tbl), }; +EXPORT_SYMBOL_GPL(imx_1443x_pll); struct imx_pll14xx_clk imx_1443x_dram_pll = { .type = PLL_1443X, @@ -75,12 +77,14 @@ struct imx_pll14xx_clk imx_1443x_dram_pll = { .rate_count = ARRAY_SIZE(imx_pll1443x_tbl), .flags = CLK_GET_RATE_NOCACHE, }; +EXPORT_SYMBOL_GPL(imx_1443x_dram_pll); struct imx_pll14xx_clk imx_1416x_pll = { .type = PLL_1416X, .rate_table = imx_pll1416x_tbl, .rate_count = ARRAY_SIZE(imx_pll1416x_tbl), }; +EXPORT_SYMBOL_GPL(imx_1416x_pll); static const struct imx_pll14xx_rate_table *imx_get_pll_settings( struct clk_pll14xx *pll, unsigned long rate) @@ -436,3 +440,4 @@ struct clk_hw *imx_dev_clk_hw_pll14xx(struct device *dev, const char *name, return hw; } +EXPORT_SYMBOL_GPL(imx_dev_clk_hw_pll14xx); diff --git a/drivers/clk/imx/clk-pllv1.c b/drivers/clk/imx/clk-pllv1.c index de4f8a41a7d0..36ffb0525735 100644 --- a/drivers/clk/imx/clk-pllv1.c +++ b/drivers/clk/imx/clk-pllv1.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include <linux/bits.h> #include <linux/clk-provider.h> #include <linux/io.h> #include <linux/slab.h> diff --git a/drivers/clk/imx/clk-pllv3.c b/drivers/clk/imx/clk-pllv3.c index b20cdea3e9cc..20ee9611ba6e 100644 --- a/drivers/clk/imx/clk-pllv3.c +++ b/drivers/clk/imx/clk-pllv3.c @@ -30,12 +30,15 @@ /** * struct clk_pllv3 - IMX PLL clock version 3 - * @clk_hw: clock source + * @hw: clock source * @base: base address of PLL registers * @power_bit: pll power bit mask * @powerup_set: set power_bit to power up the PLL * @div_mask: mask of divider bits * @div_shift: shift of divider bits + * @ref_clock: reference clock rate + * @num_offset: num register offset + * @denom_offset: denom register offset * * IMX PLL clock version 3, found on i.MX6 series. Divider for pllv3 * is actually a multiplier, and always sits at bit 0. diff --git a/drivers/clk/imx/clk-pllv4.c b/drivers/clk/imx/clk-pllv4.c index a49450431855..8ec703f27417 100644 --- a/drivers/clk/imx/clk-pllv4.c +++ b/drivers/clk/imx/clk-pllv4.c @@ -7,6 +7,7 @@ * */ +#include <linux/bits.h> #include <linux/clk-provider.h> #include <linux/err.h> #include <linux/io.h> diff --git a/drivers/clk/imx/clk-sscg-pll.c b/drivers/clk/imx/clk-sscg-pll.c index 773d8a545cdf..9d6cdff0537f 100644 --- a/drivers/clk/imx/clk-sscg-pll.c +++ b/drivers/clk/imx/clk-sscg-pll.c @@ -10,6 +10,7 @@ #include <linux/clk-provider.h> #include <linux/err.h> +#include <linux/export.h> #include <linux/io.h> #include <linux/iopoll.h> #include <linux/slab.h> @@ -537,3 +538,4 @@ struct clk_hw *imx_clk_hw_sscg_pll(const char *name, return hw; } +EXPORT_SYMBOL_GPL(imx_clk_hw_sscg_pll); diff --git a/drivers/clk/imx/clk-vf610.c b/drivers/clk/imx/clk-vf610.c index 5129ef8e1d6e..9e11f1c7c397 100644 --- a/drivers/clk/imx/clk-vf610.c +++ b/drivers/clk/imx/clk-vf610.c @@ -4,6 +4,7 @@ */ #include <linux/of_address.h> +#include <linux/bits.h> #include <linux/clk.h> #include <linux/syscore_ops.h> #include <dt-bindings/clock/vf610-clock.h> @@ -328,6 +329,7 @@ static void __init vf610_clocks_init(struct device_node *ccm_node) clk[VF610_CLK_DSPI2] = imx_clk_gate2("dspi2", "ipg_bus", CCM_CCGR6, CCM_CCGRx_CGn(12)); clk[VF610_CLK_DSPI3] = imx_clk_gate2("dspi3", "ipg_bus", CCM_CCGR6, CCM_CCGRx_CGn(13)); + clk[VF610_CLK_CRC] = imx_clk_gate2("crc", "ipg_bus", CCM_CCGR1, CCM_CCGRx_CGn(3)); clk[VF610_CLK_WDT] = imx_clk_gate2("wdt", "ipg_bus", CCM_CCGR1, CCM_CCGRx_CGn(14)); clk[VF610_CLK_ESDHC0_SEL] = imx_clk_mux("esdhc0_sel", CCM_CSCMR1, 16, 2, esdhc_sels, 4); diff --git a/drivers/clk/imx/clk.c b/drivers/clk/imx/clk.c index 87ab8db3d282..47882c51cb85 100644 --- a/drivers/clk/imx/clk.c +++ b/drivers/clk/imx/clk.c @@ -1,8 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 +#include <linux/bits.h> #include <linux/clk.h> #include <linux/clk-provider.h> #include <linux/err.h> #include <linux/io.h> +#include <linux/module.h> #include <linux/of.h> #include <linux/slab.h> #include <linux/spinlock.h> @@ -13,6 +15,7 @@ #define CCDR_MMDC_CH1_MASK BIT(16) DEFINE_SPINLOCK(imx_ccm_lock); +EXPORT_SYMBOL_GPL(imx_ccm_lock); void imx_unregister_clocks(struct clk *clks[], unsigned int count) { @@ -29,8 +32,9 @@ void imx_unregister_hw_clocks(struct clk_hw *hws[], unsigned int count) for (i = 0; i < count; i++) clk_hw_unregister(hws[i]); } +EXPORT_SYMBOL_GPL(imx_unregister_hw_clocks); -void __init imx_mmdc_mask_handshake(void __iomem *ccm_base, +void imx_mmdc_mask_handshake(void __iomem *ccm_base, unsigned int chn) { unsigned int reg; @@ -59,8 +63,9 @@ void imx_check_clk_hws(struct clk_hw *clks[], unsigned int count) pr_err("i.MX clk %u: register failed with %ld\n", i, PTR_ERR(clks[i])); } +EXPORT_SYMBOL_GPL(imx_check_clk_hws); -static struct clk * __init imx_obtain_fixed_clock_from_dt(const char *name) +static struct clk *imx_obtain_fixed_clock_from_dt(const char *name) { struct of_phandle_args phandle; struct clk *clk = ERR_PTR(-ENODEV); @@ -80,7 +85,7 @@ static struct clk * __init imx_obtain_fixed_clock_from_dt(const char *name) return clk; } -struct clk * __init imx_obtain_fixed_clock( +struct clk *imx_obtain_fixed_clock( const char *name, unsigned long rate) { struct clk *clk; @@ -91,7 +96,7 @@ struct clk * __init imx_obtain_fixed_clock( return clk; } -struct clk_hw * __init imx_obtain_fixed_clock_hw( +struct clk_hw *imx_obtain_fixed_clock_hw( const char *name, unsigned long rate) { struct clk *clk; @@ -113,6 +118,7 @@ struct clk_hw * imx_obtain_fixed_clk_hw(struct device_node *np, return __clk_get_hw(clk); } +EXPORT_SYMBOL_GPL(imx_obtain_fixed_clk_hw); /* * This fixups the register CCM_CSCMR1 write value. @@ -140,6 +146,7 @@ void imx_cscmr1_fixup(u32 *val) return; } +#ifndef MODULE static int imx_keep_uart_clocks; static struct clk ** const *imx_uart_clocks; @@ -177,3 +184,6 @@ static int __init imx_clk_disable_uart(void) return 0; } late_initcall_sync(imx_clk_disable_uart); +#endif + +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/imx/clk.h b/drivers/clk/imx/clk.h index 16adbc34e05f..3b796b3da249 100644 --- a/drivers/clk/imx/clk.h +++ b/drivers/clk/imx/clk.h @@ -2,6 +2,7 @@ #ifndef __MACH_IMX_CLK_H #define __MACH_IMX_CLK_H +#include <linux/bits.h> #include <linux/spinlock.h> #include <linux/clk-provider.h> @@ -11,7 +12,13 @@ extern spinlock_t imx_ccm_lock; void imx_check_clocks(struct clk *clks[], unsigned int count); void imx_check_clk_hws(struct clk_hw *clks[], unsigned int count); +#ifndef MODULE void imx_register_uart_clocks(struct clk ** const clks[]); +#else +static inline void imx_register_uart_clocks(struct clk ** const clks[]) +{ +} +#endif void imx_mmdc_mask_handshake(void __iomem *ccm_base, unsigned int chn); void imx_unregister_clocks(struct clk *clks[], unsigned int count); void imx_unregister_hw_clocks(struct clk_hw *hws[], unsigned int count); diff --git a/drivers/clk/ingenic/cgu.c b/drivers/clk/ingenic/cgu.c index d7981b670221..dac6edc670cc 100644 --- a/drivers/clk/ingenic/cgu.c +++ b/drivers/clk/ingenic/cgu.c @@ -12,15 +12,24 @@ #include <linux/clkdev.h> #include <linux/delay.h> #include <linux/io.h> +#include <linux/iopoll.h> #include <linux/math64.h> #include <linux/of.h> #include <linux/of_address.h> #include <linux/slab.h> #include <linux/spinlock.h> +#include <linux/time.h> + #include "cgu.h" #define MHZ (1000 * 1000) +static inline const struct ingenic_cgu_clk_info * +to_clk_info(struct ingenic_clk *clk) +{ + return &clk->cgu->clock_info[clk->idx]; +} + /** * ingenic_cgu_gate_get() - get the value of clock gate register bit * @cgu: reference to the CGU whose registers should be read @@ -71,14 +80,13 @@ static unsigned long ingenic_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw); + const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk); struct ingenic_cgu *cgu = ingenic_clk->cgu; - const struct ingenic_cgu_clk_info *clk_info; const struct ingenic_cgu_pll_info *pll_info; unsigned m, n, od_enc, od; bool bypass; u32 ctl; - clk_info = &cgu->clock_info[ingenic_clk->idx]; BUG_ON(clk_info->type != CGU_CLK_PLL); pll_info = &clk_info->pll; @@ -144,18 +152,6 @@ ingenic_pll_calc(const struct ingenic_cgu_clk_info *clk_info, n * od); } -static inline const struct ingenic_cgu_clk_info *to_clk_info( - struct ingenic_clk *ingenic_clk) -{ - struct ingenic_cgu *cgu = ingenic_clk->cgu; - const struct ingenic_cgu_clk_info *clk_info; - - clk_info = &cgu->clock_info[ingenic_clk->idx]; - BUG_ON(clk_info->type != CGU_CLK_PLL); - - return clk_info; -} - static long ingenic_pll_round_rate(struct clk_hw *hw, unsigned long req_rate, unsigned long *prate) @@ -166,6 +162,16 @@ ingenic_pll_round_rate(struct clk_hw *hw, unsigned long req_rate, return ingenic_pll_calc(clk_info, req_rate, *prate, NULL, NULL, NULL); } +static inline int ingenic_pll_check_stable(struct ingenic_cgu *cgu, + const struct ingenic_cgu_pll_info *pll_info) +{ + u32 ctl; + + return readl_poll_timeout(cgu->base + pll_info->reg, ctl, + ctl & BIT(pll_info->stable_bit), + 0, 100 * USEC_PER_MSEC); +} + static int ingenic_pll_set_rate(struct clk_hw *hw, unsigned long req_rate, unsigned long parent_rate) @@ -176,6 +182,7 @@ ingenic_pll_set_rate(struct clk_hw *hw, unsigned long req_rate, const struct ingenic_cgu_pll_info *pll_info = &clk_info->pll; unsigned long rate, flags; unsigned int m, n, od; + int ret = 0; u32 ctl; rate = ingenic_pll_calc(clk_info, req_rate, parent_rate, @@ -197,9 +204,14 @@ ingenic_pll_set_rate(struct clk_hw *hw, unsigned long req_rate, ctl |= pll_info->od_encoding[od - 1] << pll_info->od_shift; writel(ctl, cgu->base + pll_info->reg); + + /* If the PLL is enabled, verify that it's stable */ + if (ctl & BIT(pll_info->enable_bit)) + ret = ingenic_pll_check_stable(cgu, pll_info); + spin_unlock_irqrestore(&cgu->lock, flags); - return 0; + return ret; } static int ingenic_pll_enable(struct clk_hw *hw) @@ -208,9 +220,8 @@ static int ingenic_pll_enable(struct clk_hw *hw) struct ingenic_cgu *cgu = ingenic_clk->cgu; const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk); const struct ingenic_cgu_pll_info *pll_info = &clk_info->pll; - const unsigned int timeout = 100; unsigned long flags; - unsigned int i; + int ret; u32 ctl; spin_lock_irqsave(&cgu->lock, flags); @@ -226,20 +237,10 @@ static int ingenic_pll_enable(struct clk_hw *hw) writel(ctl, cgu->base + pll_info->reg); - /* wait for the PLL to stabilise */ - for (i = 0; i < timeout; i++) { - ctl = readl(cgu->base + pll_info->reg); - if (ctl & BIT(pll_info->stable_bit)) - break; - mdelay(1); - } - + ret = ingenic_pll_check_stable(cgu, pll_info); spin_unlock_irqrestore(&cgu->lock, flags); - if (i == timeout) - return -EBUSY; - - return 0; + return ret; } static void ingenic_pll_disable(struct clk_hw *hw) @@ -290,13 +291,11 @@ static const struct clk_ops ingenic_pll_ops = { static u8 ingenic_clk_get_parent(struct clk_hw *hw) { struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw); + const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk); struct ingenic_cgu *cgu = ingenic_clk->cgu; - const struct ingenic_cgu_clk_info *clk_info; u32 reg; u8 i, hw_idx, idx = 0; - clk_info = &cgu->clock_info[ingenic_clk->idx]; - if (clk_info->type & CGU_CLK_MUX) { reg = readl(cgu->base + clk_info->mux.reg); hw_idx = (reg >> clk_info->mux.shift) & @@ -318,14 +317,12 @@ static u8 ingenic_clk_get_parent(struct clk_hw *hw) static int ingenic_clk_set_parent(struct clk_hw *hw, u8 idx) { struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw); + const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk); struct ingenic_cgu *cgu = ingenic_clk->cgu; - const struct ingenic_cgu_clk_info *clk_info; unsigned long flags; u8 curr_idx, hw_idx, num_poss; u32 reg, mask; - clk_info = &cgu->clock_info[ingenic_clk->idx]; - if (clk_info->type & CGU_CLK_MUX) { /* * Convert the parent index to the hardware index by adding @@ -368,13 +365,11 @@ static unsigned long ingenic_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw); + const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk); struct ingenic_cgu *cgu = ingenic_clk->cgu; - const struct ingenic_cgu_clk_info *clk_info; unsigned long rate = parent_rate; u32 div_reg, div; - clk_info = &cgu->clock_info[ingenic_clk->idx]; - if (clk_info->type & CGU_CLK_DIV) { div_reg = readl(cgu->base + clk_info->div.reg); div = (div_reg >> clk_info->div.shift) & @@ -443,35 +438,41 @@ ingenic_clk_round_rate(struct clk_hw *hw, unsigned long req_rate, unsigned long *parent_rate) { struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw); - struct ingenic_cgu *cgu = ingenic_clk->cgu; - const struct ingenic_cgu_clk_info *clk_info; + const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk); unsigned int div = 1; - clk_info = &cgu->clock_info[ingenic_clk->idx]; - if (clk_info->type & CGU_CLK_DIV) div = ingenic_clk_calc_div(clk_info, *parent_rate, req_rate); else if (clk_info->type & CGU_CLK_FIXDIV) div = clk_info->fixdiv.div; + else if (clk_hw_can_set_rate_parent(hw)) + *parent_rate = req_rate; return DIV_ROUND_UP(*parent_rate, div); } +static inline int ingenic_clk_check_stable(struct ingenic_cgu *cgu, + const struct ingenic_cgu_clk_info *clk_info) +{ + u32 reg; + + return readl_poll_timeout(cgu->base + clk_info->div.reg, reg, + !(reg & BIT(clk_info->div.busy_bit)), + 0, 100 * USEC_PER_MSEC); +} + static int ingenic_clk_set_rate(struct clk_hw *hw, unsigned long req_rate, unsigned long parent_rate) { struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw); + const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk); struct ingenic_cgu *cgu = ingenic_clk->cgu; - const struct ingenic_cgu_clk_info *clk_info; - const unsigned timeout = 100; unsigned long rate, flags; - unsigned int hw_div, div, i; + unsigned int hw_div, div; u32 reg, mask; int ret = 0; - clk_info = &cgu->clock_info[ingenic_clk->idx]; - if (clk_info->type & CGU_CLK_DIV) { div = ingenic_clk_calc_div(clk_info, parent_rate, req_rate); rate = DIV_ROUND_UP(parent_rate, div); @@ -504,16 +505,8 @@ ingenic_clk_set_rate(struct clk_hw *hw, unsigned long req_rate, writel(reg, cgu->base + clk_info->div.reg); /* wait for the change to take effect */ - if (clk_info->div.busy_bit != -1) { - for (i = 0; i < timeout; i++) { - reg = readl(cgu->base + clk_info->div.reg); - if (!(reg & BIT(clk_info->div.busy_bit))) - break; - mdelay(1); - } - if (i == timeout) - ret = -EBUSY; - } + if (clk_info->div.busy_bit != -1) + ret = ingenic_clk_check_stable(cgu, clk_info); spin_unlock_irqrestore(&cgu->lock, flags); return ret; @@ -525,12 +518,10 @@ ingenic_clk_set_rate(struct clk_hw *hw, unsigned long req_rate, static int ingenic_clk_enable(struct clk_hw *hw) { struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw); + const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk); struct ingenic_cgu *cgu = ingenic_clk->cgu; - const struct ingenic_cgu_clk_info *clk_info; unsigned long flags; - clk_info = &cgu->clock_info[ingenic_clk->idx]; - if (clk_info->type & CGU_CLK_GATE) { /* ungate the clock */ spin_lock_irqsave(&cgu->lock, flags); @@ -547,12 +538,10 @@ static int ingenic_clk_enable(struct clk_hw *hw) static void ingenic_clk_disable(struct clk_hw *hw) { struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw); + const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk); struct ingenic_cgu *cgu = ingenic_clk->cgu; - const struct ingenic_cgu_clk_info *clk_info; unsigned long flags; - clk_info = &cgu->clock_info[ingenic_clk->idx]; - if (clk_info->type & CGU_CLK_GATE) { /* gate the clock */ spin_lock_irqsave(&cgu->lock, flags); @@ -564,12 +553,10 @@ static void ingenic_clk_disable(struct clk_hw *hw) static int ingenic_clk_is_enabled(struct clk_hw *hw) { struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw); + const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk); struct ingenic_cgu *cgu = ingenic_clk->cgu; - const struct ingenic_cgu_clk_info *clk_info; int enabled = 1; - clk_info = &cgu->clock_info[ingenic_clk->idx]; - if (clk_info->type & CGU_CLK_GATE) enabled = !ingenic_cgu_gate_get(cgu, &clk_info->gate); @@ -644,6 +631,13 @@ static int ingenic_register_clock(struct ingenic_cgu *cgu, unsigned idx) caps = clk_info->type; + if (caps & CGU_CLK_DIV) { + caps &= ~CGU_CLK_DIV; + } else if (!(caps & CGU_CLK_CUSTOM)) { + /* pass rate changes to the parent clock */ + clk_init.flags |= CLK_SET_RATE_PARENT; + } + if (caps & (CGU_CLK_MUX | CGU_CLK_CUSTOM)) { clk_init.num_parents = 0; @@ -683,7 +677,6 @@ static int ingenic_register_clock(struct ingenic_cgu *cgu, unsigned idx) } } else if (caps & CGU_CLK_PLL) { clk_init.ops = &ingenic_pll_ops; - clk_init.flags |= CLK_SET_RATE_GATE; caps &= ~CGU_CLK_PLL; @@ -706,13 +699,6 @@ static int ingenic_register_clock(struct ingenic_cgu *cgu, unsigned idx) caps &= ~(CGU_CLK_MUX | CGU_CLK_MUX_GLITCHFREE); } - if (caps & CGU_CLK_DIV) { - caps &= ~CGU_CLK_DIV; - } else { - /* pass rate changes to the parent clock */ - clk_init.flags |= CLK_SET_RATE_PARENT; - } - if (caps) { pr_err("%s: unknown clock type 0x%x\n", __func__, caps); goto out; diff --git a/drivers/clk/keystone/sci-clk.c b/drivers/clk/keystone/sci-clk.c index 2ad26cb927fd..aaf31abe1c8f 100644 --- a/drivers/clk/keystone/sci-clk.c +++ b/drivers/clk/keystone/sci-clk.c @@ -54,6 +54,8 @@ struct sci_clk_provider { * @provider: Master clock provider * @flags: Flags for the clock * @node: Link for handling clocks probed via DT + * @cached_req: Cached requested freq for determine rate calls + * @cached_res: Cached result freq for determine rate calls */ struct sci_clk { struct clk_hw hw; @@ -63,6 +65,8 @@ struct sci_clk { struct sci_clk_provider *provider; u8 flags; struct list_head node; + unsigned long cached_req; + unsigned long cached_res; }; #define to_sci_clk(_hw) container_of(_hw, struct sci_clk, hw) @@ -175,6 +179,11 @@ static int sci_clk_determine_rate(struct clk_hw *hw, int ret; u64 new_rate; + if (clk->cached_req && clk->cached_req == req->rate) { + req->rate = clk->cached_res; + return 0; + } + ret = clk->provider->ops->get_best_match_freq(clk->provider->sci, clk->dev_id, clk->clk_id, @@ -189,6 +198,9 @@ static int sci_clk_determine_rate(struct clk_hw *hw, return ret; } + clk->cached_req = req->rate; + clk->cached_res = new_rate; + req->rate = new_rate; return 0; @@ -209,7 +221,8 @@ static int sci_clk_set_rate(struct clk_hw *hw, unsigned long rate, struct sci_clk *clk = to_sci_clk(hw); return clk->provider->ops->set_freq(clk->provider->sci, clk->dev_id, - clk->clk_id, rate, rate, rate); + clk->clk_id, rate / 10 * 9, rate, + rate / 10 * 11); } /** @@ -249,6 +262,8 @@ static int sci_clk_set_parent(struct clk_hw *hw, u8 index) { struct sci_clk *clk = to_sci_clk(hw); + clk->cached_req = 0; + return clk->provider->ops->set_parent(clk->provider->sci, clk->dev_id, clk->clk_id, index + 1 + clk->clk_id); @@ -522,7 +537,7 @@ static int ti_sci_scan_clocks_from_dt(struct sci_clk_provider *provider) np = of_find_node_with_property(np, *clk_name); if (!np) { clk_name++; - break; + continue; } if (!of_device_is_available(np)) diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig index 89ceb2fbc7c4..ce8475098b31 100644 --- a/drivers/clk/mediatek/Kconfig +++ b/drivers/clk/mediatek/Kconfig @@ -352,6 +352,54 @@ config COMMON_CLK_MT8135 help This driver supports MediaTek MT8135 clocks. +config COMMON_CLK_MT8167 + bool "Clock driver for MediaTek MT8167" + depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST + select COMMON_CLK_MEDIATEK + default ARCH_MEDIATEK + help + This driver supports MediaTek MT8167 basic clocks. + +config COMMON_CLK_MT8167_AUDSYS + bool "Clock driver for MediaTek MT8167 audsys" + depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST + select COMMON_CLK_MEDIATEK + default ARCH_MEDIATEK + help + This driver supports MediaTek MT8167 audsys clocks. + +config COMMON_CLK_MT8167_IMGSYS + bool "Clock driver for MediaTek MT8167 imgsys" + depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST + select COMMON_CLK_MEDIATEK + default ARCH_MEDIATEK + help + This driver supports MediaTek MT8167 imgsys clocks. + +config COMMON_CLK_MT8167_MFGCFG + bool "Clock driver for MediaTek MT8167 mfgcfg" + depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST + select COMMON_CLK_MEDIATEK + default ARCH_MEDIATEK + help + This driver supports MediaTek MT8167 mfgcfg clocks. + +config COMMON_CLK_MT8167_MMSYS + bool "Clock driver for MediaTek MT8167 mmsys" + depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST + select COMMON_CLK_MEDIATEK + default ARCH_MEDIATEK + help + This driver supports MediaTek MT8167 mmsys clocks. + +config COMMON_CLK_MT8167_VDECSYS + bool "Clock driver for MediaTek MT8167 vdecsys" + depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST + select COMMON_CLK_MEDIATEK + default ARCH_MEDIATEK + help + This driver supports MediaTek MT8167 vdecsys clocks. + config COMMON_CLK_MT8173 bool "Clock driver for MediaTek MT8173" depends on ARCH_MEDIATEK || COMPILE_TEST diff --git a/drivers/clk/mediatek/Makefile b/drivers/clk/mediatek/Makefile index 959b556d32ea..3b0c2be73824 100644 --- a/drivers/clk/mediatek/Makefile +++ b/drivers/clk/mediatek/Makefile @@ -47,6 +47,12 @@ obj-$(CONFIG_COMMON_CLK_MT7629) += clk-mt7629.o obj-$(CONFIG_COMMON_CLK_MT7629_ETHSYS) += clk-mt7629-eth.o obj-$(CONFIG_COMMON_CLK_MT7629_HIFSYS) += clk-mt7629-hif.o obj-$(CONFIG_COMMON_CLK_MT8135) += clk-mt8135.o +obj-$(CONFIG_COMMON_CLK_MT8167) += clk-mt8167.o +obj-$(CONFIG_COMMON_CLK_MT8167_AUDSYS) += clk-mt8167-aud.o +obj-$(CONFIG_COMMON_CLK_MT8167_IMGSYS) += clk-mt8167-img.o +obj-$(CONFIG_COMMON_CLK_MT8167_MFGCFG) += clk-mt8167-mfgcfg.o +obj-$(CONFIG_COMMON_CLK_MT8167_MMSYS) += clk-mt8167-mm.o +obj-$(CONFIG_COMMON_CLK_MT8167_VDECSYS) += clk-mt8167-vdec.o obj-$(CONFIG_COMMON_CLK_MT8173) += clk-mt8173.o obj-$(CONFIG_COMMON_CLK_MT8173_MMSYS) += clk-mt8173-mm.o obj-$(CONFIG_COMMON_CLK_MT8183) += clk-mt8183.o diff --git a/drivers/clk/mediatek/clk-mt6765.c b/drivers/clk/mediatek/clk-mt6765.c index db8db1b3b79d..d77ea5aff292 100644 --- a/drivers/clk/mediatek/clk-mt6765.c +++ b/drivers/clk/mediatek/clk-mt6765.c @@ -909,7 +909,6 @@ static struct platform_driver clk_mt6765_drv = { .probe = clk_mt6765_probe, .driver = { .name = "clk-mt6765", - .owner = THIS_MODULE, .of_match_table = of_match_clk_mt6765, }, }; diff --git a/drivers/clk/mediatek/clk-mt6779.c b/drivers/clk/mediatek/clk-mt6779.c index 9766cccf5844..6e0d3a166729 100644 --- a/drivers/clk/mediatek/clk-mt6779.c +++ b/drivers/clk/mediatek/clk-mt6779.c @@ -919,6 +919,8 @@ static const struct mtk_gate infra_clks[] = { "pwm_sel", 19), GATE_INFRA0(CLK_INFRA_PWM, "infra_pwm", "pwm_sel", 21), + GATE_INFRA0(CLK_INFRA_UART0, "infra_uart0", + "uart_sel", 22), GATE_INFRA0(CLK_INFRA_UART1, "infra_uart1", "uart_sel", 23), GATE_INFRA0(CLK_INFRA_UART2, "infra_uart2", diff --git a/drivers/clk/mediatek/clk-mt6797.c b/drivers/clk/mediatek/clk-mt6797.c index f35389a11af1..428eb24ffec5 100644 --- a/drivers/clk/mediatek/clk-mt6797.c +++ b/drivers/clk/mediatek/clk-mt6797.c @@ -582,7 +582,7 @@ CLK_OF_DECLARE_DRIVER(mtk_infra, "mediatek,mt6797-infracfg", static int mtk_infrasys_init(struct platform_device *pdev) { - int r, i; + int i; struct device_node *node = pdev->dev.of_node; if (!infra_clk_data) { @@ -599,11 +599,7 @@ static int mtk_infrasys_init(struct platform_device *pdev) mtk_clk_register_factors(infra_fixed_divs, ARRAY_SIZE(infra_fixed_divs), infra_clk_data); - r = of_clk_add_provider(node, of_clk_src_onecell_get, infra_clk_data); - if (r) - return r; - - return 0; + return of_clk_add_provider(node, of_clk_src_onecell_get, infra_clk_data); } #define MT6797_PLL_FMAX (3000UL * MHZ) diff --git a/drivers/clk/mediatek/clk-mt7629.c b/drivers/clk/mediatek/clk-mt7629.c index b73bdf152836..a0ee079670c7 100644 --- a/drivers/clk/mediatek/clk-mt7629.c +++ b/drivers/clk/mediatek/clk-mt7629.c @@ -601,7 +601,6 @@ static int mtk_infrasys_init(struct platform_device *pdev) { struct device_node *node = pdev->dev.of_node; struct clk_onecell_data *clk_data; - int r; clk_data = mtk_alloc_clk_data(CLK_INFRA_NR_CLK); @@ -611,12 +610,8 @@ static int mtk_infrasys_init(struct platform_device *pdev) mtk_clk_register_cpumuxes(node, infra_muxes, ARRAY_SIZE(infra_muxes), clk_data); - r = of_clk_add_provider(node, of_clk_src_onecell_get, - clk_data); - if (r) - return r; - - return 0; + return of_clk_add_provider(node, of_clk_src_onecell_get, + clk_data); } static int mtk_pericfg_init(struct platform_device *pdev) diff --git a/drivers/clk/mediatek/clk-mt8167-aud.c b/drivers/clk/mediatek/clk-mt8167-aud.c new file mode 100644 index 000000000000..3f7bf6485792 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt8167-aud.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 MediaTek Inc. + * Copyright (c) 2020 BayLibre, SAS + * Author: James Liao <jamesjj.liao@mediatek.com> + * Fabien Parent <fparent@baylibre.com> + */ + +#include <linux/clk-provider.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> + +#include "clk-mtk.h" +#include "clk-gate.h" + +#include <dt-bindings/clock/mt8167-clk.h> + +static const struct mtk_gate_regs aud_cg_regs = { + .set_ofs = 0x0, + .clr_ofs = 0x0, + .sta_ofs = 0x0, +}; + +#define GATE_AUD(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &aud_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_no_setclr, \ + } + +static const struct mtk_gate aud_clks[] __initconst = { + GATE_AUD(CLK_AUD_AFE, "aud_afe", "clk26m_ck", 2), + GATE_AUD(CLK_AUD_I2S, "aud_i2s", "i2s_infra_bck", 6), + GATE_AUD(CLK_AUD_22M, "aud_22m", "rg_aud_engen1", 8), + GATE_AUD(CLK_AUD_24M, "aud_24m", "rg_aud_engen2", 9), + GATE_AUD(CLK_AUD_INTDIR, "aud_intdir", "rg_aud_spdif_in", 15), + GATE_AUD(CLK_AUD_APLL2_TUNER, "aud_apll2_tuner", "rg_aud_engen2", 18), + GATE_AUD(CLK_AUD_APLL_TUNER, "aud_apll_tuner", "rg_aud_engen1", 19), + GATE_AUD(CLK_AUD_HDMI, "aud_hdmi", "apll12_div4", 20), + GATE_AUD(CLK_AUD_SPDF, "aud_spdf", "apll12_div6", 21), + GATE_AUD(CLK_AUD_ADC, "aud_adc", "aud_afe", 24), + GATE_AUD(CLK_AUD_DAC, "aud_dac", "aud_afe", 25), + GATE_AUD(CLK_AUD_DAC_PREDIS, "aud_dac_predis", "aud_afe", 26), + GATE_AUD(CLK_AUD_TML, "aud_tml", "aud_afe", 27), +}; + +static void __init mtk_audsys_init(struct device_node *node) +{ + struct clk_onecell_data *clk_data; + int r; + + clk_data = mtk_alloc_clk_data(CLK_AUD_NR_CLK); + + mtk_clk_register_gates(node, aud_clks, ARRAY_SIZE(aud_clks), clk_data); + + r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data); + if (r) + pr_err("%s(): could not register clock provider: %d\n", + __func__, r); + +} +CLK_OF_DECLARE(mtk_audsys, "mediatek,mt8167-audsys", mtk_audsys_init); diff --git a/drivers/clk/mediatek/clk-mt8167-img.c b/drivers/clk/mediatek/clk-mt8167-img.c new file mode 100644 index 000000000000..3b4ec9eae432 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt8167-img.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 MediaTek Inc. + * Copyright (c) 2020 BayLibre, SAS + * Author: James Liao <jamesjj.liao@mediatek.com> + * Fabien Parent <fparent@baylibre.com> + */ + +#include <linux/clk-provider.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> + +#include "clk-mtk.h" +#include "clk-gate.h" + +#include <dt-bindings/clock/mt8167-clk.h> + +static const struct mtk_gate_regs img_cg_regs = { + .set_ofs = 0x4, + .clr_ofs = 0x8, + .sta_ofs = 0x0, +}; + +#define GATE_IMG(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &img_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +static const struct mtk_gate img_clks[] __initconst = { + GATE_IMG(CLK_IMG_LARB1_SMI, "img_larb1_smi", "smi_mm", 0), + GATE_IMG(CLK_IMG_CAM_SMI, "img_cam_smi", "smi_mm", 5), + GATE_IMG(CLK_IMG_CAM_CAM, "img_cam_cam", "smi_mm", 6), + GATE_IMG(CLK_IMG_SEN_TG, "img_sen_tg", "cam_mm", 7), + GATE_IMG(CLK_IMG_SEN_CAM, "img_sen_cam", "smi_mm", 8), + GATE_IMG(CLK_IMG_VENC, "img_venc", "smi_mm", 9), +}; + +static void __init mtk_imgsys_init(struct device_node *node) +{ + struct clk_onecell_data *clk_data; + int r; + + clk_data = mtk_alloc_clk_data(CLK_IMG_NR_CLK); + + mtk_clk_register_gates(node, img_clks, ARRAY_SIZE(img_clks), clk_data); + + r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data); + + if (r) + pr_err("%s(): could not register clock provider: %d\n", + __func__, r); + +} +CLK_OF_DECLARE(mtk_imgsys, "mediatek,mt8167-imgsys", mtk_imgsys_init); diff --git a/drivers/clk/mediatek/clk-mt8167-mfgcfg.c b/drivers/clk/mediatek/clk-mt8167-mfgcfg.c new file mode 100644 index 000000000000..90b871730f2d --- /dev/null +++ b/drivers/clk/mediatek/clk-mt8167-mfgcfg.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 MediaTek Inc. + * Copyright (c) 2020 BayLibre, SAS + * Author: James Liao <jamesjj.liao@mediatek.com> + * Fabien Parent <fparent@baylibre.com> + */ + +#include <linux/clk-provider.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> + +#include "clk-mtk.h" +#include "clk-gate.h" + +#include <dt-bindings/clock/mt8167-clk.h> + +static const struct mtk_gate_regs mfg_cg_regs = { + .set_ofs = 0x4, + .clr_ofs = 0x8, + .sta_ofs = 0x0, +}; + +#define GATE_MFG(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &mfg_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +static const struct mtk_gate mfg_clks[] __initconst = { + GATE_MFG(CLK_MFG_BAXI, "mfg_baxi", "ahb_infra_sel", 0), + GATE_MFG(CLK_MFG_BMEM, "mfg_bmem", "gfmux_emi1x_sel", 1), + GATE_MFG(CLK_MFG_BG3D, "mfg_bg3d", "mfg_mm", 2), + GATE_MFG(CLK_MFG_B26M, "mfg_b26m", "clk26m_ck", 3), +}; + +static void __init mtk_mfgcfg_init(struct device_node *node) +{ + struct clk_onecell_data *clk_data; + int r; + + clk_data = mtk_alloc_clk_data(CLK_MFG_NR_CLK); + + mtk_clk_register_gates(node, mfg_clks, ARRAY_SIZE(mfg_clks), clk_data); + + r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data); + + if (r) + pr_err("%s(): could not register clock provider: %d\n", + __func__, r); + +} +CLK_OF_DECLARE(mtk_mfgcfg, "mediatek,mt8167-mfgcfg", mtk_mfgcfg_init); diff --git a/drivers/clk/mediatek/clk-mt8167-mm.c b/drivers/clk/mediatek/clk-mt8167-mm.c new file mode 100644 index 000000000000..963b129aade1 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt8167-mm.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 MediaTek Inc. + * Copyright (c) 2020 BayLibre, SAS + * Author: James Liao <jamesjj.liao@mediatek.com> + * Fabien Parent <fparent@baylibre.com> + */ + +#include <linux/clk-provider.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> + +#include "clk-mtk.h" +#include "clk-gate.h" + +#include <dt-bindings/clock/mt8167-clk.h> + +static const struct mtk_gate_regs mm0_cg_regs = { + .set_ofs = 0x104, + .clr_ofs = 0x108, + .sta_ofs = 0x100, +}; + +static const struct mtk_gate_regs mm1_cg_regs = { + .set_ofs = 0x114, + .clr_ofs = 0x118, + .sta_ofs = 0x110, +}; + +#define GATE_MM0(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &mm0_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +#define GATE_MM1(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &mm1_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +static const struct mtk_gate mm_clks[] = { + /* MM0 */ + GATE_MM0(CLK_MM_SMI_COMMON, "mm_smi_common", "smi_mm", 0), + GATE_MM0(CLK_MM_SMI_LARB0, "mm_smi_larb0", "smi_mm", 1), + GATE_MM0(CLK_MM_CAM_MDP, "mm_cam_mdp", "smi_mm", 2), + GATE_MM0(CLK_MM_MDP_RDMA, "mm_mdp_rdma", "smi_mm", 3), + GATE_MM0(CLK_MM_MDP_RSZ0, "mm_mdp_rsz0", "smi_mm", 4), + GATE_MM0(CLK_MM_MDP_RSZ1, "mm_mdp_rsz1", "smi_mm", 5), + GATE_MM0(CLK_MM_MDP_TDSHP, "mm_mdp_tdshp", "smi_mm", 6), + GATE_MM0(CLK_MM_MDP_WDMA, "mm_mdp_wdma", "smi_mm", 7), + GATE_MM0(CLK_MM_MDP_WROT, "mm_mdp_wrot", "smi_mm", 8), + GATE_MM0(CLK_MM_FAKE_ENG, "mm_fake_eng", "smi_mm", 9), + GATE_MM0(CLK_MM_DISP_OVL0, "mm_disp_ovl0", "smi_mm", 10), + GATE_MM0(CLK_MM_DISP_RDMA0, "mm_disp_rdma0", "smi_mm", 11), + GATE_MM0(CLK_MM_DISP_RDMA1, "mm_disp_rdma1", "smi_mm", 12), + GATE_MM0(CLK_MM_DISP_WDMA, "mm_disp_wdma", "smi_mm", 13), + GATE_MM0(CLK_MM_DISP_COLOR, "mm_disp_color", "smi_mm", 14), + GATE_MM0(CLK_MM_DISP_CCORR, "mm_disp_ccorr", "smi_mm", 15), + GATE_MM0(CLK_MM_DISP_AAL, "mm_disp_aal", "smi_mm", 16), + GATE_MM0(CLK_MM_DISP_GAMMA, "mm_disp_gamma", "smi_mm", 17), + GATE_MM0(CLK_MM_DISP_DITHER, "mm_disp_dither", "smi_mm", 18), + GATE_MM0(CLK_MM_DISP_UFOE, "mm_disp_ufoe", "smi_mm", 19), + /* MM1 */ + GATE_MM1(CLK_MM_DISP_PWM_MM, "mm_disp_pwm_mm", "smi_mm", 0), + GATE_MM1(CLK_MM_DISP_PWM_26M, "mm_disp_pwm_26m", "smi_mm", 1), + GATE_MM1(CLK_MM_DSI_ENGINE, "mm_dsi_engine", "smi_mm", 2), + GATE_MM1(CLK_MM_DSI_DIGITAL, "mm_dsi_digital", "dsi0_lntc_dsick", 3), + GATE_MM1(CLK_MM_DPI0_ENGINE, "mm_dpi0_engine", "smi_mm", 4), + GATE_MM1(CLK_MM_DPI0_PXL, "mm_dpi0_pxl", "rg_fdpi0", 5), + GATE_MM1(CLK_MM_LVDS_PXL, "mm_lvds_pxl", "vpll_dpix", 14), + GATE_MM1(CLK_MM_LVDS_CTS, "mm_lvds_cts", "lvdstx_dig_cts", 15), + GATE_MM1(CLK_MM_DPI1_ENGINE, "mm_dpi1_engine", "smi_mm", 16), + GATE_MM1(CLK_MM_DPI1_PXL, "mm_dpi1_pxl", "rg_fdpi1", 17), + GATE_MM1(CLK_MM_HDMI_PXL, "mm_hdmi_pxl", "rg_fdpi1", 18), + GATE_MM1(CLK_MM_HDMI_SPDIF, "mm_hdmi_spdif", "apll12_div6", 19), + GATE_MM1(CLK_MM_HDMI_ADSP_BCK, "mm_hdmi_adsp_b", "apll12_div4b", 20), + GATE_MM1(CLK_MM_HDMI_PLL, "mm_hdmi_pll", "hdmtx_dig_cts", 21), +}; + +struct clk_mt8167_mm_driver_data { + const struct mtk_gate *gates_clk; + int gates_num; +}; + +static const struct clk_mt8167_mm_driver_data mt8167_mmsys_driver_data = { + .gates_clk = mm_clks, + .gates_num = ARRAY_SIZE(mm_clks), +}; + +static int clk_mt8167_mm_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *node = dev->parent->of_node; + const struct clk_mt8167_mm_driver_data *data; + struct clk_onecell_data *clk_data; + int ret; + + clk_data = mtk_alloc_clk_data(CLK_MM_NR_CLK); + if (!clk_data) + return -ENOMEM; + + data = &mt8167_mmsys_driver_data; + + ret = mtk_clk_register_gates(node, data->gates_clk, data->gates_num, + clk_data); + if (ret) + return ret; + + ret = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data); + if (ret) + return ret; + + return 0; +} + +static struct platform_driver clk_mt8173_mm_drv = { + .driver = { + .name = "clk-mt8167-mm", + }, + .probe = clk_mt8167_mm_probe, +}; + +builtin_platform_driver(clk_mt8173_mm_drv); diff --git a/drivers/clk/mediatek/clk-mt8167-vdec.c b/drivers/clk/mediatek/clk-mt8167-vdec.c new file mode 100644 index 000000000000..910b28355ec0 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt8167-vdec.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 MediaTek Inc. + * Copyright (c) 2020 BayLibre, SAS + * Author: James Liao <jamesjj.liao@mediatek.com> + * Fabien Parent <fparent@baylibre.com> + */ + +#include <linux/clk-provider.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> + +#include "clk-mtk.h" +#include "clk-gate.h" + +#include <dt-bindings/clock/mt8167-clk.h> + +static const struct mtk_gate_regs vdec0_cg_regs = { + .set_ofs = 0x0, + .clr_ofs = 0x4, + .sta_ofs = 0x0, +}; + +static const struct mtk_gate_regs vdec1_cg_regs = { + .set_ofs = 0x8, + .clr_ofs = 0xc, + .sta_ofs = 0x8, +}; + +#define GATE_VDEC0_I(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &vdec0_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr_inv, \ + } + +#define GATE_VDEC1_I(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &vdec1_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr_inv, \ + } + +static const struct mtk_gate vdec_clks[] __initconst = { + /* VDEC0 */ + GATE_VDEC0_I(CLK_VDEC_CKEN, "vdec_cken", "rg_vdec", 0), + /* VDEC1 */ + GATE_VDEC1_I(CLK_VDEC_LARB1_CKEN, "vdec_larb1_cken", "smi_mm", 0), +}; + +static void __init mtk_vdecsys_init(struct device_node *node) +{ + struct clk_onecell_data *clk_data; + int r; + + clk_data = mtk_alloc_clk_data(CLK_VDEC_NR_CLK); + + mtk_clk_register_gates(node, vdec_clks, ARRAY_SIZE(vdec_clks), clk_data); + + r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data); + + if (r) + pr_err("%s(): could not register clock provider: %d\n", + __func__, r); + +} +CLK_OF_DECLARE(mtk_vdecsys, "mediatek,mt8167-vdecsys", mtk_vdecsys_init); diff --git a/drivers/clk/mediatek/clk-mt8167.c b/drivers/clk/mediatek/clk-mt8167.c new file mode 100644 index 000000000000..e5ea10e31799 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt8167.c @@ -0,0 +1,1062 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 MediaTek Inc. + * Copyright (c) 2020 BayLibre, SAS + * Author: James Liao <jamesjj.liao@mediatek.com> + * Fabien Parent <fparent@baylibre.com> + */ + +#include <linux/delay.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/slab.h> +#include <linux/mfd/syscon.h> + +#include "clk-mtk.h" +#include "clk-gate.h" + +#include <dt-bindings/clock/mt8167-clk.h> + +static DEFINE_SPINLOCK(mt8167_clk_lock); + +static const struct mtk_fixed_clk fixed_clks[] __initconst = { + FIXED_CLK(CLK_TOP_CLK_NULL, "clk_null", NULL, 0), + FIXED_CLK(CLK_TOP_I2S_INFRA_BCK, "i2s_infra_bck", "clk_null", 26000000), + FIXED_CLK(CLK_TOP_MEMPLL, "mempll", "clk26m", 800000000), + FIXED_CLK(CLK_TOP_DSI0_LNTC_DSICK, "dsi0_lntc_dsick", "clk26m", 75000000), + FIXED_CLK(CLK_TOP_VPLL_DPIX, "vpll_dpix", "clk26m", 75000000), + FIXED_CLK(CLK_TOP_LVDSTX_CLKDIG_CTS, "lvdstx_dig_cts", "clk26m", 52500000), +}; + +static const struct mtk_fixed_factor top_divs[] __initconst = { + FACTOR(CLK_TOP_DMPLL, "dmpll_ck", "mempll", 1, 1), + FACTOR(CLK_TOP_MAINPLL_D2, "mainpll_d2", "mainpll", 1, 2), + FACTOR(CLK_TOP_MAINPLL_D4, "mainpll_d4", "mainpll", 1, 4), + FACTOR(CLK_TOP_MAINPLL_D8, "mainpll_d8", "mainpll", 1, 8), + FACTOR(CLK_TOP_MAINPLL_D16, "mainpll_d16", "mainpll", 1, 16), + FACTOR(CLK_TOP_MAINPLL_D11, "mainpll_d11", "mainpll", 1, 11), + FACTOR(CLK_TOP_MAINPLL_D22, "mainpll_d22", "mainpll", 1, 22), + FACTOR(CLK_TOP_MAINPLL_D3, "mainpll_d3", "mainpll", 1, 3), + FACTOR(CLK_TOP_MAINPLL_D6, "mainpll_d6", "mainpll", 1, 6), + FACTOR(CLK_TOP_MAINPLL_D12, "mainpll_d12", "mainpll", 1, 12), + FACTOR(CLK_TOP_MAINPLL_D5, "mainpll_d5", "mainpll", 1, 5), + FACTOR(CLK_TOP_MAINPLL_D10, "mainpll_d10", "mainpll", 1, 10), + FACTOR(CLK_TOP_MAINPLL_D20, "mainpll_d20", "mainpll", 1, 20), + FACTOR(CLK_TOP_MAINPLL_D40, "mainpll_d40", "mainpll", 1, 40), + FACTOR(CLK_TOP_MAINPLL_D7, "mainpll_d7", "mainpll", 1, 7), + FACTOR(CLK_TOP_MAINPLL_D14, "mainpll_d14", "mainpll", 1, 14), + FACTOR(CLK_TOP_UNIVPLL_D2, "univpll_d2", "univpll", 1, 2), + FACTOR(CLK_TOP_UNIVPLL_D4, "univpll_d4", "univpll", 1, 4), + FACTOR(CLK_TOP_UNIVPLL_D8, "univpll_d8", "univpll", 1, 8), + FACTOR(CLK_TOP_UNIVPLL_D16, "univpll_d16", "univpll", 1, 16), + FACTOR(CLK_TOP_UNIVPLL_D3, "univpll_d3", "univpll", 1, 3), + FACTOR(CLK_TOP_UNIVPLL_D6, "univpll_d6", "univpll", 1, 6), + FACTOR(CLK_TOP_UNIVPLL_D12, "univpll_d12", "univpll", 1, 12), + FACTOR(CLK_TOP_UNIVPLL_D24, "univpll_d24", "univpll", 1, 24), + FACTOR(CLK_TOP_UNIVPLL_D5, "univpll_d5", "univpll", 1, 5), + FACTOR(CLK_TOP_UNIVPLL_D20, "univpll_d20", "univpll", 1, 20), + FACTOR(CLK_TOP_MMPLL380M, "mmpll380m", "mmpll", 1, 1), + FACTOR(CLK_TOP_MMPLL_D2, "mmpll_d2", "mmpll", 1, 2), + FACTOR(CLK_TOP_MMPLL_200M, "mmpll_200m", "mmpll", 1, 3), + FACTOR(CLK_TOP_LVDSPLL, "lvdspll_ck", "lvdspll", 1, 1), + FACTOR(CLK_TOP_LVDSPLL_D2, "lvdspll_d2", "lvdspll", 1, 2), + FACTOR(CLK_TOP_LVDSPLL_D4, "lvdspll_d4", "lvdspll", 1, 4), + FACTOR(CLK_TOP_LVDSPLL_D8, "lvdspll_d8", "lvdspll", 1, 8), + FACTOR(CLK_TOP_USB_PHY48M, "usb_phy48m_ck", "univpll", 1, 26), + FACTOR(CLK_TOP_APLL1, "apll1_ck", "apll1", 1, 1), + FACTOR(CLK_TOP_APLL1_D2, "apll1_d2", "apll1_ck", 1, 2), + FACTOR(CLK_TOP_APLL1_D4, "apll1_d4", "rg_apll1_d2_en", 1, 2), + FACTOR(CLK_TOP_APLL1_D8, "apll1_d8", "rg_apll1_d4_en", 1, 2), + FACTOR(CLK_TOP_APLL2, "apll2_ck", "apll2", 1, 1), + FACTOR(CLK_TOP_APLL2_D2, "apll2_d2", "apll2_ck", 1, 2), + FACTOR(CLK_TOP_APLL2_D4, "apll2_d4", "rg_apll2_d2_en", 1, 2), + FACTOR(CLK_TOP_APLL2_D8, "apll2_d8", "rg_apll2_d4_en", 1, 2), + FACTOR(CLK_TOP_CLK26M, "clk26m_ck", "clk26m", 1, 1), + FACTOR(CLK_TOP_CLK26M_D2, "clk26m_d2", "clk26m", 1, 2), + FACTOR(CLK_TOP_MIPI_26M, "mipi_26m", "clk26m", 1, 1), + FACTOR(CLK_TOP_TVDPLL, "tvdpll_ck", "tvdpll", 1, 1), + FACTOR(CLK_TOP_TVDPLL_D2, "tvdpll_d2", "tvdpll_ck", 1, 2), + FACTOR(CLK_TOP_TVDPLL_D4, "tvdpll_d4", "tvdpll_ck", 1, 4), + FACTOR(CLK_TOP_TVDPLL_D8, "tvdpll_d8", "tvdpll_ck", 1, 8), + FACTOR(CLK_TOP_TVDPLL_D16, "tvdpll_d16", "tvdpll_ck", 1, 16), + FACTOR(CLK_TOP_AHB_INFRA_D2, "ahb_infra_d2", "ahb_infra_sel", 1, 2), + FACTOR(CLK_TOP_NFI1X, "nfi1x_ck", "nfi2x_pad_sel", 1, 2), + FACTOR(CLK_TOP_ETH_D2, "eth_d2_ck", "eth_sel", 1, 2), +}; + +static const char * const uart0_parents[] __initconst = { + "clk26m_ck", + "univpll_d24" +}; + +static const char * const gfmux_emi1x_parents[] __initconst = { + "clk26m_ck", + "dmpll_ck" +}; + +static const char * const emi_ddrphy_parents[] __initconst = { + "gfmux_emi1x_sel", + "gfmux_emi1x_sel" +}; + +static const char * const ahb_infra_parents[] __initconst = { + "clk_null", + "clk26m_ck", + "mainpll_d11", + "clk_null", + "mainpll_d12", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "mainpll_d10" +}; + +static const char * const csw_mux_mfg_parents[] __initconst = { + "clk_null", + "clk_null", + "univpll_d3", + "univpll_d2", + "clk26m_ck", + "mainpll_d4", + "univpll_d24", + "mmpll380m" +}; + +static const char * const msdc0_parents[] __initconst = { + "clk26m_ck", + "univpll_d6", + "mainpll_d8", + "univpll_d8", + "mainpll_d16", + "mmpll_200m", + "mainpll_d12", + "mmpll_d2" +}; + +static const char * const camtg_mm_parents[] __initconst = { + "clk_null", + "clk26m_ck", + "usb_phy48m_ck", + "clk_null", + "univpll_d6" +}; + +static const char * const pwm_mm_parents[] __initconst = { + "clk26m_ck", + "univpll_d12" +}; + +static const char * const uart1_parents[] __initconst = { + "clk26m_ck", + "univpll_d24" +}; + +static const char * const msdc1_parents[] __initconst = { + "clk26m_ck", + "univpll_d6", + "mainpll_d8", + "univpll_d8", + "mainpll_d16", + "mmpll_200m", + "mainpll_d12", + "mmpll_d2" +}; + +static const char * const spm_52m_parents[] __initconst = { + "clk26m_ck", + "univpll_d24" +}; + +static const char * const pmicspi_parents[] __initconst = { + "univpll_d20", + "usb_phy48m_ck", + "univpll_d16", + "clk26m_ck" +}; + +static const char * const qaxi_aud26m_parents[] __initconst = { + "clk26m_ck", + "ahb_infra_sel" +}; + +static const char * const aud_intbus_parents[] __initconst = { + "clk_null", + "clk26m_ck", + "mainpll_d22", + "clk_null", + "mainpll_d11" +}; + +static const char * const nfi2x_pad_parents[] __initconst = { + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk26m_ck", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "mainpll_d12", + "mainpll_d8", + "clk_null", + "mainpll_d6", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "mainpll_d4", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "mainpll_d10", + "mainpll_d7", + "clk_null", + "mainpll_d5" +}; + +static const char * const nfi1x_pad_parents[] __initconst = { + "ahb_infra_sel", + "nfi1x_ck" +}; + +static const char * const mfg_mm_parents[] __initconst = { + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "csw_mux_mfg_sel", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "mainpll_d3", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "mainpll_d5", + "mainpll_d7", + "clk_null", + "mainpll_d14" +}; + +static const char * const ddrphycfg_parents[] __initconst = { + "clk26m_ck", + "mainpll_d16" +}; + +static const char * const smi_mm_parents[] __initconst = { + "clk26m_ck", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "clk_null", + "univpll_d4", + "mainpll_d7", + "clk_null", + "mainpll_d14" +}; + +static const char * const usb_78m_parents[] __initconst = { + "clk_null", + "clk26m_ck", + "univpll_d16", + "clk_null", + "mainpll_d20" +}; + +static const char * const scam_mm_parents[] __initconst = { + "clk_null", + "clk26m_ck", + "mainpll_d14", + "clk_null", + "mainpll_d12" +}; + +static const char * const spinor_parents[] __initconst = { + "clk26m_d2", + "clk26m_ck", + "mainpll_d40", + "univpll_d24", + "univpll_d20", + "mainpll_d20", + "mainpll_d16", + "univpll_d12" +}; + +static const char * const msdc2_parents[] __initconst = { + "clk26m_ck", + "univpll_d6", + "mainpll_d8", + "univpll_d8", + "mainpll_d16", + "mmpll_200m", + "mainpll_d12", + "mmpll_d2" +}; + +static const char * const eth_parents[] __initconst = { + "clk26m_ck", + "mainpll_d40", + "univpll_d24", + "univpll_d20", + "mainpll_d20" +}; + +static const char * const vdec_mm_parents[] __initconst = { + "clk26m_ck", + "univpll_d4", + "mainpll_d4", + "univpll_d5", + "univpll_d6", + "mainpll_d6" +}; + +static const char * const dpi0_mm_parents[] __initconst = { + "clk26m_ck", + "lvdspll_ck", + "lvdspll_d2", + "lvdspll_d4", + "lvdspll_d8" +}; + +static const char * const dpi1_mm_parents[] __initconst = { + "clk26m_ck", + "tvdpll_d2", + "tvdpll_d4", + "tvdpll_d8", + "tvdpll_d16" +}; + +static const char * const axi_mfg_in_parents[] __initconst = { + "clk26m_ck", + "mainpll_d11", + "univpll_d24", + "mmpll380m" +}; + +static const char * const slow_mfg_parents[] __initconst = { + "clk26m_ck", + "univpll_d12", + "univpll_d24" +}; + +static const char * const aud1_parents[] __initconst = { + "clk26m_ck", + "apll1_ck" +}; + +static const char * const aud2_parents[] __initconst = { + "clk26m_ck", + "apll2_ck" +}; + +static const char * const aud_engen1_parents[] __initconst = { + "clk26m_ck", + "rg_apll1_d2_en", + "rg_apll1_d4_en", + "rg_apll1_d8_en" +}; + +static const char * const aud_engen2_parents[] __initconst = { + "clk26m_ck", + "rg_apll2_d2_en", + "rg_apll2_d4_en", + "rg_apll2_d8_en" +}; + +static const char * const i2c_parents[] __initconst = { + "clk26m_ck", + "univpll_d20", + "univpll_d16", + "univpll_d12" +}; + +static const char * const aud_i2s0_m_parents[] __initconst = { + "rg_aud1", + "rg_aud2" +}; + +static const char * const pwm_parents[] __initconst = { + "clk26m_ck", + "univpll_d12" +}; + +static const char * const spi_parents[] __initconst = { + "clk26m_ck", + "univpll_d12", + "univpll_d8", + "univpll_d6" +}; + +static const char * const aud_spdifin_parents[] __initconst = { + "clk26m_ck", + "univpll_d2" +}; + +static const char * const uart2_parents[] __initconst = { + "clk26m_ck", + "univpll_d24" +}; + +static const char * const bsi_parents[] __initconst = { + "clk26m_ck", + "mainpll_d10", + "mainpll_d12", + "mainpll_d20" +}; + +static const char * const dbg_atclk_parents[] __initconst = { + "clk_null", + "clk26m_ck", + "mainpll_d5", + "clk_null", + "univpll_d5" +}; + +static const char * const csw_nfiecc_parents[] __initconst = { + "clk_null", + "mainpll_d7", + "mainpll_d6", + "clk_null", + "mainpll_d5" +}; + +static const char * const nfiecc_parents[] __initconst = { + "clk_null", + "nfi2x_pad_sel", + "mainpll_d4", + "clk_null", + "csw_nfiecc_sel" +}; + +static struct mtk_composite top_muxes[] __initdata = { + /* CLK_MUX_SEL0 */ + MUX(CLK_TOP_UART0_SEL, "uart0_sel", uart0_parents, + 0x000, 0, 1), + MUX(CLK_TOP_GFMUX_EMI1X_SEL, "gfmux_emi1x_sel", gfmux_emi1x_parents, + 0x000, 1, 1), + MUX(CLK_TOP_EMI_DDRPHY_SEL, "emi_ddrphy_sel", emi_ddrphy_parents, + 0x000, 2, 1), + MUX(CLK_TOP_AHB_INFRA_SEL, "ahb_infra_sel", ahb_infra_parents, + 0x000, 4, 4), + MUX(CLK_TOP_CSW_MUX_MFG_SEL, "csw_mux_mfg_sel", csw_mux_mfg_parents, + 0x000, 8, 3), + MUX(CLK_TOP_MSDC0_SEL, "msdc0_sel", msdc0_parents, + 0x000, 11, 3), + MUX(CLK_TOP_CAMTG_MM_SEL, "camtg_mm_sel", camtg_mm_parents, + 0x000, 15, 3), + MUX(CLK_TOP_PWM_MM_SEL, "pwm_mm_sel", pwm_mm_parents, + 0x000, 18, 1), + MUX(CLK_TOP_UART1_SEL, "uart1_sel", uart1_parents, + 0x000, 19, 1), + MUX(CLK_TOP_MSDC1_SEL, "msdc1_sel", msdc1_parents, + 0x000, 20, 3), + MUX(CLK_TOP_SPM_52M_SEL, "spm_52m_sel", spm_52m_parents, + 0x000, 23, 1), + MUX(CLK_TOP_PMICSPI_SEL, "pmicspi_sel", pmicspi_parents, + 0x000, 24, 2), + MUX(CLK_TOP_QAXI_AUD26M_SEL, "qaxi_aud26m_sel", qaxi_aud26m_parents, + 0x000, 26, 1), + MUX(CLK_TOP_AUD_INTBUS_SEL, "aud_intbus_sel", aud_intbus_parents, + 0x000, 27, 3), + /* CLK_MUX_SEL1 */ + MUX(CLK_TOP_NFI2X_PAD_SEL, "nfi2x_pad_sel", nfi2x_pad_parents, + 0x004, 0, 7), + MUX(CLK_TOP_NFI1X_PAD_SEL, "nfi1x_pad_sel", nfi1x_pad_parents, + 0x004, 7, 1), + MUX(CLK_TOP_MFG_MM_SEL, "mfg_mm_sel", mfg_mm_parents, + 0x004, 8, 6), + MUX(CLK_TOP_DDRPHYCFG_SEL, "ddrphycfg_sel", ddrphycfg_parents, + 0x004, 15, 1), + MUX(CLK_TOP_SMI_MM_SEL, "smi_mm_sel", smi_mm_parents, + 0x004, 16, 4), + MUX(CLK_TOP_USB_78M_SEL, "usb_78m_sel", usb_78m_parents, + 0x004, 20, 3), + MUX(CLK_TOP_SCAM_MM_SEL, "scam_mm_sel", scam_mm_parents, + 0x004, 23, 3), + /* CLK_MUX_SEL8 */ + MUX(CLK_TOP_SPINOR_SEL, "spinor_sel", spinor_parents, + 0x040, 0, 3), + MUX(CLK_TOP_MSDC2_SEL, "msdc2_sel", msdc2_parents, + 0x040, 3, 3), + MUX(CLK_TOP_ETH_SEL, "eth_sel", eth_parents, + 0x040, 6, 3), + MUX(CLK_TOP_VDEC_MM_SEL, "vdec_mm_sel", vdec_mm_parents, + 0x040, 9, 3), + MUX(CLK_TOP_DPI0_MM_SEL, "dpi0_mm_sel", dpi0_mm_parents, + 0x040, 12, 3), + MUX(CLK_TOP_DPI1_MM_SEL, "dpi1_mm_sel", dpi1_mm_parents, + 0x040, 15, 3), + MUX(CLK_TOP_AXI_MFG_IN_SEL, "axi_mfg_in_sel", axi_mfg_in_parents, + 0x040, 18, 2), + MUX(CLK_TOP_SLOW_MFG_SEL, "slow_mfg_sel", slow_mfg_parents, + 0x040, 20, 2), + MUX(CLK_TOP_AUD1_SEL, "aud1_sel", aud1_parents, + 0x040, 22, 1), + MUX(CLK_TOP_AUD2_SEL, "aud2_sel", aud2_parents, + 0x040, 23, 1), + MUX(CLK_TOP_AUD_ENGEN1_SEL, "aud_engen1_sel", aud_engen1_parents, + 0x040, 24, 2), + MUX(CLK_TOP_AUD_ENGEN2_SEL, "aud_engen2_sel", aud_engen2_parents, + 0x040, 26, 2), + MUX(CLK_TOP_I2C_SEL, "i2c_sel", i2c_parents, + 0x040, 28, 2), + /* CLK_SEL_9 */ + MUX(CLK_TOP_AUD_I2S0_M_SEL, "aud_i2s0_m_sel", aud_i2s0_m_parents, + 0x044, 12, 1), + MUX(CLK_TOP_AUD_I2S1_M_SEL, "aud_i2s1_m_sel", aud_i2s0_m_parents, + 0x044, 13, 1), + MUX(CLK_TOP_AUD_I2S2_M_SEL, "aud_i2s2_m_sel", aud_i2s0_m_parents, + 0x044, 14, 1), + MUX(CLK_TOP_AUD_I2S3_M_SEL, "aud_i2s3_m_sel", aud_i2s0_m_parents, + 0x044, 15, 1), + MUX(CLK_TOP_AUD_I2S4_M_SEL, "aud_i2s4_m_sel", aud_i2s0_m_parents, + 0x044, 16, 1), + MUX(CLK_TOP_AUD_I2S5_M_SEL, "aud_i2s5_m_sel", aud_i2s0_m_parents, + 0x044, 17, 1), + MUX(CLK_TOP_AUD_SPDIF_B_SEL, "aud_spdif_b_sel", aud_i2s0_m_parents, + 0x044, 18, 1), + /* CLK_MUX_SEL13 */ + MUX(CLK_TOP_PWM_SEL, "pwm_sel", pwm_parents, + 0x07c, 0, 1), + MUX(CLK_TOP_SPI_SEL, "spi_sel", spi_parents, + 0x07c, 1, 2), + MUX(CLK_TOP_AUD_SPDIFIN_SEL, "aud_spdifin_sel", aud_spdifin_parents, + 0x07c, 3, 1), + MUX(CLK_TOP_UART2_SEL, "uart2_sel", uart2_parents, + 0x07c, 4, 1), + MUX(CLK_TOP_BSI_SEL, "bsi_sel", bsi_parents, + 0x07c, 5, 2), + MUX(CLK_TOP_DBG_ATCLK_SEL, "dbg_atclk_sel", dbg_atclk_parents, + 0x07c, 7, 3), + MUX(CLK_TOP_CSW_NFIECC_SEL, "csw_nfiecc_sel", csw_nfiecc_parents, + 0x07c, 10, 3), + MUX(CLK_TOP_NFIECC_SEL, "nfiecc_sel", nfiecc_parents, + 0x07c, 13, 3), +}; + +static const char * const ifr_mux1_parents[] __initconst = { + "clk26m_ck", + "armpll", + "univpll", + "mainpll_d2" +}; + +static const char * const ifr_eth_25m_parents[] __initconst = { + "eth_d2_ck", + "rg_eth" +}; + +static const char * const ifr_i2c0_parents[] __initconst = { + "ahb_infra_d2", + "rg_i2c" +}; + +static const struct mtk_composite ifr_muxes[] __initconst = { + MUX(CLK_IFR_MUX1_SEL, "ifr_mux1_sel", ifr_mux1_parents, 0x000, + 2, 2), + MUX(CLK_IFR_ETH_25M_SEL, "ifr_eth_25m_sel", ifr_eth_25m_parents, 0x080, + 0, 1), + MUX(CLK_IFR_I2C0_SEL, "ifr_i2c0_sel", ifr_i2c0_parents, 0x080, + 1, 1), + MUX(CLK_IFR_I2C1_SEL, "ifr_i2c1_sel", ifr_i2c0_parents, 0x080, + 2, 1), + MUX(CLK_IFR_I2C2_SEL, "ifr_i2c2_sel", ifr_i2c0_parents, 0x080, + 3, 1), +}; + +#define DIV_ADJ(_id, _name, _parent, _reg, _shift, _width) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .div_reg = _reg, \ + .div_shift = _shift, \ + .div_width = _width, \ +} + +static const struct mtk_clk_divider top_adj_divs[] = { + DIV_ADJ(CLK_TOP_APLL12_CK_DIV0, "apll12_ck_div0", "aud_i2s0_m_sel", + 0x0048, 0, 8), + DIV_ADJ(CLK_TOP_APLL12_CK_DIV1, "apll12_ck_div1", "aud_i2s1_m_sel", + 0x0048, 8, 8), + DIV_ADJ(CLK_TOP_APLL12_CK_DIV2, "apll12_ck_div2", "aud_i2s2_m_sel", + 0x0048, 16, 8), + DIV_ADJ(CLK_TOP_APLL12_CK_DIV3, "apll12_ck_div3", "aud_i2s3_m_sel", + 0x0048, 24, 8), + DIV_ADJ(CLK_TOP_APLL12_CK_DIV4, "apll12_ck_div4", "aud_i2s4_m_sel", + 0x004c, 0, 8), + DIV_ADJ(CLK_TOP_APLL12_CK_DIV4B, "apll12_ck_div4b", "apll12_div4", + 0x004c, 8, 8), + DIV_ADJ(CLK_TOP_APLL12_CK_DIV5, "apll12_ck_div5", "aud_i2s5_m_sel", + 0x004c, 16, 8), + DIV_ADJ(CLK_TOP_APLL12_CK_DIV5B, "apll12_ck_div5b", "apll12_div5", + 0x004c, 24, 8), + DIV_ADJ(CLK_TOP_APLL12_CK_DIV6, "apll12_ck_div6", "aud_spdif_b_sel", + 0x0078, 0, 8), +}; + +#define DIV_ADJ_FLAG(_id, _name, _parent, _reg, _shift, _width, _flag) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .div_reg = _reg, \ + .div_shift = _shift, \ + .div_width = _width, \ + .clk_divider_flags = _flag, \ +} + +static const struct mtk_clk_divider apmixed_adj_divs[] = { + DIV_ADJ_FLAG(CLK_APMIXED_HDMI_REF, "hdmi_ref", "tvdpll", + 0x1c4, 24, 3, CLK_DIVIDER_POWER_OF_TWO), +}; + +static const struct mtk_gate_regs top0_cg_regs = { + .set_ofs = 0x50, + .clr_ofs = 0x80, + .sta_ofs = 0x20, +}; + +static const struct mtk_gate_regs top1_cg_regs = { + .set_ofs = 0x54, + .clr_ofs = 0x84, + .sta_ofs = 0x24, +}; + +static const struct mtk_gate_regs top2_cg_regs = { + .set_ofs = 0x6c, + .clr_ofs = 0x9c, + .sta_ofs = 0x3c, +}; + +static const struct mtk_gate_regs top3_cg_regs = { + .set_ofs = 0xa0, + .clr_ofs = 0xb0, + .sta_ofs = 0x70, +}; + +static const struct mtk_gate_regs top4_cg_regs = { + .set_ofs = 0xa4, + .clr_ofs = 0xb4, + .sta_ofs = 0x74, +}; + +static const struct mtk_gate_regs top5_cg_regs = { + .set_ofs = 0x44, + .clr_ofs = 0x44, + .sta_ofs = 0x44, +}; + +#define GATE_TOP0(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &top0_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +#define GATE_TOP0_I(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &top0_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr_inv, \ + } + +#define GATE_TOP1(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &top1_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +#define GATE_TOP2(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &top2_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +#define GATE_TOP2_I(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &top2_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr_inv, \ + } + +#define GATE_TOP3(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &top3_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +#define GATE_TOP4_I(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &top4_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr_inv, \ + } + +#define GATE_TOP5(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &top5_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_no_setclr, \ + } + +static const struct mtk_gate top_clks[] __initconst = { + /* TOP0 */ + GATE_TOP0(CLK_TOP_PWM_MM, "pwm_mm", "pwm_mm_sel", 0), + GATE_TOP0(CLK_TOP_CAM_MM, "cam_mm", "camtg_mm_sel", 1), + GATE_TOP0(CLK_TOP_MFG_MM, "mfg_mm", "mfg_mm_sel", 2), + GATE_TOP0(CLK_TOP_SPM_52M, "spm_52m", "spm_52m_sel", 3), + GATE_TOP0_I(CLK_TOP_MIPI_26M_DBG, "mipi_26m_dbg", "mipi_26m", 4), + GATE_TOP0(CLK_TOP_SCAM_MM, "scam_mm", "scam_mm_sel", 5), + GATE_TOP0(CLK_TOP_SMI_MM, "smi_mm", "smi_mm_sel", 9), + /* TOP1 */ + GATE_TOP1(CLK_TOP_THEM, "them", "ahb_infra_sel", 1), + GATE_TOP1(CLK_TOP_APDMA, "apdma", "ahb_infra_sel", 2), + GATE_TOP1(CLK_TOP_I2C0, "i2c0", "ifr_i2c0_sel", 3), + GATE_TOP1(CLK_TOP_I2C1, "i2c1", "ifr_i2c1_sel", 4), + GATE_TOP1(CLK_TOP_AUXADC1, "auxadc1", "ahb_infra_sel", 5), + GATE_TOP1(CLK_TOP_NFI, "nfi", "nfi1x_pad_sel", 6), + GATE_TOP1(CLK_TOP_NFIECC, "nfiecc", "rg_nfiecc", 7), + GATE_TOP1(CLK_TOP_DEBUGSYS, "debugsys", "rg_dbg_atclk", 8), + GATE_TOP1(CLK_TOP_PWM, "pwm", "ahb_infra_sel", 9), + GATE_TOP1(CLK_TOP_UART0, "uart0", "uart0_sel", 10), + GATE_TOP1(CLK_TOP_UART1, "uart1", "uart1_sel", 11), + GATE_TOP1(CLK_TOP_BTIF, "btif", "ahb_infra_sel", 12), + GATE_TOP1(CLK_TOP_USB, "usb", "usb_78m", 13), + GATE_TOP1(CLK_TOP_FLASHIF_26M, "flashif_26m", "clk26m_ck", 14), + GATE_TOP1(CLK_TOP_AUXADC2, "auxadc2", "ahb_infra_sel", 15), + GATE_TOP1(CLK_TOP_I2C2, "i2c2", "ifr_i2c2_sel", 16), + GATE_TOP1(CLK_TOP_MSDC0, "msdc0", "msdc0_sel", 17), + GATE_TOP1(CLK_TOP_MSDC1, "msdc1", "msdc1_sel", 18), + GATE_TOP1(CLK_TOP_NFI2X, "nfi2x", "nfi2x_pad_sel", 19), + GATE_TOP1(CLK_TOP_PMICWRAP_AP, "pwrap_ap", "clk26m_ck", 20), + GATE_TOP1(CLK_TOP_SEJ, "sej", "ahb_infra_sel", 21), + GATE_TOP1(CLK_TOP_MEMSLP_DLYER, "memslp_dlyer", "clk26m_ck", 22), + GATE_TOP1(CLK_TOP_SPI, "spi", "spi_sel", 23), + GATE_TOP1(CLK_TOP_APXGPT, "apxgpt", "clk26m_ck", 24), + GATE_TOP1(CLK_TOP_AUDIO, "audio", "clk26m_ck", 25), + GATE_TOP1(CLK_TOP_PMICWRAP_MD, "pwrap_md", "clk26m_ck", 27), + GATE_TOP1(CLK_TOP_PMICWRAP_CONN, "pwrap_conn", "clk26m_ck", 28), + GATE_TOP1(CLK_TOP_PMICWRAP_26M, "pwrap_26m", "clk26m_ck", 29), + GATE_TOP1(CLK_TOP_AUX_ADC, "aux_adc", "clk26m_ck", 30), + GATE_TOP1(CLK_TOP_AUX_TP, "aux_tp", "clk26m_ck", 31), + /* TOP2 */ + GATE_TOP2(CLK_TOP_MSDC2, "msdc2", "ahb_infra_sel", 0), + GATE_TOP2(CLK_TOP_RBIST, "rbist", "univpll_d12", 1), + GATE_TOP2(CLK_TOP_NFI_BUS, "nfi_bus", "ahb_infra_sel", 2), + GATE_TOP2(CLK_TOP_GCE, "gce", "ahb_infra_sel", 4), + GATE_TOP2(CLK_TOP_TRNG, "trng", "ahb_infra_sel", 5), + GATE_TOP2(CLK_TOP_SEJ_13M, "sej_13m", "clk26m_ck", 6), + GATE_TOP2(CLK_TOP_AES, "aes", "ahb_infra_sel", 7), + GATE_TOP2(CLK_TOP_PWM_B, "pwm_b", "rg_pwm_infra", 8), + GATE_TOP2(CLK_TOP_PWM1_FB, "pwm1_fb", "rg_pwm_infra", 9), + GATE_TOP2(CLK_TOP_PWM2_FB, "pwm2_fb", "rg_pwm_infra", 10), + GATE_TOP2(CLK_TOP_PWM3_FB, "pwm3_fb", "rg_pwm_infra", 11), + GATE_TOP2(CLK_TOP_PWM4_FB, "pwm4_fb", "rg_pwm_infra", 12), + GATE_TOP2(CLK_TOP_PWM5_FB, "pwm5_fb", "rg_pwm_infra", 13), + GATE_TOP2(CLK_TOP_USB_1P, "usb_1p", "usb_78m", 14), + GATE_TOP2(CLK_TOP_FLASHIF_FREERUN, "flashif_freerun", "ahb_infra_sel", + 15), + GATE_TOP2(CLK_TOP_26M_HDMI_SIFM, "hdmi_sifm_26m", "clk26m_ck", 16), + GATE_TOP2(CLK_TOP_26M_CEC, "cec_26m", "clk26m_ck", 17), + GATE_TOP2(CLK_TOP_32K_CEC, "cec_32k", "clk32k", 18), + GATE_TOP2(CLK_TOP_66M_ETH, "eth_66m", "ahb_infra_d2", 19), + GATE_TOP2(CLK_TOP_133M_ETH, "eth_133m", "ahb_infra_sel", 20), + GATE_TOP2(CLK_TOP_FETH_25M, "feth_25m", "ifr_eth_25m_sel", 21), + GATE_TOP2(CLK_TOP_FETH_50M, "feth_50m", "rg_eth", 22), + GATE_TOP2(CLK_TOP_FLASHIF_AXI, "flashif_axi", "ahb_infra_sel", 23), + GATE_TOP2(CLK_TOP_USBIF, "usbif", "ahb_infra_sel", 24), + GATE_TOP2(CLK_TOP_UART2, "uart2", "rg_uart2", 25), + GATE_TOP2(CLK_TOP_BSI, "bsi", "ahb_infra_sel", 26), + GATE_TOP2(CLK_TOP_GCPU_B, "gcpu_b", "ahb_infra_sel", 27), + GATE_TOP2_I(CLK_TOP_MSDC0_INFRA, "msdc0_infra", "msdc0", 28), + GATE_TOP2_I(CLK_TOP_MSDC1_INFRA, "msdc1_infra", "msdc1", 29), + GATE_TOP2_I(CLK_TOP_MSDC2_INFRA, "msdc2_infra", "rg_msdc2", 30), + GATE_TOP2(CLK_TOP_USB_78M, "usb_78m", "usb_78m_sel", 31), + /* TOP3 */ + GATE_TOP3(CLK_TOP_RG_SPINOR, "rg_spinor", "spinor_sel", 0), + GATE_TOP3(CLK_TOP_RG_MSDC2, "rg_msdc2", "msdc2_sel", 1), + GATE_TOP3(CLK_TOP_RG_ETH, "rg_eth", "eth_sel", 2), + GATE_TOP3(CLK_TOP_RG_VDEC, "rg_vdec", "vdec_mm_sel", 3), + GATE_TOP3(CLK_TOP_RG_FDPI0, "rg_fdpi0", "dpi0_mm_sel", 4), + GATE_TOP3(CLK_TOP_RG_FDPI1, "rg_fdpi1", "dpi1_mm_sel", 5), + GATE_TOP3(CLK_TOP_RG_AXI_MFG, "rg_axi_mfg", "axi_mfg_in_sel", 6), + GATE_TOP3(CLK_TOP_RG_SLOW_MFG, "rg_slow_mfg", "slow_mfg_sel", 7), + GATE_TOP3(CLK_TOP_RG_AUD1, "rg_aud1", "aud1_sel", 8), + GATE_TOP3(CLK_TOP_RG_AUD2, "rg_aud2", "aud2_sel", 9), + GATE_TOP3(CLK_TOP_RG_AUD_ENGEN1, "rg_aud_engen1", "aud_engen1_sel", 10), + GATE_TOP3(CLK_TOP_RG_AUD_ENGEN2, "rg_aud_engen2", "aud_engen2_sel", 11), + GATE_TOP3(CLK_TOP_RG_I2C, "rg_i2c", "i2c_sel", 12), + GATE_TOP3(CLK_TOP_RG_PWM_INFRA, "rg_pwm_infra", "pwm_sel", 13), + GATE_TOP3(CLK_TOP_RG_AUD_SPDIF_IN, "rg_aud_spdif_in", "aud_spdifin_sel", + 14), + GATE_TOP3(CLK_TOP_RG_UART2, "rg_uart2", "uart2_sel", 15), + GATE_TOP3(CLK_TOP_RG_BSI, "rg_bsi", "bsi_sel", 16), + GATE_TOP3(CLK_TOP_RG_DBG_ATCLK, "rg_dbg_atclk", "dbg_atclk_sel", 17), + GATE_TOP3(CLK_TOP_RG_NFIECC, "rg_nfiecc", "nfiecc_sel", 18), + /* TOP4 */ + GATE_TOP4_I(CLK_TOP_RG_APLL1_D2_EN, "rg_apll1_d2_en", "apll1_d2", 8), + GATE_TOP4_I(CLK_TOP_RG_APLL1_D4_EN, "rg_apll1_d4_en", "apll1_d4", 9), + GATE_TOP4_I(CLK_TOP_RG_APLL1_D8_EN, "rg_apll1_d8_en", "apll1_d8", 10), + GATE_TOP4_I(CLK_TOP_RG_APLL2_D2_EN, "rg_apll2_d2_en", "apll2_d2", 11), + GATE_TOP4_I(CLK_TOP_RG_APLL2_D4_EN, "rg_apll2_d4_en", "apll2_d4", 12), + GATE_TOP4_I(CLK_TOP_RG_APLL2_D8_EN, "rg_apll2_d8_en", "apll2_d8", 13), + /* TOP5 */ + GATE_TOP5(CLK_TOP_APLL12_DIV0, "apll12_div0", "apll12_ck_div0", 0), + GATE_TOP5(CLK_TOP_APLL12_DIV1, "apll12_div1", "apll12_ck_div1", 1), + GATE_TOP5(CLK_TOP_APLL12_DIV2, "apll12_div2", "apll12_ck_div2", 2), + GATE_TOP5(CLK_TOP_APLL12_DIV3, "apll12_div3", "apll12_ck_div3", 3), + GATE_TOP5(CLK_TOP_APLL12_DIV4, "apll12_div4", "apll12_ck_div4", 4), + GATE_TOP5(CLK_TOP_APLL12_DIV4B, "apll12_div4b", "apll12_ck_div4b", 5), + GATE_TOP5(CLK_TOP_APLL12_DIV5, "apll12_div5", "apll12_ck_div5", 6), + GATE_TOP5(CLK_TOP_APLL12_DIV5B, "apll12_div5b", "apll12_ck_div5b", 7), + GATE_TOP5(CLK_TOP_APLL12_DIV6, "apll12_div6", "apll12_ck_div6", 8), +}; + +static void __init mtk_topckgen_init(struct device_node *node) +{ + struct clk_onecell_data *clk_data; + int r; + void __iomem *base; + + base = of_iomap(node, 0); + if (!base) { + pr_err("%s(): ioremap failed\n", __func__); + return; + } + + clk_data = mtk_alloc_clk_data(MT8167_CLK_TOP_NR_CLK); + + mtk_clk_register_fixed_clks(fixed_clks, ARRAY_SIZE(fixed_clks), + clk_data); + mtk_clk_register_gates(node, top_clks, ARRAY_SIZE(top_clks), clk_data); + + mtk_clk_register_factors(top_divs, ARRAY_SIZE(top_divs), clk_data); + mtk_clk_register_composites(top_muxes, ARRAY_SIZE(top_muxes), base, + &mt8167_clk_lock, clk_data); + mtk_clk_register_dividers(top_adj_divs, ARRAY_SIZE(top_adj_divs), + base, &mt8167_clk_lock, clk_data); + + r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data); + if (r) + pr_err("%s(): could not register clock provider: %d\n", + __func__, r); +} +CLK_OF_DECLARE(mtk_topckgen, "mediatek,mt8167-topckgen", mtk_topckgen_init); + +static void __init mtk_infracfg_init(struct device_node *node) +{ + struct clk_onecell_data *clk_data; + int r; + void __iomem *base; + + base = of_iomap(node, 0); + if (!base) { + pr_err("%s(): ioremap failed\n", __func__); + return; + } + + clk_data = mtk_alloc_clk_data(CLK_IFR_NR_CLK); + + mtk_clk_register_composites(ifr_muxes, ARRAY_SIZE(ifr_muxes), base, + &mt8167_clk_lock, clk_data); + + r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data); + if (r) + pr_err("%s(): could not register clock provider: %d\n", + __func__, r); +} +CLK_OF_DECLARE(mtk_infracfg, "mediatek,mt8167-infracfg", mtk_infracfg_init); + +#define MT8167_PLL_FMAX (2500UL * MHZ) + +#define CON0_MT8167_RST_BAR BIT(27) + +#define PLL_B(_id, _name, _reg, _pwr_reg, _en_mask, _flags, _pcwbits, \ + _pd_reg, _pd_shift, _tuner_reg, _pcw_reg, \ + _pcw_shift, _div_table) { \ + .id = _id, \ + .name = _name, \ + .reg = _reg, \ + .pwr_reg = _pwr_reg, \ + .en_mask = _en_mask, \ + .flags = _flags, \ + .rst_bar_mask = CON0_MT8167_RST_BAR, \ + .fmax = MT8167_PLL_FMAX, \ + .pcwbits = _pcwbits, \ + .pd_reg = _pd_reg, \ + .pd_shift = _pd_shift, \ + .tuner_reg = _tuner_reg, \ + .pcw_reg = _pcw_reg, \ + .pcw_shift = _pcw_shift, \ + .div_table = _div_table, \ + } + +#define PLL(_id, _name, _reg, _pwr_reg, _en_mask, _flags, _pcwbits, \ + _pd_reg, _pd_shift, _tuner_reg, _pcw_reg, \ + _pcw_shift) \ + PLL_B(_id, _name, _reg, _pwr_reg, _en_mask, _flags, _pcwbits, \ + _pd_reg, _pd_shift, _tuner_reg, _pcw_reg, _pcw_shift, \ + NULL) + +static const struct mtk_pll_div_table mmpll_div_table[] = { + { .div = 0, .freq = MT8167_PLL_FMAX }, + { .div = 1, .freq = 1000000000 }, + { .div = 2, .freq = 604500000 }, + { .div = 3, .freq = 253500000 }, + { .div = 4, .freq = 126750000 }, + { } /* sentinel */ +}; + +static const struct mtk_pll_data plls[] = { + PLL(CLK_APMIXED_ARMPLL, "armpll", 0x0100, 0x0110, 0x00000001, 0, + 21, 0x0104, 24, 0, 0x0104, 0), + PLL(CLK_APMIXED_MAINPLL, "mainpll", 0x0120, 0x0130, 0x00000001, + HAVE_RST_BAR, 21, 0x0124, 24, 0, 0x0124, 0), + PLL(CLK_APMIXED_UNIVPLL, "univpll", 0x0140, 0x0150, 0x30000001, + HAVE_RST_BAR, 7, 0x0144, 24, 0, 0x0144, 0), + PLL_B(CLK_APMIXED_MMPLL, "mmpll", 0x0160, 0x0170, 0x00000001, 0, + 21, 0x0164, 24, 0, 0x0164, 0, mmpll_div_table), + PLL(CLK_APMIXED_APLL1, "apll1", 0x0180, 0x0190, 0x00000001, 0, + 31, 0x0180, 1, 0x0194, 0x0184, 0), + PLL(CLK_APMIXED_APLL2, "apll2", 0x01A0, 0x01B0, 0x00000001, 0, + 31, 0x01A0, 1, 0x01B4, 0x01A4, 0), + PLL(CLK_APMIXED_TVDPLL, "tvdpll", 0x01C0, 0x01D0, 0x00000001, 0, + 21, 0x01C4, 24, 0, 0x01C4, 0), + PLL(CLK_APMIXED_LVDSPLL, "lvdspll", 0x01E0, 0x01F0, 0x00000001, 0, + 21, 0x01E4, 24, 0, 0x01E4, 0), +}; + +static void __init mtk_apmixedsys_init(struct device_node *node) +{ + struct clk_onecell_data *clk_data; + void __iomem *base; + int r; + + base = of_iomap(node, 0); + if (!base) { + pr_err("%s(): ioremap failed\n", __func__); + return; + } + + clk_data = mtk_alloc_clk_data(MT8167_CLK_APMIXED_NR_CLK); + + mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + mtk_clk_register_dividers(apmixed_adj_divs, ARRAY_SIZE(apmixed_adj_divs), + base, &mt8167_clk_lock, clk_data); + + r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data); + if (r) + pr_err("%s(): could not register clock provider: %d\n", + __func__, r); + +} +CLK_OF_DECLARE(mtk_apmixedsys, "mediatek,mt8167-apmixedsys", + mtk_apmixedsys_init); diff --git a/drivers/clk/meson/Kconfig b/drivers/clk/meson/Kconfig index dabeb435d067..034da203e8e0 100644 --- a/drivers/clk/meson/Kconfig +++ b/drivers/clk/meson/Kconfig @@ -1,4 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only +menu "Clock support for Amlogic platforms" + depends on ARCH_MESON || COMPILE_TEST + config COMMON_CLK_MESON_REGMAP tristate select REGMAP @@ -41,8 +44,9 @@ config COMMON_CLK_MESON_CPU_DYNDIV select COMMON_CLK_MESON_REGMAP config COMMON_CLK_MESON8B - bool - depends on ARCH_MESON + bool "Meson8 SoC Clock controller support" + depends on ARM + default y select COMMON_CLK_MESON_REGMAP select COMMON_CLK_MESON_MPLL select COMMON_CLK_MESON_PLL @@ -54,8 +58,9 @@ config COMMON_CLK_MESON8B want peripherals and CPU frequency scaling to work. config COMMON_CLK_GXBB - bool - depends on ARCH_MESON + bool "GXBB and GXL SoC clock controllers support" + depends on ARM64 + default y select COMMON_CLK_MESON_REGMAP select COMMON_CLK_MESON_DUALDIV select COMMON_CLK_MESON_VID_PLL_DIV @@ -69,8 +74,9 @@ config COMMON_CLK_GXBB Say Y if you want peripherals and CPU frequency scaling to work. config COMMON_CLK_AXG - bool - depends on ARCH_MESON + bool "AXG SoC clock controllers support" + depends on ARM64 + default y select COMMON_CLK_MESON_REGMAP select COMMON_CLK_MESON_DUALDIV select COMMON_CLK_MESON_MPLL @@ -84,7 +90,7 @@ config COMMON_CLK_AXG config COMMON_CLK_AXG_AUDIO tristate "Meson AXG Audio Clock Controller Driver" - depends on ARCH_MESON + depends on ARM64 select COMMON_CLK_MESON_REGMAP select COMMON_CLK_MESON_PHASE select COMMON_CLK_MESON_SCLK_DIV @@ -94,8 +100,9 @@ config COMMON_CLK_AXG_AUDIO aka axg, Say Y if you want audio subsystem to work. config COMMON_CLK_G12A - bool - depends on ARCH_MESON + bool "G12 and SM1 SoC clock controllers support" + depends on ARM64 + default y select COMMON_CLK_MESON_REGMAP select COMMON_CLK_MESON_DUALDIV select COMMON_CLK_MESON_MPLL @@ -107,3 +114,4 @@ config COMMON_CLK_G12A help Support for the clock controller on Amlogic S905D2, S905X2 and S905Y2 devices, aka g12a. Say Y if you want peripherals to work. +endmenu diff --git a/drivers/clk/meson/axg-audio.c b/drivers/clk/meson/axg-audio.c index 53715e36326c..7c8d02164443 100644 --- a/drivers/clk/meson/axg-audio.c +++ b/drivers/clk/meson/axg-audio.c @@ -147,6 +147,29 @@ }, \ } +#define AUD_SCLK_WS(_name, _reg, _width, _shift_ph, _shift_ws, _pname, \ + _iflags) { \ + .data = &(struct meson_sclk_ws_inv_data) { \ + .ph = { \ + .reg_off = (_reg), \ + .shift = (_shift_ph), \ + .width = (_width), \ + }, \ + .ws = { \ + .reg_off = (_reg), \ + .shift = (_shift_ws), \ + .width = (_width), \ + }, \ + }, \ + .hw.init = &(struct clk_init_data) { \ + .name = "aud_"#_name, \ + .ops = &meson_clk_phase_ops, \ + .parent_names = (const char *[]){ #_pname }, \ + .num_parents = 1, \ + .flags = (_iflags), \ + }, \ +} + /* Audio Master Clocks */ static const struct clk_parent_data mst_mux_parent_data[] = { { .fw_name = "mst_in0", }, @@ -254,6 +277,10 @@ static const struct clk_parent_data tdm_lrclk_parent_data[] = { AUD_PHASE(tdm##_name##_sclk, _reg, 1, 29, \ aud_tdm##_name##_sclk_post_en, \ CLK_DUTY_CYCLE_PARENT | CLK_SET_RATE_PARENT) +#define AUD_TDM_SCLK_WS(_name, _reg) \ + AUD_SCLK_WS(tdm##_name##_sclk, _reg, 1, 29, 28, \ + aud_tdm##_name##_sclk_post_en, \ + CLK_DUTY_CYCLE_PARENT | CLK_SET_RATE_PARENT) #define AUD_TDM_LRLCK(_name, _reg) \ AUD_MUX(tdm##_name##_lrclk, _reg, 0xf, 20, \ @@ -499,12 +526,6 @@ static struct clk_regmap tdmin_c_sclk = AUD_TDM_SCLK(in_c, AUDIO_CLK_TDMIN_C_CTRL); static struct clk_regmap tdmin_lb_sclk = AUD_TDM_SCLK(in_lb, AUDIO_CLK_TDMIN_LB_CTRL); -static struct clk_regmap tdmout_a_sclk = - AUD_TDM_SCLK(out_a, AUDIO_CLK_TDMOUT_A_CTRL); -static struct clk_regmap tdmout_b_sclk = - AUD_TDM_SCLK(out_b, AUDIO_CLK_TDMOUT_B_CTRL); -static struct clk_regmap tdmout_c_sclk = - AUD_TDM_SCLK(out_c, AUDIO_CLK_TDMOUT_C_CTRL); static struct clk_regmap tdmin_a_lrclk = AUD_TDM_LRLCK(in_a, AUDIO_CLK_TDMIN_A_CTRL); @@ -521,6 +542,14 @@ static struct clk_regmap tdmout_b_lrclk = static struct clk_regmap tdmout_c_lrclk = AUD_TDM_LRLCK(out_c, AUDIO_CLK_TDMOUT_C_CTRL); +/* AXG Clocks */ +static struct clk_regmap axg_tdmout_a_sclk = + AUD_TDM_SCLK(out_a, AUDIO_CLK_TDMOUT_A_CTRL); +static struct clk_regmap axg_tdmout_b_sclk = + AUD_TDM_SCLK(out_b, AUDIO_CLK_TDMOUT_B_CTRL); +static struct clk_regmap axg_tdmout_c_sclk = + AUD_TDM_SCLK(out_c, AUDIO_CLK_TDMOUT_C_CTRL); + /* AXG/G12A Clocks */ static struct clk_hw axg_aud_top = { .init = &(struct clk_init_data) { @@ -591,7 +620,13 @@ static struct clk_regmap g12a_tdm_sclk_pad_1 = AUD_TDM_PAD_CTRL( static struct clk_regmap g12a_tdm_sclk_pad_2 = AUD_TDM_PAD_CTRL( sclk_pad_2, AUDIO_MST_PAD_CTRL1, 8, sclk_pad_ctrl_parent_data); -/* G12a/SM1 clocks */ +static struct clk_regmap g12a_tdmout_a_sclk = + AUD_TDM_SCLK_WS(out_a, AUDIO_CLK_TDMOUT_A_CTRL); +static struct clk_regmap g12a_tdmout_b_sclk = + AUD_TDM_SCLK_WS(out_b, AUDIO_CLK_TDMOUT_B_CTRL); +static struct clk_regmap g12a_tdmout_c_sclk = + AUD_TDM_SCLK_WS(out_c, AUDIO_CLK_TDMOUT_C_CTRL); + static struct clk_regmap toram = AUD_PCLK_GATE(toram, AUDIO_CLK_GATE_EN, 20); static struct clk_regmap spdifout_b = @@ -889,9 +924,9 @@ static struct clk_hw_onecell_data axg_audio_hw_onecell_data = { [AUD_CLKID_TDMIN_B_SCLK] = &tdmin_b_sclk.hw, [AUD_CLKID_TDMIN_C_SCLK] = &tdmin_c_sclk.hw, [AUD_CLKID_TDMIN_LB_SCLK] = &tdmin_lb_sclk.hw, - [AUD_CLKID_TDMOUT_A_SCLK] = &tdmout_a_sclk.hw, - [AUD_CLKID_TDMOUT_B_SCLK] = &tdmout_b_sclk.hw, - [AUD_CLKID_TDMOUT_C_SCLK] = &tdmout_c_sclk.hw, + [AUD_CLKID_TDMOUT_A_SCLK] = &axg_tdmout_a_sclk.hw, + [AUD_CLKID_TDMOUT_B_SCLK] = &axg_tdmout_b_sclk.hw, + [AUD_CLKID_TDMOUT_C_SCLK] = &axg_tdmout_c_sclk.hw, [AUD_CLKID_TDMIN_A_LRCLK] = &tdmin_a_lrclk.hw, [AUD_CLKID_TDMIN_B_LRCLK] = &tdmin_b_lrclk.hw, [AUD_CLKID_TDMIN_C_LRCLK] = &tdmin_c_lrclk.hw, @@ -1026,9 +1061,9 @@ static struct clk_hw_onecell_data g12a_audio_hw_onecell_data = { [AUD_CLKID_TDMIN_B_SCLK] = &tdmin_b_sclk.hw, [AUD_CLKID_TDMIN_C_SCLK] = &tdmin_c_sclk.hw, [AUD_CLKID_TDMIN_LB_SCLK] = &tdmin_lb_sclk.hw, - [AUD_CLKID_TDMOUT_A_SCLK] = &tdmout_a_sclk.hw, - [AUD_CLKID_TDMOUT_B_SCLK] = &tdmout_b_sclk.hw, - [AUD_CLKID_TDMOUT_C_SCLK] = &tdmout_c_sclk.hw, + [AUD_CLKID_TDMOUT_A_SCLK] = &g12a_tdmout_a_sclk.hw, + [AUD_CLKID_TDMOUT_B_SCLK] = &g12a_tdmout_b_sclk.hw, + [AUD_CLKID_TDMOUT_C_SCLK] = &g12a_tdmout_c_sclk.hw, [AUD_CLKID_TDMIN_A_LRCLK] = &tdmin_a_lrclk.hw, [AUD_CLKID_TDMIN_B_LRCLK] = &tdmin_b_lrclk.hw, [AUD_CLKID_TDMIN_C_LRCLK] = &tdmin_c_lrclk.hw, @@ -1170,9 +1205,9 @@ static struct clk_hw_onecell_data sm1_audio_hw_onecell_data = { [AUD_CLKID_TDMIN_B_SCLK] = &tdmin_b_sclk.hw, [AUD_CLKID_TDMIN_C_SCLK] = &tdmin_c_sclk.hw, [AUD_CLKID_TDMIN_LB_SCLK] = &tdmin_lb_sclk.hw, - [AUD_CLKID_TDMOUT_A_SCLK] = &tdmout_a_sclk.hw, - [AUD_CLKID_TDMOUT_B_SCLK] = &tdmout_b_sclk.hw, - [AUD_CLKID_TDMOUT_C_SCLK] = &tdmout_c_sclk.hw, + [AUD_CLKID_TDMOUT_A_SCLK] = &g12a_tdmout_a_sclk.hw, + [AUD_CLKID_TDMOUT_B_SCLK] = &g12a_tdmout_b_sclk.hw, + [AUD_CLKID_TDMOUT_C_SCLK] = &g12a_tdmout_c_sclk.hw, [AUD_CLKID_TDMIN_A_LRCLK] = &tdmin_a_lrclk.hw, [AUD_CLKID_TDMIN_B_LRCLK] = &tdmin_b_lrclk.hw, [AUD_CLKID_TDMIN_C_LRCLK] = &tdmin_c_lrclk.hw, @@ -1209,12 +1244,7 @@ static struct clk_hw_onecell_data sm1_audio_hw_onecell_data = { }; -/* Convenience table to populate regmap in .probe() - * Note that this table is shared between both AXG and G12A, - * with spdifout_b clocks being exclusive to G12A. Since those - * clocks are not declared within the AXG onecell table, we do not - * feel the need to have separate AXG/G12A regmap tables. - */ +/* Convenience table to populate regmap in .probe(). */ static struct clk_regmap *const axg_clk_regmaps[] = { &ddr_arb, &pdm, @@ -1236,6 +1266,130 @@ static struct clk_regmap *const axg_clk_regmaps[] = { &spdifout, &resample, &power_detect, + &mst_a_mclk_sel, + &mst_b_mclk_sel, + &mst_c_mclk_sel, + &mst_d_mclk_sel, + &mst_e_mclk_sel, + &mst_f_mclk_sel, + &mst_a_mclk_div, + &mst_b_mclk_div, + &mst_c_mclk_div, + &mst_d_mclk_div, + &mst_e_mclk_div, + &mst_f_mclk_div, + &mst_a_mclk, + &mst_b_mclk, + &mst_c_mclk, + &mst_d_mclk, + &mst_e_mclk, + &mst_f_mclk, + &spdifout_clk_sel, + &spdifout_clk_div, + &spdifout_clk, + &spdifin_clk_sel, + &spdifin_clk_div, + &spdifin_clk, + &pdm_dclk_sel, + &pdm_dclk_div, + &pdm_dclk, + &pdm_sysclk_sel, + &pdm_sysclk_div, + &pdm_sysclk, + &mst_a_sclk_pre_en, + &mst_b_sclk_pre_en, + &mst_c_sclk_pre_en, + &mst_d_sclk_pre_en, + &mst_e_sclk_pre_en, + &mst_f_sclk_pre_en, + &mst_a_sclk_div, + &mst_b_sclk_div, + &mst_c_sclk_div, + &mst_d_sclk_div, + &mst_e_sclk_div, + &mst_f_sclk_div, + &mst_a_sclk_post_en, + &mst_b_sclk_post_en, + &mst_c_sclk_post_en, + &mst_d_sclk_post_en, + &mst_e_sclk_post_en, + &mst_f_sclk_post_en, + &mst_a_sclk, + &mst_b_sclk, + &mst_c_sclk, + &mst_d_sclk, + &mst_e_sclk, + &mst_f_sclk, + &mst_a_lrclk_div, + &mst_b_lrclk_div, + &mst_c_lrclk_div, + &mst_d_lrclk_div, + &mst_e_lrclk_div, + &mst_f_lrclk_div, + &mst_a_lrclk, + &mst_b_lrclk, + &mst_c_lrclk, + &mst_d_lrclk, + &mst_e_lrclk, + &mst_f_lrclk, + &tdmin_a_sclk_sel, + &tdmin_b_sclk_sel, + &tdmin_c_sclk_sel, + &tdmin_lb_sclk_sel, + &tdmout_a_sclk_sel, + &tdmout_b_sclk_sel, + &tdmout_c_sclk_sel, + &tdmin_a_sclk_pre_en, + &tdmin_b_sclk_pre_en, + &tdmin_c_sclk_pre_en, + &tdmin_lb_sclk_pre_en, + &tdmout_a_sclk_pre_en, + &tdmout_b_sclk_pre_en, + &tdmout_c_sclk_pre_en, + &tdmin_a_sclk_post_en, + &tdmin_b_sclk_post_en, + &tdmin_c_sclk_post_en, + &tdmin_lb_sclk_post_en, + &tdmout_a_sclk_post_en, + &tdmout_b_sclk_post_en, + &tdmout_c_sclk_post_en, + &tdmin_a_sclk, + &tdmin_b_sclk, + &tdmin_c_sclk, + &tdmin_lb_sclk, + &axg_tdmout_a_sclk, + &axg_tdmout_b_sclk, + &axg_tdmout_c_sclk, + &tdmin_a_lrclk, + &tdmin_b_lrclk, + &tdmin_c_lrclk, + &tdmin_lb_lrclk, + &tdmout_a_lrclk, + &tdmout_b_lrclk, + &tdmout_c_lrclk, +}; + +static struct clk_regmap *const g12a_clk_regmaps[] = { + &ddr_arb, + &pdm, + &tdmin_a, + &tdmin_b, + &tdmin_c, + &tdmin_lb, + &tdmout_a, + &tdmout_b, + &tdmout_c, + &frddr_a, + &frddr_b, + &frddr_c, + &toddr_a, + &toddr_b, + &toddr_c, + &loopback, + &spdifin, + &spdifout, + &resample, + &power_detect, &spdifout_b, &mst_a_mclk_sel, &mst_b_mclk_sel, @@ -1328,9 +1482,9 @@ static struct clk_regmap *const axg_clk_regmaps[] = { &tdmin_b_sclk, &tdmin_c_sclk, &tdmin_lb_sclk, - &tdmout_a_sclk, - &tdmout_b_sclk, - &tdmout_c_sclk, + &g12a_tdmout_a_sclk, + &g12a_tdmout_b_sclk, + &g12a_tdmout_c_sclk, &tdmin_a_lrclk, &tdmin_b_lrclk, &tdmin_c_lrclk, @@ -1465,9 +1619,9 @@ static struct clk_regmap *const sm1_clk_regmaps[] = { &tdmin_b_sclk, &tdmin_c_sclk, &tdmin_lb_sclk, - &tdmout_a_sclk, - &tdmout_b_sclk, - &tdmout_c_sclk, + &g12a_tdmout_a_sclk, + &g12a_tdmout_b_sclk, + &g12a_tdmout_c_sclk, &tdmin_a_lrclk, &tdmin_b_lrclk, &tdmin_c_lrclk, @@ -1713,8 +1867,8 @@ static const struct audioclk_data axg_audioclk_data = { }; static const struct audioclk_data g12a_audioclk_data = { - .regmap_clks = axg_clk_regmaps, - .regmap_clk_num = ARRAY_SIZE(axg_clk_regmaps), + .regmap_clks = g12a_clk_regmaps, + .regmap_clk_num = ARRAY_SIZE(g12a_clk_regmaps), .hw_onecell_data = &g12a_audio_hw_onecell_data, .reset_offset = AUDIO_SW_RESET, .reset_num = 26, diff --git a/drivers/clk/meson/clk-phase.c b/drivers/clk/meson/clk-phase.c index fe22e171121a..a6763439f7d2 100644 --- a/drivers/clk/meson/clk-phase.c +++ b/drivers/clk/meson/clk-phase.c @@ -125,6 +125,62 @@ const struct clk_ops meson_clk_triphase_ops = { }; EXPORT_SYMBOL_GPL(meson_clk_triphase_ops); +/* + * This is a special clock for the audio controller. + * This drive a bit clock inverter for which the + * opposite value of the inverter bit needs to be manually + * set into another bit + */ +static inline struct meson_sclk_ws_inv_data * +meson_sclk_ws_inv_data(struct clk_regmap *clk) +{ + return (struct meson_sclk_ws_inv_data *)clk->data; +} + +static int meson_sclk_ws_inv_sync(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_sclk_ws_inv_data *tph = meson_sclk_ws_inv_data(clk); + unsigned int val; + + /* Get phase and sync the inverted value to ws */ + val = meson_parm_read(clk->map, &tph->ph); + meson_parm_write(clk->map, &tph->ws, val ? 0 : 1); + + return 0; +} + +static int meson_sclk_ws_inv_get_phase(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_sclk_ws_inv_data *tph = meson_sclk_ws_inv_data(clk); + unsigned int val; + + val = meson_parm_read(clk->map, &tph->ph); + + return meson_clk_degrees_from_val(val, tph->ph.width); +} + +static int meson_sclk_ws_inv_set_phase(struct clk_hw *hw, int degrees) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_sclk_ws_inv_data *tph = meson_sclk_ws_inv_data(clk); + unsigned int val; + + val = meson_clk_degrees_to_val(degrees, tph->ph.width); + meson_parm_write(clk->map, &tph->ph, val); + meson_parm_write(clk->map, &tph->ws, val ? 0 : 1); + return 0; +} + +const struct clk_ops meson_sclk_ws_inv_ops = { + .init = meson_sclk_ws_inv_sync, + .get_phase = meson_sclk_ws_inv_get_phase, + .set_phase = meson_sclk_ws_inv_set_phase, +}; +EXPORT_SYMBOL_GPL(meson_sclk_ws_inv_ops); + + MODULE_DESCRIPTION("Amlogic phase driver"); MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/meson/clk-phase.h b/drivers/clk/meson/clk-phase.h index 5579f9ced142..b637b9b227bc 100644 --- a/drivers/clk/meson/clk-phase.h +++ b/drivers/clk/meson/clk-phase.h @@ -20,7 +20,13 @@ struct meson_clk_triphase_data { struct parm ph2; }; +struct meson_sclk_ws_inv_data { + struct parm ph; + struct parm ws; +}; + extern const struct clk_ops meson_clk_phase_ops; extern const struct clk_ops meson_clk_triphase_ops; +extern const struct clk_ops meson_sclk_ws_inv_ops; #endif /* __MESON_CLK_PHASE_H */ diff --git a/drivers/clk/meson/g12a.c b/drivers/clk/meson/g12a.c index 9803d44bb157..b814d44917a5 100644 --- a/drivers/clk/meson/g12a.c +++ b/drivers/clk/meson/g12a.c @@ -298,6 +298,17 @@ static struct clk_regmap g12a_fclk_div2 = { &g12a_fclk_div2_div.hw }, .num_parents = 1, + /* + * Similar to fclk_div3, it seems that this clock is used by + * the resident firmware and is required by the platform to + * operate correctly. + * Until the following condition are met, we need this clock to + * be marked as critical: + * a) Mark the clock used by a firmware resource, if possible + * b) CCF has a clock hand-off mechanism to make the sure the + * clock stays on until the proper driver comes along + */ + .flags = CLK_IS_CRITICAL, }, }; diff --git a/drivers/clk/meson/meson-aoclk.c b/drivers/clk/meson/meson-aoclk.c index bf8bea675d24..3a6d84cd6601 100644 --- a/drivers/clk/meson/meson-aoclk.c +++ b/drivers/clk/meson/meson-aoclk.c @@ -57,7 +57,7 @@ int meson_aoclkc_probe(struct platform_device *pdev) rstc->data = data; rstc->regmap = regmap; rstc->reset.ops = &meson_aoclk_reset_ops; - rstc->reset.nr_resets = data->num_reset, + rstc->reset.nr_resets = data->num_reset; rstc->reset.of_node = dev->of_node; ret = devm_reset_controller_register(dev, &rstc->reset); if (ret) { diff --git a/drivers/clk/mmp/clk-of-mmp2.c b/drivers/clk/mmp/clk-of-mmp2.c index 67208aea94c5..0839fb2049e9 100644 --- a/drivers/clk/mmp/clk-of-mmp2.c +++ b/drivers/clk/mmp/clk-of-mmp2.c @@ -347,9 +347,9 @@ static struct mmp_param_mux_clk mmp3_apmu_mux_clks[] = { }; static struct mmp_param_div_clk apmu_div_clks[] = { - {0, "disp0_div", "disp0_mux", CLK_SET_RATE_PARENT, APMU_DISP0, 8, 4, 0, &disp0_lock}, + {0, "disp0_div", "disp0_mux", CLK_SET_RATE_PARENT, APMU_DISP0, 8, 4, CLK_DIVIDER_ONE_BASED, &disp0_lock}, {0, "disp0_sphy_div", "disp0_mux", CLK_SET_RATE_PARENT, APMU_DISP0, 15, 5, 0, &disp0_lock}, - {0, "disp1_div", "disp1_mux", CLK_SET_RATE_PARENT, APMU_DISP1, 8, 4, 0, &disp1_lock}, + {0, "disp1_div", "disp1_mux", CLK_SET_RATE_PARENT, APMU_DISP1, 8, 4, CLK_DIVIDER_ONE_BASED, &disp1_lock}, {0, "ccic0_sphy_div", "ccic0_mix_clk", CLK_SET_RATE_PARENT, APMU_CCIC0, 10, 5, 0, &ccic0_lock}, {0, "ccic1_sphy_div", "ccic1_mix_clk", CLK_SET_RATE_PARENT, APMU_CCIC1, 10, 5, 0, &ccic1_lock}, }; diff --git a/drivers/clk/mmp/clk-of-pxa1928.c b/drivers/clk/mmp/clk-of-pxa1928.c index cede7b4ca3b9..998fc4207b0e 100644 --- a/drivers/clk/mmp/clk-of-pxa1928.c +++ b/drivers/clk/mmp/clk-of-pxa1928.c @@ -68,7 +68,6 @@ static struct mmp_clk_factor_tbl uart_factor_tbl[] = { static void pxa1928_pll_init(struct pxa1928_clk_unit *pxa_unit) { - struct clk *clk; struct mmp_clk_unit *unit = &pxa_unit->unit; mmp_register_fixed_rate_clks(unit, fixed_rate_clks, @@ -77,7 +76,7 @@ static void pxa1928_pll_init(struct pxa1928_clk_unit *pxa_unit) mmp_register_fixed_factor_clks(unit, fixed_factor_clks, ARRAY_SIZE(fixed_factor_clks)); - clk = mmp_clk_register_factor("uart_pll", "pll1_416", + mmp_clk_register_factor("uart_pll", "pll1_416", CLK_SET_RATE_PARENT, pxa_unit->mpmu_base + MPMU_UART_PLL, &uart_factor_masks, uart_factor_tbl, diff --git a/drivers/clk/mvebu/ap-cpu-clk.c b/drivers/clk/mvebu/ap-cpu-clk.c index 6b394302c76a..b4259b60dcfd 100644 --- a/drivers/clk/mvebu/ap-cpu-clk.c +++ b/drivers/clk/mvebu/ap-cpu-clk.c @@ -197,7 +197,7 @@ static int ap_cpu_clk_set_rate(struct clk_hw *hw, unsigned long rate, stable_bit = BIT(clk->pll_regs->ratio_state_offset + clk->cluster * - clk->pll_regs->ratio_state_cluster_offset), + clk->pll_regs->ratio_state_cluster_offset); ret = regmap_read_poll_timeout(clk->pll_cr_base, clk->pll_regs->ratio_state_reg, reg, reg & stable_bit, STATUS_POLL_PERIOD_US, diff --git a/drivers/clk/pxa/clk-pxa.h b/drivers/clk/pxa/clk-pxa.h index f131d2834af4..5768e0f728ce 100644 --- a/drivers/clk/pxa/clk-pxa.h +++ b/drivers/clk/pxa/clk-pxa.h @@ -19,11 +19,11 @@ #define MUX_RO_RATE_RO_OPS(name, clk_name) \ static struct clk_hw name ## _mux_hw; \ static struct clk_hw name ## _rate_hw; \ - static struct clk_ops name ## _mux_ops = { \ + static const struct clk_ops name ## _mux_ops = { \ .get_parent = name ## _get_parent, \ .set_parent = dummy_clk_set_parent, \ }; \ - static struct clk_ops name ## _rate_ops = { \ + static const struct clk_ops name ## _rate_ops = { \ .recalc_rate = name ## _get_rate, \ }; \ static struct clk * __init clk_register_ ## name(void) \ @@ -38,7 +38,7 @@ #define RATE_RO_OPS(name, clk_name) \ static struct clk_hw name ## _rate_hw; \ - static const struct clk_ops name ## _rate_ops = { \ + static const struct clk_ops name ## _rate_ops = { \ .recalc_rate = name ## _get_rate, \ }; \ static struct clk * __init clk_register_ ## name(void) \ @@ -53,7 +53,7 @@ #define RATE_OPS(name, clk_name) \ static struct clk_hw name ## _rate_hw; \ - static struct clk_ops name ## _rate_ops = { \ + static const struct clk_ops name ## _rate_ops = { \ .recalc_rate = name ## _get_rate, \ .set_rate = name ## _set_rate, \ .determine_rate = name ## _determine_rate, \ diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig index 058327310c25..3a965bd326d5 100644 --- a/drivers/clk/qcom/Kconfig +++ b/drivers/clk/qcom/Kconfig @@ -413,6 +413,15 @@ config SDM_LPASSCC_845 Say Y if you want to use the LPASS branch clocks of the LPASS clock controller to reset the LPASS subsystem. +config SM_DISPCC_8250 + tristate "SM8150 and SM8250 Display Clock Controller" + depends on SM_GCC_8150 || SM_GCC_8250 + help + Support for the display clock controller on Qualcomm Technologies, Inc + SM8150 and SM8250 devices. + Say Y if you want to support display devices and functionality such as + splash screen. + config SM_GCC_8150 tristate "SM8150 Global Clock Controller" help @@ -444,6 +453,24 @@ config SM_GPUCC_8250 Say Y if you want to support graphics controller devices and functionality such as 3D graphics. +config SM_VIDEOCC_8150 + tristate "SM8150 Video Clock Controller" + select SDM_GCC_8150 + select QCOM_GDSC + help + Support for the video clock controller on SM8150 devices. + Say Y if you want to support video devices and functionality such as + video encode and decode. + +config SM_VIDEOCC_8250 + tristate "SM8250 Video Clock Controller" + select SDM_GCC_8250 + select QCOM_GDSC + help + Support for the video clock controller on SM8250 devices. + Say Y if you want to support video devices and functionality such as + video encode and decode. + config SPMI_PMIC_CLKDIV tristate "SPMI PMIC clkdiv Support" depends on SPMI || COMPILE_TEST diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile index 9677e769e7e9..11ae86febe87 100644 --- a/drivers/clk/qcom/Makefile +++ b/drivers/clk/qcom/Makefile @@ -64,10 +64,13 @@ obj-$(CONFIG_SDM_GCC_845) += gcc-sdm845.o obj-$(CONFIG_SDM_GPUCC_845) += gpucc-sdm845.o obj-$(CONFIG_SDM_LPASSCC_845) += lpasscc-sdm845.o obj-$(CONFIG_SDM_VIDEOCC_845) += videocc-sdm845.o +obj-$(CONFIG_SM_DISPCC_8250) += dispcc-sm8250.o obj-$(CONFIG_SM_GCC_8150) += gcc-sm8150.o obj-$(CONFIG_SM_GCC_8250) += gcc-sm8250.o obj-$(CONFIG_SM_GPUCC_8150) += gpucc-sm8150.o obj-$(CONFIG_SM_GPUCC_8250) += gpucc-sm8250.o +obj-$(CONFIG_SM_VIDEOCC_8150) += videocc-sm8150.o +obj-$(CONFIG_SM_VIDEOCC_8250) += videocc-sm8250.o obj-$(CONFIG_SPMI_PMIC_CLKDIV) += clk-spmi-pmic-div.o obj-$(CONFIG_KPSS_XCC) += kpss-xcc.o obj-$(CONFIG_QCOM_HFPLL) += hfpll.o diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index 26139ef005e4..564431130a76 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -609,7 +609,7 @@ static unsigned long alpha_huayra_pll_calc_rate(u64 prate, u32 l, u32 a) { /* - * a contains 16 bit alpha_val in two’s compliment number in the range + * a contains 16 bit alpha_val in two’s complement number in the range * of [-0.5, 0.5). */ if (a >= BIT(PLL_HUAYRA_ALPHA_WIDTH - 1)) @@ -641,7 +641,7 @@ alpha_huayra_pll_round_rate(unsigned long rate, unsigned long prate, quotient++; /* - * alpha_val should be in two’s compliment number in the range + * alpha_val should be in two’s complement number in the range * of [-0.5, 0.5) so if quotient >= 0.5 then increment the l value * since alpha value will be subtracted in this case. */ @@ -666,7 +666,7 @@ alpha_pll_huayra_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) regmap_read(pll->clkr.regmap, PLL_ALPHA_VAL(pll), &alpha); /* * Depending upon alpha_mode, it can be treated as M/N value or - * as a two’s compliment number. When alpha_mode=1, + * as a two’s complement number. When alpha_mode=1, * pll_alpha_val<15:8>=M and pll_apla_val<7:0>=N * * Fout=FIN*(L+(M/N)) @@ -674,12 +674,12 @@ alpha_pll_huayra_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) * M is a signed number (-128 to 127) and N is unsigned * (0 to 255). M/N has to be within +/-0.5. * - * When alpha_mode=0, it is a two’s compliment number in the + * When alpha_mode=0, it is a two’s complement number in the * range [-0.5, 0.5). * * Fout=FIN*(L+(alpha_val)/2^16) * - * where alpha_val is two’s compliment number. + * where alpha_val is two’s complement number. */ if (!(ctl & PLL_ALPHA_MODE)) return alpha_huayra_pll_calc_rate(rate, l, alpha); diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c index 357159fe85b5..59a5a0f261f3 100644 --- a/drivers/clk/qcom/clk-rcg2.c +++ b/drivers/clk/qcom/clk-rcg2.c @@ -1182,14 +1182,21 @@ static int clk_rcg2_dp_set_rate_and_parent(struct clk_hw *hw, static int clk_rcg2_dp_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { - struct clk_rate_request parent_req = *req; - int ret; + struct clk_rcg2 *rcg = to_clk_rcg2(hw); + unsigned long num, den; + u64 tmp; - ret = __clk_determine_rate(clk_hw_get_parent(hw), &parent_req); - if (ret) - return ret; + /* Parent rate is a fixed phy link rate */ + rational_best_approximation(req->best_parent_rate, req->rate, + GENMASK(rcg->mnd_width - 1, 0), + GENMASK(rcg->mnd_width - 1, 0), &den, &num); + + if (!num || !den) + return -EINVAL; - req->best_parent_rate = parent_req.rate; + tmp = req->best_parent_rate * num; + do_div(tmp, den); + req->rate = tmp; return 0; } diff --git a/drivers/clk/qcom/dispcc-sc7180.c b/drivers/clk/qcom/dispcc-sc7180.c index 0a5d395bce93..f487515701e3 100644 --- a/drivers/clk/qcom/dispcc-sc7180.c +++ b/drivers/clk/qcom/dispcc-sc7180.c @@ -202,7 +202,6 @@ static struct clk_rcg2 disp_cc_mdss_dp_crypto_clk_src = { .name = "disp_cc_mdss_dp_crypto_clk_src", .parent_data = disp_cc_parent_data_1, .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), - .flags = CLK_SET_RATE_PARENT, .ops = &clk_byte2_ops, }, }; @@ -216,7 +215,6 @@ static struct clk_rcg2 disp_cc_mdss_dp_link_clk_src = { .name = "disp_cc_mdss_dp_link_clk_src", .parent_data = disp_cc_parent_data_1, .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), - .flags = CLK_SET_RATE_PARENT, .ops = &clk_byte2_ops, }, }; @@ -230,7 +228,6 @@ static struct clk_rcg2 disp_cc_mdss_dp_pixel_clk_src = { .name = "disp_cc_mdss_dp_pixel_clk_src", .parent_data = disp_cc_parent_data_1, .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), - .flags = CLK_SET_RATE_PARENT, .ops = &clk_dp_ops, }, }; diff --git a/drivers/clk/qcom/dispcc-sm8250.c b/drivers/clk/qcom/dispcc-sm8250.c new file mode 100644 index 000000000000..07a98d3f882d --- /dev/null +++ b/drivers/clk/qcom/dispcc-sm8250.c @@ -0,0 +1,1107 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. + */ + +#include <linux/clk-provider.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/reset-controller.h> + +#include <dt-bindings/clock/qcom,dispcc-sm8250.h> + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap-divider.h" +#include "common.h" +#include "gdsc.h" +#include "reset.h" + +enum { + P_BI_TCXO, + P_CHIP_SLEEP_CLK, + P_CORE_BI_PLL_TEST_SE, + P_DISP_CC_PLL0_OUT_MAIN, + P_DISP_CC_PLL1_OUT_EVEN, + P_DISP_CC_PLL1_OUT_MAIN, + P_DP_PHY_PLL_LINK_CLK, + P_DP_PHY_PLL_VCO_DIV_CLK, + P_DPTX1_PHY_PLL_LINK_CLK, + P_DPTX1_PHY_PLL_VCO_DIV_CLK, + P_DPTX2_PHY_PLL_LINK_CLK, + P_DPTX2_PHY_PLL_VCO_DIV_CLK, + P_DSI0_PHY_PLL_OUT_BYTECLK, + P_DSI0_PHY_PLL_OUT_DSICLK, + P_DSI1_PHY_PLL_OUT_BYTECLK, + P_DSI1_PHY_PLL_OUT_DSICLK, + P_EDP_PHY_PLL_LINK_CLK, + P_EDP_PHY_PLL_VCO_DIV_CLK, +}; + +static struct pll_vco vco_table[] = { + { 249600000, 2000000000, 0 }, +}; + +static struct alpha_pll_config disp_cc_pll0_config = { + .l = 0x47, + .alpha = 0xE000, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00002261, + .config_ctl_hi1_val = 0x329A699C, + .user_ctl_val = 0x00000000, + .user_ctl_hi_val = 0x00000805, + .user_ctl_hi1_val = 0x00000000, +}; + +static struct clk_init_data disp_cc_pll0_init = { + .name = "disp_cc_pll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_ops, +}; + +static struct clk_alpha_pll disp_cc_pll0 = { + .offset = 0x0, + .vco_table = vco_table, + .num_vco = ARRAY_SIZE(vco_table), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr.hw.init = &disp_cc_pll0_init +}; + +static struct alpha_pll_config disp_cc_pll1_config = { + .l = 0x1F, + .alpha = 0x4000, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00002261, + .config_ctl_hi1_val = 0x329A699C, + .user_ctl_val = 0x00000000, + .user_ctl_hi_val = 0x00000805, + .user_ctl_hi1_val = 0x00000000, +}; + +static struct clk_init_data disp_cc_pll1_init = { + .name = "disp_cc_pll1", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_ops, +}; + +static struct clk_alpha_pll disp_cc_pll1 = { + .offset = 0x1000, + .vco_table = vco_table, + .num_vco = ARRAY_SIZE(vco_table), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr.hw.init = &disp_cc_pll1_init +}; + +static const struct parent_map disp_cc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_DP_PHY_PLL_LINK_CLK, 1 }, + { P_DP_PHY_PLL_VCO_DIV_CLK, 2 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_0[] = { + { .fw_name = "bi_tcxo" }, + { .fw_name = "dp_phy_pll_link_clk" }, + { .fw_name = "dp_phy_pll_vco_div_clk" }, +}; + +static const struct parent_map disp_cc_parent_map_1[] = { + { P_BI_TCXO, 0 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_1[] = { + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map disp_cc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_DSI0_PHY_PLL_OUT_BYTECLK, 1 }, + { P_DSI1_PHY_PLL_OUT_BYTECLK, 2 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_2[] = { + { .fw_name = "bi_tcxo" }, + { .fw_name = "dsi0_phy_pll_out_byteclk" }, + { .fw_name = "dsi1_phy_pll_out_byteclk" }, +}; + +static const struct parent_map disp_cc_parent_map_3[] = { + { P_BI_TCXO, 0 }, + { P_DISP_CC_PLL1_OUT_MAIN, 4 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_3[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &disp_cc_pll1.clkr.hw }, +}; + +static const struct parent_map disp_cc_parent_map_5[] = { + { P_BI_TCXO, 0 }, + { P_DISP_CC_PLL0_OUT_MAIN, 1 }, + { P_DISP_CC_PLL1_OUT_MAIN, 4 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_5[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &disp_cc_pll0.clkr.hw }, + { .hw = &disp_cc_pll1.clkr.hw }, +}; + +static const struct parent_map disp_cc_parent_map_6[] = { + { P_BI_TCXO, 0 }, + { P_DSI0_PHY_PLL_OUT_DSICLK, 1 }, + { P_DSI1_PHY_PLL_OUT_DSICLK, 2 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_6[] = { + { .fw_name = "bi_tcxo" }, + { .fw_name = "dsi0_phy_pll_out_dsiclk" }, + { .fw_name = "dsi1_phy_pll_out_dsiclk" }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_ahb_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(37500000, P_DISP_CC_PLL1_OUT_MAIN, 16, 0, 0), + F(75000000, P_DISP_CC_PLL1_OUT_MAIN, 8, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_ahb_clk_src = { + .cmd_rcgr = 0x22bc, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_3, + .freq_tbl = ftbl_disp_cc_mdss_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_ahb_clk_src", + .parent_data = disp_cc_parent_data_3, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_byte0_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_byte0_clk_src = { + .cmd_rcgr = 0x2110, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_2, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_byte0_clk_src", + .parent_data = disp_cc_parent_data_2, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_byte2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_byte1_clk_src = { + .cmd_rcgr = 0x212c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_2, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_byte1_clk_src", + .parent_data = disp_cc_parent_data_2, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_byte2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dp_aux1_clk_src = { + .cmd_rcgr = 0x2240, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_aux1_clk_src", + .parent_data = disp_cc_parent_data_1, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dp_aux_clk_src = { + .cmd_rcgr = 0x21dc, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_aux_clk_src", + .parent_data = disp_cc_parent_data_1, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_dp_link1_clk_src[] = { + F(162000000, P_DP_PHY_PLL_LINK_CLK, 1, 0, 0), + F(270000000, P_DP_PHY_PLL_LINK_CLK, 1, 0, 0), + F(540000000, P_DP_PHY_PLL_LINK_CLK, 1, 0, 0), + F(810000000, P_DP_PHY_PLL_LINK_CLK, 1, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_dp_link1_clk_src = { + .cmd_rcgr = 0x220c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .freq_tbl = ftbl_disp_cc_mdss_dp_link1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_link1_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dp_link_clk_src = { + .cmd_rcgr = 0x2178, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .freq_tbl = ftbl_disp_cc_mdss_dp_link1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_link_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dp_pixel1_clk_src = { + .cmd_rcgr = 0x21c4, + .mnd_width = 16, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_pixel1_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + .ops = &clk_dp_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dp_pixel2_clk_src = { + .cmd_rcgr = 0x21f4, + .mnd_width = 16, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_pixel2_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + .ops = &clk_dp_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dp_pixel_clk_src = { + .cmd_rcgr = 0x21ac, + .mnd_width = 16, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_pixel_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + .ops = &clk_dp_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_esc0_clk_src = { + .cmd_rcgr = 0x2148, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_2, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_esc0_clk_src", + .parent_data = disp_cc_parent_data_2, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_esc1_clk_src = { + .cmd_rcgr = 0x2160, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_2, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_esc1_clk_src", + .parent_data = disp_cc_parent_data_2, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_mdp_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(85714286, P_DISP_CC_PLL1_OUT_MAIN, 7, 0, 0), + F(100000000, P_DISP_CC_PLL1_OUT_MAIN, 6, 0, 0), + F(150000000, P_DISP_CC_PLL1_OUT_MAIN, 4, 0, 0), + F(200000000, P_DISP_CC_PLL1_OUT_MAIN, 3, 0, 0), + F(300000000, P_DISP_CC_PLL1_OUT_MAIN, 2, 0, 0), + F(345000000, P_DISP_CC_PLL0_OUT_MAIN, 4, 0, 0), + F(460000000, P_DISP_CC_PLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_mdp_clk_src = { + .cmd_rcgr = 0x20c8, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_5, + .freq_tbl = ftbl_disp_cc_mdss_mdp_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_mdp_clk_src", + .parent_data = disp_cc_parent_data_5, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_5), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_pclk0_clk_src = { + .cmd_rcgr = 0x2098, + .mnd_width = 8, + .hid_width = 5, + .parent_map = disp_cc_parent_map_6, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_pclk0_clk_src", + .parent_data = disp_cc_parent_data_6, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_6), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_pixel_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_pclk1_clk_src = { + .cmd_rcgr = 0x20b0, + .mnd_width = 8, + .hid_width = 5, + .parent_map = disp_cc_parent_map_6, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_pclk1_clk_src", + .parent_data = disp_cc_parent_data_6, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_6), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_pixel_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_rot_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(200000000, P_DISP_CC_PLL1_OUT_MAIN, 3, 0, 0), + F(300000000, P_DISP_CC_PLL1_OUT_MAIN, 2, 0, 0), + F(345000000, P_DISP_CC_PLL0_OUT_MAIN, 4, 0, 0), + F(460000000, P_DISP_CC_PLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_rot_clk_src = { + .cmd_rcgr = 0x20e0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_5, + .freq_tbl = ftbl_disp_cc_mdss_rot_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_rot_clk_src", + .parent_data = disp_cc_parent_data_5, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_5), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_vsync_clk_src = { + .cmd_rcgr = 0x20f8, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_vsync_clk_src", + .parent_data = disp_cc_parent_data_1, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_regmap_div disp_cc_mdss_byte0_div_clk_src = { + .reg = 0x2128, + .shift = 0, + .width = 2, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_byte0_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_byte0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + }, +}; + + +static struct clk_regmap_div disp_cc_mdss_byte1_div_clk_src = { + .reg = 0x2144, + .shift = 0, + .width = 2, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_byte1_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_byte1_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + }, +}; + + +static struct clk_regmap_div disp_cc_mdss_dp_link1_div_clk_src = { + .reg = 0x2224, + .shift = 0, + .width = 2, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dp_link1_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_link1_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_regmap_div_ro_ops, + }, +}; + + +static struct clk_regmap_div disp_cc_mdss_dp_link_div_clk_src = { + .reg = 0x2190, + .shift = 0, + .width = 2, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dp_link_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_link_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_branch disp_cc_mdss_ahb_clk = { + .halt_reg = 0x2080, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_ahb_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_byte0_clk = { + .halt_reg = 0x2028, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_byte0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_byte0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_byte0_intf_clk = { + .halt_reg = 0x202c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x202c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_byte0_intf_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_byte0_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_byte1_clk = { + .halt_reg = 0x2030, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2030, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_byte1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_byte1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_byte1_intf_clk = { + .halt_reg = 0x2034, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_byte1_intf_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_byte1_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_aux1_clk = { + .halt_reg = 0x2068, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2068, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_aux1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_aux1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_aux_clk = { + .halt_reg = 0x2054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_link1_clk = { + .halt_reg = 0x205c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x205c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_link1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_link1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_link1_intf_clk = { + .halt_reg = 0x2060, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2060, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_link1_intf_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_link1_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_link_clk = { + .halt_reg = 0x2040, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_link_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_link_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_link_intf_clk = { + .halt_reg = 0x2044, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2044, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_link_intf_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_link_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_pixel1_clk = { + .halt_reg = 0x2050, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_pixel1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_pixel1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_pixel2_clk = { + .halt_reg = 0x2058, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_pixel2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_pixel2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_pixel_clk = { + .halt_reg = 0x204c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x204c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_pixel_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_pixel_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_esc0_clk = { + .halt_reg = 0x2038, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2038, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_esc0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_esc0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_esc1_clk = { + .halt_reg = 0x203c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x203c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_esc1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_esc1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_mdp_clk = { + .halt_reg = 0x200c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x200c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_mdp_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_mdp_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_mdp_lut_clk = { + .halt_reg = 0x201c, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x201c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_mdp_lut_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_mdp_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_non_gdsc_ahb_clk = { + .halt_reg = 0x4004, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x4004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_non_gdsc_ahb_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_pclk0_clk = { + .halt_reg = 0x2004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_pclk0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_pclk0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_pclk1_clk = { + .halt_reg = 0x2008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_pclk1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_pclk1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_rot_clk = { + .halt_reg = 0x2014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_rot_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_rot_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_rscc_ahb_clk = { + .halt_reg = 0x400c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x400c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_rscc_ahb_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_rscc_vsync_clk = { + .halt_reg = 0x4008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_rscc_vsync_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_vsync_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_vsync_clk = { + .halt_reg = 0x2024, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_vsync_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_vsync_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc mdss_gdsc = { + .gdscr = 0x3000, + .pd = { + .name = "mdss_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = HW_CTRL, +}; + +static struct clk_regmap *disp_cc_sm8250_clocks[] = { + [DISP_CC_MDSS_AHB_CLK] = &disp_cc_mdss_ahb_clk.clkr, + [DISP_CC_MDSS_AHB_CLK_SRC] = &disp_cc_mdss_ahb_clk_src.clkr, + [DISP_CC_MDSS_BYTE0_CLK] = &disp_cc_mdss_byte0_clk.clkr, + [DISP_CC_MDSS_BYTE0_CLK_SRC] = &disp_cc_mdss_byte0_clk_src.clkr, + [DISP_CC_MDSS_BYTE0_DIV_CLK_SRC] = &disp_cc_mdss_byte0_div_clk_src.clkr, + [DISP_CC_MDSS_BYTE0_INTF_CLK] = &disp_cc_mdss_byte0_intf_clk.clkr, + [DISP_CC_MDSS_BYTE1_CLK] = &disp_cc_mdss_byte1_clk.clkr, + [DISP_CC_MDSS_BYTE1_CLK_SRC] = &disp_cc_mdss_byte1_clk_src.clkr, + [DISP_CC_MDSS_BYTE1_DIV_CLK_SRC] = &disp_cc_mdss_byte1_div_clk_src.clkr, + [DISP_CC_MDSS_BYTE1_INTF_CLK] = &disp_cc_mdss_byte1_intf_clk.clkr, + [DISP_CC_MDSS_DP_AUX1_CLK] = &disp_cc_mdss_dp_aux1_clk.clkr, + [DISP_CC_MDSS_DP_AUX1_CLK_SRC] = &disp_cc_mdss_dp_aux1_clk_src.clkr, + [DISP_CC_MDSS_DP_AUX_CLK] = &disp_cc_mdss_dp_aux_clk.clkr, + [DISP_CC_MDSS_DP_AUX_CLK_SRC] = &disp_cc_mdss_dp_aux_clk_src.clkr, + [DISP_CC_MDSS_DP_LINK1_CLK] = &disp_cc_mdss_dp_link1_clk.clkr, + [DISP_CC_MDSS_DP_LINK1_CLK_SRC] = &disp_cc_mdss_dp_link1_clk_src.clkr, + [DISP_CC_MDSS_DP_LINK1_DIV_CLK_SRC] = &disp_cc_mdss_dp_link1_div_clk_src.clkr, + [DISP_CC_MDSS_DP_LINK1_INTF_CLK] = &disp_cc_mdss_dp_link1_intf_clk.clkr, + [DISP_CC_MDSS_DP_LINK_CLK] = &disp_cc_mdss_dp_link_clk.clkr, + [DISP_CC_MDSS_DP_LINK_CLK_SRC] = &disp_cc_mdss_dp_link_clk_src.clkr, + [DISP_CC_MDSS_DP_LINK_DIV_CLK_SRC] = &disp_cc_mdss_dp_link_div_clk_src.clkr, + [DISP_CC_MDSS_DP_LINK_INTF_CLK] = &disp_cc_mdss_dp_link_intf_clk.clkr, + [DISP_CC_MDSS_DP_PIXEL1_CLK] = &disp_cc_mdss_dp_pixel1_clk.clkr, + [DISP_CC_MDSS_DP_PIXEL1_CLK_SRC] = &disp_cc_mdss_dp_pixel1_clk_src.clkr, + [DISP_CC_MDSS_DP_PIXEL2_CLK] = &disp_cc_mdss_dp_pixel2_clk.clkr, + [DISP_CC_MDSS_DP_PIXEL2_CLK_SRC] = &disp_cc_mdss_dp_pixel2_clk_src.clkr, + [DISP_CC_MDSS_DP_PIXEL_CLK] = &disp_cc_mdss_dp_pixel_clk.clkr, + [DISP_CC_MDSS_DP_PIXEL_CLK_SRC] = &disp_cc_mdss_dp_pixel_clk_src.clkr, + [DISP_CC_MDSS_ESC0_CLK] = &disp_cc_mdss_esc0_clk.clkr, + [DISP_CC_MDSS_ESC0_CLK_SRC] = &disp_cc_mdss_esc0_clk_src.clkr, + [DISP_CC_MDSS_ESC1_CLK] = &disp_cc_mdss_esc1_clk.clkr, + [DISP_CC_MDSS_ESC1_CLK_SRC] = &disp_cc_mdss_esc1_clk_src.clkr, + [DISP_CC_MDSS_MDP_CLK] = &disp_cc_mdss_mdp_clk.clkr, + [DISP_CC_MDSS_MDP_CLK_SRC] = &disp_cc_mdss_mdp_clk_src.clkr, + [DISP_CC_MDSS_MDP_LUT_CLK] = &disp_cc_mdss_mdp_lut_clk.clkr, + [DISP_CC_MDSS_NON_GDSC_AHB_CLK] = &disp_cc_mdss_non_gdsc_ahb_clk.clkr, + [DISP_CC_MDSS_PCLK0_CLK] = &disp_cc_mdss_pclk0_clk.clkr, + [DISP_CC_MDSS_PCLK0_CLK_SRC] = &disp_cc_mdss_pclk0_clk_src.clkr, + [DISP_CC_MDSS_PCLK1_CLK] = &disp_cc_mdss_pclk1_clk.clkr, + [DISP_CC_MDSS_PCLK1_CLK_SRC] = &disp_cc_mdss_pclk1_clk_src.clkr, + [DISP_CC_MDSS_ROT_CLK] = &disp_cc_mdss_rot_clk.clkr, + [DISP_CC_MDSS_ROT_CLK_SRC] = &disp_cc_mdss_rot_clk_src.clkr, + [DISP_CC_MDSS_RSCC_AHB_CLK] = &disp_cc_mdss_rscc_ahb_clk.clkr, + [DISP_CC_MDSS_RSCC_VSYNC_CLK] = &disp_cc_mdss_rscc_vsync_clk.clkr, + [DISP_CC_MDSS_VSYNC_CLK] = &disp_cc_mdss_vsync_clk.clkr, + [DISP_CC_MDSS_VSYNC_CLK_SRC] = &disp_cc_mdss_vsync_clk_src.clkr, + [DISP_CC_PLL0] = &disp_cc_pll0.clkr, + [DISP_CC_PLL1] = &disp_cc_pll1.clkr, +}; + +static const struct qcom_reset_map disp_cc_sm8250_resets[] = { + [DISP_CC_MDSS_CORE_BCR] = { 0x2000 }, + [DISP_CC_MDSS_RSCC_BCR] = { 0x4000 }, +}; + +static struct gdsc *disp_cc_sm8250_gdscs[] = { + [MDSS_GDSC] = &mdss_gdsc, +}; + +static const struct regmap_config disp_cc_sm8250_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x10000, + .fast_io = true, +}; + +static const struct qcom_cc_desc disp_cc_sm8250_desc = { + .config = &disp_cc_sm8250_regmap_config, + .clks = disp_cc_sm8250_clocks, + .num_clks = ARRAY_SIZE(disp_cc_sm8250_clocks), + .resets = disp_cc_sm8250_resets, + .num_resets = ARRAY_SIZE(disp_cc_sm8250_resets), + .gdscs = disp_cc_sm8250_gdscs, + .num_gdscs = ARRAY_SIZE(disp_cc_sm8250_gdscs), +}; + +static const struct of_device_id disp_cc_sm8250_match_table[] = { + { .compatible = "qcom,sm8150-dispcc" }, + { .compatible = "qcom,sm8250-dispcc" }, + { } +}; +MODULE_DEVICE_TABLE(of, disp_cc_sm8250_match_table); + +static int disp_cc_sm8250_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + + regmap = qcom_cc_map(pdev, &disp_cc_sm8250_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + /* note: trion == lucid, except for the prepare() op */ + BUILD_BUG_ON(CLK_ALPHA_PLL_TYPE_TRION != CLK_ALPHA_PLL_TYPE_LUCID); + if (of_device_is_compatible(pdev->dev.of_node, "qcom,sm8150-dispcc")) { + disp_cc_pll0_config.config_ctl_hi_val = 0x00002267; + disp_cc_pll0_config.config_ctl_hi1_val = 0x00000024; + disp_cc_pll0_config.user_ctl_hi1_val = 0x000000D0; + disp_cc_pll0_init.ops = &clk_alpha_pll_trion_ops; + disp_cc_pll1_config.config_ctl_hi_val = 0x00002267; + disp_cc_pll1_config.config_ctl_hi1_val = 0x00000024; + disp_cc_pll1_config.user_ctl_hi1_val = 0x000000D0; + disp_cc_pll1_init.ops = &clk_alpha_pll_trion_ops; + } + + clk_lucid_pll_configure(&disp_cc_pll0, regmap, &disp_cc_pll0_config); + clk_lucid_pll_configure(&disp_cc_pll1, regmap, &disp_cc_pll1_config); + + /* Enable clock gating for MDP clocks */ + regmap_update_bits(regmap, 0x8000, 0x10, 0x10); + + /* DISP_CC_XO_CLK always-on */ + regmap_update_bits(regmap, 0x605c, BIT(0), BIT(0)); + + return qcom_cc_really_probe(pdev, &disp_cc_sm8250_desc, regmap); +} + +static struct platform_driver disp_cc_sm8250_driver = { + .probe = disp_cc_sm8250_probe, + .driver = { + .name = "disp_cc-sm8250", + .of_match_table = disp_cc_sm8250_match_table, + }, +}; + +static int __init disp_cc_sm8250_init(void) +{ + return platform_driver_register(&disp_cc_sm8250_driver); +} +subsys_initcall(disp_cc_sm8250_init); + +static void __exit disp_cc_sm8250_exit(void) +{ + platform_driver_unregister(&disp_cc_sm8250_driver); +} +module_exit(disp_cc_sm8250_exit); + +MODULE_DESCRIPTION("QTI DISPCC SM8250 Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/gcc-ipq8074.c b/drivers/clk/qcom/gcc-ipq8074.c index ef2c9c4cf9ab..108fe27bee10 100644 --- a/drivers/clk/qcom/gcc-ipq8074.c +++ b/drivers/clk/qcom/gcc-ipq8074.c @@ -4322,7 +4322,7 @@ static const struct freq_tbl ftbl_pcie_rchng_clk_src[] = { { } }; -struct clk_rcg2 pcie0_rchng_clk_src = { +static struct clk_rcg2 pcie0_rchng_clk_src = { .cmd_rcgr = 0x75070, .freq_tbl = ftbl_pcie_rchng_clk_src, .hid_width = 5, diff --git a/drivers/clk/qcom/gcc-msm8939.c b/drivers/clk/qcom/gcc-msm8939.c index 778354f82b1e..39ebb443ae3d 100644 --- a/drivers/clk/qcom/gcc-msm8939.c +++ b/drivers/clk/qcom/gcc-msm8939.c @@ -595,24 +595,12 @@ static const struct clk_parent_data gcc_xo_gpll1_emclk_sleep_parent_data[] = { { .fw_name = "sleep_clk", .name = "sleep_clk" }, }; -static const struct parent_map gcc_xo_gpll6_gpll0_map[] = { - { P_XO, 0 }, - { P_GPLL6, 1 }, - { P_GPLL0, 2 }, -}; - static const struct clk_parent_data gcc_xo_gpll6_gpll0_parent_data[] = { { .fw_name = "xo" }, { .hw = &gpll6_vote.hw }, { .hw = &gpll0_vote.hw }, }; -static const struct parent_map gcc_xo_gpll6_gpll0a_map[] = { - { P_XO, 0 }, - { P_GPLL6, 1 }, - { P_GPLL0_AUX, 2 }, -}; - static const struct clk_parent_data gcc_xo_gpll6_gpll0a_parent_data[] = { { .fw_name = "xo" }, { .hw = &gpll6_vote.hw }, diff --git a/drivers/clk/qcom/gcc-msm8994.c b/drivers/clk/qcom/gcc-msm8994.c index b7fc8c7ba195..144d2ba7a9be 100644 --- a/drivers/clk/qcom/gcc-msm8994.c +++ b/drivers/clk/qcom/gcc-msm8994.c @@ -20,6 +20,7 @@ #include "clk-rcg.h" #include "clk-branch.h" #include "reset.h" +#include "gdsc.h" enum { P_XO, @@ -1772,6 +1773,32 @@ static struct clk_branch gcc_gp3_clk = { }, }; +static struct clk_branch gcc_lpass_q6_axi_clk = { + .halt_reg = 0x0280, + .clkr = { + .enable_reg = 0x0280, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_lpass_q6_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mss_q6_bimc_axi_clk = { + .halt_reg = 0x0284, + .clkr = { + .enable_reg = 0x0284, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_mss_q6_bimc_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_pcie_0_aux_clk = { .halt_reg = 0x1ad4, .clkr = { @@ -1790,6 +1817,32 @@ static struct clk_branch gcc_pcie_0_aux_clk = { }, }; +static struct clk_branch gcc_pcie_0_cfg_ahb_clk = { + .halt_reg = 0x1ad0, + .clkr = { + .enable_reg = 0x1ad0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_pcie_0_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_mstr_axi_clk = { + .halt_reg = 0x1acc, + .clkr = { + .enable_reg = 0x1acc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_pcie_0_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_pcie_0_pipe_clk = { .halt_reg = 0x1ad8, .halt_check = BRANCH_HALT_DELAY, @@ -1809,6 +1862,20 @@ static struct clk_branch gcc_pcie_0_pipe_clk = { }, }; +static struct clk_branch gcc_pcie_0_slv_axi_clk = { + .halt_reg = 0x1ac8, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x1ac8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_pcie_0_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_pcie_1_aux_clk = { .halt_reg = 0x1b54, .clkr = { @@ -1827,6 +1894,32 @@ static struct clk_branch gcc_pcie_1_aux_clk = { }, }; +static struct clk_branch gcc_pcie_1_cfg_ahb_clk = { + .halt_reg = 0x1b54, + .clkr = { + .enable_reg = 0x1b54, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_pcie_1_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_mstr_axi_clk = { + .halt_reg = 0x1b50, + .clkr = { + .enable_reg = 0x1b50, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_pcie_1_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_pcie_1_pipe_clk = { .halt_reg = 0x1b58, .halt_check = BRANCH_HALT_DELAY, @@ -1846,6 +1939,19 @@ static struct clk_branch gcc_pcie_1_pipe_clk = { }, }; +static struct clk_branch gcc_pcie_1_slv_axi_clk = { + .halt_reg = 0x1b48, + .clkr = { + .enable_reg = 0x1b48, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_pcie_1_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_pdm2_clk = { .halt_reg = 0x0ccc, .clkr = { @@ -1864,6 +1970,19 @@ static struct clk_branch gcc_pdm2_clk = { }, }; +static struct clk_branch gcc_pdm_ahb_clk = { + .halt_reg = 0x0cc4, + .clkr = { + .enable_reg = 0x0cc4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_pdm_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_sdcc1_apps_clk = { .halt_reg = 0x04c4, .clkr = { @@ -1899,6 +2018,23 @@ static struct clk_branch gcc_sdcc1_ahb_clk = { }, }; +static struct clk_branch gcc_sdcc2_ahb_clk = { + .halt_reg = 0x0508, + .clkr = { + .enable_reg = 0x0508, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_sdcc2_ahb_clk", + .parent_names = (const char *[]){ + "periph_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_sdcc2_apps_clk = { .halt_reg = 0x0504, .clkr = { @@ -1917,6 +2053,23 @@ static struct clk_branch gcc_sdcc2_apps_clk = { }, }; +static struct clk_branch gcc_sdcc3_ahb_clk = { + .halt_reg = 0x0548, + .clkr = { + .enable_reg = 0x0548, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_sdcc3_ahb_clk", + .parent_names = (const char *[]){ + "periph_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_sdcc3_apps_clk = { .halt_reg = 0x0544, .clkr = { @@ -1935,6 +2088,23 @@ static struct clk_branch gcc_sdcc3_apps_clk = { }, }; +static struct clk_branch gcc_sdcc4_ahb_clk = { + .halt_reg = 0x0588, + .clkr = { + .enable_reg = 0x0588, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_sdcc4_ahb_clk", + .parent_names = (const char *[]){ + "periph_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_sdcc4_apps_clk = { .halt_reg = 0x0584, .clkr = { @@ -1989,6 +2159,19 @@ static struct clk_branch gcc_sys_noc_usb3_axi_clk = { }, }; +static struct clk_branch gcc_tsif_ahb_clk = { + .halt_reg = 0x0d84, + .clkr = { + .enable_reg = 0x0d84, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_tsif_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_tsif_ref_clk = { .halt_reg = 0x0d88, .clkr = { @@ -2007,6 +2190,19 @@ static struct clk_branch gcc_tsif_ref_clk = { }, }; +static struct clk_branch gcc_ufs_ahb_clk = { + .halt_reg = 0x1d4c, + .clkr = { + .enable_reg = 0x1d4c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_ufs_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_ufs_axi_clk = { .halt_reg = 0x1d48, .clkr = { @@ -2043,6 +2239,34 @@ static struct clk_branch gcc_ufs_rx_cfg_clk = { }, }; +static struct clk_branch gcc_ufs_rx_symbol_0_clk = { + .halt_reg = 0x1d60, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x1d60, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_ufs_rx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_rx_symbol_1_clk = { + .halt_reg = 0x1d64, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x1d64, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_ufs_rx_symbol_1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_ufs_tx_cfg_clk = { .halt_reg = 0x1d50, .clkr = { @@ -2061,6 +2285,47 @@ static struct clk_branch gcc_ufs_tx_cfg_clk = { }, }; +static struct clk_branch gcc_ufs_tx_symbol_0_clk = { + .halt_reg = 0x1d58, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x1d58, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_ufs_tx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_tx_symbol_1_clk = { + .halt_reg = 0x1d5c, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x1d5c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_ufs_tx_symbol_1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb2_hs_phy_sleep_clk = { + .halt_reg = 0x04ac, + .clkr = { + .enable_reg = 0x04ac, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_usb2_hs_phy_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_usb30_master_clk = { .halt_reg = 0x03c8, .clkr = { @@ -2097,6 +2362,19 @@ static struct clk_branch gcc_usb30_mock_utmi_clk = { }, }; +static struct clk_branch gcc_usb30_sleep_clk = { + .halt_reg = 0x03cc, + .clkr = { + .enable_reg = 0x03cc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_usb30_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_usb3_phy_aux_clk = { .halt_reg = 0x1408, .clkr = { @@ -2115,6 +2393,19 @@ static struct clk_branch gcc_usb3_phy_aux_clk = { }, }; +static struct clk_branch gcc_usb_hs_ahb_clk = { + .halt_reg = 0x0488, + .clkr = { + .enable_reg = 0x0488, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_usb_hs_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_usb_hs_system_clk = { .halt_reg = 0x0484, .clkr = { @@ -2133,6 +2424,59 @@ static struct clk_branch gcc_usb_hs_system_clk = { }, }; +static struct clk_branch gcc_usb_phy_cfg_ahb2phy_clk = { + .halt_reg = 0x1a84, + .clkr = { + .enable_reg = 0x1a84, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) + { + .name = "gcc_usb_phy_cfg_ahb2phy_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc pcie_gdsc = { + .gdscr = 0x1e18, + .pd = { + .name = "pcie", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc pcie_0_gdsc = { + .gdscr = 0x1ac4, + .pd = { + .name = "pcie_0", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc pcie_1_gdsc = { + .gdscr = 0x1b44, + .pd = { + .name = "pcie_1", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc usb30_gdsc = { + .gdscr = 0x3c4, + .pd = { + .name = "usb30", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc ufs_gdsc = { + .gdscr = 0x1d44, + .pd = { + .name = "ufs", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + static struct clk_regmap *gcc_msm8994_clocks[] = { [GPLL0_EARLY] = &gpll0_early.clkr, [GPLL0] = &gpll0.clkr, @@ -2233,26 +2577,64 @@ static struct clk_regmap *gcc_msm8994_clocks[] = { [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_LPASS_Q6_AXI_CLK] = &gcc_lpass_q6_axi_clk.clkr, + [GCC_MSS_Q6_BIMC_AXI_CLK] = &gcc_mss_q6_bimc_axi_clk.clkr, [GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr, + [GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr, + [GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr, [GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr, + [GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr, [GCC_PCIE_1_AUX_CLK] = &gcc_pcie_1_aux_clk.clkr, + [GCC_PCIE_1_CFG_AHB_CLK] = &gcc_pcie_1_cfg_ahb_clk.clkr, + [GCC_PCIE_1_MSTR_AXI_CLK] = &gcc_pcie_1_mstr_axi_clk.clkr, [GCC_PCIE_1_PIPE_CLK] = &gcc_pcie_1_pipe_clk.clkr, + [GCC_PCIE_1_SLV_AXI_CLK] = &gcc_pcie_1_slv_axi_clk.clkr, [GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr, + [GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr, + [GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr, [GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr, + [GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr, [GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr, + [GCC_SDCC3_AHB_CLK] = &gcc_sdcc3_ahb_clk.clkr, [GCC_SDCC3_APPS_CLK] = &gcc_sdcc3_apps_clk.clkr, + [GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr, [GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr, - [GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr, [GCC_SYS_NOC_UFS_AXI_CLK] = &gcc_sys_noc_ufs_axi_clk.clkr, [GCC_SYS_NOC_USB3_AXI_CLK] = &gcc_sys_noc_usb3_axi_clk.clkr, + [GCC_TSIF_AHB_CLK] = &gcc_tsif_ahb_clk.clkr, [GCC_TSIF_REF_CLK] = &gcc_tsif_ref_clk.clkr, + [GCC_UFS_AHB_CLK] = &gcc_ufs_ahb_clk.clkr, [GCC_UFS_AXI_CLK] = &gcc_ufs_axi_clk.clkr, [GCC_UFS_RX_CFG_CLK] = &gcc_ufs_rx_cfg_clk.clkr, + [GCC_UFS_RX_SYMBOL_0_CLK] = &gcc_ufs_rx_symbol_0_clk.clkr, + [GCC_UFS_RX_SYMBOL_1_CLK] = &gcc_ufs_rx_symbol_1_clk.clkr, [GCC_UFS_TX_CFG_CLK] = &gcc_ufs_tx_cfg_clk.clkr, + [GCC_UFS_TX_SYMBOL_0_CLK] = &gcc_ufs_tx_symbol_0_clk.clkr, + [GCC_UFS_TX_SYMBOL_1_CLK] = &gcc_ufs_tx_symbol_1_clk.clkr, + [GCC_USB2_HS_PHY_SLEEP_CLK] = &gcc_usb2_hs_phy_sleep_clk.clkr, [GCC_USB30_MASTER_CLK] = &gcc_usb30_master_clk.clkr, [GCC_USB30_MOCK_UTMI_CLK] = &gcc_usb30_mock_utmi_clk.clkr, + [GCC_USB30_SLEEP_CLK] = &gcc_usb30_sleep_clk.clkr, [GCC_USB3_PHY_AUX_CLK] = &gcc_usb3_phy_aux_clk.clkr, + [GCC_USB_HS_AHB_CLK] = &gcc_usb_hs_ahb_clk.clkr, [GCC_USB_HS_SYSTEM_CLK] = &gcc_usb_hs_system_clk.clkr, + [GCC_USB_PHY_CFG_AHB2PHY_CLK] = &gcc_usb_phy_cfg_ahb2phy_clk.clkr, +}; + +static struct gdsc *gcc_msm8994_gdscs[] = { + [PCIE_GDSC] = &pcie_gdsc, + [PCIE_0_GDSC] = &pcie_0_gdsc, + [PCIE_1_GDSC] = &pcie_1_gdsc, + [USB30_GDSC] = &usb30_gdsc, + [UFS_GDSC] = &ufs_gdsc, +}; + +static const struct qcom_reset_map gcc_msm8994_resets[] = { + [USB3_PHY_RESET] = { 0x1400 }, + [USB3PHY_PHY_RESET] = { 0x1404 }, + [PCIE_PHY_0_RESET] = { 0x1b18 }, + [PCIE_PHY_1_RESET] = { 0x1b98 }, + [QUSB2_PHY_RESET] = { 0x04b8 }, }; static const struct regmap_config gcc_msm8994_regmap_config = { @@ -2267,6 +2649,10 @@ static const struct qcom_cc_desc gcc_msm8994_desc = { .config = &gcc_msm8994_regmap_config, .clks = gcc_msm8994_clocks, .num_clks = ARRAY_SIZE(gcc_msm8994_clocks), + .resets = gcc_msm8994_resets, + .num_resets = ARRAY_SIZE(gcc_msm8994_resets), + .gdscs = gcc_msm8994_gdscs, + .num_gdscs = ARRAY_SIZE(gcc_msm8994_gdscs), }; static const struct of_device_id gcc_msm8994_match_table[] = { diff --git a/drivers/clk/qcom/gcc-sdm660.c b/drivers/clk/qcom/gcc-sdm660.c index f0b47b7d50ca..31258795e7b8 100644 --- a/drivers/clk/qcom/gcc-sdm660.c +++ b/drivers/clk/qcom/gcc-sdm660.c @@ -666,7 +666,7 @@ static struct clk_rcg2 hmss_rbcpr_clk_src = { .cmd_rcgr = 0x48044, .mnd_width = 0, .hid_width = 5, - .parent_map = gcc_parent_map_xo_gpll0_gpll0_early_div, + .parent_map = gcc_parent_map_xo_gpll0, .freq_tbl = ftbl_hmss_rbcpr_clk_src, .clkr.hw.init = &(struct clk_init_data){ .name = "hmss_rbcpr_clk_src", diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c index bfc4ac02f9ea..af26e0695b86 100644 --- a/drivers/clk/qcom/gdsc.c +++ b/drivers/clk/qcom/gdsc.c @@ -358,6 +358,14 @@ static int gdsc_init(struct gdsc *sc) if ((sc->flags & VOTABLE) && on) gdsc_enable(&sc->pd); + /* + * Make sure the retain bit is set if the GDSC is already on, otherwise + * we end up turning off the GDSC and destroying all the register + * contents that we thought we were saving. + */ + if ((sc->flags & RETAIN_FF_ENABLE) && on) + gdsc_retain_ff_on(sc); + /* If ALWAYS_ON GDSCs are not ON, turn them ON */ if (sc->flags & ALWAYS_ON) { if (!on) diff --git a/drivers/clk/qcom/videocc-sm8150.c b/drivers/clk/qcom/videocc-sm8150.c new file mode 100644 index 000000000000..3087e2ec8fd4 --- /dev/null +++ b/drivers/clk/qcom/videocc-sm8150.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved. + */ + +#include <linux/clk-provider.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> + +#include <dt-bindings/clock/qcom,videocc-sm8150.h> + +#include "common.h" +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "reset.h" +#include "gdsc.h" + +enum { + P_BI_TCXO, + P_CHIP_SLEEP_CLK, + P_CORE_BI_PLL_TEST_SE, + P_VIDEO_PLL0_OUT_EVEN, + P_VIDEO_PLL0_OUT_MAIN, + P_VIDEO_PLL0_OUT_ODD, +}; + +static struct pll_vco trion_vco[] = { + { 249600000, 2000000000, 0 }, +}; + +static struct alpha_pll_config video_pll0_config = { + .l = 0x14, + .alpha = 0xD555, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00002267, + .config_ctl_hi1_val = 0x00000024, + .user_ctl_val = 0x00000000, + .user_ctl_hi_val = 0x00000805, + .user_ctl_hi1_val = 0x000000D0, +}; + +static struct clk_alpha_pll video_pll0 = { + .offset = 0x42c, + .vco_table = trion_vco, + .num_vco = ARRAY_SIZE(trion_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION], + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "video_pll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_trion_ops, + }, + }, +}; + +static const struct parent_map video_cc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_VIDEO_PLL0_OUT_MAIN, 1 }, +}; + +static const struct clk_parent_data video_cc_parent_data_0[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &video_pll0.clkr.hw }, +}; + +static const struct freq_tbl ftbl_video_cc_iris_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(200000000, P_VIDEO_PLL0_OUT_MAIN, 2, 0, 0), + F(240000000, P_VIDEO_PLL0_OUT_MAIN, 2, 0, 0), + F(338000000, P_VIDEO_PLL0_OUT_MAIN, 2, 0, 0), + F(365000000, P_VIDEO_PLL0_OUT_MAIN, 2, 0, 0), + F(444000000, P_VIDEO_PLL0_OUT_MAIN, 2, 0, 0), + F(533000000, P_VIDEO_PLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 video_cc_iris_clk_src = { + .cmd_rcgr = 0x7f0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = video_cc_parent_map_0, + .freq_tbl = ftbl_video_cc_iris_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "video_cc_iris_clk_src", + .parent_data = video_cc_parent_data_0, + .num_parents = ARRAY_SIZE(video_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_branch video_cc_iris_ahb_clk = { + .halt_reg = 0x8f4, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x8f4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_cc_iris_ahb_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &video_cc_iris_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_mvs0_core_clk = { + .halt_reg = 0x890, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x890, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_cc_mvs0_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &video_cc_iris_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_mvs1_core_clk = { + .halt_reg = 0x8d0, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x8d0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_cc_mvs1_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &video_cc_iris_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_mvsc_core_clk = { + .halt_reg = 0x850, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x850, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_cc_mvsc_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &video_cc_iris_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc venus_gdsc = { + .gdscr = 0x814, + .pd = { + .name = "venus_gdsc", + }, + .flags = 0, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc vcodec0_gdsc = { + .gdscr = 0x874, + .pd = { + .name = "vcodec0_gdsc", + }, + .flags = HW_CTRL, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc vcodec1_gdsc = { + .gdscr = 0x8b4, + .pd = { + .name = "vcodec1_gdsc", + }, + .flags = HW_CTRL, + .pwrsts = PWRSTS_OFF_ON, +}; +static struct clk_regmap *video_cc_sm8150_clocks[] = { + [VIDEO_CC_IRIS_AHB_CLK] = &video_cc_iris_ahb_clk.clkr, + [VIDEO_CC_IRIS_CLK_SRC] = &video_cc_iris_clk_src.clkr, + [VIDEO_CC_MVS0_CORE_CLK] = &video_cc_mvs0_core_clk.clkr, + [VIDEO_CC_MVS1_CORE_CLK] = &video_cc_mvs1_core_clk.clkr, + [VIDEO_CC_MVSC_CORE_CLK] = &video_cc_mvsc_core_clk.clkr, + [VIDEO_CC_PLL0] = &video_pll0.clkr, +}; + +static struct gdsc *video_cc_sm8150_gdscs[] = { + [VENUS_GDSC] = &venus_gdsc, + [VCODEC0_GDSC] = &vcodec0_gdsc, + [VCODEC1_GDSC] = &vcodec1_gdsc, +}; + +static const struct regmap_config video_cc_sm8150_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0xb94, + .fast_io = true, +}; + +static const struct qcom_reset_map video_cc_sm8150_resets[] = { + [VIDEO_CC_MVSC_CORE_CLK_BCR] = { 0x850, 2 }, +}; + +static const struct qcom_cc_desc video_cc_sm8150_desc = { + .config = &video_cc_sm8150_regmap_config, + .clks = video_cc_sm8150_clocks, + .num_clks = ARRAY_SIZE(video_cc_sm8150_clocks), + .resets = video_cc_sm8150_resets, + .num_resets = ARRAY_SIZE(video_cc_sm8150_resets), + .gdscs = video_cc_sm8150_gdscs, + .num_gdscs = ARRAY_SIZE(video_cc_sm8150_gdscs), +}; + +static const struct of_device_id video_cc_sm8150_match_table[] = { + { .compatible = "qcom,sm8150-videocc" }, + { } +}; +MODULE_DEVICE_TABLE(of, video_cc_sm8150_match_table); + +static int video_cc_sm8150_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + + regmap = qcom_cc_map(pdev, &video_cc_sm8150_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + clk_trion_pll_configure(&video_pll0, regmap, &video_pll0_config); + + /* Keep VIDEO_CC_XO_CLK ALWAYS-ON */ + regmap_update_bits(regmap, 0x984, 0x1, 0x1); + + return qcom_cc_really_probe(pdev, &video_cc_sm8150_desc, regmap); +} + +static struct platform_driver video_cc_sm8150_driver = { + .probe = video_cc_sm8150_probe, + .driver = { + .name = "video_cc-sm8150", + .of_match_table = video_cc_sm8150_match_table, + }, +}; + +static int __init video_cc_sm8150_init(void) +{ + return platform_driver_register(&video_cc_sm8150_driver); +} +subsys_initcall(video_cc_sm8150_init); + +static void __exit video_cc_sm8150_exit(void) +{ + platform_driver_unregister(&video_cc_sm8150_driver); +} +module_exit(video_cc_sm8150_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("QTI VIDEOCC SM8150 Driver"); diff --git a/drivers/clk/qcom/videocc-sm8250.c b/drivers/clk/qcom/videocc-sm8250.c new file mode 100644 index 000000000000..2797c61f5938 --- /dev/null +++ b/drivers/clk/qcom/videocc-sm8250.c @@ -0,0 +1,369 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. + */ + +#include <linux/clk-provider.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> + +#include <dt-bindings/clock/qcom,videocc-sm8250.h> + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "clk-regmap-divider.h" +#include "common.h" +#include "reset.h" +#include "gdsc.h" + +enum { + P_BI_TCXO, + P_CHIP_SLEEP_CLK, + P_CORE_BI_PLL_TEST_SE, + P_VIDEO_PLL0_OUT_MAIN, + P_VIDEO_PLL1_OUT_MAIN, +}; + +static struct pll_vco lucid_vco[] = { + { 249600000, 2000000000, 0 }, +}; + +static const struct alpha_pll_config video_pll0_config = { + .l = 0x25, + .alpha = 0x8000, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00002261, + .config_ctl_hi1_val = 0x329A699C, + .user_ctl_val = 0x00000000, + .user_ctl_hi_val = 0x00000805, + .user_ctl_hi1_val = 0x00000000, +}; + +static struct clk_alpha_pll video_pll0 = { + .offset = 0x42c, + .vco_table = lucid_vco, + .num_vco = ARRAY_SIZE(lucid_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "video_pll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_ops, + }, + }, +}; + +static const struct alpha_pll_config video_pll1_config = { + .l = 0x2B, + .alpha = 0xC000, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00002261, + .config_ctl_hi1_val = 0x329A699C, + .user_ctl_val = 0x00000000, + .user_ctl_hi_val = 0x00000805, + .user_ctl_hi1_val = 0x00000000, +}; + +static struct clk_alpha_pll video_pll1 = { + .offset = 0x7d0, + .vco_table = lucid_vco, + .num_vco = ARRAY_SIZE(lucid_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "video_pll1", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_ops, + }, + }, +}; + +static const struct parent_map video_cc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_VIDEO_PLL0_OUT_MAIN, 1 }, +}; + +static const struct clk_parent_data video_cc_parent_data_1[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &video_pll0.clkr.hw }, +}; + +static const struct parent_map video_cc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_VIDEO_PLL1_OUT_MAIN, 1 }, +}; + +static const struct clk_parent_data video_cc_parent_data_2[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &video_pll1.clkr.hw }, +}; + +static const struct freq_tbl ftbl_video_cc_mvs0_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(720000000, P_VIDEO_PLL0_OUT_MAIN, 1, 0, 0), + F(1014000000, P_VIDEO_PLL0_OUT_MAIN, 1, 0, 0), + F(1098000000, P_VIDEO_PLL0_OUT_MAIN, 1, 0, 0), + F(1332000000, P_VIDEO_PLL0_OUT_MAIN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 video_cc_mvs0_clk_src = { + .cmd_rcgr = 0xb94, + .mnd_width = 0, + .hid_width = 5, + .parent_map = video_cc_parent_map_1, + .freq_tbl = ftbl_video_cc_mvs0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "video_cc_mvs0_clk_src", + .parent_data = video_cc_parent_data_1, + .num_parents = ARRAY_SIZE(video_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_video_cc_mvs1_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(840000000, P_VIDEO_PLL1_OUT_MAIN, 1, 0, 0), + F(1098000000, P_VIDEO_PLL1_OUT_MAIN, 1, 0, 0), + F(1332000000, P_VIDEO_PLL1_OUT_MAIN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 video_cc_mvs1_clk_src = { + .cmd_rcgr = 0xbb4, + .mnd_width = 0, + .hid_width = 5, + .parent_map = video_cc_parent_map_2, + .freq_tbl = ftbl_video_cc_mvs1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "video_cc_mvs1_clk_src", + .parent_data = video_cc_parent_data_2, + .num_parents = ARRAY_SIZE(video_cc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_regmap_div video_cc_mvs0c_div2_div_clk_src = { + .reg = 0xc54, + .shift = 0, + .width = 2, + .clkr.hw.init = &(struct clk_init_data) { + .name = "video_cc_mvs0c_div2_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &video_cc_mvs0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div video_cc_mvs1c_div2_div_clk_src = { + .reg = 0xcf4, + .shift = 0, + .width = 2, + .clkr.hw.init = &(struct clk_init_data) { + .name = "video_cc_mvs1c_div2_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &video_cc_mvs1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_branch video_cc_mvs0c_clk = { + .halt_reg = 0xc34, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xc34, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_cc_mvs0c_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &video_cc_mvs0c_div2_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_mvs1_div2_clk = { + .halt_reg = 0xdf4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0xdf4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_cc_mvs1_div2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &video_cc_mvs1c_div2_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_mvs1c_clk = { + .halt_reg = 0xcd4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0xcd4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_cc_mvs1c_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &video_cc_mvs1c_div2_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc mvs0c_gdsc = { + .gdscr = 0xbf8, + .pd = { + .name = "mvs0c_gdsc", + }, + .flags = 0, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc mvs1c_gdsc = { + .gdscr = 0xc98, + .pd = { + .name = "mvs1c_gdsc", + }, + .flags = 0, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc mvs0_gdsc = { + .gdscr = 0xd18, + .pd = { + .name = "mvs0_gdsc", + }, + .flags = HW_CTRL, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc mvs1_gdsc = { + .gdscr = 0xd98, + .pd = { + .name = "mvs1_gdsc", + }, + .flags = HW_CTRL, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct clk_regmap *video_cc_sm8250_clocks[] = { + [VIDEO_CC_MVS0_CLK_SRC] = &video_cc_mvs0_clk_src.clkr, + [VIDEO_CC_MVS0C_CLK] = &video_cc_mvs0c_clk.clkr, + [VIDEO_CC_MVS0C_DIV2_DIV_CLK_SRC] = &video_cc_mvs0c_div2_div_clk_src.clkr, + [VIDEO_CC_MVS1_CLK_SRC] = &video_cc_mvs1_clk_src.clkr, + [VIDEO_CC_MVS1_DIV2_CLK] = &video_cc_mvs1_div2_clk.clkr, + [VIDEO_CC_MVS1C_CLK] = &video_cc_mvs1c_clk.clkr, + [VIDEO_CC_MVS1C_DIV2_DIV_CLK_SRC] = &video_cc_mvs1c_div2_div_clk_src.clkr, + [VIDEO_CC_PLL0] = &video_pll0.clkr, + [VIDEO_CC_PLL1] = &video_pll1.clkr, +}; + +static const struct qcom_reset_map video_cc_sm8250_resets[] = { + [VIDEO_CC_CVP_INTERFACE_BCR] = { 0xe54 }, + [VIDEO_CC_CVP_MVS0_BCR] = { 0xd14 }, + [VIDEO_CC_MVS0C_CLK_ARES] = { 0xc34, 2 }, + [VIDEO_CC_CVP_MVS0C_BCR] = { 0xbf4 }, + [VIDEO_CC_CVP_MVS1_BCR] = { 0xd94 }, + [VIDEO_CC_MVS1C_CLK_ARES] = { 0xcd4, 2 }, + [VIDEO_CC_CVP_MVS1C_BCR] = { 0xc94 }, +}; + +static struct gdsc *video_cc_sm8250_gdscs[] = { + [MVS0C_GDSC] = &mvs0c_gdsc, + [MVS1C_GDSC] = &mvs1c_gdsc, + [MVS0_GDSC] = &mvs0_gdsc, + [MVS1_GDSC] = &mvs1_gdsc, +}; + +static const struct regmap_config video_cc_sm8250_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0xf4c, + .fast_io = true, +}; + +static const struct qcom_cc_desc video_cc_sm8250_desc = { + .config = &video_cc_sm8250_regmap_config, + .clks = video_cc_sm8250_clocks, + .num_clks = ARRAY_SIZE(video_cc_sm8250_clocks), + .resets = video_cc_sm8250_resets, + .num_resets = ARRAY_SIZE(video_cc_sm8250_resets), + .gdscs = video_cc_sm8250_gdscs, + .num_gdscs = ARRAY_SIZE(video_cc_sm8250_gdscs), +}; + +static const struct of_device_id video_cc_sm8250_match_table[] = { + { .compatible = "qcom,sm8250-videocc" }, + { } +}; +MODULE_DEVICE_TABLE(of, video_cc_sm8250_match_table); + +static int video_cc_sm8250_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + + regmap = qcom_cc_map(pdev, &video_cc_sm8250_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + clk_lucid_pll_configure(&video_pll0, regmap, &video_pll0_config); + clk_lucid_pll_configure(&video_pll1, regmap, &video_pll1_config); + + /* Keep VIDEO_CC_AHB_CLK and VIDEO_CC_XO_CLK ALWAYS-ON */ + regmap_update_bits(regmap, 0xe58, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0xeec, BIT(0), BIT(0)); + + return qcom_cc_really_probe(pdev, &video_cc_sm8250_desc, regmap); +} + +static struct platform_driver video_cc_sm8250_driver = { + .probe = video_cc_sm8250_probe, + .driver = { + .name = "sm8250-videocc", + .of_match_table = video_cc_sm8250_match_table, + }, +}; + +static int __init video_cc_sm8250_init(void) +{ + return platform_driver_register(&video_cc_sm8250_driver); +} +subsys_initcall(video_cc_sm8250_init); + +static void __exit video_cc_sm8250_exit(void) +{ + platform_driver_unregister(&video_cc_sm8250_driver); +} +module_exit(video_cc_sm8250_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("QTI VIDEOCC SM8250 Driver"); diff --git a/drivers/clk/renesas/Kconfig b/drivers/clk/renesas/Kconfig index 28e8730ce263..18915d668a30 100644 --- a/drivers/clk/renesas/Kconfig +++ b/drivers/clk/renesas/Kconfig @@ -30,6 +30,7 @@ config CLK_RENESAS select CLK_R8A77980 if ARCH_R8A77980 select CLK_R8A77990 if ARCH_R8A77990 select CLK_R8A77995 if ARCH_R8A77995 + select CLK_R8A779A0 if ARCH_R8A779A0 select CLK_R9A06G032 if ARCH_R9A06G032 select CLK_SH73A0 if ARCH_SH73A0 @@ -145,6 +146,10 @@ config CLK_R8A77995 bool "R-Car D3 clock support" if COMPILE_TEST select CLK_RCAR_GEN3_CPG +config CLK_R8A779A0 + bool "R-Car V3U clock support" if COMPILE_TEST + select CLK_RENESAS_CPG_MSSR + config CLK_R9A06G032 bool "Renesas R9A06G032 clock driver" help @@ -162,7 +167,7 @@ config CLK_RCAR_GEN2_CPG select CLK_RENESAS_CPG_MSSR config CLK_RCAR_GEN3_CPG - bool "R-Car Gen3 CPG clock support" if COMPILE_TEST + bool "R-Car Gen3 and RZ/G2 CPG clock support" if COMPILE_TEST select CLK_RENESAS_CPG_MSSR config CLK_RCAR_USB2_CLOCK_SEL diff --git a/drivers/clk/renesas/Makefile b/drivers/clk/renesas/Makefile index c7c03ab9a6a3..c803912ef2ce 100644 --- a/drivers/clk/renesas/Makefile +++ b/drivers/clk/renesas/Makefile @@ -27,6 +27,7 @@ obj-$(CONFIG_CLK_R8A77970) += r8a77970-cpg-mssr.o obj-$(CONFIG_CLK_R8A77980) += r8a77980-cpg-mssr.o obj-$(CONFIG_CLK_R8A77990) += r8a77990-cpg-mssr.o obj-$(CONFIG_CLK_R8A77995) += r8a77995-cpg-mssr.o +obj-$(CONFIG_CLK_R8A779A0) += r8a779a0-cpg-mssr.o obj-$(CONFIG_CLK_R9A06G032) += r9a06g032-clocks.o obj-$(CONFIG_CLK_SH73A0) += clk-sh73a0.o diff --git a/drivers/clk/renesas/r7s9210-cpg-mssr.c b/drivers/clk/renesas/r7s9210-cpg-mssr.c index 443bff08df4c..a85227c248f3 100644 --- a/drivers/clk/renesas/r7s9210-cpg-mssr.c +++ b/drivers/clk/renesas/r7s9210-cpg-mssr.c @@ -214,7 +214,7 @@ const struct cpg_mssr_info r7s9210_cpg_mssr_info __initconst = { .cpg_clk_register = rza2_cpg_clk_register, /* RZ/A2 has Standby Control Registers */ - .stbyctrl = true, + .reg_layout = CLK_REG_LAYOUT_RZ_A, }; static void __init r7s9210_cpg_mssr_early_init(struct device_node *np) diff --git a/drivers/clk/renesas/r8a7742-cpg-mssr.c b/drivers/clk/renesas/r8a7742-cpg-mssr.c index e919828668a4..e541489bd1cd 100644 --- a/drivers/clk/renesas/r8a7742-cpg-mssr.c +++ b/drivers/clk/renesas/r8a7742-cpg-mssr.c @@ -97,7 +97,8 @@ static const struct mssr_mod_clk r8a7742_mod_clks[] __initconst = { DEF_MOD("tmu0", 125, R8A7742_CLK_CP), DEF_MOD("vsp1du1", 127, R8A7742_CLK_ZS), DEF_MOD("vsp1du0", 128, R8A7742_CLK_ZS), - DEF_MOD("vsp1-sy", 131, R8A7742_CLK_ZS), + DEF_MOD("vspr", 130, R8A7742_CLK_ZS), + DEF_MOD("vsps", 131, R8A7742_CLK_ZS), DEF_MOD("scifa2", 202, R8A7742_CLK_MP), DEF_MOD("scifa1", 203, R8A7742_CLK_MP), DEF_MOD("scifa0", 204, R8A7742_CLK_MP), diff --git a/drivers/clk/renesas/r8a7743-cpg-mssr.c b/drivers/clk/renesas/r8a7743-cpg-mssr.c index c01d9af2525a..0bba12a48d22 100644 --- a/drivers/clk/renesas/r8a7743-cpg-mssr.c +++ b/drivers/clk/renesas/r8a7743-cpg-mssr.c @@ -92,7 +92,7 @@ static const struct mssr_mod_clk r8a7743_mod_clks[] __initconst = { DEF_MOD("tmu0", 125, R8A7743_CLK_CP), DEF_MOD("vsp1du1", 127, R8A7743_CLK_ZS), DEF_MOD("vsp1du0", 128, R8A7743_CLK_ZS), - DEF_MOD("vsp1-sy", 131, R8A7743_CLK_ZS), + DEF_MOD("vsps", 131, R8A7743_CLK_ZS), DEF_MOD("scifa2", 202, R8A7743_CLK_MP), DEF_MOD("scifa1", 203, R8A7743_CLK_MP), DEF_MOD("scifa0", 204, R8A7743_CLK_MP), diff --git a/drivers/clk/renesas/r8a7745-cpg-mssr.c b/drivers/clk/renesas/r8a7745-cpg-mssr.c index 493874e5ebee..dc4a64e8dfb5 100644 --- a/drivers/clk/renesas/r8a7745-cpg-mssr.c +++ b/drivers/clk/renesas/r8a7745-cpg-mssr.c @@ -90,7 +90,7 @@ static const struct mssr_mod_clk r8a7745_mod_clks[] __initconst = { DEF_MOD("cmt0", 124, R8A7745_CLK_R), DEF_MOD("tmu0", 125, R8A7745_CLK_CP), DEF_MOD("vsp1du0", 128, R8A7745_CLK_ZS), - DEF_MOD("vsp1-sy", 131, R8A7745_CLK_ZS), + DEF_MOD("vsps", 131, R8A7745_CLK_ZS), DEF_MOD("scifa2", 202, R8A7745_CLK_MP), DEF_MOD("scifa1", 203, R8A7745_CLK_MP), DEF_MOD("scifa0", 204, R8A7745_CLK_MP), diff --git a/drivers/clk/renesas/r8a77470-cpg-mssr.c b/drivers/clk/renesas/r8a77470-cpg-mssr.c index d81ae65f0d18..f3d6e65011d7 100644 --- a/drivers/clk/renesas/r8a77470-cpg-mssr.c +++ b/drivers/clk/renesas/r8a77470-cpg-mssr.c @@ -85,7 +85,7 @@ static const struct mssr_mod_clk r8a77470_mod_clks[] __initconst = { DEF_MOD("tmu2", 122, R8A77470_CLK_P), DEF_MOD("cmt0", 124, R8A77470_CLK_R), DEF_MOD("vsp1du0", 128, R8A77470_CLK_ZS), - DEF_MOD("vsp1-sy", 131, R8A77470_CLK_ZS), + DEF_MOD("vsps", 131, R8A77470_CLK_ZS), DEF_MOD("msiof2", 205, R8A77470_CLK_MP), DEF_MOD("msiof1", 208, R8A77470_CLK_MP), DEF_MOD("sys-dmac1", 218, R8A77470_CLK_ZS), diff --git a/drivers/clk/renesas/r8a7790-cpg-mssr.c b/drivers/clk/renesas/r8a7790-cpg-mssr.c index c57cb93f8315..f7d233e0c142 100644 --- a/drivers/clk/renesas/r8a7790-cpg-mssr.c +++ b/drivers/clk/renesas/r8a7790-cpg-mssr.c @@ -108,8 +108,8 @@ static const struct mssr_mod_clk r8a7790_mod_clks[] __initconst = { DEF_MOD("tmu0", 125, R8A7790_CLK_CP), DEF_MOD("vsp1du1", 127, R8A7790_CLK_ZS), DEF_MOD("vsp1du0", 128, R8A7790_CLK_ZS), - DEF_MOD("vsp1-rt", 130, R8A7790_CLK_ZS), - DEF_MOD("vsp1-sy", 131, R8A7790_CLK_ZS), + DEF_MOD("vspr", 130, R8A7790_CLK_ZS), + DEF_MOD("vsps", 131, R8A7790_CLK_ZS), DEF_MOD("scifa2", 202, R8A7790_CLK_MP), DEF_MOD("scifa1", 203, R8A7790_CLK_MP), DEF_MOD("scifa0", 204, R8A7790_CLK_MP), diff --git a/drivers/clk/renesas/r8a7791-cpg-mssr.c b/drivers/clk/renesas/r8a7791-cpg-mssr.c index 65702debcabb..a0de784868da 100644 --- a/drivers/clk/renesas/r8a7791-cpg-mssr.c +++ b/drivers/clk/renesas/r8a7791-cpg-mssr.c @@ -102,7 +102,7 @@ static const struct mssr_mod_clk r8a7791_mod_clks[] __initconst = { DEF_MOD("tmu0", 125, R8A7791_CLK_CP), DEF_MOD("vsp1du1", 127, R8A7791_CLK_ZS), DEF_MOD("vsp1du0", 128, R8A7791_CLK_ZS), - DEF_MOD("vsp1-sy", 131, R8A7791_CLK_ZS), + DEF_MOD("vsps", 131, R8A7791_CLK_ZS), DEF_MOD("scifa2", 202, R8A7791_CLK_MP), DEF_MOD("scifa1", 203, R8A7791_CLK_MP), DEF_MOD("scifa0", 204, R8A7791_CLK_MP), diff --git a/drivers/clk/renesas/r8a7792-cpg-mssr.c b/drivers/clk/renesas/r8a7792-cpg-mssr.c index cf8b84a3a060..77af250876a5 100644 --- a/drivers/clk/renesas/r8a7792-cpg-mssr.c +++ b/drivers/clk/renesas/r8a7792-cpg-mssr.c @@ -88,7 +88,7 @@ static const struct mssr_mod_clk r8a7792_mod_clks[] __initconst = { DEF_MOD("tmu0", 125, R8A7792_CLK_CP), DEF_MOD("vsp1du1", 127, R8A7792_CLK_ZS), DEF_MOD("vsp1du0", 128, R8A7792_CLK_ZS), - DEF_MOD("vsp1-sy", 131, R8A7792_CLK_ZS), + DEF_MOD("vsps", 131, R8A7792_CLK_ZS), DEF_MOD("msiof1", 208, R8A7792_CLK_MP), DEF_MOD("sys-dmac1", 218, R8A7792_CLK_ZS), DEF_MOD("sys-dmac0", 219, R8A7792_CLK_ZS), diff --git a/drivers/clk/renesas/r8a7794-cpg-mssr.c b/drivers/clk/renesas/r8a7794-cpg-mssr.c index c1948693c5c1..4d7fa26a72c9 100644 --- a/drivers/clk/renesas/r8a7794-cpg-mssr.c +++ b/drivers/clk/renesas/r8a7794-cpg-mssr.c @@ -97,7 +97,7 @@ static const struct mssr_mod_clk r8a7794_mod_clks[] __initconst = { DEF_MOD("cmt0", 124, R8A7794_CLK_R), DEF_MOD("tmu0", 125, R8A7794_CLK_CP), DEF_MOD("vsp1du0", 128, R8A7794_CLK_ZS), - DEF_MOD("vsp1-sy", 131, R8A7794_CLK_ZS), + DEF_MOD("vsps", 131, R8A7794_CLK_ZS), DEF_MOD("scifa2", 202, R8A7794_CLK_MP), DEF_MOD("scifa1", 203, R8A7794_CLK_MP), DEF_MOD("scifa0", 204, R8A7794_CLK_MP), diff --git a/drivers/clk/renesas/r8a779a0-cpg-mssr.c b/drivers/clk/renesas/r8a779a0-cpg-mssr.c new file mode 100644 index 000000000000..17ebbac7ddfb --- /dev/null +++ b/drivers/clk/renesas/r8a779a0-cpg-mssr.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * r8a779a0 Clock Pulse Generator / Module Standby and Software Reset + * + * Copyright (C) 2020 Renesas Electronics Corp. + * + * Based on r8a7795-cpg-mssr.c + * + * Copyright (C) 2015 Glider bvba + * Copyright (C) 2015 Renesas Electronics Corp. + */ + +#include <linux/bug.h> +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/clk-provider.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/pm.h> +#include <linux/slab.h> +#include <linux/soc/renesas/rcar-rst.h> + +#include <dt-bindings/clock/r8a779a0-cpg-mssr.h> + +#include "renesas-cpg-mssr.h" +#include "rcar-gen3-cpg.h" + +enum rcar_r8a779a0_clk_types { + CLK_TYPE_R8A779A0_MAIN = CLK_TYPE_CUSTOM, + CLK_TYPE_R8A779A0_PLL1, + CLK_TYPE_R8A779A0_PLL2X_3X, /* PLL[23][01] */ + CLK_TYPE_R8A779A0_PLL5, + CLK_TYPE_R8A779A0_MDSEL, /* Select parent/divider using mode pin */ + CLK_TYPE_R8A779A0_OSC, /* OSC EXTAL predivider and fixed divider */ +}; + +struct rcar_r8a779a0_cpg_pll_config { + u8 extal_div; + u8 pll1_mult; + u8 pll1_div; + u8 pll5_mult; + u8 pll5_div; + u8 osc_prediv; +}; + +enum clk_ids { + /* Core Clock Outputs exported to DT */ + LAST_DT_CORE_CLK = R8A779A0_CLK_OSC, + + /* External Input Clocks */ + CLK_EXTAL, + CLK_EXTALR, + + /* Internal Core Clocks */ + CLK_MAIN, + CLK_PLL1, + CLK_PLL20, + CLK_PLL21, + CLK_PLL30, + CLK_PLL31, + CLK_PLL5, + CLK_PLL1_DIV2, + CLK_PLL20_DIV2, + CLK_PLL21_DIV2, + CLK_PLL30_DIV2, + CLK_PLL31_DIV2, + CLK_PLL5_DIV2, + CLK_PLL5_DIV4, + CLK_S1, + CLK_S2, + CLK_S3, + CLK_SDSRC, + CLK_RPCSRC, + CLK_OCO, + + /* Module Clocks */ + MOD_CLK_BASE +}; + +#define DEF_PLL(_name, _id, _offset) \ + DEF_BASE(_name, _id, CLK_TYPE_R8A779A0_PLL2X_3X, CLK_MAIN, \ + .offset = _offset) + +static const struct cpg_core_clk r8a779a0_core_clks[] __initconst = { + /* External Clock Inputs */ + DEF_INPUT("extal", CLK_EXTAL), + DEF_INPUT("extalr", CLK_EXTALR), + + /* Internal Core Clocks */ + DEF_BASE(".main", CLK_MAIN, CLK_TYPE_R8A779A0_MAIN, CLK_EXTAL), + DEF_BASE(".pll1", CLK_PLL1, CLK_TYPE_R8A779A0_PLL1, CLK_MAIN), + DEF_BASE(".pll5", CLK_PLL5, CLK_TYPE_R8A779A0_PLL5, CLK_MAIN), + DEF_PLL(".pll20", CLK_PLL20, 0x0834), + DEF_PLL(".pll21", CLK_PLL21, 0x0838), + DEF_PLL(".pll30", CLK_PLL30, 0x083c), + DEF_PLL(".pll31", CLK_PLL31, 0x0840), + + DEF_FIXED(".pll1_div2", CLK_PLL1_DIV2, CLK_PLL1, 2, 1), + DEF_FIXED(".pll20_div2", CLK_PLL20_DIV2, CLK_PLL20, 2, 1), + DEF_FIXED(".pll21_div2", CLK_PLL21_DIV2, CLK_PLL21, 2, 1), + DEF_FIXED(".pll30_div2", CLK_PLL30_DIV2, CLK_PLL30, 2, 1), + DEF_FIXED(".pll31_div2", CLK_PLL31_DIV2, CLK_PLL31, 2, 1), + DEF_FIXED(".pll5_div2", CLK_PLL5_DIV2, CLK_PLL5, 2, 1), + DEF_FIXED(".pll5_div4", CLK_PLL5_DIV4, CLK_PLL5_DIV2, 2, 1), + DEF_FIXED(".s1", CLK_S1, CLK_PLL1_DIV2, 2, 1), + DEF_FIXED(".s3", CLK_S3, CLK_PLL1_DIV2, 4, 1), + DEF_RATE(".oco", CLK_OCO, 32768), + + /* Core Clock Outputs */ + DEF_FIXED("zx", R8A779A0_CLK_ZX, CLK_PLL20_DIV2, 2, 1), + DEF_FIXED("s1d1", R8A779A0_CLK_S1D1, CLK_S1, 1, 1), + DEF_FIXED("s1d2", R8A779A0_CLK_S1D2, CLK_S1, 2, 1), + DEF_FIXED("s1d4", R8A779A0_CLK_S1D4, CLK_S1, 4, 1), + DEF_FIXED("s1d8", R8A779A0_CLK_S1D8, CLK_S1, 8, 1), + DEF_FIXED("s1d12", R8A779A0_CLK_S1D12, CLK_S1, 12, 1), + DEF_FIXED("s3d1", R8A779A0_CLK_S3D1, CLK_S3, 1, 1), + DEF_FIXED("s3d2", R8A779A0_CLK_S3D2, CLK_S3, 2, 1), + DEF_FIXED("s3d4", R8A779A0_CLK_S3D4, CLK_S3, 4, 1), + DEF_FIXED("zs", R8A779A0_CLK_ZS, CLK_PLL1_DIV2, 4, 1), + DEF_FIXED("zt", R8A779A0_CLK_ZT, CLK_PLL1_DIV2, 2, 1), + DEF_FIXED("ztr", R8A779A0_CLK_ZTR, CLK_PLL1_DIV2, 2, 1), + DEF_FIXED("zr", R8A779A0_CLK_ZR, CLK_PLL1_DIV2, 1, 1), + DEF_FIXED("dsi", R8A779A0_CLK_DSI, CLK_PLL5_DIV4, 1, 1), + DEF_FIXED("cnndsp", R8A779A0_CLK_CNNDSP, CLK_PLL5_DIV4, 1, 1), + DEF_FIXED("vip", R8A779A0_CLK_VIP, CLK_PLL5, 5, 1), + DEF_FIXED("adgh", R8A779A0_CLK_ADGH, CLK_PLL5_DIV4, 1, 1), + DEF_FIXED("icu", R8A779A0_CLK_ICU, CLK_PLL5_DIV4, 2, 1), + DEF_FIXED("icud2", R8A779A0_CLK_ICUD2, CLK_PLL5_DIV4, 4, 1), + DEF_FIXED("vcbus", R8A779A0_CLK_VCBUS, CLK_PLL5_DIV4, 1, 1), + DEF_FIXED("cbfusa", R8A779A0_CLK_CBFUSA, CLK_MAIN, 2, 1), + + DEF_DIV6P1("mso", R8A779A0_CLK_MSO, CLK_PLL5_DIV4, 0x87c), + DEF_DIV6P1("canfd", R8A779A0_CLK_CANFD, CLK_PLL5_DIV4, 0x878), + DEF_DIV6P1("csi0", R8A779A0_CLK_CSI0, CLK_PLL5_DIV4, 0x880), + + DEF_GEN3_OSC("osc", R8A779A0_CLK_OSC, CLK_EXTAL, 8), + DEF_GEN3_MDSEL("r", R8A779A0_CLK_R, 29, CLK_EXTALR, 1, CLK_OCO, 1), +}; + +static const struct mssr_mod_clk r8a779a0_mod_clks[] __initconst = { + DEF_MOD("scif0", 702, R8A779A0_CLK_S1D8), + DEF_MOD("scif1", 703, R8A779A0_CLK_S1D8), + DEF_MOD("scif3", 704, R8A779A0_CLK_S1D8), + DEF_MOD("scif4", 705, R8A779A0_CLK_S1D8), +}; + +static spinlock_t cpg_lock; + +static const struct rcar_r8a779a0_cpg_pll_config *cpg_pll_config __initdata; +static unsigned int cpg_clk_extalr __initdata; +static u32 cpg_mode __initdata; + +struct clk * __init rcar_r8a779a0_cpg_clk_register(struct device *dev, + const struct cpg_core_clk *core, const struct cpg_mssr_info *info, + struct clk **clks, void __iomem *base, + struct raw_notifier_head *notifiers) +{ + const struct clk *parent; + unsigned int mult = 1; + unsigned int div = 1; + u32 value; + + parent = clks[core->parent & 0xffff]; /* some types use high bits */ + if (IS_ERR(parent)) + return ERR_CAST(parent); + + switch (core->type) { + case CLK_TYPE_R8A779A0_MAIN: + div = cpg_pll_config->extal_div; + break; + + case CLK_TYPE_R8A779A0_PLL1: + mult = cpg_pll_config->pll1_mult; + div = cpg_pll_config->pll1_div; + break; + + case CLK_TYPE_R8A779A0_PLL2X_3X: + value = readl(base + core->offset); + mult = (((value >> 24) & 0x7f) + 1) * 2; + break; + + case CLK_TYPE_R8A779A0_PLL5: + mult = cpg_pll_config->pll5_mult; + div = cpg_pll_config->pll5_div; + break; + + case CLK_TYPE_R8A779A0_MDSEL: + /* + * Clock selectable between two parents and two fixed dividers + * using a mode pin + */ + if (cpg_mode & BIT(core->offset)) { + div = core->div & 0xffff; + } else { + parent = clks[core->parent >> 16]; + if (IS_ERR(parent)) + return ERR_CAST(parent); + div = core->div >> 16; + } + mult = 1; + break; + + case CLK_TYPE_R8A779A0_OSC: + /* + * Clock combining OSC EXTAL predivider and a fixed divider + */ + div = cpg_pll_config->osc_prediv * core->div; + break; + + default: + return ERR_PTR(-EINVAL); + } + + return clk_register_fixed_factor(NULL, core->name, + __clk_get_name(parent), 0, mult, div); +} + +/* + * CPG Clock Data + */ +/* + * MD EXTAL PLL1 PLL20 PLL30 PLL4 PLL5 OSC + * 14 13 (MHz) 21 31 + * -------------------------------------------------------- + * 0 0 16.66 x 1 x128 x216 x128 x144 x192 /16 + * 0 1 20 x 1 x106 x180 x106 x120 x160 /19 + * 1 0 Prohibited setting + * 1 1 33.33 / 2 x128 x216 x128 x144 x192 /32 + */ +#define CPG_PLL_CONFIG_INDEX(md) ((((md) & BIT(14)) >> 13) | \ + (((md) & BIT(13)) >> 13)) + +static const struct rcar_r8a779a0_cpg_pll_config cpg_pll_configs[4] = { + /* EXTAL div PLL1 mult/div PLL5 mult/div OSC prediv */ + { 1, 128, 1, 192, 1, 16, }, + { 1, 106, 1, 160, 1, 19, }, + { 0, 0, 0, 0, 0, 0, }, + { 2, 128, 1, 192, 1, 32, }, +}; + +static int __init r8a779a0_cpg_mssr_init(struct device *dev) +{ + int error; + + error = rcar_rst_read_mode_pins(&cpg_mode); + if (error) + return error; + + cpg_pll_config = &cpg_pll_configs[CPG_PLL_CONFIG_INDEX(cpg_mode)]; + cpg_clk_extalr = CLK_EXTALR; + spin_lock_init(&cpg_lock); + + return 0; +} + +const struct cpg_mssr_info r8a779a0_cpg_mssr_info __initconst = { + /* Core Clocks */ + .core_clks = r8a779a0_core_clks, + .num_core_clks = ARRAY_SIZE(r8a779a0_core_clks), + .last_dt_core_clk = LAST_DT_CORE_CLK, + .num_total_core_clks = MOD_CLK_BASE, + + /* Module Clocks */ + .mod_clks = r8a779a0_mod_clks, + .num_mod_clks = ARRAY_SIZE(r8a779a0_mod_clks), + .num_hw_mod_clks = 15 * 32, + + /* Callbacks */ + .init = r8a779a0_cpg_mssr_init, + .cpg_clk_register = rcar_r8a779a0_cpg_clk_register, + + .reg_layout = CLK_REG_LAYOUT_RCAR_V3U, +}; diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c index 5a306d28738c..94db88370337 100644 --- a/drivers/clk/renesas/renesas-cpg-mssr.c +++ b/drivers/clk/renesas/renesas-cpg-mssr.c @@ -57,8 +57,10 @@ static const u16 mstpsr[] = { 0x9A0, 0x9A4, 0x9A8, 0x9AC, }; -#define MSTPSR(i) mstpsr[i] - +static const u16 mstpsr_for_v3u[] = { + 0x2E00, 0x2E04, 0x2E08, 0x2E0C, 0x2E10, 0x2E14, 0x2E18, 0x2E1C, + 0x2E20, 0x2E24, 0x2E28, 0x2E2C, 0x2E30, 0x2E34, 0x2E38, +}; /* * System Module Stop Control Register offsets @@ -69,7 +71,10 @@ static const u16 smstpcr[] = { 0x990, 0x994, 0x998, 0x99C, }; -#define SMSTPCR(i) smstpcr[i] +static const u16 mstpcr_for_v3u[] = { + 0x2D00, 0x2D04, 0x2D08, 0x2D0C, 0x2D10, 0x2D14, 0x2D18, 0x2D1C, + 0x2D20, 0x2D24, 0x2D28, 0x2D2C, 0x2D30, 0x2D34, 0x2D38, +}; /* * Standby Control Register offsets (RZ/A) @@ -81,8 +86,6 @@ static const u16 stbcr[] = { 0x424, 0x428, 0x42C, }; -#define STBCR(i) stbcr[i] - /* * Software Reset Register offsets */ @@ -92,8 +95,10 @@ static const u16 srcr[] = { 0x920, 0x924, 0x928, 0x92C, }; -#define SRCR(i) srcr[i] - +static const u16 srcr_for_v3u[] = { + 0x2C00, 0x2C04, 0x2C08, 0x2C0C, 0x2C10, 0x2C14, 0x2C18, 0x2C1C, + 0x2C20, 0x2C24, 0x2C28, 0x2C2C, 0x2C30, 0x2C34, 0x2C38, +}; /* Realtime Module Stop Control Register offsets */ #define RMSTPCR(i) (smstpcr[i] - 0x20) @@ -102,8 +107,16 @@ static const u16 srcr[] = { #define MMSTPCR(i) (smstpcr[i] + 0x20) /* Software Reset Clearing Register offsets */ -#define SRSTCLR(i) (0x940 + (i) * 4) +static const u16 srstclr[] = { + 0x940, 0x944, 0x948, 0x94C, 0x950, 0x954, 0x958, 0x95C, + 0x960, 0x964, 0x968, 0x96C, +}; + +static const u16 srstclr_for_v3u[] = { + 0x2C80, 0x2C84, 0x2C88, 0x2C8C, 0x2C90, 0x2C94, 0x2C98, 0x2C9C, + 0x2CA0, 0x2CA4, 0x2CA8, 0x2CAC, 0x2CB0, 0x2CB4, 0x2CB8, +}; /** * Clock Pulse Generator / Module Standby and Software Reset Private Data @@ -111,13 +124,17 @@ static const u16 srcr[] = { * @rcdev: Optional reset controller entity * @dev: CPG/MSSR device * @base: CPG/MSSR register block base address + * @reg_layout: CPG/MSSR register layout * @rmw_lock: protects RMW register accesses * @np: Device node in DT for this CPG/MSSR module * @num_core_clks: Number of Core Clocks in clks[] * @num_mod_clks: Number of Module Clocks in clks[] * @last_dt_core_clk: ID of the last Core Clock exported to DT - * @stbyctrl: This device has Standby Control Registers * @notifiers: Notifier chain to save/restore clock state for system resume + * @status_regs: Pointer to status registers array + * @control_regs: Pointer to control registers array + * @reset_regs: Pointer to reset registers array + * @reset_clear_regs: Pointer to reset clearing registers array * @smstpcr_saved[].mask: Mask of SMSTPCR[] bits under our control * @smstpcr_saved[].val: Saved values of SMSTPCR[] * @clks: Array containing all Core and Module Clocks @@ -128,19 +145,23 @@ struct cpg_mssr_priv { #endif struct device *dev; void __iomem *base; + enum clk_reg_layout reg_layout; spinlock_t rmw_lock; struct device_node *np; unsigned int num_core_clks; unsigned int num_mod_clks; unsigned int last_dt_core_clk; - bool stbyctrl; struct raw_notifier_head notifiers; + const u16 *status_regs; + const u16 *control_regs; + const u16 *reset_regs; + const u16 *reset_clear_regs; struct { u32 mask; u32 val; - } smstpcr_saved[ARRAY_SIZE(smstpcr)]; + } smstpcr_saved[ARRAY_SIZE(mstpsr_for_v3u)]; struct clk *clks[]; }; @@ -177,40 +198,40 @@ static int cpg_mstp_clock_endisable(struct clk_hw *hw, bool enable) enable ? "ON" : "OFF"); spin_lock_irqsave(&priv->rmw_lock, flags); - if (priv->stbyctrl) { - value = readb(priv->base + STBCR(reg)); + if (priv->reg_layout == CLK_REG_LAYOUT_RZ_A) { + value = readb(priv->base + priv->control_regs[reg]); if (enable) value &= ~bitmask; else value |= bitmask; - writeb(value, priv->base + STBCR(reg)); + writeb(value, priv->base + priv->control_regs[reg]); /* dummy read to ensure write has completed */ - readb(priv->base + STBCR(reg)); - barrier_data(priv->base + STBCR(reg)); + readb(priv->base + priv->control_regs[reg]); + barrier_data(priv->base + priv->control_regs[reg]); } else { - value = readl(priv->base + SMSTPCR(reg)); + value = readl(priv->base + priv->control_regs[reg]); if (enable) value &= ~bitmask; else value |= bitmask; - writel(value, priv->base + SMSTPCR(reg)); + writel(value, priv->base + priv->control_regs[reg]); } spin_unlock_irqrestore(&priv->rmw_lock, flags); - if (!enable || priv->stbyctrl) + if (!enable || priv->reg_layout == CLK_REG_LAYOUT_RZ_A) return 0; for (i = 1000; i > 0; --i) { - if (!(readl(priv->base + MSTPSR(reg)) & bitmask)) + if (!(readl(priv->base + priv->status_regs[reg]) & bitmask)) break; cpu_relax(); } if (!i) { dev_err(dev, "Failed to enable SMSTP %p[%d]\n", - priv->base + SMSTPCR(reg), bit); + priv->base + priv->control_regs[reg], bit); return -ETIMEDOUT; } @@ -233,10 +254,10 @@ static int cpg_mstp_clock_is_enabled(struct clk_hw *hw) struct cpg_mssr_priv *priv = clock->priv; u32 value; - if (priv->stbyctrl) - value = readb(priv->base + STBCR(clock->index / 32)); + if (priv->reg_layout == CLK_REG_LAYOUT_RZ_A) + value = readb(priv->base + priv->control_regs[clock->index / 32]); else - value = readl(priv->base + MSTPSR(clock->index / 32)); + value = readl(priv->base + priv->status_regs[clock->index / 32]); return !(value & BIT(clock->index % 32)); } @@ -272,7 +293,7 @@ struct clk *cpg_mssr_clk_src_twocell_get(struct of_phandle_args *clkspec, case CPG_MOD: type = "module"; - if (priv->stbyctrl) { + if (priv->reg_layout == CLK_REG_LAYOUT_RZ_A) { idx = MOD_CLK_PACK_10(clkidx); range_check = 7 - (clkidx % 10); } else { @@ -578,13 +599,13 @@ static int cpg_mssr_reset(struct reset_controller_dev *rcdev, dev_dbg(priv->dev, "reset %u%02u\n", reg, bit); /* Reset module */ - writel(bitmask, priv->base + SRCR(reg)); + writel(bitmask, priv->base + priv->reset_regs[reg]); /* Wait for at least one cycle of the RCLK clock (@ ca. 32 kHz) */ udelay(35); /* Release module from reset state */ - writel(bitmask, priv->base + SRSTCLR(reg)); + writel(bitmask, priv->base + priv->reset_clear_regs[reg]); return 0; } @@ -598,7 +619,7 @@ static int cpg_mssr_assert(struct reset_controller_dev *rcdev, unsigned long id) dev_dbg(priv->dev, "assert %u%02u\n", reg, bit); - writel(bitmask, priv->base + SRCR(reg)); + writel(bitmask, priv->base + priv->reset_regs[reg]); return 0; } @@ -612,7 +633,7 @@ static int cpg_mssr_deassert(struct reset_controller_dev *rcdev, dev_dbg(priv->dev, "deassert %u%02u\n", reg, bit); - writel(bitmask, priv->base + SRSTCLR(reg)); + writel(bitmask, priv->base + priv->reset_clear_regs[reg]); return 0; } @@ -624,7 +645,7 @@ static int cpg_mssr_status(struct reset_controller_dev *rcdev, unsigned int bit = id % 32; u32 bitmask = BIT(bit); - return !!(readl(priv->base + SRCR(reg)) & bitmask); + return !!(readl(priv->base + priv->reset_regs[reg]) & bitmask); } static const struct reset_control_ops cpg_mssr_reset_ops = { @@ -804,6 +825,12 @@ static const struct of_device_id cpg_mssr_match[] = { .data = &r8a77995_cpg_mssr_info, }, #endif +#ifdef CONFIG_CLK_R8A779A0 + { + .compatible = "renesas,r8a779a0-cpg-mssr", + .data = &r8a779a0_cpg_mssr_info, + }, +#endif { /* sentinel */ } }; @@ -825,9 +852,10 @@ static int cpg_mssr_suspend_noirq(struct device *dev) /* Save module registers with bits under our control */ for (reg = 0; reg < ARRAY_SIZE(priv->smstpcr_saved); reg++) { if (priv->smstpcr_saved[reg].mask) - priv->smstpcr_saved[reg].val = priv->stbyctrl ? - readb(priv->base + STBCR(reg)) : - readl(priv->base + SMSTPCR(reg)); + priv->smstpcr_saved[reg].val = + priv->reg_layout == CLK_REG_LAYOUT_RZ_A ? + readb(priv->base + priv->control_regs[reg]) : + readl(priv->base + priv->control_regs[reg]); } /* Save core clocks */ @@ -855,23 +883,23 @@ static int cpg_mssr_resume_noirq(struct device *dev) if (!mask) continue; - if (priv->stbyctrl) - oldval = readb(priv->base + STBCR(reg)); + if (priv->reg_layout == CLK_REG_LAYOUT_RZ_A) + oldval = readb(priv->base + priv->control_regs[reg]); else - oldval = readl(priv->base + SMSTPCR(reg)); + oldval = readl(priv->base + priv->control_regs[reg]); newval = oldval & ~mask; newval |= priv->smstpcr_saved[reg].val & mask; if (newval == oldval) continue; - if (priv->stbyctrl) { - writeb(newval, priv->base + STBCR(reg)); + if (priv->reg_layout == CLK_REG_LAYOUT_RZ_A) { + writeb(newval, priv->base + priv->control_regs[reg]); /* dummy read to ensure write has completed */ - readb(priv->base + STBCR(reg)); - barrier_data(priv->base + STBCR(reg)); + readb(priv->base + priv->control_regs[reg]); + barrier_data(priv->base + priv->control_regs[reg]); continue; } else - writel(newval, priv->base + SMSTPCR(reg)); + writel(newval, priv->base + priv->control_regs[reg]); /* Wait until enabled clocks are really enabled */ mask &= ~priv->smstpcr_saved[reg].val; @@ -879,7 +907,7 @@ static int cpg_mssr_resume_noirq(struct device *dev) continue; for (i = 1000; i > 0; --i) { - oldval = readl(priv->base + MSTPSR(reg)); + oldval = readl(priv->base + priv->status_regs[reg]); if (!(oldval & mask)) break; cpu_relax(); @@ -887,8 +915,8 @@ static int cpg_mssr_resume_noirq(struct device *dev) if (!i) dev_warn(dev, "Failed to enable %s%u[0x%x]\n", - priv->stbyctrl ? "STB" : "SMSTP", reg, - oldval & mask); + priv->reg_layout == CLK_REG_LAYOUT_RZ_A ? + "STB" : "SMSTP", reg, oldval & mask); } return 0; @@ -937,7 +965,23 @@ static int __init cpg_mssr_common_init(struct device *dev, priv->num_mod_clks = info->num_hw_mod_clks; priv->last_dt_core_clk = info->last_dt_core_clk; RAW_INIT_NOTIFIER_HEAD(&priv->notifiers); - priv->stbyctrl = info->stbyctrl; + priv->reg_layout = info->reg_layout; + if (priv->reg_layout == CLK_REG_LAYOUT_RCAR_GEN2_AND_GEN3) { + priv->status_regs = mstpsr; + priv->control_regs = smstpcr; + priv->reset_regs = srcr; + priv->reset_clear_regs = srstclr; + } else if (priv->reg_layout == CLK_REG_LAYOUT_RZ_A) { + priv->control_regs = stbcr; + } else if (priv->reg_layout == CLK_REG_LAYOUT_RCAR_V3U) { + priv->status_regs = mstpsr_for_v3u; + priv->control_regs = mstpcr_for_v3u; + priv->reset_regs = srcr_for_v3u; + priv->reset_clear_regs = srstclr_for_v3u; + } else { + error = -EINVAL; + goto out_err; + } for (i = 0; i < nclks; i++) priv->clks[i] = ERR_PTR(-ENOENT); @@ -1015,7 +1059,7 @@ static int __init cpg_mssr_probe(struct platform_device *pdev) return error; /* Reset Controller not supported for Standby Control SoCs */ - if (info->stbyctrl) + if (priv->reg_layout == CLK_REG_LAYOUT_RZ_A) return 0; error = cpg_mssr_reset_controller_register(priv); diff --git a/drivers/clk/renesas/renesas-cpg-mssr.h b/drivers/clk/renesas/renesas-cpg-mssr.h index 1cc569484250..6b2a0ade482e 100644 --- a/drivers/clk/renesas/renesas-cpg-mssr.h +++ b/drivers/clk/renesas/renesas-cpg-mssr.h @@ -85,6 +85,12 @@ struct mssr_mod_clk { struct device_node; +enum clk_reg_layout { + CLK_REG_LAYOUT_RCAR_GEN2_AND_GEN3 = 0, + CLK_REG_LAYOUT_RZ_A, + CLK_REG_LAYOUT_RCAR_V3U, +}; + /** * SoC-specific CPG/MSSR Description * @@ -105,6 +111,7 @@ struct device_node; * @crit_mod_clks: Array with Module Clock IDs of critical clocks that * should not be disabled without a knowledgeable driver * @num_crit_mod_clks: Number of entries in crit_mod_clks[] + * @reg_layout: CPG/MSSR register layout from enum clk_reg_layout * * @core_pm_clks: Array with IDs of Core Clocks that are suitable for Power * Management, in addition to Module Clocks @@ -112,10 +119,6 @@ struct device_node; * * @init: Optional callback to perform SoC-specific initialization * @cpg_clk_register: Optional callback to handle special Core Clock types - * - * @stbyctrl: This device has Standby Control Registers which are 8-bits - * wide, no status registers (MSTPSR) and have different address - * offsets. */ struct cpg_mssr_info { @@ -130,7 +133,7 @@ struct cpg_mssr_info { unsigned int num_core_clks; unsigned int last_dt_core_clk; unsigned int num_total_core_clks; - bool stbyctrl; + enum clk_reg_layout reg_layout; /* Module Clocks */ const struct mssr_mod_clk *mod_clks; @@ -174,6 +177,7 @@ extern const struct cpg_mssr_info r8a77970_cpg_mssr_info; extern const struct cpg_mssr_info r8a77980_cpg_mssr_info; extern const struct cpg_mssr_info r8a77990_cpg_mssr_info; extern const struct cpg_mssr_info r8a77995_cpg_mssr_info; +extern const struct cpg_mssr_info r8a779a0_cpg_mssr_info; void __init cpg_mssr_early_init(struct device_node *np, const struct cpg_mssr_info *info); diff --git a/drivers/clk/rockchip/Kconfig b/drivers/clk/rockchip/Kconfig new file mode 100644 index 000000000000..47cd6c5de837 --- /dev/null +++ b/drivers/clk/rockchip/Kconfig @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: GPL-2.0 +# common clock support for ROCKCHIP SoC family. + +config COMMON_CLK_ROCKCHIP + bool "Rockchip clock controller common support" + depends on ARCH_ROCKCHIP + default ARCH_ROCKCHIP + help + Say y here to enable common clock controller for Rockchip platforms. + +if COMMON_CLK_ROCKCHIP +config CLK_PX30 + bool "Rockchip PX30 clock controller support" + default y + help + Build the driver for PX30 Clock Driver. + +config CLK_RV110X + bool "Rockchip RV110x clock controller support" + default y + help + Build the driver for RV110x Clock Driver. + +config CLK_RK3036 + bool "Rockchip RK3036 clock controller support" + default y + help + Build the driver for RK3036 Clock Driver. + +config CLK_RK312X + bool "Rockchip RK312x clock controller support" + default y + help + Build the driver for RK312x Clock Driver. + +config CLK_RK3188 + bool "Rockchip RK3188 clock controller support" + default y + help + Build the driver for RK3188 Clock Driver. + +config CLK_RK322X + bool "Rockchip RK322x clock controller support" + default y + help + Build the driver for RK322x Clock Driver. + +config CLK_RK3288 + bool "Rockchip RK3288 clock controller support" + depends on ARM + default y + help + Build the driver for RK3288 Clock Driver. + +config CLK_RK3308 + bool "Rockchip RK3308 clock controller support" + default y + help + Build the driver for RK3308 Clock Driver. + +config CLK_RK3328 + bool "Rockchip RK3328 clock controller support" + default y + help + Build the driver for RK3328 Clock Driver. + +config CLK_RK3368 + bool "Rockchip RK3368 clock controller support" + default y + help + Build the driver for RK3368 Clock Driver. + +config CLK_RK3399 + tristate "Rockchip RK3399 clock controller support" + default y + help + Build the driver for RK3399 Clock Driver. +endif diff --git a/drivers/clk/rockchip/Makefile b/drivers/clk/rockchip/Makefile index 7c5b5813a87c..a99e4d9bbae1 100644 --- a/drivers/clk/rockchip/Makefile +++ b/drivers/clk/rockchip/Makefile @@ -3,24 +3,26 @@ # Rockchip Clock specific Makefile # -obj-y += clk.o -obj-y += clk-pll.o -obj-y += clk-cpu.o -obj-y += clk-half-divider.o -obj-y += clk-inverter.o -obj-y += clk-mmc-phase.o -obj-y += clk-muxgrf.o -obj-y += clk-ddr.o -obj-$(CONFIG_RESET_CONTROLLER) += softrst.o +obj-$(CONFIG_COMMON_CLK_ROCKCHIP) += clk-rockchip.o -obj-y += clk-px30.o -obj-y += clk-rv1108.o -obj-y += clk-rk3036.o -obj-y += clk-rk3128.o -obj-y += clk-rk3188.o -obj-y += clk-rk3228.o -obj-y += clk-rk3288.o -obj-y += clk-rk3308.o -obj-y += clk-rk3328.o -obj-y += clk-rk3368.o -obj-y += clk-rk3399.o +clk-rockchip-y += clk.o +clk-rockchip-y += clk-pll.o +clk-rockchip-y += clk-cpu.o +clk-rockchip-y += clk-half-divider.o +clk-rockchip-y += clk-inverter.o +clk-rockchip-y += clk-mmc-phase.o +clk-rockchip-y += clk-muxgrf.o +clk-rockchip-y += clk-ddr.o +clk-rockchip-$(CONFIG_RESET_CONTROLLER) += softrst.o + +obj-$(CONFIG_CLK_PX30) += clk-px30.o +obj-$(CONFIG_CLK_RV110X) += clk-rv1108.o +obj-$(CONFIG_CLK_RK3036) += clk-rk3036.o +obj-$(CONFIG_CLK_RK312X) += clk-rk3128.o +obj-$(CONFIG_CLK_RK3188) += clk-rk3188.o +obj-$(CONFIG_CLK_RK322X) += clk-rk3228.o +obj-$(CONFIG_CLK_RK3288) += clk-rk3288.o +obj-$(CONFIG_CLK_RK3308) += clk-rk3308.o +obj-$(CONFIG_CLK_RK3328) += clk-rk3328.o +obj-$(CONFIG_CLK_RK3368) += clk-rk3368.o +obj-$(CONFIG_CLK_RK3399) += clk-rk3399.o diff --git a/drivers/clk/rockchip/clk-ddr.c b/drivers/clk/rockchip/clk-ddr.c index 9273bce4d7b6..86718c54e56b 100644 --- a/drivers/clk/rockchip/clk-ddr.c +++ b/drivers/clk/rockchip/clk-ddr.c @@ -136,3 +136,4 @@ struct clk *rockchip_clk_register_ddrclk(const char *name, int flags, return clk; } +EXPORT_SYMBOL_GPL(rockchip_clk_register_ddrclk); diff --git a/drivers/clk/rockchip/clk-half-divider.c b/drivers/clk/rockchip/clk-half-divider.c index b333fc28c94b..ccd5c270c213 100644 --- a/drivers/clk/rockchip/clk-half-divider.c +++ b/drivers/clk/rockchip/clk-half-divider.c @@ -166,7 +166,7 @@ struct clk *rockchip_clk_register_halfdiv(const char *name, unsigned long flags, spinlock_t *lock) { - struct clk *clk; + struct clk_hw *hw = ERR_PTR(-ENOMEM); struct clk_mux *mux = NULL; struct clk_gate *gate = NULL; struct clk_divider *div = NULL; @@ -212,16 +212,18 @@ struct clk *rockchip_clk_register_halfdiv(const char *name, div_ops = &clk_half_divider_ops; } - clk = clk_register_composite(NULL, name, parent_names, num_parents, - mux ? &mux->hw : NULL, mux_ops, - div ? &div->hw : NULL, div_ops, - gate ? &gate->hw : NULL, gate_ops, - flags); + hw = clk_hw_register_composite(NULL, name, parent_names, num_parents, + mux ? &mux->hw : NULL, mux_ops, + div ? &div->hw : NULL, div_ops, + gate ? &gate->hw : NULL, gate_ops, + flags); + if (IS_ERR(hw)) + goto err_div; - return clk; + return hw->clk; err_div: kfree(gate); err_gate: kfree(mux); - return ERR_PTR(-ENOMEM); + return ERR_CAST(hw); } diff --git a/drivers/clk/rockchip/clk-rk3308.c b/drivers/clk/rockchip/clk-rk3308.c index b0baf87a283e..5bf15f2a44b7 100644 --- a/drivers/clk/rockchip/clk-rk3308.c +++ b/drivers/clk/rockchip/clk-rk3308.c @@ -133,7 +133,6 @@ PNAME(mux_uart1_p) = { "clk_uart1_src", "dummy", "clk_uart1_frac" }; PNAME(mux_uart2_p) = { "clk_uart2_src", "dummy", "clk_uart2_frac" }; PNAME(mux_uart3_p) = { "clk_uart3_src", "dummy", "clk_uart3_frac" }; PNAME(mux_uart4_p) = { "clk_uart4_src", "dummy", "clk_uart4_frac" }; -PNAME(mux_timer_src_p) = { "xin24m", "clk_rtc32k" }; PNAME(mux_dclk_vop_p) = { "dclk_vop_src", "dclk_vop_frac", "xin24m" }; PNAME(mux_nandc_p) = { "clk_nandc_div", "clk_nandc_div50" }; PNAME(mux_sdmmc_p) = { "clk_sdmmc_div", "clk_sdmmc_div50" }; diff --git a/drivers/clk/rockchip/clk-rk3399.c b/drivers/clk/rockchip/clk-rk3399.c index ce1d2446f142..7df2f1e00347 100644 --- a/drivers/clk/rockchip/clk-rk3399.c +++ b/drivers/clk/rockchip/clk-rk3399.c @@ -5,9 +5,11 @@ */ #include <linux/clk-provider.h> +#include <linux/module.h> #include <linux/io.h> #include <linux/of.h> #include <linux/of_address.h> +#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/regmap.h> #include <dt-bindings/clock/rk3399-cru.h> @@ -1600,3 +1602,57 @@ static void __init rk3399_pmu_clk_init(struct device_node *np) rockchip_clk_of_add_provider(np, ctx); } CLK_OF_DECLARE(rk3399_cru_pmu, "rockchip,rk3399-pmucru", rk3399_pmu_clk_init); + +struct clk_rk3399_inits { + void (*inits)(struct device_node *np); +}; + +static const struct clk_rk3399_inits clk_rk3399_pmucru_init = { + .inits = rk3399_pmu_clk_init, +}; + +static const struct clk_rk3399_inits clk_rk3399_cru_init = { + .inits = rk3399_clk_init, +}; + +static const struct of_device_id clk_rk3399_match_table[] = { + { + .compatible = "rockchip,rk3399-cru", + .data = &clk_rk3399_cru_init, + }, { + .compatible = "rockchip,rk3399-pmucru", + .data = &clk_rk3399_pmucru_init, + }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_rk3399_match_table); + +static int __init clk_rk3399_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + const struct of_device_id *match; + const struct clk_rk3399_inits *init_data; + + match = of_match_device(clk_rk3399_match_table, &pdev->dev); + if (!match || !match->data) + return -EINVAL; + + init_data = match->data; + if (init_data->inits) + init_data->inits(np); + + return 0; +} + +static struct platform_driver clk_rk3399_driver = { + .driver = { + .name = "clk-rk3399", + .of_match_table = clk_rk3399_match_table, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver_probe(clk_rk3399_driver, clk_rk3399_probe); + +MODULE_DESCRIPTION("Rockchip RK3399 Clock Driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:clk-rk3399"); diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c index 546e810c3560..b443169dd408 100644 --- a/drivers/clk/rockchip/clk.c +++ b/drivers/clk/rockchip/clk.c @@ -43,7 +43,7 @@ static struct clk *rockchip_clk_register_branch(const char *name, u8 gate_shift, u8 gate_flags, unsigned long flags, spinlock_t *lock) { - struct clk *clk; + struct clk_hw *hw; struct clk_mux *mux = NULL; struct clk_gate *gate = NULL; struct clk_divider *div = NULL; @@ -100,20 +100,18 @@ static struct clk *rockchip_clk_register_branch(const char *name, : &clk_divider_ops; } - clk = clk_register_composite(NULL, name, parent_names, num_parents, - mux ? &mux->hw : NULL, mux_ops, - div ? &div->hw : NULL, div_ops, - gate ? &gate->hw : NULL, gate_ops, - flags); - - if (IS_ERR(clk)) { - ret = PTR_ERR(clk); - goto err_composite; + hw = clk_hw_register_composite(NULL, name, parent_names, num_parents, + mux ? &mux->hw : NULL, mux_ops, + div ? &div->hw : NULL, div_ops, + gate ? &gate->hw : NULL, gate_ops, + flags); + if (IS_ERR(hw)) { + kfree(div); + kfree(gate); + return ERR_CAST(hw); } - return clk; -err_composite: - kfree(div); + return hw->clk; err_div: kfree(gate); err_gate: @@ -214,8 +212,8 @@ static struct clk *rockchip_clk_register_frac_branch( unsigned long flags, struct rockchip_clk_branch *child, spinlock_t *lock) { + struct clk_hw *hw; struct rockchip_clk_frac *frac; - struct clk *clk; struct clk_gate *gate = NULL; struct clk_fractional_divider *div = NULL; const struct clk_ops *div_ops = NULL, *gate_ops = NULL; @@ -255,14 +253,14 @@ static struct clk *rockchip_clk_register_frac_branch( div->approximation = rockchip_fractional_approximation; div_ops = &clk_fractional_divider_ops; - clk = clk_register_composite(NULL, name, parent_names, num_parents, - NULL, NULL, - &div->hw, div_ops, - gate ? &gate->hw : NULL, gate_ops, - flags | CLK_SET_RATE_UNGATE); - if (IS_ERR(clk)) { + hw = clk_hw_register_composite(NULL, name, parent_names, num_parents, + NULL, NULL, + &div->hw, div_ops, + gate ? &gate->hw : NULL, gate_ops, + flags | CLK_SET_RATE_UNGATE); + if (IS_ERR(hw)) { kfree(frac); - return clk; + return ERR_CAST(hw); } if (child) { @@ -292,7 +290,7 @@ static struct clk *rockchip_clk_register_frac_branch( mux_clk = clk_register(NULL, &frac_mux->hw); if (IS_ERR(mux_clk)) { kfree(frac); - return clk; + return mux_clk; } rockchip_clk_add_lookup(ctx, mux_clk, child->id); @@ -301,7 +299,7 @@ static struct clk *rockchip_clk_register_frac_branch( if (frac->mux_frac_idx >= 0) { pr_debug("%s: found fractional parent in mux at pos %d\n", __func__, frac->mux_frac_idx); - ret = clk_notifier_register(clk, &frac->clk_nb); + ret = clk_notifier_register(hw->clk, &frac->clk_nb); if (ret) pr_err("%s: failed to register clock notifier for %s\n", __func__, name); @@ -311,7 +309,7 @@ static struct clk *rockchip_clk_register_frac_branch( } } - return clk; + return hw->clk; } static struct clk *rockchip_clk_register_factor_branch(const char *name, @@ -320,7 +318,7 @@ static struct clk *rockchip_clk_register_factor_branch(const char *name, int gate_offset, u8 gate_shift, u8 gate_flags, unsigned long flags, spinlock_t *lock) { - struct clk *clk; + struct clk_hw *hw; struct clk_gate *gate = NULL; struct clk_fixed_factor *fix = NULL; @@ -349,20 +347,22 @@ static struct clk *rockchip_clk_register_factor_branch(const char *name, fix->mult = mult; fix->div = div; - clk = clk_register_composite(NULL, name, parent_names, num_parents, - NULL, NULL, - &fix->hw, &clk_fixed_factor_ops, - &gate->hw, &clk_gate_ops, flags); - if (IS_ERR(clk)) { + hw = clk_hw_register_composite(NULL, name, parent_names, num_parents, + NULL, NULL, + &fix->hw, &clk_fixed_factor_ops, + &gate->hw, &clk_gate_ops, flags); + if (IS_ERR(hw)) { kfree(fix); kfree(gate); + return ERR_CAST(hw); } - return clk; + return hw->clk; } -struct rockchip_clk_provider * __init rockchip_clk_init(struct device_node *np, - void __iomem *base, unsigned long nr_clks) +struct rockchip_clk_provider *rockchip_clk_init(struct device_node *np, + void __iomem *base, + unsigned long nr_clks) { struct rockchip_clk_provider *ctx; struct clk **clk_table; @@ -394,14 +394,16 @@ err_free: kfree(ctx); return ERR_PTR(-ENOMEM); } +EXPORT_SYMBOL_GPL(rockchip_clk_init); -void __init rockchip_clk_of_add_provider(struct device_node *np, - struct rockchip_clk_provider *ctx) +void rockchip_clk_of_add_provider(struct device_node *np, + struct rockchip_clk_provider *ctx) { if (of_clk_add_provider(np, of_clk_src_onecell_get, &ctx->clk_data)) pr_err("%s: could not register clk provider\n", __func__); } +EXPORT_SYMBOL_GPL(rockchip_clk_of_add_provider); void rockchip_clk_add_lookup(struct rockchip_clk_provider *ctx, struct clk *clk, unsigned int id) @@ -409,8 +411,9 @@ void rockchip_clk_add_lookup(struct rockchip_clk_provider *ctx, if (ctx->clk_data.clks && id) ctx->clk_data.clks[id] = clk; } +EXPORT_SYMBOL_GPL(rockchip_clk_add_lookup); -void __init rockchip_clk_register_plls(struct rockchip_clk_provider *ctx, +void rockchip_clk_register_plls(struct rockchip_clk_provider *ctx, struct rockchip_pll_clock *list, unsigned int nr_pll, int grf_lock_offset) { @@ -433,11 +436,11 @@ void __init rockchip_clk_register_plls(struct rockchip_clk_provider *ctx, rockchip_clk_add_lookup(ctx, clk, list->id); } } +EXPORT_SYMBOL_GPL(rockchip_clk_register_plls); -void __init rockchip_clk_register_branches( - struct rockchip_clk_provider *ctx, - struct rockchip_clk_branch *list, - unsigned int nr_clk) +void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx, + struct rockchip_clk_branch *list, + unsigned int nr_clk) { struct clk *clk = NULL; unsigned int idx; @@ -566,14 +569,15 @@ void __init rockchip_clk_register_branches( rockchip_clk_add_lookup(ctx, clk, list->id); } } - -void __init rockchip_clk_register_armclk(struct rockchip_clk_provider *ctx, - unsigned int lookup_id, - const char *name, const char *const *parent_names, - u8 num_parents, - const struct rockchip_cpuclk_reg_data *reg_data, - const struct rockchip_cpuclk_rate_table *rates, - int nrates) +EXPORT_SYMBOL_GPL(rockchip_clk_register_branches); + +void rockchip_clk_register_armclk(struct rockchip_clk_provider *ctx, + unsigned int lookup_id, + const char *name, const char *const *parent_names, + u8 num_parents, + const struct rockchip_cpuclk_reg_data *reg_data, + const struct rockchip_cpuclk_rate_table *rates, + int nrates) { struct clk *clk; @@ -588,9 +592,10 @@ void __init rockchip_clk_register_armclk(struct rockchip_clk_provider *ctx, rockchip_clk_add_lookup(ctx, clk, lookup_id); } +EXPORT_SYMBOL_GPL(rockchip_clk_register_armclk); -void __init rockchip_clk_protect_critical(const char *const clocks[], - int nclocks) +void rockchip_clk_protect_critical(const char *const clocks[], + int nclocks) { int i; @@ -602,6 +607,7 @@ void __init rockchip_clk_protect_critical(const char *const clocks[], clk_prepare_enable(clk); } } +EXPORT_SYMBOL_GPL(rockchip_clk_protect_critical); static void __iomem *rst_base; static unsigned int reg_restart; @@ -621,10 +627,10 @@ static struct notifier_block rockchip_restart_handler = { .priority = 128, }; -void __init +void rockchip_register_restart_notifier(struct rockchip_clk_provider *ctx, - unsigned int reg, - void (*cb)(void)) + unsigned int reg, + void (*cb)(void)) { int ret; @@ -636,3 +642,4 @@ rockchip_register_restart_notifier(struct rockchip_clk_provider *ctx, pr_err("%s: cannot register restart handler, %d\n", __func__, ret); } +EXPORT_SYMBOL_GPL(rockchip_register_restart_notifier); diff --git a/drivers/clk/rockchip/softrst.c b/drivers/clk/rockchip/softrst.c index 5f1ff5e47c4f..5d07266745b8 100644 --- a/drivers/clk/rockchip/softrst.c +++ b/drivers/clk/rockchip/softrst.c @@ -77,9 +77,9 @@ static const struct reset_control_ops rockchip_softrst_ops = { .deassert = rockchip_softrst_deassert, }; -void __init rockchip_register_softrst(struct device_node *np, - unsigned int num_regs, - void __iomem *base, u8 flags) +void rockchip_register_softrst(struct device_node *np, + unsigned int num_regs, + void __iomem *base, u8 flags) { struct rockchip_softrst *softrst; int ret; @@ -107,3 +107,4 @@ void __init rockchip_register_softrst(struct device_node *np, kfree(softrst); } }; +EXPORT_SYMBOL_GPL(rockchip_register_softrst); diff --git a/drivers/clk/samsung/clk-cpu.c b/drivers/clk/samsung/clk-cpu.c index efc4fa61fbaf..00ef4d1b0888 100644 --- a/drivers/clk/samsung/clk-cpu.c +++ b/drivers/clk/samsung/clk-cpu.c @@ -401,26 +401,34 @@ static int exynos5433_cpuclk_notifier_cb(struct notifier_block *nb, /* helper function to register a CPU clock */ int __init exynos_register_cpu_clock(struct samsung_clk_provider *ctx, - unsigned int lookup_id, const char *name, const char *parent, - const char *alt_parent, unsigned long offset, - const struct exynos_cpuclk_cfg_data *cfg, + unsigned int lookup_id, const char *name, + const struct clk_hw *parent, const struct clk_hw *alt_parent, + unsigned long offset, const struct exynos_cpuclk_cfg_data *cfg, unsigned long num_cfgs, unsigned long flags) { struct exynos_cpuclk *cpuclk; struct clk_init_data init; - struct clk *parent_clk; + const char *parent_name; int ret = 0; + if (IS_ERR(parent) || IS_ERR(alt_parent)) { + pr_err("%s: invalid parent clock(s)\n", __func__); + return -EINVAL; + } + cpuclk = kzalloc(sizeof(*cpuclk), GFP_KERNEL); if (!cpuclk) return -ENOMEM; + parent_name = clk_hw_get_name(parent); + init.name = name; init.flags = CLK_SET_RATE_PARENT; - init.parent_names = &parent; + init.parent_names = &parent_name; init.num_parents = 1; init.ops = &exynos_cpuclk_clk_ops; + cpuclk->alt_parent = alt_parent; cpuclk->hw.init = &init; cpuclk->ctrl_base = ctx->reg_base + offset; cpuclk->lock = &ctx->lock; @@ -430,23 +438,8 @@ int __init exynos_register_cpu_clock(struct samsung_clk_provider *ctx, else cpuclk->clk_nb.notifier_call = exynos_cpuclk_notifier_cb; - cpuclk->alt_parent = __clk_get_hw(__clk_lookup(alt_parent)); - if (!cpuclk->alt_parent) { - pr_err("%s: could not lookup alternate parent %s\n", - __func__, alt_parent); - ret = -EINVAL; - goto free_cpuclk; - } - - parent_clk = __clk_lookup(parent); - if (!parent_clk) { - pr_err("%s: could not lookup parent clock %s\n", - __func__, parent); - ret = -EINVAL; - goto free_cpuclk; - } - ret = clk_notifier_register(parent_clk, &cpuclk->clk_nb); + ret = clk_notifier_register(parent->clk, &cpuclk->clk_nb); if (ret) { pr_err("%s: failed to register clock notifier for %s\n", __func__, name); @@ -471,7 +464,7 @@ int __init exynos_register_cpu_clock(struct samsung_clk_provider *ctx, free_cpuclk_data: kfree(cpuclk->cfg); unregister_clk_nb: - clk_notifier_unregister(parent_clk, &cpuclk->clk_nb); + clk_notifier_unregister(parent->clk, &cpuclk->clk_nb); free_cpuclk: kfree(cpuclk); return ret; diff --git a/drivers/clk/samsung/clk-cpu.h b/drivers/clk/samsung/clk-cpu.h index ad38cc27f3df..af74686db9ef 100644 --- a/drivers/clk/samsung/clk-cpu.h +++ b/drivers/clk/samsung/clk-cpu.h @@ -46,7 +46,7 @@ struct exynos_cpuclk_cfg_data { */ struct exynos_cpuclk { struct clk_hw hw; - struct clk_hw *alt_parent; + const struct clk_hw *alt_parent; void __iomem *ctrl_base; spinlock_t *lock; const struct exynos_cpuclk_cfg_data *cfg; @@ -62,9 +62,9 @@ struct exynos_cpuclk { #define CLK_CPU_HAS_E5433_REGS_LAYOUT (1 << 2) }; -extern int __init exynos_register_cpu_clock(struct samsung_clk_provider *ctx, +int __init exynos_register_cpu_clock(struct samsung_clk_provider *ctx, unsigned int lookup_id, const char *name, - const char *parent, const char *alt_parent, + const struct clk_hw *parent, const struct clk_hw *alt_parent, unsigned long offset, const struct exynos_cpuclk_cfg_data *cfg, unsigned long num_cfgs, unsigned long flags); diff --git a/drivers/clk/samsung/clk-exynos3250.c b/drivers/clk/samsung/clk-exynos3250.c index 17897c7a84d4..17df7f9755aa 100644 --- a/drivers/clk/samsung/clk-exynos3250.c +++ b/drivers/clk/samsung/clk-exynos3250.c @@ -808,14 +808,16 @@ static const struct exynos_cpuclk_cfg_data e3250_armclk_d[] __initconst = { static void __init exynos3250_cmu_init(struct device_node *np) { struct samsung_clk_provider *ctx; + struct clk_hw **hws; ctx = samsung_cmu_register_one(np, &cmu_info); if (!ctx) return; + hws = ctx->clk_data.hws; exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk", - mout_core_p[0], mout_core_p[1], 0x14200, - e3250_armclk_d, ARRAY_SIZE(e3250_armclk_d), + hws[CLK_MOUT_APLL], hws[CLK_MOUT_MPLL_USER_C], + 0x14200, e3250_armclk_d, ARRAY_SIZE(e3250_armclk_d), CLK_CPU_HAS_DIV1); exynos3_core_down_clock(ctx->reg_base); diff --git a/drivers/clk/samsung/clk-exynos4.c b/drivers/clk/samsung/clk-exynos4.c index f4086287bb71..bf13e29a655c 100644 --- a/drivers/clk/samsung/clk-exynos4.c +++ b/drivers/clk/samsung/clk-exynos4.c @@ -1233,6 +1233,8 @@ static void __init exynos4_clk_init(struct device_node *np, enum exynos4_soc soc) { struct samsung_clk_provider *ctx; + struct clk_hw **hws; + exynos4_soc = soc; reg_base = of_iomap(np, 0); @@ -1240,6 +1242,7 @@ static void __init exynos4_clk_init(struct device_node *np, panic("%s: failed to map registers\n", __func__); ctx = samsung_clk_init(np, reg_base, CLK_NR_CLKS); + hws = ctx->clk_data.hws; samsung_clk_of_register_fixed_ext(ctx, exynos4_fixed_rate_ext_clks, ARRAY_SIZE(exynos4_fixed_rate_ext_clks), @@ -1302,7 +1305,7 @@ static void __init exynos4_clk_init(struct device_node *np, exynos4210_fixed_factor_clks, ARRAY_SIZE(exynos4210_fixed_factor_clks)); exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk", - mout_core_p4210[0], mout_core_p4210[1], 0x14200, + hws[CLK_MOUT_APLL], hws[CLK_SCLK_MPLL], 0x14200, e4210_armclk_d, ARRAY_SIZE(e4210_armclk_d), CLK_CPU_NEEDS_DEBUG_ALT_DIV | CLK_CPU_HAS_DIV1); } else { @@ -1317,7 +1320,7 @@ static void __init exynos4_clk_init(struct device_node *np, ARRAY_SIZE(exynos4x12_fixed_factor_clks)); exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk", - mout_core_p4x12[0], mout_core_p4x12[1], 0x14200, + hws[CLK_MOUT_APLL], hws[CLK_MOUT_MPLL_USER_C], 0x14200, e4412_armclk_d, ARRAY_SIZE(e4412_armclk_d), CLK_CPU_NEEDS_DEBUG_ALT_DIV | CLK_CPU_HAS_DIV1); } diff --git a/drivers/clk/samsung/clk-exynos5250.c b/drivers/clk/samsung/clk-exynos5250.c index 931c70a4da19..06588fab408a 100644 --- a/drivers/clk/samsung/clk-exynos5250.c +++ b/drivers/clk/samsung/clk-exynos5250.c @@ -253,14 +253,14 @@ static const struct samsung_mux_clock exynos5250_mux_clks[] __initconst = { /* * CMU_CPU */ - MUX_F(0, "mout_apll", mout_apll_p, SRC_CPU, 0, 1, + MUX_F(CLK_MOUT_APLL, "mout_apll", mout_apll_p, SRC_CPU, 0, 1, CLK_SET_RATE_PARENT, 0), MUX(0, "mout_cpu", mout_cpu_p, SRC_CPU, 16, 1), /* * CMU_CORE */ - MUX(0, "mout_mpll", mout_mpll_p, SRC_CORE1, 8, 1), + MUX(CLK_MOUT_MPLL, "mout_mpll", mout_mpll_p, SRC_CORE1, 8, 1), /* * CMU_TOP @@ -782,6 +782,7 @@ static void __init exynos5250_clk_init(struct device_node *np) { struct samsung_clk_provider *ctx; unsigned int tmp; + struct clk_hw **hws; if (np) { reg_base = of_iomap(np, 0); @@ -792,6 +793,7 @@ static void __init exynos5250_clk_init(struct device_node *np) } ctx = samsung_clk_init(np, reg_base, CLK_NR_CLKS); + hws = ctx->clk_data.hws; samsung_clk_of_register_fixed_ext(ctx, exynos5250_fixed_rate_ext_clks, ARRAY_SIZE(exynos5250_fixed_rate_ext_clks), @@ -821,7 +823,7 @@ static void __init exynos5250_clk_init(struct device_node *np) samsung_clk_register_gate(ctx, exynos5250_gate_clks, ARRAY_SIZE(exynos5250_gate_clks)); exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk", - mout_cpu_p[0], mout_cpu_p[1], 0x200, + hws[CLK_MOUT_APLL], hws[CLK_MOUT_MPLL], 0x200, exynos5250_armclk_d, ARRAY_SIZE(exynos5250_armclk_d), CLK_CPU_HAS_DIV1); diff --git a/drivers/clk/samsung/clk-exynos5420.c b/drivers/clk/samsung/clk-exynos5420.c index bd620876544d..3ccd4eabd2a6 100644 --- a/drivers/clk/samsung/clk-exynos5420.c +++ b/drivers/clk/samsung/clk-exynos5420.c @@ -596,13 +596,14 @@ static const struct samsung_gate_clock exynos5420_gate_clks[] __initconst = { static const struct samsung_mux_clock exynos5x_mux_clks[] __initconst = { MUX(0, "mout_user_pclk66_gpio", mout_user_pclk66_gpio_p, SRC_TOP7, 4, 1), - MUX(0, "mout_mspll_kfc", mout_mspll_cpu_p, SRC_TOP7, 8, 2), - MUX(0, "mout_mspll_cpu", mout_mspll_cpu_p, SRC_TOP7, 12, 2), - - MUX_F(0, "mout_apll", mout_apll_p, SRC_CPU, 0, 1, + MUX(CLK_MOUT_MSPLL_KFC, "mout_mspll_kfc", mout_mspll_cpu_p, + SRC_TOP7, 8, 2), + MUX(CLK_MOUT_MSPLL_CPU, "mout_mspll_cpu", mout_mspll_cpu_p, + SRC_TOP7, 12, 2), + MUX_F(CLK_MOUT_APLL, "mout_apll", mout_apll_p, SRC_CPU, 0, 1, CLK_SET_RATE_PARENT | CLK_RECALC_NEW_RATES, 0), MUX(0, "mout_cpu", mout_cpu_p, SRC_CPU, 16, 1), - MUX_F(0, "mout_kpll", mout_kpll_p, SRC_KFC, 0, 1, + MUX_F(CLK_MOUT_KPLL, "mout_kpll", mout_kpll_p, SRC_KFC, 0, 1, CLK_SET_RATE_PARENT | CLK_RECALC_NEW_RATES, 0), MUX(0, "mout_kfc", mout_kfc_p, SRC_KFC, 16, 1), @@ -712,8 +713,8 @@ static const struct samsung_mux_clock exynos5x_mux_clks[] __initconst = { SRC_TOP12, 8, 1), MUX(0, "mout_sw_aclk266_g2d", mout_sw_aclk266_g2d_p, SRC_TOP12, 12, 1), - MUX_F(0, "mout_sw_aclk_g3d", mout_sw_aclk_g3d_p, SRC_TOP12, 16, 1, - CLK_SET_RATE_PARENT, 0), + MUX_F(CLK_MOUT_SW_ACLK_G3D, "mout_sw_aclk_g3d", mout_sw_aclk_g3d_p, + SRC_TOP12, 16, 1, CLK_SET_RATE_PARENT, 0), MUX(0, "mout_sw_aclk300_jpeg", mout_sw_aclk300_jpeg_p, SRC_TOP12, 20, 1), MUX(CLK_MOUT_SW_ACLK300, "mout_sw_aclk300_disp1", @@ -1560,6 +1561,7 @@ static void __init exynos5x_clk_init(struct device_node *np, enum exynos5x_soc soc) { struct samsung_clk_provider *ctx; + struct clk_hw **hws; if (np) { reg_base = of_iomap(np, 0); @@ -1572,6 +1574,7 @@ static void __init exynos5x_clk_init(struct device_node *np, exynos5x_soc = soc; ctx = samsung_clk_init(np, reg_base, CLK_NR_CLKS); + hws = ctx->clk_data.hws; samsung_clk_of_register_fixed_ext(ctx, exynos5x_fixed_rate_ext_clks, ARRAY_SIZE(exynos5x_fixed_rate_ext_clks), @@ -1623,15 +1626,15 @@ static void __init exynos5x_clk_init(struct device_node *np, if (soc == EXYNOS5420) { exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk", - mout_cpu_p[0], mout_cpu_p[1], 0x200, + hws[CLK_MOUT_APLL], hws[CLK_MOUT_MSPLL_CPU], 0x200, exynos5420_eglclk_d, ARRAY_SIZE(exynos5420_eglclk_d), 0); } else { exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk", - mout_cpu_p[0], mout_cpu_p[1], 0x200, + hws[CLK_MOUT_APLL], hws[CLK_MOUT_MSPLL_CPU], 0x200, exynos5800_eglclk_d, ARRAY_SIZE(exynos5800_eglclk_d), 0); } exynos_register_cpu_clock(ctx, CLK_KFC_CLK, "kfcclk", - mout_kfc_p[0], mout_kfc_p[1], 0x28200, + hws[CLK_MOUT_KPLL], hws[CLK_MOUT_MSPLL_KFC], 0x28200, exynos5420_kfcclk_d, ARRAY_SIZE(exynos5420_kfcclk_d), 0); samsung_clk_extended_sleep_init(reg_base, @@ -1654,12 +1657,12 @@ static void __init exynos5x_clk_init(struct device_node *np, * that the internal busses get their clock regardless of the * main G3D clock enablement status. */ - clk_prepare_enable(__clk_lookup("mout_sw_aclk_g3d")); + clk_prepare_enable(hws[CLK_MOUT_SW_ACLK_G3D]->clk); /* * Keep top BPLL mux enabled permanently to ensure that DRAM operates * properly. */ - clk_prepare_enable(__clk_lookup("mout_bpll")); + clk_prepare_enable(hws[CLK_MOUT_BPLL]->clk); samsung_clk_of_add_provider(np, ctx); } diff --git a/drivers/clk/samsung/clk-exynos5433.c b/drivers/clk/samsung/clk-exynos5433.c index 6f29ecd0442e..f203074d858b 100644 --- a/drivers/clk/samsung/clk-exynos5433.c +++ b/drivers/clk/samsung/clk-exynos5433.c @@ -3679,6 +3679,7 @@ static void __init exynos5433_cmu_apollo_init(struct device_node *np) { void __iomem *reg_base; struct samsung_clk_provider *ctx; + struct clk_hw **hws; reg_base = of_iomap(np, 0); if (!reg_base) { @@ -3701,8 +3702,10 @@ static void __init exynos5433_cmu_apollo_init(struct device_node *np) samsung_clk_register_gate(ctx, apollo_gate_clks, ARRAY_SIZE(apollo_gate_clks)); + hws = ctx->clk_data.hws; + exynos_register_cpu_clock(ctx, CLK_SCLK_APOLLO, "apolloclk", - mout_apollo_p[0], mout_apollo_p[1], 0x200, + hws[CLK_MOUT_APOLLO_PLL], hws[CLK_MOUT_BUS_PLL_APOLLO_USER], 0x200, exynos5433_apolloclk_d, ARRAY_SIZE(exynos5433_apolloclk_d), CLK_CPU_HAS_E5433_REGS_LAYOUT); @@ -3933,6 +3936,7 @@ static void __init exynos5433_cmu_atlas_init(struct device_node *np) { void __iomem *reg_base; struct samsung_clk_provider *ctx; + struct clk_hw **hws; reg_base = of_iomap(np, 0); if (!reg_base) { @@ -3955,8 +3959,10 @@ static void __init exynos5433_cmu_atlas_init(struct device_node *np) samsung_clk_register_gate(ctx, atlas_gate_clks, ARRAY_SIZE(atlas_gate_clks)); + hws = ctx->clk_data.hws; + exynos_register_cpu_clock(ctx, CLK_SCLK_ATLAS, "atlasclk", - mout_atlas_p[0], mout_atlas_p[1], 0x200, + hws[CLK_MOUT_ATLAS_PLL], hws[CLK_MOUT_BUS_PLL_ATLAS_USER], 0x200, exynos5433_atlasclk_d, ARRAY_SIZE(exynos5433_atlasclk_d), CLK_CPU_HAS_E5433_REGS_LAYOUT); diff --git a/drivers/clk/samsung/clk-s3c2410-dclk.c b/drivers/clk/samsung/clk-s3c2410-dclk.c index 7dad9098e897..f5e0a6ba2d12 100644 --- a/drivers/clk/samsung/clk-s3c2410-dclk.c +++ b/drivers/clk/samsung/clk-s3c2410-dclk.c @@ -11,13 +11,10 @@ #include <linux/clk-provider.h> #include <linux/io.h> #include <linux/platform_device.h> +#include <linux/platform_data/clk-s3c2410.h> #include <linux/module.h> #include "clk.h" -/* legacy access to misccr, until dt conversion is finished */ -#include <mach/hardware.h> -#include <mach/regs-gpio.h> - #define MUX_DCLK0 0 #define MUX_DCLK1 1 #define DIV_DCLK0 2 @@ -52,6 +49,7 @@ struct s3c24xx_clkout { struct clk_hw hw; u32 mask; u8 shift; + unsigned int (*modify_misccr)(unsigned int clr, unsigned int chg); }; #define to_s3c24xx_clkout(_hw) container_of(_hw, struct s3c24xx_clkout, hw) @@ -62,7 +60,7 @@ static u8 s3c24xx_clkout_get_parent(struct clk_hw *hw) int num_parents = clk_hw_get_num_parents(hw); u32 val; - val = readl_relaxed(S3C24XX_MISCCR) >> clkout->shift; + val = clkout->modify_misccr(0, 0) >> clkout->shift; val >>= clkout->shift; val &= clkout->mask; @@ -76,7 +74,7 @@ static int s3c24xx_clkout_set_parent(struct clk_hw *hw, u8 index) { struct s3c24xx_clkout *clkout = to_s3c24xx_clkout(hw); - s3c2410_modify_misccr((clkout->mask << clkout->shift), + clkout->modify_misccr((clkout->mask << clkout->shift), (index << clkout->shift)); return 0; @@ -92,10 +90,14 @@ static struct clk_hw *s3c24xx_register_clkout(struct device *dev, const char *name, const char **parent_names, u8 num_parents, u8 shift, u32 mask) { + struct s3c2410_clk_platform_data *pdata = dev_get_platdata(dev); struct s3c24xx_clkout *clkout; struct clk_init_data init; int ret; + if (!pdata) + return ERR_PTR(-EINVAL); + /* allocate the clkout */ clkout = kzalloc(sizeof(*clkout), GFP_KERNEL); if (!clkout) @@ -110,6 +112,7 @@ static struct clk_hw *s3c24xx_register_clkout(struct device *dev, clkout->shift = shift; clkout->mask = mask; clkout->hw.init = &init; + clkout->modify_misccr = pdata->modify_misccr; ret = clk_hw_register(dev, &clkout->hw); if (ret) diff --git a/drivers/clk/samsung/clk-s3c2410.c b/drivers/clk/samsung/clk-s3c2410.c index fcf6764693cc..5831d0606077 100644 --- a/drivers/clk/samsung/clk-s3c2410.c +++ b/drivers/clk/samsung/clk-s3c2410.c @@ -6,6 +6,7 @@ */ #include <linux/clk-provider.h> +#include <linux/clk/samsung.h> #include <linux/of.h> #include <linux/of_address.h> diff --git a/drivers/clk/samsung/clk-s3c2412.c b/drivers/clk/samsung/clk-s3c2412.c index a95ab5f75163..724ef642f048 100644 --- a/drivers/clk/samsung/clk-s3c2412.c +++ b/drivers/clk/samsung/clk-s3c2412.c @@ -6,6 +6,7 @@ */ #include <linux/clk-provider.h> +#include <linux/clk/samsung.h> #include <linux/io.h> #include <linux/of.h> #include <linux/of_address.h> diff --git a/drivers/clk/samsung/clk-s3c2443.c b/drivers/clk/samsung/clk-s3c2443.c index c7aba1e1af70..a827d63766d1 100644 --- a/drivers/clk/samsung/clk-s3c2443.c +++ b/drivers/clk/samsung/clk-s3c2443.c @@ -6,6 +6,7 @@ */ #include <linux/clk-provider.h> +#include <linux/clk/samsung.h> #include <linux/io.h> #include <linux/of.h> #include <linux/of_address.h> diff --git a/drivers/clk/samsung/clk-s3c64xx.c b/drivers/clk/samsung/clk-s3c64xx.c index b96d33e5eb45..56f95b63f71f 100644 --- a/drivers/clk/samsung/clk-s3c64xx.c +++ b/drivers/clk/samsung/clk-s3c64xx.c @@ -7,6 +7,7 @@ #include <linux/slab.h> #include <linux/clk-provider.h> +#include <linux/clk/samsung.h> #include <linux/of.h> #include <linux/of_address.h> diff --git a/drivers/clk/sirf/clk-prima2.c b/drivers/clk/sirf/clk-prima2.c index 45dcbc9e0302..d17b345f4d2d 100644 --- a/drivers/clk/sirf/clk-prima2.c +++ b/drivers/clk/sirf/clk-prima2.c @@ -134,7 +134,7 @@ static void __init prima2_clk_init(struct device_node *np) for (i = pll1; i < maxclk; i++) { prima2_clks[i] = clk_register(NULL, prima2_clk_hw_array[i]); - BUG_ON(!prima2_clks[i]); + BUG_ON(IS_ERR(prima2_clks[i])); } clk_register_clkdev(prima2_clks[cpu], NULL, "cpu"); clk_register_clkdev(prima2_clks[io], NULL, "io"); diff --git a/drivers/clk/socfpga/clk-agilex.c b/drivers/clk/socfpga/clk-agilex.c index 8fb12cbe0208..bb3e80928ebe 100644 --- a/drivers/clk/socfpga/clk-agilex.c +++ b/drivers/clk/socfpga/clk-agilex.c @@ -21,19 +21,6 @@ static const struct clk_parent_data pll_mux[] = { .name = "f2s-free-clk", }, }; -static const struct clk_parent_data cntr_mux[] = { - { .fw_name = "main_pll", - .name = "main_pll", }, - { .fw_name = "periph_pll", - .name = "periph_pll", }, - { .fw_name = "osc1", - .name = "osc1", }, - { .fw_name = "cb-intosc-hs-div2-clk", - .name = "cb-intosc-hs-div2-clk", }, - { .fw_name = "f2s-free-clk", - .name = "f2s-free-clk", }, -}; - static const struct clk_parent_data boot_mux[] = { { .fw_name = "osc1", .name = "osc1", }, diff --git a/drivers/clk/sunxi-ng/Kconfig b/drivers/clk/sunxi-ng/Kconfig index cdf333003c30..ce5f5847d5d3 100644 --- a/drivers/clk/sunxi-ng/Kconfig +++ b/drivers/clk/sunxi-ng/Kconfig @@ -17,6 +17,16 @@ config SUN50I_A64_CCU default ARM64 && ARCH_SUNXI depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST +config SUN50I_A100_CCU + bool "Support for the Allwinner A100 CCU" + default ARM64 && ARCH_SUNXI + depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST + +config SUN50I_A100_R_CCU + bool "Support for the Allwinner A100 PRCM CCU" + default ARM64 && ARCH_SUNXI + depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST + config SUN50I_H6_CCU bool "Support for the Allwinner H6 CCU" default ARM64 && ARCH_SUNXI diff --git a/drivers/clk/sunxi-ng/Makefile b/drivers/clk/sunxi-ng/Makefile index 4c7bee883f2f..3eb5cff40eac 100644 --- a/drivers/clk/sunxi-ng/Makefile +++ b/drivers/clk/sunxi-ng/Makefile @@ -23,6 +23,8 @@ obj-y += ccu_mp.o # SoC support obj-$(CONFIG_SUNIV_F1C100S_CCU) += ccu-suniv-f1c100s.o obj-$(CONFIG_SUN50I_A64_CCU) += ccu-sun50i-a64.o +obj-$(CONFIG_SUN50I_A100_CCU) += ccu-sun50i-a100.o +obj-$(CONFIG_SUN50I_A100_R_CCU) += ccu-sun50i-a100-r.o obj-$(CONFIG_SUN50I_H6_CCU) += ccu-sun50i-h6.o obj-$(CONFIG_SUN50I_H6_R_CCU) += ccu-sun50i-h6-r.o obj-$(CONFIG_SUN4I_A10_CCU) += ccu-sun4i-a10.o diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c b/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c new file mode 100644 index 000000000000..a56142b90993 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 Yangtao Li <frank@allwinnertech.com> + */ + +#include <linux/clk-provider.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/platform_device.h> + +#include "ccu_common.h" +#include "ccu_reset.h" + +#include "ccu_div.h" +#include "ccu_gate.h" +#include "ccu_mp.h" +#include "ccu_nm.h" + +#include "ccu-sun50i-a100-r.h" + +static const char * const cpus_r_apb2_parents[] = { "dcxo24M", "osc32k", + "iosc", "pll-periph0" }; +static const struct ccu_mux_var_prediv cpus_r_apb2_predivs[] = { + { .index = 3, .shift = 0, .width = 5 }, +}; + +static struct ccu_div r_cpus_clk = { + .div = _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO), + + .mux = { + .shift = 24, + .width = 2, + + .var_predivs = cpus_r_apb2_predivs, + .n_var_predivs = ARRAY_SIZE(cpus_r_apb2_predivs), + }, + + .common = { + .reg = 0x000, + .features = CCU_FEATURE_VARIABLE_PREDIV, + .hw.init = CLK_HW_INIT_PARENTS("cpus", + cpus_r_apb2_parents, + &ccu_div_ops, + 0), + }, +}; + +static CLK_FIXED_FACTOR_HW(r_ahb_clk, "r-ahb", &r_cpus_clk.common.hw, 1, 1, 0); + +static struct ccu_div r_apb1_clk = { + .div = _SUNXI_CCU_DIV(0, 2), + + .common = { + .reg = 0x00c, + .hw.init = CLK_HW_INIT("r-apb1", + "r-ahb", + &ccu_div_ops, + 0), + }, +}; + +static struct ccu_div r_apb2_clk = { + .div = _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO), + + .mux = { + .shift = 24, + .width = 2, + + .var_predivs = cpus_r_apb2_predivs, + .n_var_predivs = ARRAY_SIZE(cpus_r_apb2_predivs), + }, + + .common = { + .reg = 0x010, + .features = CCU_FEATURE_VARIABLE_PREDIV, + .hw.init = CLK_HW_INIT_PARENTS("r-apb2", + cpus_r_apb2_parents, + &ccu_div_ops, + 0), + }, +}; + +static const struct clk_parent_data clk_parent_r_apb1[] = { + { .hw = &r_apb1_clk.common.hw }, +}; + +static const struct clk_parent_data clk_parent_r_apb2[] = { + { .hw = &r_apb2_clk.common.hw }, +}; + +static SUNXI_CCU_GATE_DATA(r_apb1_timer_clk, "r-apb1-timer", clk_parent_r_apb1, + 0x11c, BIT(0), 0); + +static SUNXI_CCU_GATE_DATA(r_apb1_twd_clk, "r-apb1-twd", clk_parent_r_apb1, + 0x12c, BIT(0), 0); + +static const char * const r_apb1_pwm_clk_parents[] = { "dcxo24M", "osc32k", + "iosc" }; +static SUNXI_CCU_MUX(r_apb1_pwm_clk, "r-apb1-pwm", r_apb1_pwm_clk_parents, + 0x130, 24, 2, 0); + +static SUNXI_CCU_GATE_DATA(r_apb1_bus_pwm_clk, "r-apb1-bus-pwm", + clk_parent_r_apb1, 0x13c, BIT(0), 0); + +static SUNXI_CCU_GATE_DATA(r_apb1_ppu_clk, "r-apb1-ppu", clk_parent_r_apb1, + 0x17c, BIT(0), 0); + +static SUNXI_CCU_GATE_DATA(r_apb2_uart_clk, "r-apb2-uart", clk_parent_r_apb2, + 0x18c, BIT(0), 0); + +static SUNXI_CCU_GATE_DATA(r_apb2_i2c0_clk, "r-apb2-i2c0", clk_parent_r_apb2, + 0x19c, BIT(0), 0); + +static SUNXI_CCU_GATE_DATA(r_apb2_i2c1_clk, "r-apb2-i2c1", clk_parent_r_apb2, + 0x19c, BIT(1), 0); + +static const char * const r_apb1_ir_rx_parents[] = { "osc32k", "dcxo24M" }; +static SUNXI_CCU_MP_WITH_MUX_GATE(r_apb1_ir_rx_clk, "r-apb1-ir-rx", + r_apb1_ir_rx_parents, 0x1c0, + 0, 5, /* M */ + 8, 2, /* P */ + 24, 1, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_DATA(r_apb1_bus_ir_rx_clk, "r-apb1-bus-ir-rx", + clk_parent_r_apb1, 0x1cc, BIT(0), 0); + +static SUNXI_CCU_GATE(r_ahb_bus_rtc_clk, "r-ahb-rtc", "r-ahb", + 0x20c, BIT(0), 0); + +static struct ccu_common *sun50i_a100_r_ccu_clks[] = { + &r_cpus_clk.common, + &r_apb1_clk.common, + &r_apb2_clk.common, + &r_apb1_timer_clk.common, + &r_apb1_twd_clk.common, + &r_apb1_pwm_clk.common, + &r_apb1_bus_pwm_clk.common, + &r_apb1_ppu_clk.common, + &r_apb2_uart_clk.common, + &r_apb2_i2c0_clk.common, + &r_apb2_i2c1_clk.common, + &r_apb1_ir_rx_clk.common, + &r_apb1_bus_ir_rx_clk.common, + &r_ahb_bus_rtc_clk.common, +}; + +static struct clk_hw_onecell_data sun50i_a100_r_hw_clks = { + .hws = { + [CLK_R_CPUS] = &r_cpus_clk.common.hw, + [CLK_R_AHB] = &r_ahb_clk.hw, + [CLK_R_APB1] = &r_apb1_clk.common.hw, + [CLK_R_APB2] = &r_apb2_clk.common.hw, + [CLK_R_APB1_TIMER] = &r_apb1_timer_clk.common.hw, + [CLK_R_APB1_TWD] = &r_apb1_twd_clk.common.hw, + [CLK_R_APB1_PWM] = &r_apb1_pwm_clk.common.hw, + [CLK_R_APB1_BUS_PWM] = &r_apb1_bus_pwm_clk.common.hw, + [CLK_R_APB1_PPU] = &r_apb1_ppu_clk.common.hw, + [CLK_R_APB2_UART] = &r_apb2_uart_clk.common.hw, + [CLK_R_APB2_I2C0] = &r_apb2_i2c0_clk.common.hw, + [CLK_R_APB2_I2C1] = &r_apb2_i2c1_clk.common.hw, + [CLK_R_APB1_IR] = &r_apb1_ir_rx_clk.common.hw, + [CLK_R_APB1_BUS_IR] = &r_apb1_bus_ir_rx_clk.common.hw, + [CLK_R_AHB_BUS_RTC] = &r_ahb_bus_rtc_clk.common.hw, + }, + .num = CLK_NUMBER, +}; + +static struct ccu_reset_map sun50i_a100_r_ccu_resets[] = { + [RST_R_APB1_TIMER] = { 0x11c, BIT(16) }, + [RST_R_APB1_BUS_PWM] = { 0x13c, BIT(16) }, + [RST_R_APB1_PPU] = { 0x17c, BIT(16) }, + [RST_R_APB2_UART] = { 0x18c, BIT(16) }, + [RST_R_APB2_I2C0] = { 0x19c, BIT(16) }, + [RST_R_APB2_I2C1] = { 0x19c, BIT(17) }, + [RST_R_APB1_BUS_IR] = { 0x1cc, BIT(16) }, + [RST_R_AHB_BUS_RTC] = { 0x20c, BIT(16) }, +}; + +static const struct sunxi_ccu_desc sun50i_a100_r_ccu_desc = { + .ccu_clks = sun50i_a100_r_ccu_clks, + .num_ccu_clks = ARRAY_SIZE(sun50i_a100_r_ccu_clks), + + .hw_clks = &sun50i_a100_r_hw_clks, + + .resets = sun50i_a100_r_ccu_resets, + .num_resets = ARRAY_SIZE(sun50i_a100_r_ccu_resets), +}; + +static int sun50i_a100_r_ccu_probe(struct platform_device *pdev) +{ + void __iomem *reg; + + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); + + return sunxi_ccu_probe(pdev->dev.of_node, reg, &sun50i_a100_r_ccu_desc); +} + +static const struct of_device_id sun50i_a100_r_ccu_ids[] = { + { .compatible = "allwinner,sun50i-a100-r-ccu" }, + { } +}; + +static struct platform_driver sun50i_a100_r_ccu_driver = { + .probe = sun50i_a100_r_ccu_probe, + .driver = { + .name = "sun50i-a100-r-ccu", + .of_match_table = sun50i_a100_r_ccu_ids, + }, +}; +module_platform_driver(sun50i_a100_r_ccu_driver); diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.h b/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.h new file mode 100644 index 000000000000..3a8f187a51b7 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2020 Yangtao Li <frank@allwinnertech.com> + */ + +#ifndef _CCU_SUN50I_A100_R_H +#define _CCU_SUN50I_A100_R_H + +#include <dt-bindings/clock/sun50i-a100-r-ccu.h> +#include <dt-bindings/reset/sun50i-a100-r-ccu.h> + +#define CLK_R_CPUS 0 +#define CLK_R_AHB 1 + +/* exported except APB1 for R_PIO */ + +#define CLK_R_APB2 3 + +#define CLK_NUMBER (CLK_R_AHB_BUS_RTC + 1) + +#endif /* _CCU_SUN50I_A100_R_H */ diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a100.c b/drivers/clk/sunxi-ng/ccu-sun50i-a100.c new file mode 100644 index 000000000000..81b48c73d389 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a100.c @@ -0,0 +1,1276 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 Yangtao Li <frank@allwinnertech.com> + */ + +#include <linux/clk-provider.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/platform_device.h> + +#include "ccu_common.h" +#include "ccu_reset.h" + +#include "ccu_div.h" +#include "ccu_gate.h" +#include "ccu_mp.h" +#include "ccu_mult.h" +#include "ccu_nk.h" +#include "ccu_nkm.h" +#include "ccu_nkmp.h" +#include "ccu_nm.h" + +#include "ccu-sun50i-a100.h" + +#define SUN50I_A100_PLL_SDM_ENABLE BIT(24) +#define SUN50I_A100_PLL_OUTPUT_ENABLE BIT(27) +#define SUN50I_A100_PLL_LOCK BIT(28) +#define SUN50I_A100_PLL_LOCK_ENABLE BIT(29) +#define SUN50I_A100_PLL_ENABLE BIT(31) + +#define SUN50I_A100_PLL_PERIPH1_PATTERN0 0xd1303333 + +/* + * The CPU PLL is actually NP clock, with P being /1, /2 or /4. However + * P should only be used for output frequencies lower than 288 MHz. + * + * For now we can just model it as a multiplier clock, and force P to /1. + * + * The M factor is present in the register's description, but not in the + * frequency formula, and it's documented as "M is only used for backdoor + * testing", so it's not modelled and then force to 0. + */ +#define SUN50I_A100_PLL_CPUX_REG 0x000 +static struct ccu_mult pll_cpux_clk = { + .enable = SUN50I_A100_PLL_OUTPUT_ENABLE, + .lock = SUN50I_A100_PLL_LOCK, + .mult = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .common = { + .reg = 0x000, + .hw.init = CLK_HW_INIT("pll-cpux", "dcxo24M", + &ccu_mult_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +/* Some PLLs are input * N / div1 / P. Model them as NKMP with no K */ +#define SUN50I_A100_PLL_DDR0_REG 0x010 +static struct ccu_nkmp pll_ddr0_clk = { + .enable = SUN50I_A100_PLL_OUTPUT_ENABLE, + .lock = SUN50I_A100_PLL_LOCK, + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .common = { + .reg = 0x010, + .hw.init = CLK_HW_INIT("pll-ddr0", "dcxo24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE | + CLK_IS_CRITICAL), + }, +}; + +#define SUN50I_A100_PLL_PERIPH0_REG 0x020 +static struct ccu_nkmp pll_periph0_clk = { + .enable = SUN50I_A100_PLL_OUTPUT_ENABLE, + .lock = SUN50I_A100_PLL_LOCK, + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .fixed_post_div = 2, + .common = { + .reg = 0x020, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-periph0", "dcxo24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_A100_PLL_PERIPH1_REG 0x028 +static struct ccu_nkmp pll_periph1_clk = { + .enable = SUN50I_A100_PLL_OUTPUT_ENABLE, + .lock = SUN50I_A100_PLL_LOCK, + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .fixed_post_div = 2, + .common = { + .reg = 0x028, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-periph1", "dcxo24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; +#define SUN50I_A100_PLL_PERIPH1_PATTERN0_REG 0x128 + +#define SUN50I_A100_PLL_GPU_REG 0x030 +static struct ccu_nkmp pll_gpu_clk = { + .enable = SUN50I_A100_PLL_OUTPUT_ENABLE, + .lock = SUN50I_A100_PLL_LOCK, + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .common = { + .reg = 0x030, + .hw.init = CLK_HW_INIT("pll-gpu", "dcxo24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +/* + * For Video PLLs, the output divider is described as "used for testing" + * in the user manual. So it's not modelled and forced to 0. + */ +#define SUN50I_A100_PLL_VIDEO0_REG 0x040 +static struct ccu_nm pll_video0_clk = { + .enable = SUN50I_A100_PLL_OUTPUT_ENABLE, + .lock = SUN50I_A100_PLL_LOCK, + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .fixed_post_div = 4, + .common = { + .reg = 0x040, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-video0", "dcxo24M", + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_A100_PLL_VIDEO1_REG 0x048 +static struct ccu_nm pll_video1_clk = { + .enable = SUN50I_A100_PLL_OUTPUT_ENABLE, + .lock = SUN50I_A100_PLL_LOCK, + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .fixed_post_div = 4, + .common = { + .reg = 0x048, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-video1", "dcxo24M", + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_A100_PLL_VIDEO2_REG 0x050 +static struct ccu_nm pll_video2_clk = { + .enable = SUN50I_A100_PLL_OUTPUT_ENABLE, + .lock = SUN50I_A100_PLL_LOCK, + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .fixed_post_div = 4, + .common = { + .reg = 0x050, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-video2", "dcxo24M", + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_A100_PLL_VE_REG 0x058 +static struct ccu_nkmp pll_ve_clk = { + .enable = SUN50I_A100_PLL_OUTPUT_ENABLE, + .lock = SUN50I_A100_PLL_LOCK, + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .common = { + .reg = 0x058, + .hw.init = CLK_HW_INIT("pll-ve", "dcxo24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +/* + * The COM PLL has m0 dividers in addition to the usual N, M + * factors. Since we only need 1 frequencies from this PLL: 45.1584 MHz, + * ignore it for now. + */ +#define SUN50I_A100_PLL_COM_REG 0x060 +static struct ccu_sdm_setting pll_com_sdm_table[] = { + { .rate = 451584000, .pattern = 0xc0014396, .m = 2, .n = 37 }, +}; + +static struct ccu_nm pll_com_clk = { + .enable = SUN50I_A100_PLL_OUTPUT_ENABLE, + .lock = SUN50I_A100_PLL_LOCK, + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(0, 1), + .sdm = _SUNXI_CCU_SDM(pll_com_sdm_table, BIT(24), + 0x160, BIT(31)), + .common = { + .reg = 0x060, + .features = CCU_FEATURE_SIGMA_DELTA_MOD, + .hw.init = CLK_HW_INIT("pll-com", "dcxo24M", + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_A100_PLL_VIDEO3_REG 0x068 +static struct ccu_nm pll_video3_clk = { + .enable = SUN50I_A100_PLL_OUTPUT_ENABLE, + .lock = SUN50I_A100_PLL_LOCK, + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .fixed_post_div = 4, + .common = { + .reg = 0x068, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-video3", "dcxo24M", + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +/* + * The Audio PLL has m0, m1 dividers in addition to the usual N, M + * factors. Since we only need 4 frequencies from this PLL: 22.5792 MHz, + * 24.576 MHz, 90.3168MHz and 98.304MHz ignore them for now. + * Enforce the default for them, which is m0 = 1, m1 = 0. + */ +#define SUN50I_A100_PLL_AUDIO_REG 0x078 +static struct ccu_sdm_setting pll_audio_sdm_table[] = { + { .rate = 45158400, .pattern = 0xc001bcd3, .m = 18, .n = 33 }, + { .rate = 49152000, .pattern = 0xc001eb85, .m = 20, .n = 40 }, + { .rate = 180633600, .pattern = 0xc001288d, .m = 3, .n = 22 }, + { .rate = 196608000, .pattern = 0xc001eb85, .m = 5, .n = 40 }, +}; + +static struct ccu_nm pll_audio_clk = { + .enable = SUN50I_A100_PLL_OUTPUT_ENABLE, + .lock = SUN50I_A100_PLL_LOCK, + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(16, 6), + .fixed_post_div = 2, + .sdm = _SUNXI_CCU_SDM(pll_audio_sdm_table, BIT(24), + 0x178, BIT(31)), + .common = { + .reg = 0x078, + .features = CCU_FEATURE_FIXED_POSTDIV | + CCU_FEATURE_SIGMA_DELTA_MOD, + .hw.init = CLK_HW_INIT("pll-audio", "dcxo24M", + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +static const char * const cpux_parents[] = { "dcxo24M", "osc32k", + "iosc", "pll-cpux", + "pll-periph0" }; +static SUNXI_CCU_MUX(cpux_clk, "cpux", cpux_parents, + 0x500, 24, 3, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL); +static SUNXI_CCU_M(axi_clk, "axi", "cpux", 0x500, 0, 2, 0); +static SUNXI_CCU_M(cpux_apb_clk, "cpux-apb", "cpux", 0x500, 8, 2, 0); + +static const char * const psi_ahb1_ahb2_parents[] = { "dcxo24M", "osc32k", + "iosc", "pll-periph0", + "pll-periph0-2x" }; +static SUNXI_CCU_MP_WITH_MUX(psi_ahb1_ahb2_clk, "psi-ahb1-ahb2", + psi_ahb1_ahb2_parents, 0x510, + 0, 2, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + 0); + +static const char * const ahb3_apb1_apb2_parents[] = { "dcxo24M", "osc32k", + "psi-ahb1-ahb2", + "pll-periph0", + "pll-periph0-2x" }; +static SUNXI_CCU_MP_WITH_MUX(ahb3_clk, "ahb3", ahb3_apb1_apb2_parents, 0x51c, + 0, 2, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + 0); + +static SUNXI_CCU_MP_WITH_MUX(apb1_clk, "apb1", ahb3_apb1_apb2_parents, 0x520, + 0, 2, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + 0); + +static SUNXI_CCU_MP_WITH_MUX(apb2_clk, "apb2", ahb3_apb1_apb2_parents, 0x524, + 0, 2, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + 0); + +static const char * const mbus_parents[] = { "dcxo24M", "pll-ddr0", + "pll-periph0", + "pll-periph0-2x" }; +static SUNXI_CCU_M_WITH_MUX_GATE(mbus_clk, "mbus", mbus_parents, 0x540, + 0, 3, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + CLK_IS_CRITICAL); + +static const char * const de_parents[] = { "pll-com", "pll-periph0-2x" }; +static SUNXI_CCU_M_WITH_MUX_GATE(de_clk, "de0", de_parents, 0x600, + 0, 4, /* M */ + 24, 1, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE(bus_de_clk, "bus-de", "psi-ahb1-ahb2", + 0x60c, BIT(0), 0); + +static const char * const g2d_parents[] = { "pll-com", "pll-periph0-2x", + "pll-video0-2x", "pll-video1-2x", + "pll-video2-2x"}; +static SUNXI_CCU_M_WITH_MUX_GATE(g2d_clk, "g2d", + g2d_parents, + 0x630, + 0, 4, /* M */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_g2d_clk, "bus-g2d", "psi-ahb1-ahb2", + 0x63c, BIT(0), 0); + +static const char * const gpu_parents[] = { "pll-gpu" }; +static SUNXI_CCU_M_WITH_MUX_GATE(gpu_clk, "gpu", gpu_parents, 0x670, + 0, 2, /* M */ + 24, 1, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_gpu_clk, "bus-gpu", "psi-ahb1-ahb2", + 0x67c, BIT(0), 0); + +static const char * const ce_parents[] = { "dcxo24M", "pll-periph0-2x" }; +static SUNXI_CCU_MP_WITH_MUX_GATE(ce_clk, "ce", ce_parents, 0x680, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 1, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_ce_clk, "bus-ce", "psi-ahb1-ahb2", + 0x68c, BIT(0), 0); + +static const char * const ve_parents[] = { "pll-ve" }; +static SUNXI_CCU_M_WITH_MUX_GATE(ve_clk, "ve", ve_parents, 0x690, + 0, 3, /* M */ + 24, 1, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE(bus_ve_clk, "bus-ve", "psi-ahb1-ahb2", + 0x69c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_dma_clk, "bus-dma", "psi-ahb1-ahb2", + 0x70c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_msgbox_clk, "bus-msgbox", "psi-ahb1-ahb2", + 0x71c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_spinlock_clk, "bus-spinlock", "psi-ahb1-ahb2", + 0x72c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_hstimer_clk, "bus-hstimer", "psi-ahb1-ahb2", + 0x73c, BIT(0), 0); + +static SUNXI_CCU_GATE(avs_clk, "avs", "dcxo24M", 0x740, BIT(31), 0); + +static SUNXI_CCU_GATE(bus_dbg_clk, "bus-dbg", "psi-ahb1-ahb2", + 0x78c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_psi_clk, "bus-psi", "psi-ahb1-ahb2", + 0x79c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_pwm_clk, "bus-pwm", "apb1", 0x7ac, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_iommu_clk, "bus-iommu", "apb1", 0x7bc, BIT(0), 0); + +static SUNXI_CCU_GATE(mbus_dma_clk, "mbus-dma", "mbus", + 0x804, BIT(0), 0); +static SUNXI_CCU_GATE(mbus_ve_clk, "mbus-ve", "mbus", + 0x804, BIT(1), 0); +static SUNXI_CCU_GATE(mbus_ce_clk, "mbus-ce", "mbus", + 0x804, BIT(2), 0); +static SUNXI_CCU_GATE(mbus_nand_clk, "mbus-nand", "mbus", + 0x804, BIT(5), 0); +static SUNXI_CCU_GATE(mbus_csi_clk, "mbus-csi", "mbus", + 0x804, BIT(8), 0); +static SUNXI_CCU_GATE(mbus_isp_clk, "mbus-isp", "mbus", + 0x804, BIT(9), 0); +static SUNXI_CCU_GATE(mbus_g2d_clk, "mbus-g2d", "mbus", + 0x804, BIT(10), 0); + +static SUNXI_CCU_GATE(bus_dram_clk, "bus-dram", "psi-ahb1-ahb2", + 0x80c, BIT(0), CLK_IS_CRITICAL); + +static const char * const nand_spi_parents[] = { "dcxo24M", + "pll-periph0", + "pll-periph1", + "pll-periph0-2x", + "pll-periph1-2x" }; +static SUNXI_CCU_MP_WITH_MUX_GATE(nand0_clk, "nand0", nand_spi_parents, 0x810, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(nand1_clk, "nand1", nand_spi_parents, 0x814, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_nand_clk, "bus-nand", "ahb3", 0x82c, BIT(0), 0); + +static const char * const mmc_parents[] = { "dcxo24M", "pll-periph0-2x", + "pll-periph1-2x" }; +static SUNXI_CCU_MP_WITH_MUX_GATE_POSTDIV(mmc0_clk, "mmc0", mmc_parents, 0x830, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 2, /* post-div */ + CLK_SET_RATE_NO_REPARENT); + +static SUNXI_CCU_MP_WITH_MUX_GATE_POSTDIV(mmc1_clk, "mmc1", mmc_parents, 0x834, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 2, /* post-div */ + CLK_SET_RATE_NO_REPARENT); + +static SUNXI_CCU_MP_WITH_MUX_GATE_POSTDIV(mmc2_clk, "mmc2", mmc_parents, 0x838, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 2, /* post-div */ + CLK_SET_RATE_NO_REPARENT); + +static SUNXI_CCU_GATE(bus_mmc0_clk, "bus-mmc0", "ahb3", 0x84c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_mmc1_clk, "bus-mmc1", "ahb3", 0x84c, BIT(1), 0); +static SUNXI_CCU_GATE(bus_mmc2_clk, "bus-mmc2", "ahb3", 0x84c, BIT(2), 0); + +static SUNXI_CCU_GATE(bus_uart0_clk, "bus-uart0", "apb2", 0x90c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_uart1_clk, "bus-uart1", "apb2", 0x90c, BIT(1), 0); +static SUNXI_CCU_GATE(bus_uart2_clk, "bus-uart2", "apb2", 0x90c, BIT(2), 0); +static SUNXI_CCU_GATE(bus_uart3_clk, "bus-uart3", "apb2", 0x90c, BIT(3), 0); +static SUNXI_CCU_GATE(bus_uart4_clk, "bus-uart4", "apb2", 0x90c, BIT(4), 0); + +static SUNXI_CCU_GATE(bus_i2c0_clk, "bus-i2c0", "apb2", 0x91c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_i2c1_clk, "bus-i2c1", "apb2", 0x91c, BIT(1), 0); +static SUNXI_CCU_GATE(bus_i2c2_clk, "bus-i2c2", "apb2", 0x91c, BIT(2), 0); +static SUNXI_CCU_GATE(bus_i2c3_clk, "bus-i2c3", "apb2", 0x91c, BIT(3), 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(spi0_clk, "spi0", nand_spi_parents, 0x940, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(spi1_clk, "spi1", nand_spi_parents, 0x944, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(spi2_clk, "spi2", nand_spi_parents, 0x948, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_spi0_clk, "bus-spi0", "ahb3", 0x96c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_spi1_clk, "bus-spi1", "ahb3", 0x96c, BIT(1), 0); +static SUNXI_CCU_GATE(bus_spi2_clk, "bus-spi2", "ahb3", 0x96c, BIT(2), 0); + +static SUNXI_CCU_GATE(emac_25m_clk, "emac-25m", "ahb3", 0x970, + BIT(31) | BIT(30), 0); + +static SUNXI_CCU_GATE(bus_emac_clk, "bus-emac", "ahb3", 0x97c, BIT(0), 0); + +static const char * const ir_parents[] = { "osc32k", "iosc", + "pll-periph0", "pll-periph1" }; +static SUNXI_CCU_MP_WITH_MUX_GATE(ir_rx_clk, "ir-rx", ir_parents, 0x990, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_ir_rx_clk, "bus-ir-rx", "ahb3", 0x99c, BIT(0), 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(ir_tx_clk, "ir-tx", ir_parents, 0x9c0, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_ir_tx_clk, "bus-ir-tx", "apb1", 0x9cc, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_gpadc_clk, "bus-gpadc", "apb1", 0x9ec, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_ths_clk, "bus-ths", "apb1", 0x9fc, BIT(0), 0); + +static const char * const audio_parents[] = { "pll-audio", "pll-com-audio" }; +static struct ccu_div i2s0_clk = { + .enable = BIT(31), + .div = _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO), + .mux = _SUNXI_CCU_MUX(24, 2), + .common = { + .reg = 0xa10, + .hw.init = CLK_HW_INIT_PARENTS("i2s0", + audio_parents, + &ccu_div_ops, + CLK_SET_RATE_PARENT), + }, +}; + +static struct ccu_div i2s1_clk = { + .enable = BIT(31), + .div = _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO), + .mux = _SUNXI_CCU_MUX(24, 2), + .common = { + .reg = 0xa14, + .hw.init = CLK_HW_INIT_PARENTS("i2s1", + audio_parents, + &ccu_div_ops, + CLK_SET_RATE_PARENT), + }, +}; + +static struct ccu_div i2s2_clk = { + .enable = BIT(31), + .div = _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO), + .mux = _SUNXI_CCU_MUX(24, 2), + .common = { + .reg = 0xa18, + .hw.init = CLK_HW_INIT_PARENTS("i2s2", + audio_parents, + &ccu_div_ops, + CLK_SET_RATE_PARENT), + }, +}; + +static struct ccu_div i2s3_clk = { + .enable = BIT(31), + .div = _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO), + .mux = _SUNXI_CCU_MUX(24, 2), + .common = { + .reg = 0xa1c, + .hw.init = CLK_HW_INIT_PARENTS("i2s3", + audio_parents, + &ccu_div_ops, + CLK_SET_RATE_PARENT), + }, +}; + +static SUNXI_CCU_GATE(bus_i2s0_clk, "bus-i2s0", "apb1", 0xa20, BIT(0), 0); +static SUNXI_CCU_GATE(bus_i2s1_clk, "bus-i2s1", "apb1", 0xa20, BIT(1), 0); +static SUNXI_CCU_GATE(bus_i2s2_clk, "bus-i2s2", "apb1", 0xa20, BIT(2), 0); +static SUNXI_CCU_GATE(bus_i2s3_clk, "bus-i2s3", "apb1", 0xa20, BIT(3), 0); + +static struct ccu_div spdif_clk = { + .enable = BIT(31), + .div = _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO), + .mux = _SUNXI_CCU_MUX(24, 2), + .common = { + .reg = 0xa24, + .hw.init = CLK_HW_INIT_PARENTS("spdif", + audio_parents, + &ccu_div_ops, + 0), + }, +}; + +static SUNXI_CCU_GATE(bus_spdif_clk, "bus-spdif", "apb1", 0xa2c, BIT(0), 0); + +static struct ccu_div dmic_clk = { + .enable = BIT(31), + .div = _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO), + .mux = _SUNXI_CCU_MUX(24, 2), + .common = { + .reg = 0xa40, + .hw.init = CLK_HW_INIT_PARENTS("dmic", + audio_parents, + &ccu_div_ops, + 0), + }, +}; + +static SUNXI_CCU_GATE(bus_dmic_clk, "bus-dmic", "apb1", 0xa4c, BIT(0), 0); + +static SUNXI_CCU_M_WITH_MUX_GATE(audio_codec_dac_clk, "audio-codec-dac", + audio_parents, 0xa50, + 0, 4, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_M_WITH_MUX_GATE(audio_codec_adc_clk, "audio-codec-adc", + audio_parents, 0xa54, + 0, 4, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_M_WITH_MUX_GATE(audio_codec_4x_clk, "audio-codec-4x", + audio_parents, 0xa58, + 0, 4, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_audio_codec_clk, "bus-audio-codec", "apb1", 0xa5c, + BIT(0), 0); + +/* + * There are OHCI 12M clock source selection bits for 2 USB 2.0 ports. + * We will force them to 0 (12M divided from 48M). + */ +#define SUN50I_A100_USB0_CLK_REG 0xa70 +#define SUN50I_A100_USB1_CLK_REG 0xa74 + +static SUNXI_CCU_GATE(usb_ohci0_clk, "usb-ohci0", "osc12M", 0xa70, BIT(31), 0); +static SUNXI_CCU_GATE(usb_phy0_clk, "usb-phy0", "dcxo24M", 0xa70, BIT(29), 0); + +static SUNXI_CCU_GATE(usb_ohci1_clk, "usb-ohci1", "osc12M", 0xa74, BIT(31), 0); +static SUNXI_CCU_GATE(usb_phy1_clk, "usb-phy1", "dcxo24M", 0xa74, BIT(29), 0); + +static SUNXI_CCU_GATE(bus_ohci0_clk, "bus-ohci0", "ahb3", 0xa8c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_ohci1_clk, "bus-ohci1", "ahb3", 0xa8c, BIT(1), 0); +static SUNXI_CCU_GATE(bus_ehci0_clk, "bus-ehci0", "ahb3", 0xa8c, BIT(4), 0); +static SUNXI_CCU_GATE(bus_ehci1_clk, "bus-ehci1", "ahb3", 0xa8c, BIT(5), 0); +static SUNXI_CCU_GATE(bus_otg_clk, "bus-otg", "ahb3", 0xa8c, BIT(8), 0); + +static SUNXI_CCU_GATE(bus_lradc_clk, "bus-lradc", "ahb3", 0xa9c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_dpss_top0_clk, "bus-dpss-top0", "ahb3", + 0xabc, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_dpss_top1_clk, "bus-dpss-top1", "ahb3", + 0xacc, BIT(0), 0); + +static const char * const mipi_dsi_parents[] = { "dcxo24M", "pll-periph0-2x", + "pll-periph0" }; +static SUNXI_CCU_M_WITH_MUX_GATE(mipi_dsi_clk, "mipi-dsi", + mipi_dsi_parents, + 0xb24, + 0, 4, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_mipi_dsi_clk, "bus-mipi-dsi", "ahb3", + 0xb4c, BIT(0), 0); + +static const char * const tcon_lcd_parents[] = { "pll-video0-4x", + "pll-video1-4x", + "pll-video2-4x", + "pll-video3-4x", + "pll-periph0-2x" }; +static SUNXI_CCU_MP_WITH_MUX_GATE(tcon_lcd_clk, "tcon-lcd0", + tcon_lcd_parents, 0xb60, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_tcon_lcd_clk, "bus-tcon-lcd0", "ahb3", + 0xb7c, BIT(0), 0); + +static const char * const ledc_parents[] = { "dcxo24M", + "pll-periph0" }; +static SUNXI_CCU_MP_WITH_MUX_GATE(ledc_clk, "ledc", + ledc_parents, 0xbf0, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_ledc_clk, "bus-ledc", "ahb3", 0xbfc, BIT(0), 0); + +static const char * const csi_top_parents[] = { "pll-periph0-2x", + "pll-video0-2x", + "pll-video1-2x", + "pll-video2-2x", + "pll-video3-2x" }; +static SUNXI_CCU_M_WITH_MUX_GATE(csi_top_clk, "csi-top", + csi_top_parents, 0xc04, + 0, 4, /* M */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static const char * const csi0_mclk_parents[] = { "dcxo24M", "pll-video2", + "pll-video3", "pll-video0", + "pll-video1" }; +static SUNXI_CCU_M_WITH_MUX_GATE(csi0_mclk_clk, "csi0-mclk", + csi0_mclk_parents, 0xc08, + 0, 5, /* M */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static const char * const csi1_mclk_parents[] = { "dcxo24M", "pll-video3", + "pll-video0", "pll-video1", + "pll-video2" }; +static SUNXI_CCU_M_WITH_MUX_GATE(csi1_mclk_clk, "csi1-mclk", + csi1_mclk_parents, 0xc0c, + 0, 5, /* M */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_csi_clk, "bus-csi", "ahb3", 0xc1c, BIT(0), 0); + +static const char * const csi_isp_parents[] = { "pll-periph0-2x", + "pll-video0-2x", + "pll-video1-2x", + "pll-video2-2x", + "pll-video3-2x" }; +static SUNXI_CCU_M_WITH_MUX_GATE(csi_isp_clk, "csi-isp", + csi_isp_parents, 0xc20, + 0, 5, /* M */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +/* Fixed factor clocks */ +static CLK_FIXED_FACTOR_FW_NAME(osc12M_clk, "osc12M", "hosc", 2, 1, 0); + +static CLK_FIXED_FACTOR_HW(pll_com_audio_clk, "pll-com-audio", + &pll_com_clk.common.hw, + 5, 1, CLK_SET_RATE_PARENT); + +static CLK_FIXED_FACTOR_HW(pll_periph0_2x_clk, "pll-periph0-2x", + &pll_periph0_clk.common.hw, + 1, 2, 0); + +static CLK_FIXED_FACTOR_HW(pll_periph1_2x_clk, "pll-periph1-2x", + &pll_periph1_clk.common.hw, + 1, 2, 0); + +static const struct clk_hw *pll_video0_parents[] = { + &pll_video0_clk.common.hw +}; +static CLK_FIXED_FACTOR_HWS(pll_video0_4x_clk, "pll-video0-4x", + pll_video0_parents, + 1, 4, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_video0_2x_clk, "pll-video0-2x", + pll_video0_parents, + 1, 2, CLK_SET_RATE_PARENT); + +static const struct clk_hw *pll_video1_parents[] = { + &pll_video1_clk.common.hw +}; +static CLK_FIXED_FACTOR_HWS(pll_video1_4x_clk, "pll-video1-4x", + pll_video1_parents, + 1, 4, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_video1_2x_clk, "pll-video1-2x", + pll_video1_parents, + 1, 2, CLK_SET_RATE_PARENT); + +static const struct clk_hw *pll_video2_parents[] = { + &pll_video2_clk.common.hw +}; +static CLK_FIXED_FACTOR_HWS(pll_video2_4x_clk, "pll-video2-4x", + pll_video2_parents, + 1, 4, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_video2_2x_clk, "pll-video2-2x", + pll_video2_parents, + 1, 2, CLK_SET_RATE_PARENT); + +static const struct clk_hw *pll_video3_parents[] = { + &pll_video3_clk.common.hw +}; +static CLK_FIXED_FACTOR_HWS(pll_video3_4x_clk, "pll-video3-4x", + pll_video3_parents, + 1, 4, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_video3_2x_clk, "pll-video3-2x", + pll_video3_parents, + 1, 2, CLK_SET_RATE_PARENT); + +static struct ccu_common *sun50i_a100_ccu_clks[] = { + &pll_cpux_clk.common, + &pll_ddr0_clk.common, + &pll_periph0_clk.common, + &pll_periph1_clk.common, + &pll_gpu_clk.common, + &pll_video0_clk.common, + &pll_video1_clk.common, + &pll_video2_clk.common, + &pll_video3_clk.common, + &pll_ve_clk.common, + &pll_com_clk.common, + &pll_audio_clk.common, + &cpux_clk.common, + &axi_clk.common, + &cpux_apb_clk.common, + &psi_ahb1_ahb2_clk.common, + &ahb3_clk.common, + &apb1_clk.common, + &apb2_clk.common, + &mbus_clk.common, + &de_clk.common, + &bus_de_clk.common, + &g2d_clk.common, + &bus_g2d_clk.common, + &gpu_clk.common, + &bus_gpu_clk.common, + &ce_clk.common, + &bus_ce_clk.common, + &ve_clk.common, + &bus_ve_clk.common, + &bus_dma_clk.common, + &bus_msgbox_clk.common, + &bus_spinlock_clk.common, + &bus_hstimer_clk.common, + &avs_clk.common, + &bus_dbg_clk.common, + &bus_psi_clk.common, + &bus_pwm_clk.common, + &bus_iommu_clk.common, + &mbus_dma_clk.common, + &mbus_ve_clk.common, + &mbus_ce_clk.common, + &mbus_nand_clk.common, + &mbus_csi_clk.common, + &mbus_isp_clk.common, + &mbus_g2d_clk.common, + &bus_dram_clk.common, + &nand0_clk.common, + &nand1_clk.common, + &bus_nand_clk.common, + &mmc0_clk.common, + &mmc1_clk.common, + &mmc2_clk.common, + &bus_mmc0_clk.common, + &bus_mmc1_clk.common, + &bus_mmc2_clk.common, + &bus_uart0_clk.common, + &bus_uart1_clk.common, + &bus_uart2_clk.common, + &bus_uart3_clk.common, + &bus_uart4_clk.common, + &bus_i2c0_clk.common, + &bus_i2c1_clk.common, + &bus_i2c2_clk.common, + &bus_i2c3_clk.common, + &spi0_clk.common, + &spi1_clk.common, + &spi2_clk.common, + &bus_spi0_clk.common, + &bus_spi1_clk.common, + &bus_spi2_clk.common, + &emac_25m_clk.common, + &bus_emac_clk.common, + &ir_rx_clk.common, + &bus_ir_rx_clk.common, + &ir_tx_clk.common, + &bus_ir_tx_clk.common, + &bus_gpadc_clk.common, + &bus_ths_clk.common, + &i2s0_clk.common, + &i2s1_clk.common, + &i2s2_clk.common, + &i2s3_clk.common, + &bus_i2s0_clk.common, + &bus_i2s1_clk.common, + &bus_i2s2_clk.common, + &bus_i2s3_clk.common, + &spdif_clk.common, + &bus_spdif_clk.common, + &dmic_clk.common, + &bus_dmic_clk.common, + &audio_codec_dac_clk.common, + &audio_codec_adc_clk.common, + &audio_codec_4x_clk.common, + &bus_audio_codec_clk.common, + &usb_ohci0_clk.common, + &usb_phy0_clk.common, + &usb_ohci1_clk.common, + &usb_phy1_clk.common, + &bus_ohci0_clk.common, + &bus_ohci1_clk.common, + &bus_ehci0_clk.common, + &bus_ehci1_clk.common, + &bus_otg_clk.common, + &bus_lradc_clk.common, + &bus_dpss_top0_clk.common, + &bus_dpss_top1_clk.common, + &mipi_dsi_clk.common, + &bus_mipi_dsi_clk.common, + &tcon_lcd_clk.common, + &bus_tcon_lcd_clk.common, + &ledc_clk.common, + &bus_ledc_clk.common, + &csi_top_clk.common, + &csi0_mclk_clk.common, + &csi1_mclk_clk.common, + &bus_csi_clk.common, + &csi_isp_clk.common, +}; + +static struct clk_hw_onecell_data sun50i_a100_hw_clks = { + .hws = { + [CLK_OSC12M] = &osc12M_clk.hw, + [CLK_PLL_CPUX] = &pll_cpux_clk.common.hw, + [CLK_PLL_DDR0] = &pll_ddr0_clk.common.hw, + [CLK_PLL_PERIPH0] = &pll_periph0_clk.common.hw, + [CLK_PLL_PERIPH0_2X] = &pll_periph0_2x_clk.hw, + [CLK_PLL_PERIPH1] = &pll_periph1_clk.common.hw, + [CLK_PLL_PERIPH1_2X] = &pll_periph1_2x_clk.hw, + [CLK_PLL_GPU] = &pll_gpu_clk.common.hw, + [CLK_PLL_VIDEO0] = &pll_video0_clk.common.hw, + [CLK_PLL_VIDEO0_2X] = &pll_video0_2x_clk.hw, + [CLK_PLL_VIDEO0_4X] = &pll_video0_4x_clk.hw, + [CLK_PLL_VIDEO1] = &pll_video1_clk.common.hw, + [CLK_PLL_VIDEO1_2X] = &pll_video1_2x_clk.hw, + [CLK_PLL_VIDEO1_4X] = &pll_video1_4x_clk.hw, + [CLK_PLL_VIDEO2] = &pll_video2_clk.common.hw, + [CLK_PLL_VIDEO2_2X] = &pll_video2_2x_clk.hw, + [CLK_PLL_VIDEO2_4X] = &pll_video2_4x_clk.hw, + [CLK_PLL_VIDEO3] = &pll_video3_clk.common.hw, + [CLK_PLL_VIDEO3_2X] = &pll_video3_2x_clk.hw, + [CLK_PLL_VIDEO3_4X] = &pll_video3_4x_clk.hw, + [CLK_PLL_VE] = &pll_ve_clk.common.hw, + [CLK_PLL_COM] = &pll_com_clk.common.hw, + [CLK_PLL_COM_AUDIO] = &pll_com_audio_clk.hw, + [CLK_PLL_AUDIO] = &pll_audio_clk.common.hw, + [CLK_CPUX] = &cpux_clk.common.hw, + [CLK_AXI] = &axi_clk.common.hw, + [CLK_CPUX_APB] = &cpux_apb_clk.common.hw, + [CLK_PSI_AHB1_AHB2] = &psi_ahb1_ahb2_clk.common.hw, + [CLK_AHB3] = &ahb3_clk.common.hw, + [CLK_APB1] = &apb1_clk.common.hw, + [CLK_APB2] = &apb2_clk.common.hw, + [CLK_MBUS] = &mbus_clk.common.hw, + [CLK_DE] = &de_clk.common.hw, + [CLK_BUS_DE] = &bus_de_clk.common.hw, + [CLK_G2D] = &g2d_clk.common.hw, + [CLK_BUS_G2D] = &bus_g2d_clk.common.hw, + [CLK_GPU] = &gpu_clk.common.hw, + [CLK_BUS_GPU] = &bus_gpu_clk.common.hw, + [CLK_CE] = &ce_clk.common.hw, + [CLK_BUS_CE] = &bus_ce_clk.common.hw, + [CLK_VE] = &ve_clk.common.hw, + [CLK_BUS_VE] = &bus_ve_clk.common.hw, + [CLK_BUS_DMA] = &bus_dma_clk.common.hw, + [CLK_BUS_MSGBOX] = &bus_msgbox_clk.common.hw, + [CLK_BUS_SPINLOCK] = &bus_spinlock_clk.common.hw, + [CLK_BUS_HSTIMER] = &bus_hstimer_clk.common.hw, + [CLK_AVS] = &avs_clk.common.hw, + [CLK_BUS_DBG] = &bus_dbg_clk.common.hw, + [CLK_BUS_PSI] = &bus_psi_clk.common.hw, + [CLK_BUS_PWM] = &bus_pwm_clk.common.hw, + [CLK_BUS_IOMMU] = &bus_iommu_clk.common.hw, + [CLK_MBUS_DMA] = &mbus_dma_clk.common.hw, + [CLK_MBUS_VE] = &mbus_ve_clk.common.hw, + [CLK_MBUS_CE] = &mbus_ce_clk.common.hw, + [CLK_MBUS_NAND] = &mbus_nand_clk.common.hw, + [CLK_MBUS_CSI] = &mbus_csi_clk.common.hw, + [CLK_MBUS_ISP] = &mbus_isp_clk.common.hw, + [CLK_MBUS_G2D] = &mbus_g2d_clk.common.hw, + [CLK_BUS_DRAM] = &bus_dram_clk.common.hw, + [CLK_NAND0] = &nand0_clk.common.hw, + [CLK_NAND1] = &nand1_clk.common.hw, + [CLK_BUS_NAND] = &bus_nand_clk.common.hw, + [CLK_MMC0] = &mmc0_clk.common.hw, + [CLK_MMC1] = &mmc1_clk.common.hw, + [CLK_MMC2] = &mmc2_clk.common.hw, + [CLK_BUS_MMC0] = &bus_mmc0_clk.common.hw, + [CLK_BUS_MMC1] = &bus_mmc1_clk.common.hw, + [CLK_BUS_MMC2] = &bus_mmc2_clk.common.hw, + [CLK_BUS_UART0] = &bus_uart0_clk.common.hw, + [CLK_BUS_UART1] = &bus_uart1_clk.common.hw, + [CLK_BUS_UART2] = &bus_uart2_clk.common.hw, + [CLK_BUS_UART3] = &bus_uart3_clk.common.hw, + [CLK_BUS_UART4] = &bus_uart4_clk.common.hw, + [CLK_BUS_I2C0] = &bus_i2c0_clk.common.hw, + [CLK_BUS_I2C1] = &bus_i2c1_clk.common.hw, + [CLK_BUS_I2C2] = &bus_i2c2_clk.common.hw, + [CLK_BUS_I2C3] = &bus_i2c3_clk.common.hw, + [CLK_SPI0] = &spi0_clk.common.hw, + [CLK_SPI1] = &spi1_clk.common.hw, + [CLK_SPI2] = &spi2_clk.common.hw, + [CLK_BUS_SPI0] = &bus_spi0_clk.common.hw, + [CLK_BUS_SPI1] = &bus_spi1_clk.common.hw, + [CLK_BUS_SPI2] = &bus_spi2_clk.common.hw, + [CLK_EMAC_25M] = &emac_25m_clk.common.hw, + [CLK_BUS_EMAC] = &bus_emac_clk.common.hw, + [CLK_IR_RX] = &ir_rx_clk.common.hw, + [CLK_BUS_IR_RX] = &bus_ir_rx_clk.common.hw, + [CLK_IR_TX] = &ir_tx_clk.common.hw, + [CLK_BUS_IR_TX] = &bus_ir_tx_clk.common.hw, + [CLK_BUS_GPADC] = &bus_gpadc_clk.common.hw, + [CLK_BUS_THS] = &bus_ths_clk.common.hw, + [CLK_I2S0] = &i2s0_clk.common.hw, + [CLK_I2S1] = &i2s1_clk.common.hw, + [CLK_I2S2] = &i2s2_clk.common.hw, + [CLK_I2S3] = &i2s3_clk.common.hw, + [CLK_BUS_I2S0] = &bus_i2s0_clk.common.hw, + [CLK_BUS_I2S1] = &bus_i2s1_clk.common.hw, + [CLK_BUS_I2S2] = &bus_i2s2_clk.common.hw, + [CLK_BUS_I2S3] = &bus_i2s3_clk.common.hw, + [CLK_SPDIF] = &spdif_clk.common.hw, + [CLK_BUS_SPDIF] = &bus_spdif_clk.common.hw, + [CLK_DMIC] = &dmic_clk.common.hw, + [CLK_BUS_DMIC] = &bus_dmic_clk.common.hw, + [CLK_AUDIO_DAC] = &audio_codec_dac_clk.common.hw, + [CLK_AUDIO_ADC] = &audio_codec_adc_clk.common.hw, + [CLK_AUDIO_4X] = &audio_codec_4x_clk.common.hw, + [CLK_BUS_AUDIO_CODEC] = &bus_audio_codec_clk.common.hw, + [CLK_USB_OHCI0] = &usb_ohci0_clk.common.hw, + [CLK_USB_PHY0] = &usb_phy0_clk.common.hw, + [CLK_USB_OHCI1] = &usb_ohci1_clk.common.hw, + [CLK_USB_PHY1] = &usb_phy1_clk.common.hw, + [CLK_BUS_OHCI0] = &bus_ohci0_clk.common.hw, + [CLK_BUS_OHCI1] = &bus_ohci1_clk.common.hw, + [CLK_BUS_EHCI0] = &bus_ehci0_clk.common.hw, + [CLK_BUS_EHCI1] = &bus_ehci1_clk.common.hw, + [CLK_BUS_OTG] = &bus_otg_clk.common.hw, + [CLK_BUS_LRADC] = &bus_lradc_clk.common.hw, + [CLK_BUS_DPSS_TOP0] = &bus_dpss_top0_clk.common.hw, + [CLK_BUS_DPSS_TOP1] = &bus_dpss_top1_clk.common.hw, + [CLK_MIPI_DSI] = &mipi_dsi_clk.common.hw, + [CLK_BUS_MIPI_DSI] = &bus_mipi_dsi_clk.common.hw, + [CLK_TCON_LCD] = &tcon_lcd_clk.common.hw, + [CLK_BUS_TCON_LCD] = &bus_tcon_lcd_clk.common.hw, + [CLK_LEDC] = &ledc_clk.common.hw, + [CLK_BUS_LEDC] = &bus_ledc_clk.common.hw, + [CLK_CSI_TOP] = &csi_top_clk.common.hw, + [CLK_CSI0_MCLK] = &csi0_mclk_clk.common.hw, + [CLK_CSI1_MCLK] = &csi1_mclk_clk.common.hw, + [CLK_BUS_CSI] = &bus_csi_clk.common.hw, + [CLK_CSI_ISP] = &csi_isp_clk.common.hw, + }, + .num = CLK_NUMBER, +}; + +static struct ccu_reset_map sun50i_a100_ccu_resets[] = { + [RST_MBUS] = { 0x540, BIT(30) }, + + [RST_BUS_DE] = { 0x60c, BIT(16) }, + [RST_BUS_G2D] = { 0x63c, BIT(16) }, + [RST_BUS_GPU] = { 0x67c, BIT(16) }, + [RST_BUS_CE] = { 0x68c, BIT(16) }, + [RST_BUS_VE] = { 0x69c, BIT(16) }, + [RST_BUS_DMA] = { 0x70c, BIT(16) }, + [RST_BUS_MSGBOX] = { 0x71c, BIT(16) }, + [RST_BUS_SPINLOCK] = { 0x72c, BIT(16) }, + [RST_BUS_HSTIMER] = { 0x73c, BIT(16) }, + [RST_BUS_DBG] = { 0x78c, BIT(16) }, + [RST_BUS_PSI] = { 0x79c, BIT(16) }, + [RST_BUS_PWM] = { 0x7ac, BIT(16) }, + [RST_BUS_DRAM] = { 0x80c, BIT(16) }, + [RST_BUS_NAND] = { 0x82c, BIT(16) }, + [RST_BUS_MMC0] = { 0x84c, BIT(16) }, + [RST_BUS_MMC1] = { 0x84c, BIT(17) }, + [RST_BUS_MMC2] = { 0x84c, BIT(18) }, + [RST_BUS_UART0] = { 0x90c, BIT(16) }, + [RST_BUS_UART1] = { 0x90c, BIT(17) }, + [RST_BUS_UART2] = { 0x90c, BIT(18) }, + [RST_BUS_UART3] = { 0x90c, BIT(19) }, + [RST_BUS_UART4] = { 0x90c, BIT(20) }, + [RST_BUS_I2C0] = { 0x91c, BIT(16) }, + [RST_BUS_I2C1] = { 0x91c, BIT(17) }, + [RST_BUS_I2C2] = { 0x91c, BIT(18) }, + [RST_BUS_I2C3] = { 0x91c, BIT(19) }, + [RST_BUS_SPI0] = { 0x96c, BIT(16) }, + [RST_BUS_SPI1] = { 0x96c, BIT(17) }, + [RST_BUS_SPI2] = { 0x96c, BIT(18) }, + [RST_BUS_EMAC] = { 0x97c, BIT(16) }, + [RST_BUS_IR_RX] = { 0x99c, BIT(16) }, + [RST_BUS_IR_TX] = { 0x9cc, BIT(16) }, + [RST_BUS_GPADC] = { 0x9ec, BIT(16) }, + [RST_BUS_THS] = { 0x9fc, BIT(16) }, + [RST_BUS_I2S0] = { 0xa20, BIT(16) }, + [RST_BUS_I2S1] = { 0xa20, BIT(17) }, + [RST_BUS_I2S2] = { 0xa20, BIT(18) }, + [RST_BUS_I2S3] = { 0xa20, BIT(19) }, + [RST_BUS_SPDIF] = { 0xa2c, BIT(16) }, + [RST_BUS_DMIC] = { 0xa4c, BIT(16) }, + [RST_BUS_AUDIO_CODEC] = { 0xa5c, BIT(16) }, + + [RST_USB_PHY0] = { 0xa70, BIT(30) }, + [RST_USB_PHY1] = { 0xa74, BIT(30) }, + + [RST_BUS_OHCI0] = { 0xa8c, BIT(16) }, + [RST_BUS_OHCI1] = { 0xa8c, BIT(17) }, + [RST_BUS_EHCI0] = { 0xa8c, BIT(20) }, + [RST_BUS_EHCI1] = { 0xa8c, BIT(21) }, + [RST_BUS_OTG] = { 0xa8c, BIT(24) }, + + [RST_BUS_LRADC] = { 0xa9c, BIT(16) }, + [RST_BUS_DPSS_TOP0] = { 0xabc, BIT(16) }, + [RST_BUS_DPSS_TOP1] = { 0xacc, BIT(16) }, + [RST_BUS_MIPI_DSI] = { 0xb4c, BIT(16) }, + [RST_BUS_TCON_LCD] = { 0xb7c, BIT(16) }, + [RST_BUS_LVDS] = { 0xbac, BIT(16) }, + [RST_BUS_LEDC] = { 0xbfc, BIT(16) }, + [RST_BUS_CSI] = { 0xc1c, BIT(16) }, + [RST_BUS_CSI_ISP] = { 0xc2c, BIT(16) }, +}; + +static const struct sunxi_ccu_desc sun50i_a100_ccu_desc = { + .ccu_clks = sun50i_a100_ccu_clks, + .num_ccu_clks = ARRAY_SIZE(sun50i_a100_ccu_clks), + + .hw_clks = &sun50i_a100_hw_clks, + + .resets = sun50i_a100_ccu_resets, + .num_resets = ARRAY_SIZE(sun50i_a100_ccu_resets), +}; + +static const u32 sun50i_a100_pll_regs[] = { + SUN50I_A100_PLL_CPUX_REG, + SUN50I_A100_PLL_DDR0_REG, + SUN50I_A100_PLL_PERIPH0_REG, + SUN50I_A100_PLL_PERIPH1_REG, + SUN50I_A100_PLL_GPU_REG, + SUN50I_A100_PLL_VIDEO0_REG, + SUN50I_A100_PLL_VIDEO1_REG, + SUN50I_A100_PLL_VIDEO2_REG, + SUN50I_A100_PLL_VIDEO3_REG, + SUN50I_A100_PLL_VE_REG, + SUN50I_A100_PLL_COM_REG, + SUN50I_A100_PLL_AUDIO_REG, +}; + +static const u32 sun50i_a100_pll_video_regs[] = { + SUN50I_A100_PLL_VIDEO0_REG, + SUN50I_A100_PLL_VIDEO1_REG, + SUN50I_A100_PLL_VIDEO2_REG, + SUN50I_A100_PLL_VIDEO3_REG, +}; + +static const u32 sun50i_a100_usb2_clk_regs[] = { + SUN50I_A100_USB0_CLK_REG, + SUN50I_A100_USB1_CLK_REG, +}; + +static struct ccu_pll_nb sun50i_a100_pll_cpu_nb = { + .common = &pll_cpux_clk.common, + /* copy from pll_cpux_clk */ + .enable = BIT(27), + .lock = BIT(28), +}; + +static struct ccu_mux_nb sun50i_a100_cpu_nb = { + .common = &cpux_clk.common, + .cm = &cpux_clk.mux, + .delay_us = 1, + .bypass_index = 4, /* index of pll periph0 */ +}; + +static int sun50i_a100_ccu_probe(struct platform_device *pdev) +{ + void __iomem *reg; + u32 val; + int i, ret; + + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); + + /* + * Enable lock and enable bits on all PLLs. + * + * Due to the current design, multiple PLLs share one power switch, + * so switching PLL is easy to cause stability problems. + * When initializing, we enable them by default. When disable, + * we only turn off the output of PLL. + */ + for (i = 0; i < ARRAY_SIZE(sun50i_a100_pll_regs); i++) { + val = readl(reg + sun50i_a100_pll_regs[i]); + val |= SUN50I_A100_PLL_LOCK_ENABLE | SUN50I_A100_PLL_ENABLE; + writel(val, reg + sun50i_a100_pll_regs[i]); + } + + /* + * In order to pass the EMI certification, the SDM function of + * the peripheral 1 bus is enabled, and the frequency is still + * calculated using the previous division factor. + */ + writel(SUN50I_A100_PLL_PERIPH1_PATTERN0, + reg + SUN50I_A100_PLL_PERIPH1_PATTERN0_REG); + + val = readl(reg + SUN50I_A100_PLL_PERIPH1_REG); + val |= SUN50I_A100_PLL_SDM_ENABLE; + writel(val, reg + SUN50I_A100_PLL_PERIPH1_REG); + + /* + * Force the output divider of video PLLs to 0. + * + * See the comment before pll-video0 definition for the reason. + */ + for (i = 0; i < ARRAY_SIZE(sun50i_a100_pll_video_regs); i++) { + val = readl(reg + sun50i_a100_pll_video_regs[i]); + val &= ~BIT(0); + writel(val, reg + sun50i_a100_pll_video_regs[i]); + } + + /* + * Enforce m1 = 0, m0 = 1 for Audio PLL + * + * See the comment before pll-audio definition for the reason. + */ + val = readl(reg + SUN50I_A100_PLL_AUDIO_REG); + val &= ~BIT(1); + val |= BIT(0); + writel(val, reg + SUN50I_A100_PLL_AUDIO_REG); + + /* + * Force OHCI 12M clock sources to 00 (12MHz divided from 48MHz) + * + * This clock mux is still mysterious, and the code just enforces + * it to have a valid clock parent. + */ + for (i = 0; i < ARRAY_SIZE(sun50i_a100_usb2_clk_regs); i++) { + val = readl(reg + sun50i_a100_usb2_clk_regs[i]); + val &= ~GENMASK(25, 24); + writel(val, reg + sun50i_a100_usb2_clk_regs[i]); + } + + ret = sunxi_ccu_probe(pdev->dev.of_node, reg, &sun50i_a100_ccu_desc); + if (ret) + return ret; + + /* Gate then ungate PLL CPU after any rate changes */ + ccu_pll_notifier_register(&sun50i_a100_pll_cpu_nb); + + /* Reparent CPU during PLL CPU rate changes */ + ccu_mux_notifier_register(pll_cpux_clk.common.hw.clk, + &sun50i_a100_cpu_nb); + + return 0; +} + +static const struct of_device_id sun50i_a100_ccu_ids[] = { + { .compatible = "allwinner,sun50i-a100-ccu" }, + { } +}; + +static struct platform_driver sun50i_a100_ccu_driver = { + .probe = sun50i_a100_ccu_probe, + .driver = { + .name = "sun50i-a100-ccu", + .of_match_table = sun50i_a100_ccu_ids, + }, +}; +module_platform_driver(sun50i_a100_ccu_driver); diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a100.h b/drivers/clk/sunxi-ng/ccu-sun50i-a100.h new file mode 100644 index 000000000000..21ce92bb1d5f --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a100.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2020 Yangtao Li <frank@allwinnertech.com> + */ + +#ifndef _CCU_SUN50I_A100_H_ +#define _CCU_SUN50I_A100_H_ + +#include <dt-bindings/clock/sun50i-a100-ccu.h> +#include <dt-bindings/reset/sun50i-a100-ccu.h> + +#define CLK_OSC12M 0 +#define CLK_PLL_CPUX 1 +#define CLK_PLL_DDR0 2 + +/* PLL_PERIPH0 exported for PRCM */ + +#define CLK_PLL_PERIPH0_2X 4 +#define CLK_PLL_PERIPH1 5 +#define CLK_PLL_PERIPH1_2X 6 +#define CLK_PLL_GPU 7 +#define CLK_PLL_VIDEO0 8 +#define CLK_PLL_VIDEO0_2X 9 +#define CLK_PLL_VIDEO0_4X 10 +#define CLK_PLL_VIDEO1 11 +#define CLK_PLL_VIDEO1_2X 12 +#define CLK_PLL_VIDEO1_4X 13 +#define CLK_PLL_VIDEO2 14 +#define CLK_PLL_VIDEO2_2X 15 +#define CLK_PLL_VIDEO2_4X 16 +#define CLK_PLL_VIDEO3 17 +#define CLK_PLL_VIDEO3_2X 18 +#define CLK_PLL_VIDEO3_4X 19 +#define CLK_PLL_VE 20 +#define CLK_PLL_COM 21 +#define CLK_PLL_COM_AUDIO 22 +#define CLK_PLL_AUDIO 23 + +/* CPUX clock exported for DVFS */ + +#define CLK_AXI 25 +#define CLK_CPUX_APB 26 +#define CLK_PSI_AHB1_AHB2 27 +#define CLK_AHB3 28 + +/* APB1 clock exported for PIO */ + +#define CLK_APB2 30 + +/* All module clocks and bus gates are exported except DRAM */ + +#define CLK_BUS_DRAM 58 + +#define CLK_NUMBER (CLK_CSI_ISP + 1) + +#endif /* _CCU_SUN50I_A100_H_ */ diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c index 23bfe1d12f21..84153418453f 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c @@ -45,18 +45,29 @@ static struct ccu_nkmp pll_cpu_clk = { * the base (2x, 4x and 8x), and one variable divider (the one true * pll audio). * - * We don't have any need for the variable divider for now, so we just - * hardcode it to match with the clock names + * With sigma-delta modulation for fractional-N on the audio PLL, + * we have to use specific dividers. This means the variable divider + * can no longer be used, as the audio codec requests the exact clock + * rates we support through this mechanism. So we now hard code the + * variable divider to 1. This means the clock rates will no longer + * match the clock names. */ #define SUN8I_R40_PLL_AUDIO_REG 0x008 -static SUNXI_CCU_NM_WITH_GATE_LOCK(pll_audio_base_clk, "pll-audio-base", - "osc24M", 0x008, - 8, 7, /* N */ - 0, 5, /* M */ - BIT(31), /* gate */ - BIT(28), /* lock */ - CLK_SET_RATE_UNGATE); +static struct ccu_sdm_setting pll_audio_sdm_table[] = { + { .rate = 22579200, .pattern = 0xc0010d84, .m = 8, .n = 7 }, + { .rate = 24576000, .pattern = 0xc000ac02, .m = 14, .n = 14 }, +}; + +static SUNXI_CCU_NM_WITH_SDM_GATE_LOCK(pll_audio_base_clk, "pll-audio-base", + "osc24M", 0x008, + 8, 7, /* N */ + 0, 5, /* M */ + pll_audio_sdm_table, BIT(24), + 0x284, BIT(31), + BIT(31), /* gate */ + BIT(28), /* lock */ + CLK_SET_RATE_UNGATE); static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK_MIN_MAX(pll_video0_clk, "pll-video0", "osc24M", 0x0010, @@ -952,10 +963,10 @@ static const struct clk_hw *clk_parent_pll_audio[] = { &pll_audio_base_clk.common.hw }; -/* We hardcode the divider to 4 for now */ +/* We hardcode the divider to 1 for now */ static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio", clk_parent_pll_audio, - 4, 1, CLK_SET_RATE_PARENT); + 1, 1, CLK_SET_RATE_PARENT); static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x", clk_parent_pll_audio, 2, 1, CLK_SET_RATE_PARENT); @@ -1307,10 +1318,10 @@ static int sun8i_r40_ccu_probe(struct platform_device *pdev) if (IS_ERR(reg)) return PTR_ERR(reg); - /* Force the PLL-Audio-1x divider to 4 */ + /* Force the PLL-Audio-1x divider to 1 */ val = readl(reg + SUN8I_R40_PLL_AUDIO_REG); val &= ~GENMASK(19, 16); - writel(val | (3 << 16), reg + SUN8I_R40_PLL_AUDIO_REG); + writel(val | (0 << 16), reg + SUN8I_R40_PLL_AUDIO_REG); /* Force PLL-MIPI to MIPI mode */ val = readl(reg + SUN8I_R40_PLL_MIPI_REG); diff --git a/drivers/clk/tegra/clk-tegra210-emc.c b/drivers/clk/tegra/clk-tegra210-emc.c index 51fd0ec2a2d0..672ca8c184d2 100644 --- a/drivers/clk/tegra/clk-tegra210-emc.c +++ b/drivers/clk/tegra/clk-tegra210-emc.c @@ -128,7 +128,7 @@ static int tegra210_clk_emc_set_rate(struct clk_hw *hw, unsigned long rate, unsigned int i; int err; - if (!provider || !provider->configs || provider->num_configs == 0) + if (!provider->configs || provider->num_configs == 0) return -EINVAL; for (i = 0; i < provider->num_configs; i++) { diff --git a/drivers/clk/ti/autoidle.c b/drivers/clk/ti/autoidle.c index 1cae226759dd..f6f8a409f148 100644 --- a/drivers/clk/ti/autoidle.c +++ b/drivers/clk/ti/autoidle.c @@ -82,7 +82,12 @@ static int _omap2_clk_allow_idle(struct clk_hw_omap *clk) */ int omap2_clk_deny_idle(struct clk *clk) { - struct clk_hw *hw = __clk_get_hw(clk); + struct clk_hw *hw; + + if (!clk) + return -EINVAL; + + hw = __clk_get_hw(clk); if (omap2_clk_is_hw_omap(hw)) { struct clk_hw_omap *c = to_clk_hw_omap(hw); @@ -101,7 +106,12 @@ int omap2_clk_deny_idle(struct clk *clk) */ int omap2_clk_allow_idle(struct clk *clk) { - struct clk_hw *hw = __clk_get_hw(clk); + struct clk_hw *hw; + + if (!clk) + return -EINVAL; + + hw = __clk_get_hw(clk); if (omap2_clk_is_hw_omap(hw)) { struct clk_hw_omap *c = to_clk_hw_omap(hw); diff --git a/drivers/clk/ti/clk-7xx.c b/drivers/clk/ti/clk-7xx.c index b4cf578a69e1..4e27f88062e7 100644 --- a/drivers/clk/ti/clk-7xx.c +++ b/drivers/clk/ti/clk-7xx.c @@ -637,6 +637,7 @@ static const struct omap_clkctrl_reg_data dra7_l4sec_clkctrl_regs[] __initconst { DRA7_L4SEC_DES_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" }, { DRA7_L4SEC_RNG_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_NONSEC, "l4_root_clk_div" }, { DRA7_L4SEC_SHAM_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" }, + { DRA7_L4SEC_SHAM2_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" }, { 0 }, }; diff --git a/drivers/clk/ti/clockdomain.c b/drivers/clk/ti/clockdomain.c index ee56306f79d5..700b7f44f671 100644 --- a/drivers/clk/ti/clockdomain.c +++ b/drivers/clk/ti/clockdomain.c @@ -148,10 +148,12 @@ static void __init of_ti_clockdomain_setup(struct device_node *node) if (!omap2_clk_is_hw_omap(clk_hw)) { pr_warn("can't setup clkdm for basic clk %s\n", __clk_get_name(clk)); + clk_put(clk); continue; } to_clk_hw_omap(clk_hw)->clkdm_name = clkdm_name; omap2_init_clk_clkdm(clk_hw); + clk_put(clk); } } diff --git a/drivers/clk/uniphier/clk-uniphier-cpugear.c b/drivers/clk/uniphier/clk-uniphier-cpugear.c index 1a33a08abf2f..a2f01a4da127 100644 --- a/drivers/clk/uniphier/clk-uniphier-cpugear.c +++ b/drivers/clk/uniphier/clk-uniphier-cpugear.c @@ -90,7 +90,7 @@ struct clk_hw *uniphier_clk_register_cpugear(struct device *dev, init.ops = &uniphier_clk_cpugear_ops; init.flags = CLK_SET_RATE_PARENT; init.parent_names = data->parent_names; - init.num_parents = data->num_parents, + init.num_parents = data->num_parents; gear->regmap = regmap; gear->regbase = data->regbase; diff --git a/drivers/clk/uniphier/clk-uniphier-mux.c b/drivers/clk/uniphier/clk-uniphier-mux.c index c0f4631601e2..462c84321b2d 100644 --- a/drivers/clk/uniphier/clk-uniphier-mux.c +++ b/drivers/clk/uniphier/clk-uniphier-mux.c @@ -70,7 +70,7 @@ struct clk_hw *uniphier_clk_register_mux(struct device *dev, init.ops = &uniphier_clk_mux_ops; init.flags = CLK_SET_RATE_PARENT; init.parent_names = data->parent_names; - init.num_parents = data->num_parents, + init.num_parents = data->num_parents; mux->regmap = regmap; mux->reg = data->reg; diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm index bf5830eb664f..015ec0c02835 100644 --- a/drivers/cpufreq/Kconfig.arm +++ b/drivers/cpufreq/Kconfig.arm @@ -196,7 +196,6 @@ config ARM_S3C24XX_CPUFREQ_DEBUGFS config ARM_S3C2410_CPUFREQ bool depends on ARM_S3C24XX_CPUFREQ && CPU_S3C2410 - select S3C2410_CPUFREQ_UTILS help CPU Frequency scaling support for S3C2410 @@ -233,7 +232,6 @@ config ARM_S3C2416_CPUFREQ_VCORESCALE config ARM_S3C2440_CPUFREQ bool "S3C2440/S3C2442 CPU Frequency scaling support" depends on ARM_S3C24XX_CPUFREQ && (CPU_S3C2440 || CPU_S3C2442) - select S3C2410_CPUFREQ_UTILS default y help CPU Frequency scaling support for S3C2440 and S3C2442 SoC CPUs. diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index e4ff681faaaa..1e4fbb002a31 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -691,7 +691,8 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) cpumask_copy(policy->cpus, topology_core_cpumask(cpu)); } - if (check_amd_hwpstate_cpu(cpu) && !acpi_pstate_strict) { + if (check_amd_hwpstate_cpu(cpu) && boot_cpu_data.x86 < 0x19 && + !acpi_pstate_strict) { cpumask_clear(policy->cpus); cpumask_set_cpu(cpu, policy->cpus); cpumask_copy(data->freqdomain_cpus, diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 1877f5e2e5b0..f4b60663efe6 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1454,14 +1454,13 @@ static int cpufreq_online(unsigned int cpu) */ if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK) && has_target()) { + unsigned int old_freq = policy->cur; + /* Are we running at unknown frequency ? */ - ret = cpufreq_frequency_table_get_index(policy, policy->cur); + ret = cpufreq_frequency_table_get_index(policy, old_freq); if (ret == -EINVAL) { - /* Warn user and fix it */ - pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n", - __func__, policy->cpu, policy->cur); - ret = __cpufreq_driver_target(policy, policy->cur - 1, - CPUFREQ_RELATION_L); + ret = __cpufreq_driver_target(policy, old_freq - 1, + CPUFREQ_RELATION_L); /* * Reaching here after boot in a few seconds may not @@ -1469,8 +1468,8 @@ static int cpufreq_online(unsigned int cpu) * frequency for longer duration. Hence, a BUG_ON(). */ BUG_ON(ret); - pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n", - __func__, policy->cpu, policy->cur); + pr_info("%s: CPU%d: Running at unlisted initial frequency: %u KHz, changing to: %u KHz\n", + __func__, policy->cpu, old_freq, policy->cur); } } diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 9a515c460a00..3c1455518738 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1420,6 +1420,24 @@ static void __init intel_pstate_sysfs_expose_params(void) } } +static void __init intel_pstate_sysfs_remove(void) +{ + if (!intel_pstate_kobject) + return; + + sysfs_remove_group(intel_pstate_kobject, &intel_pstate_attr_group); + + if (!per_cpu_limits) { + sysfs_remove_file(intel_pstate_kobject, &max_perf_pct.attr); + sysfs_remove_file(intel_pstate_kobject, &min_perf_pct.attr); + + if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids)) + sysfs_remove_file(intel_pstate_kobject, &energy_efficiency.attr); + } + + kobject_put(intel_pstate_kobject); +} + static void intel_pstate_sysfs_expose_hwp_dynamic_boost(void) { int rc; @@ -3063,8 +3081,10 @@ hwp_cpu_matched: mutex_lock(&intel_pstate_driver_lock); rc = intel_pstate_register_driver(default_driver); mutex_unlock(&intel_pstate_driver_lock); - if (rc) + if (rc) { + intel_pstate_sysfs_remove(); return rc; + } if (hwp_active) { const struct x86_cpu_id *id; diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c index a9af15e994cc..e439b43c19eb 100644 --- a/drivers/cpufreq/powernv-cpufreq.c +++ b/drivers/cpufreq/powernv-cpufreq.c @@ -885,12 +885,15 @@ static int powernv_cpufreq_reboot_notifier(struct notifier_block *nb, unsigned long action, void *unused) { int cpu; - struct cpufreq_policy cpu_policy; + struct cpufreq_policy *cpu_policy; rebooting = true; for_each_online_cpu(cpu) { - cpufreq_get_policy(&cpu_policy, cpu); - powernv_cpufreq_target_index(&cpu_policy, get_nominal_index()); + cpu_policy = cpufreq_cpu_get(cpu); + if (!cpu_policy) + continue; + powernv_cpufreq_target_index(cpu_policy, get_nominal_index()); + cpufreq_cpu_put(cpu_policy); } return NOTIFY_DONE; diff --git a/drivers/cpufreq/s3c2410-cpufreq.c b/drivers/cpufreq/s3c2410-cpufreq.c index 0c4f2ccd7e22..5dcfbf0bfb74 100644 --- a/drivers/cpufreq/s3c2410-cpufreq.c +++ b/drivers/cpufreq/s3c2410-cpufreq.c @@ -16,14 +16,14 @@ #include <linux/clk.h> #include <linux/err.h> #include <linux/io.h> +#include <linux/soc/samsung/s3c-cpufreq-core.h> +#include <linux/soc/samsung/s3c-pm.h> #include <asm/mach/arch.h> #include <asm/mach/map.h> -#include <mach/regs-clock.h> - -#include <plat/cpu.h> -#include <plat/cpu-freq-core.h> +#define S3C2410_CLKDIVN_PDIVN (1<<0) +#define S3C2410_CLKDIVN_HDIVN (1<<1) /* Note, 2410A has an extra mode for 1:4:4 ratio, bit 2 of CLKDIV */ @@ -37,7 +37,7 @@ static void s3c2410_cpufreq_setdivs(struct s3c_cpufreq_config *cfg) if (cfg->divs.p_divisor != cfg->divs.h_divisor) clkdiv |= S3C2410_CLKDIVN_PDIVN; - __raw_writel(clkdiv, S3C2410_CLKDIVN); + s3c24xx_write_clkdivn(clkdiv); } static int s3c2410_cpufreq_calcdivs(struct s3c_cpufreq_config *cfg) diff --git a/drivers/cpufreq/s3c2412-cpufreq.c b/drivers/cpufreq/s3c2412-cpufreq.c index 53385a9ab957..5945945ead7c 100644 --- a/drivers/cpufreq/s3c2412-cpufreq.c +++ b/drivers/cpufreq/s3c2412-cpufreq.c @@ -19,15 +19,24 @@ #include <linux/clk.h> #include <linux/err.h> #include <linux/io.h> +#include <linux/soc/samsung/s3c-cpufreq-core.h> +#include <linux/soc/samsung/s3c-pm.h> #include <asm/mach/arch.h> #include <asm/mach/map.h> -#include <mach/regs-clock.h> -#include <mach/s3c2412.h> - -#include <plat/cpu.h> -#include <plat/cpu-freq-core.h> +#define S3C2412_CLKDIVN_PDIVN (1<<2) +#define S3C2412_CLKDIVN_HDIVN_MASK (3<<0) +#define S3C2412_CLKDIVN_ARMDIVN (1<<3) +#define S3C2412_CLKDIVN_DVSEN (1<<4) +#define S3C2412_CLKDIVN_HALFHCLK (1<<5) +#define S3C2412_CLKDIVN_USB48DIV (1<<6) +#define S3C2412_CLKDIVN_UARTDIV_MASK (15<<8) +#define S3C2412_CLKDIVN_UARTDIV_SHIFT (8) +#define S3C2412_CLKDIVN_I2SDIV_MASK (15<<12) +#define S3C2412_CLKDIVN_I2SDIV_SHIFT (12) +#define S3C2412_CLKDIVN_CAMDIV_MASK (15<<16) +#define S3C2412_CLKDIVN_CAMDIV_SHIFT (16) /* our clock resources. */ static struct clk *xtal; @@ -117,7 +126,7 @@ static void s3c2412_cpufreq_setdivs(struct s3c_cpufreq_config *cfg) unsigned long clkdiv; unsigned long olddiv; - olddiv = clkdiv = __raw_readl(S3C2410_CLKDIVN); + olddiv = clkdiv = s3c24xx_read_clkdivn(); /* clear off current clock info */ @@ -134,32 +143,11 @@ static void s3c2412_cpufreq_setdivs(struct s3c_cpufreq_config *cfg) clkdiv |= S3C2412_CLKDIVN_PDIVN; s3c_freq_dbg("%s: div %08lx => %08lx\n", __func__, olddiv, clkdiv); - __raw_writel(clkdiv, S3C2410_CLKDIVN); + s3c24xx_write_clkdivn(clkdiv); clk_set_parent(armclk, cfg->divs.dvs ? hclk : fclk); } -static void s3c2412_cpufreq_setrefresh(struct s3c_cpufreq_config *cfg) -{ - struct s3c_cpufreq_board *board = cfg->board; - unsigned long refresh; - - s3c_freq_dbg("%s: refresh %u ns, hclk %lu\n", __func__, - board->refresh, cfg->freq.hclk); - - /* Reduce both the refresh time (in ns) and the frequency (in MHz) - * by 10 each to ensure that we do not overflow 32 bit numbers. This - * should work for HCLK up to 133MHz and refresh period up to 30usec. - */ - - refresh = (board->refresh / 10); - refresh *= (cfg->freq.hclk / 100); - refresh /= (1 * 1000 * 1000); /* 10^6 */ - - s3c_freq_dbg("%s: setting refresh 0x%08lx\n", __func__, refresh); - __raw_writel(refresh, S3C2412_REFRESH); -} - /* set the default cpu frequency information, based on an 200MHz part * as we have no other way of detecting the speed rating in software. */ diff --git a/drivers/cpufreq/s3c2440-cpufreq.c b/drivers/cpufreq/s3c2440-cpufreq.c index 3f772ba8896e..148e8aedefa9 100644 --- a/drivers/cpufreq/s3c2440-cpufreq.c +++ b/drivers/cpufreq/s3c2440-cpufreq.c @@ -20,14 +20,27 @@ #include <linux/clk.h> #include <linux/err.h> #include <linux/io.h> +#include <linux/soc/samsung/s3c-cpufreq-core.h> +#include <linux/soc/samsung/s3c-pm.h> #include <asm/mach/arch.h> #include <asm/mach/map.h> -#include <mach/regs-clock.h> +#define S3C2440_CLKDIVN_PDIVN (1<<0) +#define S3C2440_CLKDIVN_HDIVN_MASK (3<<1) +#define S3C2440_CLKDIVN_HDIVN_1 (0<<1) +#define S3C2440_CLKDIVN_HDIVN_2 (1<<1) +#define S3C2440_CLKDIVN_HDIVN_4_8 (2<<1) +#define S3C2440_CLKDIVN_HDIVN_3_6 (3<<1) +#define S3C2440_CLKDIVN_UCLK (1<<3) -#include <plat/cpu.h> -#include <plat/cpu-freq-core.h> +#define S3C2440_CAMDIVN_CAMCLK_MASK (0xf<<0) +#define S3C2440_CAMDIVN_CAMCLK_SEL (1<<4) +#define S3C2440_CAMDIVN_HCLK3_HALF (1<<8) +#define S3C2440_CAMDIVN_HCLK4_HALF (1<<9) +#define S3C2440_CAMDIVN_DVSEN (1<<12) + +#define S3C2442_CAMDIVN_CAMCLK_DIV3 (1<<5) static struct clk *xtal; static struct clk *fclk; @@ -143,8 +156,8 @@ static void s3c2440_cpufreq_setdivs(struct s3c_cpufreq_config *cfg) s3c_freq_dbg("%s: divisors: h=%d, p=%d\n", __func__, cfg->divs.h_divisor, cfg->divs.p_divisor); - clkdiv = __raw_readl(S3C2410_CLKDIVN); - camdiv = __raw_readl(S3C2440_CAMDIVN); + clkdiv = s3c24xx_read_clkdivn(); + camdiv = s3c2440_read_camdivn(); clkdiv &= ~(S3C2440_CLKDIVN_HDIVN_MASK | S3C2440_CLKDIVN_PDIVN); camdiv &= ~CAMDIVN_HCLK_HALF; @@ -184,11 +197,11 @@ static void s3c2440_cpufreq_setdivs(struct s3c_cpufreq_config *cfg) * then make a short delay and remove the hclk halving if necessary. */ - __raw_writel(camdiv | CAMDIVN_HCLK_HALF, S3C2440_CAMDIVN); - __raw_writel(clkdiv, S3C2410_CLKDIVN); + s3c2440_write_camdivn(camdiv | CAMDIVN_HCLK_HALF); + s3c24xx_write_clkdivn(clkdiv); ndelay(20); - __raw_writel(camdiv, S3C2440_CAMDIVN); + s3c2440_write_camdivn(camdiv); clk_set_parent(armclk, cfg->divs.dvs ? hclk : fclk); } diff --git a/drivers/cpufreq/s3c24xx-cpufreq-debugfs.c b/drivers/cpufreq/s3c24xx-cpufreq-debugfs.c index 290e3539d03e..93971dfe7c75 100644 --- a/drivers/cpufreq/s3c24xx-cpufreq-debugfs.c +++ b/drivers/cpufreq/s3c24xx-cpufreq-debugfs.c @@ -18,7 +18,7 @@ #include <linux/seq_file.h> #include <linux/err.h> -#include <plat/cpu-freq-core.h> +#include <linux/soc/samsung/s3c-cpufreq-core.h> static struct dentry *dbgfs_root; static struct dentry *dbgfs_file_io; diff --git a/drivers/cpufreq/s3c24xx-cpufreq.c b/drivers/cpufreq/s3c24xx-cpufreq.c index ed0e713b1b57..37efc0dc3f91 100644 --- a/drivers/cpufreq/s3c24xx-cpufreq.c +++ b/drivers/cpufreq/s3c24xx-cpufreq.c @@ -21,17 +21,13 @@ #include <linux/device.h> #include <linux/sysfs.h> #include <linux/slab.h> +#include <linux/soc/samsung/s3c-cpufreq-core.h> +#include <linux/soc/samsung/s3c-pm.h> #include <asm/mach/arch.h> #include <asm/mach/map.h> -#include <plat/cpu.h> -#include <plat/cpu-freq-core.h> - -#include <mach/regs-clock.h> - /* note, cpufreq support deals in kHz, no Hz */ - static struct cpufreq_driver s3c24xx_driver; static struct s3c_cpufreq_config cpu_cur; static struct s3c_iotimings s3c24xx_iotiming; @@ -68,7 +64,7 @@ static void s3c_cpufreq_getcur(struct s3c_cpufreq_config *cfg) cfg->freq.pclk = pclk = clk_get_rate(clk_pclk); cfg->freq.armclk = armclk = clk_get_rate(clk_arm); - cfg->pll.driver_data = __raw_readl(S3C2410_MPLLCON); + cfg->pll.driver_data = s3c24xx_read_mpllcon(); cfg->pll.frequency = fclk; cfg->freq.hclk_tns = 1000000000 / (cfg->freq.hclk / 10); @@ -386,7 +382,7 @@ static unsigned int suspend_freq; static int s3c_cpufreq_suspend(struct cpufreq_policy *policy) { suspend_pll.frequency = clk_get_rate(_clk_mpll); - suspend_pll.driver_data = __raw_readl(S3C2410_MPLLCON); + suspend_pll.driver_data = s3c24xx_read_mpllcon(); suspend_freq = clk_get_rate(clk_arm); return 0; @@ -547,7 +543,7 @@ static void s3c_cpufreq_update_loctkime(void) val |= calc_locktime(rate, cpu_cur.info->locktime_m); pr_info("%s: new locktime is 0x%08x\n", __func__, val); - __raw_writel(val, S3C2410_LOCKTIME); + s3c24xx_write_locktime(val); } static int s3c_cpufreq_build_freq(void) diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c index 6dd1311660b5..e855e8612a67 100644 --- a/drivers/cpufreq/scmi-cpufreq.c +++ b/drivers/cpufreq/scmi-cpufreq.c @@ -29,7 +29,7 @@ static const struct scmi_handle *handle; static unsigned int scmi_cpufreq_get_rate(unsigned int cpu) { struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); - struct scmi_perf_ops *perf_ops = handle->perf_ops; + const struct scmi_perf_ops *perf_ops = handle->perf_ops; struct scmi_data *priv = policy->driver_data; unsigned long rate; int ret; @@ -49,7 +49,7 @@ static int scmi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index) { struct scmi_data *priv = policy->driver_data; - struct scmi_perf_ops *perf_ops = handle->perf_ops; + const struct scmi_perf_ops *perf_ops = handle->perf_ops; u64 freq = policy->freq_table[index].frequency; return perf_ops->freq_set(handle, priv->domain_id, freq * 1000, false); @@ -59,7 +59,7 @@ static unsigned int scmi_cpufreq_fast_switch(struct cpufreq_policy *policy, unsigned int target_freq) { struct scmi_data *priv = policy->driver_data; - struct scmi_perf_ops *perf_ops = handle->perf_ops; + const struct scmi_perf_ops *perf_ops = handle->perf_ops; if (!perf_ops->freq_set(handle, priv->domain_id, target_freq * 1000, true)) diff --git a/drivers/cpuidle/cpuidle-powernv.c b/drivers/cpuidle/cpuidle-powernv.c index addaa6e6718b..c32c600b3cf8 100644 --- a/drivers/cpuidle/cpuidle-powernv.c +++ b/drivers/cpuidle/cpuidle-powernv.c @@ -141,7 +141,7 @@ static int stop_loop(struct cpuidle_device *dev, struct cpuidle_driver *drv, int index) { - power9_idle_type(stop_psscr_table[index].val, + arch300_idle_type(stop_psscr_table[index].val, stop_psscr_table[index].mask); return index; } diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 37593387164a..37da0c070a88 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -71,10 +71,26 @@ config ZCRYPT help Select this option if you want to enable support for s390 cryptographic adapters like: - + PCI-X Cryptographic Coprocessor (PCIXCC) - + Crypto Express 2,3,4 or 5 Coprocessor (CEXxC) - + Crypto Express 2,3,4 or 5 Accelerator (CEXxA) - + Crypto Express 4 or 5 EP11 Coprocessor (CEXxP) + + Crypto Express 2 up to 7 Coprocessor (CEXxC) + + Crypto Express 2 up to 7 Accelerator (CEXxA) + + Crypto Express 4 up to 7 EP11 Coprocessor (CEXxP) + +config ZCRYPT_DEBUG + bool "Enable debug features for s390 cryptographic adapters" + default n + depends on DEBUG_KERNEL + depends on ZCRYPT + help + Say 'Y' here to enable some additional debug features on the + s390 cryptographic adapters driver. + + There will be some more sysfs attributes displayed for ap cards + and queues and some flags on crypto requests are interpreted as + debugging messages to force error injection. + + Do not enable on production level kernel build. + + If unsure, say N. config ZCRYPT_MULTIDEVNODES bool "Support for multiple zcrypt device nodes" diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c index 6c933f2b604e..b4368c5b6a0c 100644 --- a/drivers/dax/kmem.c +++ b/drivers/dax/kmem.c @@ -35,11 +35,17 @@ static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r) return 0; } +struct dax_kmem_data { + const char *res_name; + struct resource *res[]; +}; + static int dev_dax_kmem_probe(struct dev_dax *dev_dax) { struct device *dev = &dev_dax->dev; + struct dax_kmem_data *data; + int rc = -ENOMEM; int i, mapped = 0; - char *res_name; int numa_node; /* @@ -55,14 +61,17 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) return -EINVAL; } - res_name = kstrdup(dev_name(dev), GFP_KERNEL); - if (!res_name) + data = kzalloc(sizeof(*data) + sizeof(struct resource *) * dev_dax->nr_range, GFP_KERNEL); + if (!data) return -ENOMEM; + data->res_name = kstrdup(dev_name(dev), GFP_KERNEL); + if (!data->res_name) + goto err_res_name; + for (i = 0; i < dev_dax->nr_range; i++) { struct resource *res; struct range range; - int rc; rc = dax_kmem_range(dev_dax, i, &range); if (rc) { @@ -72,7 +81,7 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) } /* Region is permanently reserved if hotremove fails. */ - res = request_mem_region(range.start, range_len(&range), res_name); + res = request_mem_region(range.start, range_len(&range), data->res_name); if (!res) { dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve region\n", i, range.start, range.end); @@ -82,9 +91,10 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) */ if (mapped) continue; - kfree(res_name); - return -EBUSY; + rc = -EBUSY; + goto err_request_mem; } + data->res[i] = res; /* * Set flags appropriate for System RAM. Leave ..._BUSY clear @@ -99,23 +109,30 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) * this as RAM automatically. */ rc = add_memory_driver_managed(numa_node, range.start, - range_len(&range), kmem_name); + range_len(&range), kmem_name, MHP_NONE); if (rc) { dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n", i, range.start, range.end); - release_mem_region(range.start, range_len(&range)); + release_resource(res); + kfree(res); + data->res[i] = NULL; if (mapped) continue; - kfree(res_name); - return rc; + goto err_request_mem; } mapped++; } - dev_set_drvdata(dev, res_name); + dev_set_drvdata(dev, data); return 0; + +err_request_mem: + kfree(data->res_name); +err_res_name: + kfree(data); + return rc; } #ifdef CONFIG_MEMORY_HOTREMOVE @@ -123,7 +140,7 @@ static int dev_dax_kmem_remove(struct dev_dax *dev_dax) { int i, success = 0; struct device *dev = &dev_dax->dev; - const char *res_name = dev_get_drvdata(dev); + struct dax_kmem_data *data = dev_get_drvdata(dev); /* * We have one shot for removing memory, if some memory blocks were not @@ -142,7 +159,9 @@ static int dev_dax_kmem_remove(struct dev_dax *dev_dax) rc = remove_memory(dev_dax->target_node, range.start, range_len(&range)); if (rc == 0) { - release_mem_region(range.start, range_len(&range)); + release_resource(data->res[i]); + kfree(data->res[i]); + data->res[i] = NULL; success++; continue; } @@ -153,7 +172,8 @@ static int dev_dax_kmem_remove(struct dev_dax *dev_dax) } if (success >= dev_dax->nr_range) { - kfree(res_name); + kfree(data->res_name); + kfree(data); dev_set_drvdata(dev, NULL); } diff --git a/drivers/dax/super.c b/drivers/dax/super.c index e84070b55463..edc279be3e59 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -46,7 +46,8 @@ EXPORT_SYMBOL_GPL(dax_read_unlock); int bdev_dax_pgoff(struct block_device *bdev, sector_t sector, size_t size, pgoff_t *pgoff) { - phys_addr_t phys_off = (get_start_sect(bdev) + sector) * 512; + sector_t start_sect = bdev ? get_start_sect(bdev) : 0; + phys_addr_t phys_off = (start_sect + sector) * 512; if (pgoff) *pgoff = PHYS_PFN(phys_off); diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index 42c8ad10d75e..a367584f0d7b 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -573,8 +573,8 @@ static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, /* request and cfg rings */ ret = k3_ringacc_request_rings_pair(rx_chn->common.ringacc, - flow_cfg->ring_rxq_id, flow_cfg->ring_rxfdq0_id, + flow_cfg->ring_rxq_id, &flow->ringrxfdq, &flow->ringrx); if (ret) { diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index d78dd3c82e9d..3315e3c21586 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -7,7 +7,7 @@ menu "Firmware Drivers" config ARM_SCMI_PROTOCOL - bool "ARM System Control and Management Interface (SCMI) Message Protocol" + tristate "ARM System Control and Management Interface (SCMI) Message Protocol" depends on ARM || ARM64 || COMPILE_TEST depends on MAILBOX help diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile index 99510be9f5ed..5e013b6a3692 100644 --- a/drivers/firmware/Makefile +++ b/drivers/firmware/Makefile @@ -22,7 +22,7 @@ obj-$(CONFIG_TI_SCI_PROTOCOL) += ti_sci.o obj-$(CONFIG_TRUSTED_FOUNDATIONS) += trusted_foundations.o obj-$(CONFIG_TURRIS_MOX_RWTM) += turris-mox-rwtm.o -obj-$(CONFIG_ARM_SCMI_PROTOCOL) += arm_scmi/ +obj-y += arm_scmi/ obj-y += broadcom/ obj-y += meson/ obj-$(CONFIG_GOOGLE_FIRMWARE) += google/ diff --git a/drivers/firmware/arm_scmi/Makefile b/drivers/firmware/arm_scmi/Makefile index 6f9cbc4aef22..bc0d54f8e861 100644 --- a/drivers/firmware/arm_scmi/Makefile +++ b/drivers/firmware/arm_scmi/Makefile @@ -1,9 +1,11 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-y = scmi-bus.o scmi-driver.o scmi-protocols.o scmi-transport.o scmi-bus-y = bus.o scmi-driver-y = driver.o notify.o scmi-transport-y = shmem.o scmi-transport-$(CONFIG_MAILBOX) += mailbox.o scmi-transport-$(CONFIG_HAVE_ARM_SMCCC_DISCOVERY) += smc.o -scmi-protocols-y = base.o clock.o perf.o power.o reset.o sensors.o +scmi-protocols-y = base.o clock.o perf.o power.o reset.o sensors.o system.o +scmi-module-objs := $(scmi-bus-y) $(scmi-driver-y) $(scmi-protocols-y) \ + $(scmi-transport-y) +obj-$(CONFIG_ARM_SCMI_PROTOCOL) += scmi-module.o obj-$(CONFIG_ARM_SCMI_POWER_DOMAIN) += scmi_pm_domain.o diff --git a/drivers/firmware/arm_scmi/bus.c b/drivers/firmware/arm_scmi/bus.c index db55c43a2cbd..1377ec76a45d 100644 --- a/drivers/firmware/arm_scmi/bus.c +++ b/drivers/firmware/arm_scmi/bus.c @@ -230,7 +230,7 @@ static void scmi_devices_unregister(void) bus_for_each_dev(&scmi_bus_type, NULL, NULL, __scmi_devices_unregister); } -static int __init scmi_bus_init(void) +int __init scmi_bus_init(void) { int retval; @@ -240,12 +240,10 @@ static int __init scmi_bus_init(void) return retval; } -subsys_initcall(scmi_bus_init); -static void __exit scmi_bus_exit(void) +void __exit scmi_bus_exit(void) { scmi_devices_unregister(); bus_unregister(&scmi_bus_type); ida_destroy(&scmi_bus_id); } -module_exit(scmi_bus_exit); diff --git a/drivers/firmware/arm_scmi/clock.c b/drivers/firmware/arm_scmi/clock.c index 75e39882746e..c1cfe3ee3d55 100644 --- a/drivers/firmware/arm_scmi/clock.c +++ b/drivers/firmware/arm_scmi/clock.c @@ -318,7 +318,7 @@ scmi_clock_info_get(const struct scmi_handle *handle, u32 clk_id) return clk; } -static struct scmi_clk_ops clk_ops = { +static const struct scmi_clk_ops clk_ops = { .count_get = scmi_clock_count_get, .info_get = scmi_clock_info_get, .rate_get = scmi_clock_rate_get, @@ -364,9 +364,4 @@ static int scmi_clock_protocol_init(struct scmi_handle *handle) return 0; } -static int __init scmi_clock_init(void) -{ - return scmi_protocol_register(SCMI_PROTOCOL_CLOCK, - &scmi_clock_protocol_init); -} -subsys_initcall(scmi_clock_init); +DEFINE_SCMI_PROTOCOL_REGISTER_UNREGISTER(SCMI_PROTOCOL_CLOCK, clock) diff --git a/drivers/firmware/arm_scmi/common.h b/drivers/firmware/arm_scmi/common.h index c113e578cc6c..37fb583f1bf5 100644 --- a/drivers/firmware/arm_scmi/common.h +++ b/drivers/firmware/arm_scmi/common.h @@ -156,6 +156,30 @@ void scmi_setup_protocol_implemented(const struct scmi_handle *handle, int scmi_base_protocol_init(struct scmi_handle *h); +int __init scmi_bus_init(void); +void __exit scmi_bus_exit(void); + +#define DECLARE_SCMI_REGISTER_UNREGISTER(func) \ + int __init scmi_##func##_register(void); \ + void __exit scmi_##func##_unregister(void) +DECLARE_SCMI_REGISTER_UNREGISTER(clock); +DECLARE_SCMI_REGISTER_UNREGISTER(perf); +DECLARE_SCMI_REGISTER_UNREGISTER(power); +DECLARE_SCMI_REGISTER_UNREGISTER(reset); +DECLARE_SCMI_REGISTER_UNREGISTER(sensors); +DECLARE_SCMI_REGISTER_UNREGISTER(system); + +#define DEFINE_SCMI_PROTOCOL_REGISTER_UNREGISTER(id, name) \ +int __init scmi_##name##_register(void) \ +{ \ + return scmi_protocol_register((id), &scmi_##name##_protocol_init); \ +} \ +\ +void __exit scmi_##name##_unregister(void) \ +{ \ + scmi_protocol_unregister((id)); \ +} + /* SCMI Transport */ /** * struct scmi_chan_info - Structure representing a SCMI channel information @@ -210,7 +234,7 @@ struct scmi_transport_ops { * @max_msg_size: Maximum size of data per message that can be handled. */ struct scmi_desc { - struct scmi_transport_ops *ops; + const struct scmi_transport_ops *ops; int max_rx_timeout_ms; int max_msg; int max_msg_size; diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c index 03ec74242c14..c5dea87edf8f 100644 --- a/drivers/firmware/arm_scmi/driver.c +++ b/drivers/firmware/arm_scmi/driver.c @@ -730,6 +730,7 @@ struct scmi_prot_devnames { static struct scmi_prot_devnames devnames[] = { { SCMI_PROTOCOL_POWER, { "genpd" },}, + { SCMI_PROTOCOL_SYSTEM, { "syspower" },}, { SCMI_PROTOCOL_PERF, { "cpufreq" },}, { SCMI_PROTOCOL_CLOCK, { "clocks" },}, { SCMI_PROTOCOL_SENSOR, { "hwmon" },}, @@ -928,7 +929,35 @@ static struct platform_driver scmi_driver = { .remove = scmi_remove, }; -module_platform_driver(scmi_driver); +static int __init scmi_driver_init(void) +{ + scmi_bus_init(); + + scmi_clock_register(); + scmi_perf_register(); + scmi_power_register(); + scmi_reset_register(); + scmi_sensors_register(); + scmi_system_register(); + + return platform_driver_register(&scmi_driver); +} +subsys_initcall(scmi_driver_init); + +static void __exit scmi_driver_exit(void) +{ + scmi_bus_exit(); + + scmi_clock_unregister(); + scmi_perf_unregister(); + scmi_power_unregister(); + scmi_reset_unregister(); + scmi_sensors_unregister(); + scmi_system_unregister(); + + platform_driver_unregister(&scmi_driver); +} +module_exit(scmi_driver_exit); MODULE_ALIAS("platform: arm-scmi"); MODULE_AUTHOR("Sudeep Holla <sudeep.holla@arm.com>"); diff --git a/drivers/firmware/arm_scmi/mailbox.c b/drivers/firmware/arm_scmi/mailbox.c index 6998dc86b5ce..4626404be541 100644 --- a/drivers/firmware/arm_scmi/mailbox.c +++ b/drivers/firmware/arm_scmi/mailbox.c @@ -110,7 +110,7 @@ static int mailbox_chan_free(int id, void *p, void *data) struct scmi_chan_info *cinfo = p; struct scmi_mailbox *smbox = cinfo->transport_info; - if (!IS_ERR(smbox->chan)) { + if (smbox && !IS_ERR(smbox->chan)) { mbox_free_channel(smbox->chan); cinfo->transport_info = NULL; smbox->chan = NULL; @@ -181,7 +181,7 @@ mailbox_poll_done(struct scmi_chan_info *cinfo, struct scmi_xfer *xfer) return shmem_poll_done(smbox->shmem, xfer); } -static struct scmi_transport_ops scmi_mailbox_ops = { +static const struct scmi_transport_ops scmi_mailbox_ops = { .chan_available = mailbox_chan_available, .chan_setup = mailbox_chan_setup, .chan_free = mailbox_chan_free, diff --git a/drivers/firmware/arm_scmi/notify.c b/drivers/firmware/arm_scmi/notify.c index 4731daaacd19..2754f9d01636 100644 --- a/drivers/firmware/arm_scmi/notify.c +++ b/drivers/firmware/arm_scmi/notify.c @@ -1421,7 +1421,7 @@ static void scmi_protocols_late_init(struct work_struct *work) * notify_ops are attached to the handle so that can be accessed * directly from an scmi_driver to register its own notifiers. */ -static struct scmi_notify_ops notify_ops = { +static const struct scmi_notify_ops notify_ops = { .register_event_notifier = scmi_register_notifier, .unregister_event_notifier = scmi_unregister_notifier, }; diff --git a/drivers/firmware/arm_scmi/perf.c b/drivers/firmware/arm_scmi/perf.c index 3e1e87012c95..ed475b40bd08 100644 --- a/drivers/firmware/arm_scmi/perf.c +++ b/drivers/firmware/arm_scmi/perf.c @@ -748,7 +748,7 @@ static bool scmi_fast_switch_possible(const struct scmi_handle *handle, return dom->fc_info && dom->fc_info->level_set_addr; } -static struct scmi_perf_ops perf_ops = { +static const struct scmi_perf_ops perf_ops = { .limits_set = scmi_perf_limits_set, .limits_get = scmi_perf_limits_get, .level_set = scmi_perf_level_set, @@ -890,9 +890,4 @@ static int scmi_perf_protocol_init(struct scmi_handle *handle) return 0; } -static int __init scmi_perf_init(void) -{ - return scmi_protocol_register(SCMI_PROTOCOL_PERF, - &scmi_perf_protocol_init); -} -subsys_initcall(scmi_perf_init); +DEFINE_SCMI_PROTOCOL_REGISTER_UNREGISTER(SCMI_PROTOCOL_PERF, perf) diff --git a/drivers/firmware/arm_scmi/power.c b/drivers/firmware/arm_scmi/power.c index 46f213644c49..1f37258e9bee 100644 --- a/drivers/firmware/arm_scmi/power.c +++ b/drivers/firmware/arm_scmi/power.c @@ -184,7 +184,7 @@ static char *scmi_power_name_get(const struct scmi_handle *handle, u32 domain) return dom->name; } -static struct scmi_power_ops power_ops = { +static const struct scmi_power_ops power_ops = { .num_domains_get = scmi_power_num_domains_get, .name_get = scmi_power_name_get, .state_set = scmi_power_state_set, @@ -301,9 +301,4 @@ static int scmi_power_protocol_init(struct scmi_handle *handle) return 0; } -static int __init scmi_power_init(void) -{ - return scmi_protocol_register(SCMI_PROTOCOL_POWER, - &scmi_power_protocol_init); -} -subsys_initcall(scmi_power_init); +DEFINE_SCMI_PROTOCOL_REGISTER_UNREGISTER(SCMI_PROTOCOL_POWER, power) diff --git a/drivers/firmware/arm_scmi/reset.c b/drivers/firmware/arm_scmi/reset.c index 3691bafca057..f063cfe17e02 100644 --- a/drivers/firmware/arm_scmi/reset.c +++ b/drivers/firmware/arm_scmi/reset.c @@ -194,7 +194,7 @@ scmi_reset_domain_deassert(const struct scmi_handle *handle, u32 domain) return scmi_domain_reset(handle, domain, 0, ARCH_COLD_RESET); } -static struct scmi_reset_ops reset_ops = { +static const struct scmi_reset_ops reset_ops = { .num_domains_get = scmi_reset_num_domains_get, .name_get = scmi_reset_name_get, .latency_get = scmi_reset_latency_get, @@ -313,9 +313,4 @@ static int scmi_reset_protocol_init(struct scmi_handle *handle) return 0; } -static int __init scmi_reset_init(void) -{ - return scmi_protocol_register(SCMI_PROTOCOL_RESET, - &scmi_reset_protocol_init); -} -subsys_initcall(scmi_reset_init); +DEFINE_SCMI_PROTOCOL_REGISTER_UNREGISTER(SCMI_PROTOCOL_RESET, reset) diff --git a/drivers/firmware/arm_scmi/sensors.c b/drivers/firmware/arm_scmi/sensors.c index 1af0ad362e82..9703cf6356a0 100644 --- a/drivers/firmware/arm_scmi/sensors.c +++ b/drivers/firmware/arm_scmi/sensors.c @@ -275,7 +275,7 @@ static int scmi_sensor_count_get(const struct scmi_handle *handle) return si->num_sensors; } -static struct scmi_sensor_ops sensor_ops = { +static const struct scmi_sensor_ops sensor_ops = { .count_get = scmi_sensor_count_get, .info_get = scmi_sensor_info_get, .trip_point_config = scmi_sensor_trip_point_config, @@ -365,9 +365,4 @@ static int scmi_sensors_protocol_init(struct scmi_handle *handle) return 0; } -static int __init scmi_sensors_init(void) -{ - return scmi_protocol_register(SCMI_PROTOCOL_SENSOR, - &scmi_sensors_protocol_init); -} -subsys_initcall(scmi_sensors_init); +DEFINE_SCMI_PROTOCOL_REGISTER_UNREGISTER(SCMI_PROTOCOL_SENSOR, sensors) diff --git a/drivers/firmware/arm_scmi/smc.c b/drivers/firmware/arm_scmi/smc.c index a1537d123e38..1a03c3ec0230 100644 --- a/drivers/firmware/arm_scmi/smc.c +++ b/drivers/firmware/arm_scmi/smc.c @@ -137,7 +137,7 @@ smc_poll_done(struct scmi_chan_info *cinfo, struct scmi_xfer *xfer) return shmem_poll_done(scmi_info->shmem, xfer); } -static struct scmi_transport_ops scmi_smc_ops = { +static const struct scmi_transport_ops scmi_smc_ops = { .chan_available = smc_chan_available, .chan_setup = smc_chan_setup, .chan_free = smc_chan_free, diff --git a/drivers/firmware/arm_scmi/system.c b/drivers/firmware/arm_scmi/system.c new file mode 100644 index 000000000000..283e12d5f24b --- /dev/null +++ b/drivers/firmware/arm_scmi/system.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * System Control and Management Interface (SCMI) System Power Protocol + * + * Copyright (C) 2020 ARM Ltd. + */ + +#define pr_fmt(fmt) "SCMI Notifications SYSTEM - " fmt + +#include <linux/scmi_protocol.h> + +#include "common.h" +#include "notify.h" + +#define SCMI_SYSTEM_NUM_SOURCES 1 + +enum scmi_system_protocol_cmd { + SYSTEM_POWER_STATE_NOTIFY = 0x5, +}; + +struct scmi_system_power_state_notify { + __le32 notify_enable; +}; + +struct scmi_system_power_state_notifier_payld { + __le32 agent_id; + __le32 flags; + __le32 system_state; +}; + +struct scmi_system_info { + u32 version; +}; + +static int scmi_system_request_notify(const struct scmi_handle *handle, + bool enable) +{ + int ret; + struct scmi_xfer *t; + struct scmi_system_power_state_notify *notify; + + ret = scmi_xfer_get_init(handle, SYSTEM_POWER_STATE_NOTIFY, + SCMI_PROTOCOL_SYSTEM, sizeof(*notify), 0, &t); + if (ret) + return ret; + + notify = t->tx.buf; + notify->notify_enable = enable ? cpu_to_le32(BIT(0)) : 0; + + ret = scmi_do_xfer(handle, t); + + scmi_xfer_put(handle, t); + return ret; +} + +static int scmi_system_set_notify_enabled(const struct scmi_handle *handle, + u8 evt_id, u32 src_id, bool enable) +{ + int ret; + + ret = scmi_system_request_notify(handle, enable); + if (ret) + pr_debug("FAIL_ENABLE - evt[%X] - ret:%d\n", evt_id, ret); + + return ret; +} + +static void *scmi_system_fill_custom_report(const struct scmi_handle *handle, + u8 evt_id, ktime_t timestamp, + const void *payld, size_t payld_sz, + void *report, u32 *src_id) +{ + const struct scmi_system_power_state_notifier_payld *p = payld; + struct scmi_system_power_state_notifier_report *r = report; + + if (evt_id != SCMI_EVENT_SYSTEM_POWER_STATE_NOTIFIER || + sizeof(*p) != payld_sz) + return NULL; + + r->timestamp = timestamp; + r->agent_id = le32_to_cpu(p->agent_id); + r->flags = le32_to_cpu(p->flags); + r->system_state = le32_to_cpu(p->system_state); + *src_id = 0; + + return r; +} + +static const struct scmi_event system_events[] = { + { + .id = SCMI_EVENT_SYSTEM_POWER_STATE_NOTIFIER, + .max_payld_sz = + sizeof(struct scmi_system_power_state_notifier_payld), + .max_report_sz = + sizeof(struct scmi_system_power_state_notifier_report), + }, +}; + +static const struct scmi_event_ops system_event_ops = { + .set_notify_enabled = scmi_system_set_notify_enabled, + .fill_custom_report = scmi_system_fill_custom_report, +}; + +static int scmi_system_protocol_init(struct scmi_handle *handle) +{ + u32 version; + struct scmi_system_info *pinfo; + + scmi_version_get(handle, SCMI_PROTOCOL_SYSTEM, &version); + + dev_dbg(handle->dev, "System Power Version %d.%d\n", + PROTOCOL_REV_MAJOR(version), PROTOCOL_REV_MINOR(version)); + + pinfo = devm_kzalloc(handle->dev, sizeof(*pinfo), GFP_KERNEL); + if (!pinfo) + return -ENOMEM; + + scmi_register_protocol_events(handle, + SCMI_PROTOCOL_SYSTEM, SCMI_PROTO_QUEUE_SZ, + &system_event_ops, + system_events, + ARRAY_SIZE(system_events), + SCMI_SYSTEM_NUM_SOURCES); + + pinfo->version = version; + handle->system_priv = pinfo; + + return 0; +} + +DEFINE_SCMI_PROTOCOL_REGISTER_UNREGISTER(SCMI_PROTOCOL_SYSTEM, system) diff --git a/drivers/firmware/broadcom/bcm47xx_sprom.c b/drivers/firmware/broadcom/bcm47xx_sprom.c index 4787f86c8ac1..14fbcd11657c 100644 --- a/drivers/firmware/broadcom/bcm47xx_sprom.c +++ b/drivers/firmware/broadcom/bcm47xx_sprom.c @@ -27,6 +27,7 @@ */ #include <linux/bcm47xx_nvram.h> +#include <linux/bcm47xx_sprom.h> #include <linux/bcma/bcma.h> #include <linux/etherdevice.h> #include <linux/if_ether.h> diff --git a/drivers/firmware/dmi_scan.c b/drivers/firmware/dmi_scan.c index 5066d1f1d687..d51ca0428bb8 100644 --- a/drivers/firmware/dmi_scan.c +++ b/drivers/firmware/dmi_scan.c @@ -21,7 +21,7 @@ EXPORT_SYMBOL_GPL(dmi_kobj); /* * DMI stands for "Desktop Management Interface". It is part * of and an antecedent to, SMBIOS, which stands for System - * Management BIOS. See further: http://www.dmtf.org/standards + * Management BIOS. See further: https://www.dmtf.org/standards */ static const char dmi_empty_string[] = ""; diff --git a/drivers/firmware/efi/Kconfig b/drivers/firmware/efi/Kconfig index da1887f72a51..36ec1f718893 100644 --- a/drivers/firmware/efi/Kconfig +++ b/drivers/firmware/efi/Kconfig @@ -106,7 +106,7 @@ config EFI_GENERIC_STUB config EFI_ARMSTUB_DTB_LOADER bool "Enable the DTB loader" - depends on EFI_GENERIC_STUB + depends on EFI_GENERIC_STUB && !RISCV default y help Select this config option to add support for the dtb= command @@ -123,6 +123,7 @@ config EFI_GENERIC_STUB_INITRD_CMDLINE_LOADER bool "Enable the command line initrd loader" if !X86 depends on EFI_STUB && (EFI_GENERIC_STUB || X86) default y + depends on !RISCV help Select this config option to add support for the initrd= command line parameter, allowing an initrd that resides on the same volume diff --git a/drivers/firmware/efi/Makefile b/drivers/firmware/efi/Makefile index e8da782280b6..d6ca2da19339 100644 --- a/drivers/firmware/efi/Makefile +++ b/drivers/firmware/efi/Makefile @@ -36,6 +36,8 @@ fake_map-$(CONFIG_X86) += x86_fake_mem.o arm-obj-$(CONFIG_EFI) := efi-init.o arm-runtime.o obj-$(CONFIG_ARM) += $(arm-obj-y) obj-$(CONFIG_ARM64) += $(arm-obj-y) +riscv-obj-$(CONFIG_EFI) := efi-init.o riscv-runtime.o +obj-$(CONFIG_RISCV) += $(riscv-obj-y) obj-$(CONFIG_EFI_CAPSULE_LOADER) += capsule-loader.o obj-$(CONFIG_EFI_EARLYCON) += earlycon.o obj-$(CONFIG_UEFI_CPER_ARM) += cper-arm.o diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index 039a9acab817..8a94388e38b3 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -23,6 +23,8 @@ cflags-$(CONFIG_ARM64) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \ cflags-$(CONFIG_ARM) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \ -fno-builtin -fpic \ $(call cc-option,-mno-single-pic-base) +cflags-$(CONFIG_RISCV) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \ + -fpic cflags-$(CONFIG_EFI_GENERIC_STUB) += -I$(srctree)/scripts/dtc/libfdt @@ -64,6 +66,7 @@ lib-$(CONFIG_EFI_GENERIC_STUB) += efi-stub.o fdt.o string.o \ lib-$(CONFIG_ARM) += arm32-stub.o lib-$(CONFIG_ARM64) += arm64-stub.o lib-$(CONFIG_X86) += x86-stub.o +lib-$(CONFIG_RISCV) += riscv-stub.o CFLAGS_arm32-stub.o := -DTEXT_OFFSET=$(TEXT_OFFSET) # Even when -mbranch-protection=none is set, Clang will generate a @@ -112,6 +115,13 @@ STUBCOPY_FLAGS-$(CONFIG_ARM64) += --prefix-alloc-sections=.init \ --prefix-symbols=__efistub_ STUBCOPY_RELOC-$(CONFIG_ARM64) := R_AARCH64_ABS +# For RISC-V, we don't need anything special other than arm64. Keep all the +# symbols in .init section and make sure that no absolute symbols references +# doesn't exist. +STUBCOPY_FLAGS-$(CONFIG_RISCV) += --prefix-alloc-sections=.init \ + --prefix-symbols=__efistub_ +STUBCOPY_RELOC-$(CONFIG_RISCV) := R_RISCV_HI20 + $(obj)/%.stub.o: $(obj)/%.o FORCE $(call if_changed,stubcopy) diff --git a/drivers/firmware/efi/libstub/efi-stub.c b/drivers/firmware/efi/libstub/efi-stub.c index 311a16802dd6..914a343c7785 100644 --- a/drivers/firmware/efi/libstub/efi-stub.c +++ b/drivers/firmware/efi/libstub/efi-stub.c @@ -17,7 +17,10 @@ /* * This is the base address at which to start allocating virtual memory ranges - * for UEFI Runtime Services. This is in the low TTBR0 range so that we can use + * for UEFI Runtime Services. + * + * For ARM/ARM64: + * This is in the low TTBR0 range so that we can use * any allocation we choose, and eliminate the risk of a conflict after kexec. * The value chosen is the largest non-zero power of 2 suitable for this purpose * both on 32-bit and 64-bit ARM CPUs, to maximize the likelihood that it can @@ -25,6 +28,12 @@ * Since 32-bit ARM could potentially execute with a 1G/3G user/kernel split, * map everything below 1 GB. (512 MB is a reasonable upper bound for the * entire footprint of the UEFI runtime services memory regions) + * + * For RISC-V: + * There is no specific reason for which, this address (512MB) can't be used + * EFI runtime virtual address for RISC-V. It also helps to use EFI runtime + * services on both RV32/RV64. Keep the same runtime virtual address for RISC-V + * as well to minimize the code churn. */ #define EFI_RT_VIRTUAL_BASE SZ_512M #define EFI_RT_VIRTUAL_SIZE SZ_512M diff --git a/drivers/firmware/efi/libstub/riscv-stub.c b/drivers/firmware/efi/libstub/riscv-stub.c new file mode 100644 index 000000000000..380e4e251399 --- /dev/null +++ b/drivers/firmware/efi/libstub/riscv-stub.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +#include <linux/efi.h> +#include <linux/libfdt.h> + +#include <asm/efi.h> +#include <asm/sections.h> + +#include "efistub.h" + +/* + * RISC-V requires the kernel image to placed 2 MB aligned base for 64 bit and + * 4MB for 32 bit. + */ +#ifdef CONFIG_64BIT +#define MIN_KIMG_ALIGN SZ_2M +#else +#define MIN_KIMG_ALIGN SZ_4M +#endif + +typedef void __noreturn (*jump_kernel_func)(unsigned int, unsigned long); + +static u32 hartid; + +static u32 get_boot_hartid_from_fdt(void) +{ + const void *fdt; + int chosen_node, len; + const fdt32_t *prop; + + fdt = get_efi_config_table(DEVICE_TREE_GUID); + if (!fdt) + return U32_MAX; + + chosen_node = fdt_path_offset(fdt, "/chosen"); + if (chosen_node < 0) + return U32_MAX; + + prop = fdt_getprop((void *)fdt, chosen_node, "boot-hartid", &len); + if (!prop || len != sizeof(u32)) + return U32_MAX; + + return fdt32_to_cpu(*prop); +} + +efi_status_t check_platform_features(void) +{ + hartid = get_boot_hartid_from_fdt(); + if (hartid == U32_MAX) { + efi_err("/chosen/boot-hartid missing or invalid!\n"); + return EFI_UNSUPPORTED; + } + return EFI_SUCCESS; +} + +void __noreturn efi_enter_kernel(unsigned long entrypoint, unsigned long fdt, + unsigned long fdt_size) +{ + unsigned long stext_offset = _start_kernel - _start; + unsigned long kernel_entry = entrypoint + stext_offset; + jump_kernel_func jump_kernel = (jump_kernel_func)kernel_entry; + + /* + * Jump to real kernel here with following constraints. + * 1. MMU should be disabled. + * 2. a0 should contain hartid + * 3. a1 should DT address + */ + csr_write(CSR_SATP, 0); + jump_kernel(hartid, fdt); +} + +efi_status_t handle_kernel_image(unsigned long *image_addr, + unsigned long *image_size, + unsigned long *reserve_addr, + unsigned long *reserve_size, + efi_loaded_image_t *image) +{ + unsigned long kernel_size = 0; + unsigned long preferred_addr; + efi_status_t status; + + kernel_size = _edata - _start; + *image_addr = (unsigned long)_start; + *image_size = kernel_size + (_end - _edata); + + /* + * RISC-V kernel maps PAGE_OFFSET virtual address to the same physical + * address where kernel is booted. That's why kernel should boot from + * as low as possible to avoid wastage of memory. Currently, dram_base + * is occupied by the firmware. So the preferred address for kernel to + * boot is next aligned address. If preferred address is not available, + * relocate_kernel will fall back to efi_low_alloc_above to allocate + * lowest possible memory region as long as the address and size meets + * the alignment constraints. + */ + preferred_addr = MIN_KIMG_ALIGN; + status = efi_relocate_kernel(image_addr, kernel_size, *image_size, + preferred_addr, MIN_KIMG_ALIGN, 0x0); + + if (status != EFI_SUCCESS) { + efi_err("Failed to relocate kernel\n"); + *image_size = 0; + } + return status; +} diff --git a/drivers/firmware/efi/riscv-runtime.c b/drivers/firmware/efi/riscv-runtime.c new file mode 100644 index 000000000000..d28e715d2bcc --- /dev/null +++ b/drivers/firmware/efi/riscv-runtime.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Extensible Firmware Interface + * + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + * + * Based on Extensible Firmware Interface Specification version 2.4 + * Adapted from drivers/firmware/efi/arm-runtime.c + * + */ + +#include <linux/dmi.h> +#include <linux/efi.h> +#include <linux/io.h> +#include <linux/memblock.h> +#include <linux/mm_types.h> +#include <linux/preempt.h> +#include <linux/rbtree.h> +#include <linux/rwsem.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/pgtable.h> + +#include <asm/cacheflush.h> +#include <asm/efi.h> +#include <asm/mmu.h> +#include <asm/pgalloc.h> + +static bool __init efi_virtmap_init(void) +{ + efi_memory_desc_t *md; + + efi_mm.pgd = pgd_alloc(&efi_mm); + mm_init_cpumask(&efi_mm); + init_new_context(NULL, &efi_mm); + + for_each_efi_memory_desc(md) { + phys_addr_t phys = md->phys_addr; + int ret; + + if (!(md->attribute & EFI_MEMORY_RUNTIME)) + continue; + if (md->virt_addr == 0) + return false; + + ret = efi_create_mapping(&efi_mm, md); + if (ret) { + pr_warn(" EFI remap %pa: failed to create mapping (%d)\n", + &phys, ret); + return false; + } + } + + if (efi_memattr_apply_permissions(&efi_mm, efi_set_mapping_permissions)) + return false; + + return true; +} + +/* + * Enable the UEFI Runtime Services if all prerequisites are in place, i.e., + * non-early mapping of the UEFI system table and virtual mappings for all + * EFI_MEMORY_RUNTIME regions. + */ +static int __init riscv_enable_runtime_services(void) +{ + u64 mapsize; + + if (!efi_enabled(EFI_BOOT)) { + pr_info("EFI services will not be available.\n"); + return 0; + } + + efi_memmap_unmap(); + + mapsize = efi.memmap.desc_size * efi.memmap.nr_map; + + if (efi_memmap_init_late(efi.memmap.phys_map, mapsize)) { + pr_err("Failed to remap EFI memory map\n"); + return 0; + } + + if (efi_soft_reserve_enabled()) { + efi_memory_desc_t *md; + + for_each_efi_memory_desc(md) { + int md_size = md->num_pages << EFI_PAGE_SHIFT; + struct resource *res; + + if (!(md->attribute & EFI_MEMORY_SP)) + continue; + + res = kzalloc(sizeof(*res), GFP_KERNEL); + if (WARN_ON(!res)) + break; + + res->start = md->phys_addr; + res->end = md->phys_addr + md_size - 1; + res->name = "Soft Reserved"; + res->flags = IORESOURCE_MEM; + res->desc = IORES_DESC_SOFT_RESERVED; + + insert_resource(&iomem_resource, res); + } + } + + if (efi_runtime_disabled()) { + pr_info("EFI runtime services will be disabled.\n"); + return 0; + } + + if (efi_enabled(EFI_RUNTIME_SERVICES)) { + pr_info("EFI runtime services access via paravirt.\n"); + return 0; + } + + pr_info("Remapping and enabling EFI services.\n"); + + if (!efi_virtmap_init()) { + pr_err("UEFI virtual mapping missing or invalid -- runtime services will not be available\n"); + return -ENOMEM; + } + + /* Set up runtime services function pointers */ + efi_native_runtime_setup(); + set_bit(EFI_RUNTIME_SERVICES, &efi.flags); + + return 0; +} +early_initcall(riscv_enable_runtime_services); + +void efi_virtmap_load(void) +{ + preempt_disable(); + switch_mm(current->active_mm, &efi_mm, NULL); +} + +void efi_virtmap_unload(void) +{ + switch_mm(&efi_mm, current->active_mm, NULL); + preempt_enable(); +} diff --git a/drivers/firmware/imx/scu-pd.c b/drivers/firmware/imx/scu-pd.c index af3d6d9ead28..946eea292b52 100644 --- a/drivers/firmware/imx/scu-pd.c +++ b/drivers/firmware/imx/scu-pd.c @@ -46,6 +46,7 @@ #include <dt-bindings/firmware/imx/rsrc.h> #include <linux/firmware/imx/sci.h> +#include <linux/firmware/imx/svc/rm.h> #include <linux/io.h> #include <linux/module.h> #include <linux/of.h> @@ -256,6 +257,9 @@ imx_scu_add_pm_domain(struct device *dev, int idx, struct imx_sc_pm_domain *sc_pd; int ret; + if (!imx_sc_rm_is_resource_owned(pm_ipc_handle, pd_ranges->rsrc + idx)) + return NULL; + sc_pd = devm_kzalloc(dev, sizeof(*sc_pd), GFP_KERNEL); if (!sc_pd) return ERR_PTR(-ENOMEM); diff --git a/drivers/firmware/smccc/smccc.c b/drivers/firmware/smccc/smccc.c index 4e80921ee212..00c88b809c0c 100644 --- a/drivers/firmware/smccc/smccc.c +++ b/drivers/firmware/smccc/smccc.c @@ -24,8 +24,10 @@ enum arm_smccc_conduit arm_smccc_1_1_get_conduit(void) return smccc_conduit; } +EXPORT_SYMBOL_GPL(arm_smccc_1_1_get_conduit); u32 arm_smccc_get_version(void) { return smccc_version; } +EXPORT_SYMBOL_GPL(arm_smccc_get_version); diff --git a/drivers/firmware/tegra/bpmp.c b/drivers/firmware/tegra/bpmp.c index 4d93d8925e14..0742a90cb844 100644 --- a/drivers/firmware/tegra/bpmp.c +++ b/drivers/firmware/tegra/bpmp.c @@ -856,7 +856,8 @@ static const struct tegra_bpmp_soc tegra210_soc = { static const struct of_device_id tegra_bpmp_match[] = { #if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \ - IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) + IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \ + IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC) { .compatible = "nvidia,tegra186-bpmp", .data = &tegra186_soc }, #endif #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC) diff --git a/drivers/firmware/ti_sci.c b/drivers/firmware/ti_sci.c index 722af9ee53d6..896f53ec7857 100644 --- a/drivers/firmware/ti_sci.c +++ b/drivers/firmware/ti_sci.c @@ -1106,7 +1106,8 @@ static int ti_sci_cmd_get_clock(const struct ti_sci_handle *handle, u32 dev_id, static int ti_sci_cmd_idle_clock(const struct ti_sci_handle *handle, u32 dev_id, u32 clk_id) { - return ti_sci_set_clock_state(handle, dev_id, clk_id, 0, + return ti_sci_set_clock_state(handle, dev_id, clk_id, + MSG_FLAG_CLOCK_ALLOW_FREQ_CHANGE, MSG_CLOCK_SW_STATE_UNREQ); } @@ -1125,7 +1126,8 @@ static int ti_sci_cmd_idle_clock(const struct ti_sci_handle *handle, static int ti_sci_cmd_put_clock(const struct ti_sci_handle *handle, u32 dev_id, u32 clk_id) { - return ti_sci_set_clock_state(handle, dev_id, clk_id, 0, + return ti_sci_set_clock_state(handle, dev_id, clk_id, + MSG_FLAG_CLOCK_ALLOW_FREQ_CHANGE, MSG_CLOCK_SW_STATE_AUTO); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index e8b41756c9f9..37da3537ba2e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -4625,7 +4625,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, retry: /* Rest of adevs pre asic reset from XGMI hive. */ list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { r = amdgpu_device_pre_asic_reset(tmp_adev, - NULL, + (tmp_adev == adev) ? job : NULL, &need_full_reset); /*TODO Should we stop ?*/ if (r) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 18be544d8c1e..96a9699f87ba 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -208,7 +208,8 @@ static int psp_sw_fini(void *handle) adev->psp.ta_fw = NULL; } - if (adev->asic_type == CHIP_NAVI10) + if (adev->asic_type == CHIP_NAVI10 || + adev->asic_type == CHIP_SIENNA_CICHLID) psp_sysfs_fini(adev); return 0; @@ -1750,6 +1751,12 @@ static int psp_get_fw_type(struct amdgpu_firmware_info *ucode, case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM: *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM; break; + case AMDGPU_UCODE_ID_RLC_IRAM: + *type = GFX_FW_TYPE_RLC_IRAM; + break; + case AMDGPU_UCODE_ID_RLC_DRAM: + *type = GFX_FW_TYPE_RLC_DRAM_BOOT; + break; case AMDGPU_UCODE_ID_SMC: *type = GFX_FW_TYPE_SMU; break; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index 8bf6a7c056bc..4e36551ab50b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -1986,7 +1986,8 @@ static int amdgpu_ras_check_asic_type(struct amdgpu_device *adev) { if (adev->asic_type != CHIP_VEGA10 && adev->asic_type != CHIP_VEGA20 && - adev->asic_type != CHIP_ARCTURUS) + adev->asic_type != CHIP_ARCTURUS && + adev->asic_type != CHIP_SIENNA_CICHLID) return 1; else return 0; @@ -2030,7 +2031,6 @@ static void amdgpu_ras_check_supported(struct amdgpu_device *adev, *supported = amdgpu_ras_enable == 0 ? 0 : *hw_supported & amdgpu_ras_mask; - adev->ras_features = *supported; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h index 60bb3e8b3118..aeaaae713c59 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h @@ -168,12 +168,16 @@ struct amdgpu_rlc { u32 save_restore_list_cntl_size_bytes; u32 save_restore_list_gpm_size_bytes; u32 save_restore_list_srm_size_bytes; + u32 rlc_iram_ucode_size_bytes; + u32 rlc_dram_ucode_size_bytes; u32 *register_list_format; u32 *register_restore; u8 *save_restore_list_cntl; u8 *save_restore_list_gpm; u8 *save_restore_list_srm; + u8 *rlc_iram_ucode; + u8 *rlc_dram_ucode; bool is_rlc_v2_1; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c index 55fe19a2f332..b313ce4c3e97 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c @@ -500,6 +500,8 @@ static int amdgpu_ucode_init_single_fw(struct amdgpu_device *adev, ucode->ucode_id != AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL && ucode->ucode_id != AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM && ucode->ucode_id != AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM && + ucode->ucode_id != AMDGPU_UCODE_ID_RLC_IRAM && + ucode->ucode_id != AMDGPU_UCODE_ID_RLC_DRAM && ucode->ucode_id != AMDGPU_UCODE_ID_DMCU_ERAM && ucode->ucode_id != AMDGPU_UCODE_ID_DMCU_INTV && ucode->ucode_id != AMDGPU_UCODE_ID_DMCUB)) { @@ -556,6 +558,14 @@ static int amdgpu_ucode_init_single_fw(struct amdgpu_device *adev, ucode->ucode_size = adev->gfx.rlc.save_restore_list_srm_size_bytes; memcpy(ucode->kaddr, adev->gfx.rlc.save_restore_list_srm, ucode->ucode_size); + } else if (ucode->ucode_id == AMDGPU_UCODE_ID_RLC_IRAM) { + ucode->ucode_size = adev->gfx.rlc.rlc_iram_ucode_size_bytes; + memcpy(ucode->kaddr, adev->gfx.rlc.rlc_iram_ucode, + ucode->ucode_size); + } else if (ucode->ucode_id == AMDGPU_UCODE_ID_RLC_DRAM) { + ucode->ucode_size = adev->gfx.rlc.rlc_dram_ucode_size_bytes; + memcpy(ucode->kaddr, adev->gfx.rlc.rlc_dram_ucode, + ucode->ucode_size); } else if (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MES) { ucode->ucode_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes); memcpy(ucode->kaddr, (void *)((uint8_t *)adev->mes.fw->data + diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h index 3c23c6293ff9..0e43b46d3ab5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h @@ -222,6 +222,15 @@ struct rlc_firmware_header_v2_1 { uint32_t save_restore_list_srm_offset_bytes; }; +/* version_major=2, version_minor=1 */ +struct rlc_firmware_header_v2_2 { + struct rlc_firmware_header_v2_1 v2_1; + uint32_t rlc_iram_ucode_size_bytes; + uint32_t rlc_iram_ucode_offset_bytes; + uint32_t rlc_dram_ucode_size_bytes; + uint32_t rlc_dram_ucode_offset_bytes; +}; + /* version_major=1, version_minor=0 */ struct sdma_firmware_header_v1_0 { struct common_firmware_header header; @@ -339,6 +348,8 @@ enum AMDGPU_UCODE_ID { AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL, AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM, AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM, + AMDGPU_UCODE_ID_RLC_IRAM, + AMDGPU_UCODE_ID_RLC_DRAM, AMDGPU_UCODE_ID_RLC_G, AMDGPU_UCODE_ID_STORAGE, AMDGPU_UCODE_ID_SMC, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c index 495c3d7bb2b2..f3b7287e84c4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c @@ -68,6 +68,7 @@ int amdgpu_vcn_sw_init(struct amdgpu_device *adev) INIT_DELAYED_WORK(&adev->vcn.idle_work, amdgpu_vcn_idle_work_handler); mutex_init(&adev->vcn.vcn_pg_lock); + mutex_init(&adev->vcn.vcn1_jpeg1_workaround); atomic_set(&adev->vcn.total_submission_cnt, 0); for (i = 0; i < adev->vcn.num_vcn_inst; i++) atomic_set(&adev->vcn.inst[i].dpg_enc_submission_cnt, 0); @@ -237,6 +238,7 @@ int amdgpu_vcn_sw_fini(struct amdgpu_device *adev) } release_firmware(adev->vcn.fw); + mutex_destroy(&adev->vcn.vcn1_jpeg1_workaround); mutex_destroy(&adev->vcn.vcn_pg_lock); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h index 7a9b804bc988..17691158f783 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h @@ -220,6 +220,7 @@ struct amdgpu_vcn { struct amdgpu_vcn_inst inst[AMDGPU_MAX_VCN_INSTANCES]; struct amdgpu_vcn_reg internal; struct mutex vcn_pg_lock; + struct mutex vcn1_jpeg1_workaround; atomic_t total_submission_cnt; unsigned harvest_config; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 9792ec737029..56fdbe626d30 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -112,6 +112,22 @@ #define mmCP_HYP_ME_UCODE_DATA 0x5817 #define mmCP_HYP_ME_UCODE_DATA_BASE_IDX 1 +//CC_GC_SA_UNIT_DISABLE +#define mmCC_GC_SA_UNIT_DISABLE 0x0fe9 +#define mmCC_GC_SA_UNIT_DISABLE_BASE_IDX 0 +#define CC_GC_SA_UNIT_DISABLE__SA_DISABLE__SHIFT 0x8 +#define CC_GC_SA_UNIT_DISABLE__SA_DISABLE_MASK 0x0000FF00L +//GC_USER_SA_UNIT_DISABLE +#define mmGC_USER_SA_UNIT_DISABLE 0x0fea +#define mmGC_USER_SA_UNIT_DISABLE_BASE_IDX 0 +#define GC_USER_SA_UNIT_DISABLE__SA_DISABLE__SHIFT 0x8 +#define GC_USER_SA_UNIT_DISABLE__SA_DISABLE_MASK 0x0000FF00L +//PA_SC_ENHANCE_3 +#define mmPA_SC_ENHANCE_3 0x1085 +#define mmPA_SC_ENHANCE_3_BASE_IDX 0 +#define PA_SC_ENHANCE_3__FORCE_PBB_WORKLOAD_MODE_TO_ZERO__SHIFT 0x3 +#define PA_SC_ENHANCE_3__FORCE_PBB_WORKLOAD_MODE_TO_ZERO_MASK 0x00000008L + MODULE_FIRMWARE("amdgpu/navi10_ce.bin"); MODULE_FIRMWARE("amdgpu/navi10_pfp.bin"); MODULE_FIRMWARE("amdgpu/navi10_me.bin"); @@ -3091,6 +3107,7 @@ static const struct soc15_reg_golden golden_settings_gc_10_3[] = SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_ADDR_MATCH_MASK, 0xffffffff, 0xffffffcf), SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_CM_CTRL1, 0xff8fff0f, 0x580f1008), SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_CTRL3, 0xf7ffffff, 0x10f80988), + SOC15_REG_GOLDEN_VALUE(GC, 0, mmLDS_CONFIG, 0x00000020, 0x00000020), SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_CL_ENHANCE, 0xf17fffff, 0x01200007), SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_TIMEOUT_COUNTER, 0xffffffff, 0x00000800), SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_2, 0xffffffbf, 0x00000820), @@ -3188,6 +3205,8 @@ static int gfx_v10_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev); static void gfx_v10_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume); static void gfx_v10_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume); static void gfx_v10_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure); +static u32 gfx_v10_3_get_disabled_sa(struct amdgpu_device *adev); +static void gfx_v10_3_program_pbb_mode(struct amdgpu_device *adev); static void gfx10_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask) { @@ -3586,6 +3605,17 @@ static void gfx_v10_0_init_rlc_ext_microcode(struct amdgpu_device *adev) le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); } +static void gfx_v10_0_init_rlc_iram_dram_microcode(struct amdgpu_device *adev) +{ + const struct rlc_firmware_header_v2_2 *rlc_hdr; + + rlc_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; + adev->gfx.rlc.rlc_iram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_iram_ucode_size_bytes); + adev->gfx.rlc.rlc_iram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_iram_ucode_offset_bytes); + adev->gfx.rlc.rlc_dram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_dram_ucode_size_bytes); + adev->gfx.rlc.rlc_dram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_dram_ucode_offset_bytes); +} + static bool gfx_v10_0_navi10_gfxoff_should_enable(struct amdgpu_device *adev) { bool ret = false; @@ -3701,8 +3731,6 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev) rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; version_major = le16_to_cpu(rlc_hdr->header.header_version_major); version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); - if (version_major == 2 && version_minor == 1) - adev->gfx.rlc.is_rlc_v2_1 = true; adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); @@ -3744,8 +3772,12 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev) for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++) adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); - if (adev->gfx.rlc.is_rlc_v2_1) - gfx_v10_0_init_rlc_ext_microcode(adev); + if (version_major == 2) { + if (version_minor >= 1) + gfx_v10_0_init_rlc_ext_microcode(adev); + if (version_minor == 2) + gfx_v10_0_init_rlc_iram_dram_microcode(adev); + } } snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec%s.bin", chip_name, wks); @@ -3806,8 +3838,7 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev) adev->firmware.fw_size += ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); } - if (adev->gfx.rlc.is_rlc_v2_1 && - adev->gfx.rlc.save_restore_list_cntl_size_bytes && + if (adev->gfx.rlc.save_restore_list_cntl_size_bytes && adev->gfx.rlc.save_restore_list_gpm_size_bytes && adev->gfx.rlc.save_restore_list_srm_size_bytes) { info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; @@ -3827,6 +3858,21 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev) info->fw = adev->gfx.rlc_fw; adev->firmware.fw_size += ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); + + if (adev->gfx.rlc.rlc_iram_ucode_size_bytes && + adev->gfx.rlc.rlc_dram_ucode_size_bytes) { + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_IRAM]; + info->ucode_id = AMDGPU_UCODE_ID_RLC_IRAM; + info->fw = adev->gfx.rlc_fw; + adev->firmware.fw_size += + ALIGN(adev->gfx.rlc.rlc_iram_ucode_size_bytes, PAGE_SIZE); + + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_DRAM]; + info->ucode_id = AMDGPU_UCODE_ID_RLC_DRAM; + info->fw = adev->gfx.rlc_fw; + adev->firmware.fw_size += + ALIGN(adev->gfx.rlc.rlc_dram_ucode_size_bytes, PAGE_SIZE); + } } info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; @@ -4536,12 +4582,17 @@ static void gfx_v10_0_setup_rb(struct amdgpu_device *adev) int i, j; u32 data; u32 active_rbs = 0; + u32 bitmap; u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / adev->gfx.config.max_sh_per_se; mutex_lock(&adev->grbm_idx_mutex); for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { + bitmap = i * adev->gfx.config.max_sh_per_se + j; + if ((adev->asic_type == CHIP_SIENNA_CICHLID) && + ((gfx_v10_3_get_disabled_sa(adev) >> bitmap) & 1)) + continue; gfx_v10_0_select_se_sh(adev, i, j, 0xffffffff); data = gfx_v10_0_get_rb_active_bitmap(adev); active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * @@ -6950,6 +7001,9 @@ static int gfx_v10_0_hw_init(void *handle) if (r) return r; + if (adev->asic_type == CHIP_SIENNA_CICHLID) + gfx_v10_3_program_pbb_mode(adev); + return r; } @@ -8763,6 +8817,10 @@ static int gfx_v10_0_get_cu_info(struct amdgpu_device *adev, mutex_lock(&adev->grbm_idx_mutex); for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { + bitmap = i * adev->gfx.config.max_sh_per_se + j; + if ((adev->asic_type == CHIP_SIENNA_CICHLID) && + ((gfx_v10_3_get_disabled_sa(adev) >> bitmap) & 1)) + continue; mask = 1; ao_bitmap = 0; counter = 0; @@ -8797,6 +8855,47 @@ static int gfx_v10_0_get_cu_info(struct amdgpu_device *adev, return 0; } +static u32 gfx_v10_3_get_disabled_sa(struct amdgpu_device *adev) +{ + uint32_t efuse_setting, vbios_setting, disabled_sa, max_sa_mask; + + efuse_setting = RREG32_SOC15(GC, 0, mmCC_GC_SA_UNIT_DISABLE); + efuse_setting &= CC_GC_SA_UNIT_DISABLE__SA_DISABLE_MASK; + efuse_setting >>= CC_GC_SA_UNIT_DISABLE__SA_DISABLE__SHIFT; + + vbios_setting = RREG32_SOC15(GC, 0, mmGC_USER_SA_UNIT_DISABLE); + vbios_setting &= GC_USER_SA_UNIT_DISABLE__SA_DISABLE_MASK; + vbios_setting >>= GC_USER_SA_UNIT_DISABLE__SA_DISABLE__SHIFT; + + max_sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se * + adev->gfx.config.max_shader_engines); + disabled_sa = efuse_setting | vbios_setting; + disabled_sa &= max_sa_mask; + + return disabled_sa; +} + +static void gfx_v10_3_program_pbb_mode(struct amdgpu_device *adev) +{ + uint32_t max_sa_per_se, max_sa_per_se_mask, max_shader_engines; + uint32_t disabled_sa_mask, se_index, disabled_sa_per_se; + + disabled_sa_mask = gfx_v10_3_get_disabled_sa(adev); + + max_sa_per_se = adev->gfx.config.max_sh_per_se; + max_sa_per_se_mask = (1 << max_sa_per_se) - 1; + max_shader_engines = adev->gfx.config.max_shader_engines; + + for (se_index = 0; max_shader_engines > se_index; se_index++) { + disabled_sa_per_se = disabled_sa_mask >> (se_index * max_sa_per_se); + disabled_sa_per_se &= max_sa_per_se_mask; + if (disabled_sa_per_se == max_sa_per_se_mask) { + WREG32_FIELD15(GC, 0, PA_SC_ENHANCE_3, FORCE_PBB_WORKLOAD_MODE_TO_ZERO, 1); + break; + } + } +} + const struct amdgpu_ip_block_version gfx_v10_0_ip_block = { .type = AMD_IP_BLOCK_TYPE_GFX, diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v1_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v1_0.c index bc300283b6ab..c600b61b5f45 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v1_0.c @@ -33,6 +33,7 @@ static void jpeg_v1_0_set_dec_ring_funcs(struct amdgpu_device *adev); static void jpeg_v1_0_set_irq_funcs(struct amdgpu_device *adev); +static void jpeg_v1_0_ring_begin_use(struct amdgpu_ring *ring); static void jpeg_v1_0_decode_ring_patch_wreg(struct amdgpu_ring *ring, uint32_t *ptr, uint32_t reg_offset, uint32_t val) { @@ -564,8 +565,8 @@ static const struct amdgpu_ring_funcs jpeg_v1_0_decode_ring_vm_funcs = { .insert_start = jpeg_v1_0_decode_ring_insert_start, .insert_end = jpeg_v1_0_decode_ring_insert_end, .pad_ib = amdgpu_ring_generic_pad_ib, - .begin_use = vcn_v1_0_ring_begin_use, - .end_use = amdgpu_vcn_ring_end_use, + .begin_use = jpeg_v1_0_ring_begin_use, + .end_use = vcn_v1_0_ring_end_use, .emit_wreg = jpeg_v1_0_decode_ring_emit_wreg, .emit_reg_wait = jpeg_v1_0_decode_ring_emit_reg_wait, .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, @@ -586,3 +587,22 @@ static void jpeg_v1_0_set_irq_funcs(struct amdgpu_device *adev) { adev->jpeg.inst->irq.funcs = &jpeg_v1_0_irq_funcs; } + +static void jpeg_v1_0_ring_begin_use(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + bool set_clocks = !cancel_delayed_work_sync(&adev->vcn.idle_work); + int cnt = 0; + + mutex_lock(&adev->vcn.vcn1_jpeg1_workaround); + + if (amdgpu_fence_wait_empty(&adev->vcn.inst->ring_dec)) + DRM_ERROR("JPEG dec: vcn dec ring may not be empty\n"); + + for (cnt = 0; cnt < adev->vcn.num_enc_rings; cnt++) { + if (amdgpu_fence_wait_empty(&adev->vcn.inst->ring_enc[cnt])) + DRM_ERROR("JPEG dec: vcn enc ring[%d] may not be empty\n", cnt); + } + + vcn_v1_0_set_pg_for_begin_use(ring, set_clocks); +} diff --git a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h index 1ef2f5b1d828..4137dc710aaf 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h +++ b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h @@ -201,7 +201,7 @@ enum psp_gfx_fw_type { GFX_FW_TYPE_UVD1 = 23, /* UVD1 VG-20 */ GFX_FW_TYPE_TOC = 24, /* TOC NV-10 */ GFX_FW_TYPE_RLC_P = 25, /* RLC P NV */ - GFX_FW_TYPE_RLX6 = 26, /* RLX6 NV */ + GFX_FW_TYPE_RLC_IRAM = 26, /* RLC_IRAM NV */ GFX_FW_TYPE_GLOBAL_TAP_DELAYS = 27, /* GLOBAL TAP DELAYS NV */ GFX_FW_TYPE_SE0_TAP_DELAYS = 28, /* SE0 TAP DELAYS NV */ GFX_FW_TYPE_SE1_TAP_DELAYS = 29, /* SE1 TAP DELAYS NV */ @@ -223,7 +223,7 @@ enum psp_gfx_fw_type { GFX_FW_TYPE_ACCUM_CTRL_RAM = 45, /* ACCUM CTRL RAM NV */ GFX_FW_TYPE_RLCP_CAM = 46, /* RLCP CAM NV */ GFX_FW_TYPE_RLC_SPP_CAM_EXT = 47, /* RLC SPP CAM EXT NV */ - GFX_FW_TYPE_RLX6_DRAM_BOOT = 48, /* RLX6 DRAM BOOT NV */ + GFX_FW_TYPE_RLC_DRAM_BOOT = 48, /* RLC DRAM BOOT NV */ GFX_FW_TYPE_VCN0_RAM = 49, /* VCN_RAM NV + RN */ GFX_FW_TYPE_VCN1_RAM = 50, /* VCN_RAM NV + RN */ GFX_FW_TYPE_DMUB = 51, /* DMUB RN */ diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c index 73699eafb51e..86e1ef732ebe 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c @@ -54,6 +54,7 @@ static int vcn_v1_0_pause_dpg_mode(struct amdgpu_device *adev, int inst_idx, struct dpg_pause_state *new_state); static void vcn_v1_0_idle_work_handler(struct work_struct *work); +static void vcn_v1_0_ring_begin_use(struct amdgpu_ring *ring); /** * vcn_v1_0_early_init - set function pointers @@ -1804,11 +1805,24 @@ static void vcn_v1_0_idle_work_handler(struct work_struct *work) } } -void vcn_v1_0_ring_begin_use(struct amdgpu_ring *ring) +static void vcn_v1_0_ring_begin_use(struct amdgpu_ring *ring) { - struct amdgpu_device *adev = ring->adev; + struct amdgpu_device *adev = ring->adev; bool set_clocks = !cancel_delayed_work_sync(&adev->vcn.idle_work); + mutex_lock(&adev->vcn.vcn1_jpeg1_workaround); + + if (amdgpu_fence_wait_empty(&ring->adev->jpeg.inst->ring_dec)) + DRM_ERROR("VCN dec: jpeg dec ring may not be empty\n"); + + vcn_v1_0_set_pg_for_begin_use(ring, set_clocks); + +} + +void vcn_v1_0_set_pg_for_begin_use(struct amdgpu_ring *ring, bool set_clocks) +{ + struct amdgpu_device *adev = ring->adev; + if (set_clocks) { amdgpu_gfx_off_ctrl(adev, false); if (adev->pm.dpm_enabled) @@ -1844,6 +1858,12 @@ void vcn_v1_0_ring_begin_use(struct amdgpu_ring *ring) } } +void vcn_v1_0_ring_end_use(struct amdgpu_ring *ring) +{ + schedule_delayed_work(&ring->adev->vcn.idle_work, VCN_IDLE_TIMEOUT); + mutex_unlock(&ring->adev->vcn.vcn1_jpeg1_workaround); +} + static const struct amd_ip_funcs vcn_v1_0_ip_funcs = { .name = "vcn_v1_0", .early_init = vcn_v1_0_early_init, @@ -1891,7 +1911,7 @@ static const struct amdgpu_ring_funcs vcn_v1_0_dec_ring_vm_funcs = { .insert_end = vcn_v1_0_dec_ring_insert_end, .pad_ib = amdgpu_ring_generic_pad_ib, .begin_use = vcn_v1_0_ring_begin_use, - .end_use = amdgpu_vcn_ring_end_use, + .end_use = vcn_v1_0_ring_end_use, .emit_wreg = vcn_v1_0_dec_ring_emit_wreg, .emit_reg_wait = vcn_v1_0_dec_ring_emit_reg_wait, .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, @@ -1923,7 +1943,7 @@ static const struct amdgpu_ring_funcs vcn_v1_0_enc_ring_vm_funcs = { .insert_end = vcn_v1_0_enc_ring_insert_end, .pad_ib = amdgpu_ring_generic_pad_ib, .begin_use = vcn_v1_0_ring_begin_use, - .end_use = amdgpu_vcn_ring_end_use, + .end_use = vcn_v1_0_ring_end_use, .emit_wreg = vcn_v1_0_enc_ring_emit_wreg, .emit_reg_wait = vcn_v1_0_enc_ring_emit_reg_wait, .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.h b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.h index f67d7391fc21..1f1cc7f0ece7 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.h +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.h @@ -24,7 +24,8 @@ #ifndef __VCN_V1_0_H__ #define __VCN_V1_0_H__ -void vcn_v1_0_ring_begin_use(struct amdgpu_ring *ring); +void vcn_v1_0_ring_end_use(struct amdgpu_ring *ring); +void vcn_v1_0_set_pg_for_begin_use(struct amdgpu_ring *ring, bool set_clocks); extern const struct amdgpu_ip_block_version vcn_v1_0_ip_block; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index d2981524dba0..5e2254b9e931 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -1426,5 +1426,5 @@ int kfd_create_crat_image_virtual(void **crat_image, size_t *size, */ void kfd_destroy_crat_image(void *crat_image) { - kfree(crat_image); + kvfree(crat_image); } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c index 72e4d61ac752..ad0593342333 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c @@ -58,8 +58,9 @@ static int update_qpd_v10(struct device_queue_manager *dqm, /* check if sh_mem_config register already configured */ if (qpd->sh_mem_config == 0) { qpd->sh_mem_config = - SH_MEM_ALIGNMENT_MODE_UNALIGNED << - SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; + (SH_MEM_ALIGNMENT_MODE_UNALIGNED << + SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | + (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT); #if 0 /* TODO: * This shouldn't be an issue with Navi10. Verify. diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index bb1bc7f5d149..e2b23486ba4c 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -5063,7 +5063,13 @@ static void amdgpu_dm_connector_destroy(struct drm_connector *connector) struct amdgpu_device *adev = drm_to_adev(connector->dev); struct amdgpu_display_manager *dm = &adev->dm; - drm_atomic_private_obj_fini(&aconnector->mst_mgr.base); + /* + * Call only if mst_mgr was iniitalized before since it's not done + * for all connector types. + */ + if (aconnector->mst_mgr.dev) + drm_dp_mst_topology_mgr_destroy(&aconnector->mst_mgr); + #if defined(CONFIG_BACKLIGHT_CLASS_DEVICE) ||\ defined(CONFIG_BACKLIGHT_CLASS_DEVICE_MODULE) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index 9c1e003d9c29..34f6369bf51f 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -149,6 +149,8 @@ struct amdgpu_dm_backlight_caps { * @cached_state: Caches device atomic state for suspend/resume * @cached_dc_state: Cached state of content streams * @compressor: Frame buffer compression buffer. See &struct dm_comressor_info + * @force_timing_sync: set via debugfs. When set, indicates that all connected + * displays will be forced to synchronize. */ struct amdgpu_display_manager { diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index db741e47d194..eee19edeeee5 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -647,7 +647,7 @@ static void try_disable_dsc(struct drm_atomic_state *state, for (i = 0; i < count; i++) { if (vars[i].dsc_enabled && vars[i].bpp_x16 == params[i].bw_range.max_target_bpp_x16 - && !params[i].clock_force_enable == DSC_CLK_FORCE_DEFAULT) { + && params[i].clock_force_enable == DSC_CLK_FORCE_DEFAULT) { kbps_increase[i] = params[i].bw_range.stream_kbps - params[i].bw_range.max_kbps; tried[i] = false; remaining_to_try += 1; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 2a725a5fba40..1eb29c362122 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -848,7 +848,7 @@ static void disable_vbios_mode_if_required( struct dc *dc, struct dc_state *context) { - unsigned int i; + unsigned int i, j; /* check if timing_changed, disable stream*/ for (i = 0; i < dc->res_pool->pipe_count; i++) { @@ -872,10 +872,10 @@ static void disable_vbios_mode_if_required( enc_inst = link->link_enc->funcs->get_dig_frontend(link->link_enc); if (enc_inst != ENGINE_ID_UNKNOWN) { - for (i = 0; i < dc->res_pool->stream_enc_count; i++) { - if (dc->res_pool->stream_enc[i]->id == enc_inst) { - tg_inst = dc->res_pool->stream_enc[i]->funcs->dig_source_otg( - dc->res_pool->stream_enc[i]); + for (j = 0; j < dc->res_pool->stream_enc_count; j++) { + if (dc->res_pool->stream_enc[j]->id == enc_inst) { + tg_inst = dc->res_pool->stream_enc[j]->funcs->dig_source_otg( + dc->res_pool->stream_enc[j]); break; } } diff --git a/drivers/gpu/drm/amd/pm/inc/smu_types.h b/drivers/gpu/drm/amd/pm/inc/smu_types.h index 35fc46d3c9c0..cbf4a58b77d9 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu_types.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_types.h @@ -220,6 +220,7 @@ enum smu_clk_type { __SMU_DUMMY_MAP(DPM_MP0CLK), \ __SMU_DUMMY_MAP(DPM_LINK), \ __SMU_DUMMY_MAP(DPM_DCEFCLK), \ + __SMU_DUMMY_MAP(DPM_XGMI), \ __SMU_DUMMY_MAP(DS_GFXCLK), \ __SMU_DUMMY_MAP(DS_SOCCLK), \ __SMU_DUMMY_MAP(DS_LCLK), \ diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c index 3bf8be4d107b..1e8919b0acdb 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c @@ -2883,7 +2883,7 @@ static int smu7_vblank_too_short(struct pp_hwmgr *hwmgr, if (hwmgr->is_kicker) switch_limit_us = data->is_memory_gddr5 ? 450 : 150; else - switch_limit_us = data->is_memory_gddr5 ? 190 : 150; + switch_limit_us = data->is_memory_gddr5 ? 200 : 150; break; case CHIP_VEGAM: switch_limit_us = 30; diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index 942793947b4b..fc4f95fa87cf 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -417,6 +417,9 @@ static int smu_early_init(void *handle) smu->pm_enabled = !!amdgpu_dpm; smu->is_apu = false; mutex_init(&smu->mutex); + mutex_init(&smu->smu_baco.mutex); + smu->smu_baco.state = SMU_BACO_STATE_EXIT; + smu->smu_baco.platform_support = false; return smu_set_funcs(adev); } @@ -795,10 +798,6 @@ static int smu_sw_init(void *handle) bitmap_zero(smu->smu_feature.enabled, SMU_FEATURE_MAX); bitmap_zero(smu->smu_feature.allowed, SMU_FEATURE_MAX); - mutex_init(&smu->smu_baco.mutex); - smu->smu_baco.state = SMU_BACO_STATE_EXIT; - smu->smu_baco.platform_support = false; - mutex_init(&smu->sensor_lock); mutex_init(&smu->metrics_lock); mutex_init(&smu->message_lock); diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c index c27806fd07e0..685a8a3b25d4 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c @@ -151,14 +151,17 @@ static struct cmn2asic_mapping sienna_cichlid_feature_mask_map[SMU_FEATURE_COUNT FEA_MAP(DPM_GFXCLK), FEA_MAP(DPM_GFX_GPO), FEA_MAP(DPM_UCLK), + FEA_MAP(DPM_FCLK), FEA_MAP(DPM_SOCCLK), FEA_MAP(DPM_MP0CLK), FEA_MAP(DPM_LINK), FEA_MAP(DPM_DCEFCLK), + FEA_MAP(DPM_XGMI), FEA_MAP(MEM_VDDCI_SCALING), FEA_MAP(MEM_MVDD_SCALING), FEA_MAP(DS_GFXCLK), FEA_MAP(DS_SOCCLK), + FEA_MAP(DS_FCLK), FEA_MAP(DS_LCLK), FEA_MAP(DS_DCEFCLK), FEA_MAP(DS_UCLK), @@ -452,6 +455,9 @@ static int sienna_cichlid_get_smu_metrics_data(struct smu_context *smu, case METRICS_CURR_DCEFCLK: *value = metrics->CurrClock[PPCLK_DCEFCLK]; break; + case METRICS_CURR_FCLK: + *value = metrics->CurrClock[PPCLK_FCLK]; + break; case METRICS_AVERAGE_GFXCLK: if (metrics->AverageGfxActivity <= SMU_11_0_7_GFX_BUSY_THRESHOLD) *value = metrics->AverageGfxclkFrequencyPostDs; @@ -948,19 +954,23 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu, freq_values[1] = cur_value; mark_index = cur_value == freq_values[0] ? 0 : cur_value == freq_values[2] ? 2 : 1; - if (mark_index != 1) - freq_values[1] = (freq_values[0] + freq_values[2]) / 2; - for (i = 0; i < 3; i++) { + count = 3; + if (mark_index != 1) { + count = 2; + freq_values[1] = freq_values[2]; + } + + for (i = 0; i < count; i++) { size += sprintf(buf + size, "%d: %uMhz %s\n", i, freq_values[i], - i == mark_index ? "*" : ""); + cur_value == freq_values[i] ? "*" : ""); } } break; case SMU_PCIE: - gen_speed = smu_v11_0_get_current_pcie_link_speed(smu); - lane_width = smu_v11_0_get_current_pcie_link_width(smu); + gen_speed = smu_v11_0_get_current_pcie_link_speed_level(smu); + lane_width = smu_v11_0_get_current_pcie_link_width_level(smu); for (i = 0; i < NUM_LINK_LEVELS; i++) size += sprintf(buf + size, "%d: %s %s %dMhz %s\n", i, (dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 0) ? "2.5GT/s," : diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index c30d3338825f..92b2ea4c197b 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -431,10 +431,9 @@ size_t smu_cmn_get_pp_feature_mask(struct smu_context *smu, char *buf) { uint32_t feature_mask[2] = { 0 }; - int32_t feature_index = 0; + int feature_index = 0; uint32_t count = 0; - uint32_t sort_feature[SMU_FEATURE_COUNT]; - uint64_t hw_feature_count = 0; + int8_t sort_feature[SMU_FEATURE_COUNT]; size_t size = 0; int ret = 0, i; @@ -447,23 +446,31 @@ size_t smu_cmn_get_pp_feature_mask(struct smu_context *smu, size = sprintf(buf + size, "features high: 0x%08x low: 0x%08x\n", feature_mask[1], feature_mask[0]); + memset(sort_feature, -1, sizeof(sort_feature)); + for (i = 0; i < SMU_FEATURE_COUNT; i++) { feature_index = smu_cmn_to_asic_specific_index(smu, CMN2ASIC_MAPPING_FEATURE, i); if (feature_index < 0) continue; + sort_feature[feature_index] = i; - hw_feature_count++; } - for (i = 0; i < hw_feature_count; i++) { + size += sprintf(buf + size, "%-2s. %-20s %-3s : %-s\n", + "No", "Feature", "Bit", "State"); + + for (i = 0; i < SMU_FEATURE_COUNT; i++) { + if (sort_feature[i] < 0) + continue; + size += sprintf(buf + size, "%02d. %-20s (%2d) : %s\n", - count++, - smu_get_feature_name(smu, sort_feature[i]), - i, - !!smu_cmn_feature_is_enabled(smu, sort_feature[i]) ? - "enabled" : "disabled"); + count++, + smu_get_feature_name(smu, sort_feature[i]), + i, + !!smu_cmn_feature_is_enabled(smu, sort_feature[i]) ? + "enabled" : "disabled"); } return size; diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c index 11fe9ff76fd5..d6808f678db5 100644 --- a/drivers/gpu/drm/drm_prime.c +++ b/drivers/gpu/drm/drm_prime.c @@ -806,30 +806,27 @@ static const struct dma_buf_ops drm_gem_prime_dmabuf_ops = { struct sg_table *drm_prime_pages_to_sg(struct drm_device *dev, struct page **pages, unsigned int nr_pages) { - struct sg_table *sg = NULL; + struct sg_table *sg; + struct scatterlist *sge; size_t max_segment = 0; - int ret; sg = kmalloc(sizeof(struct sg_table), GFP_KERNEL); - if (!sg) { - ret = -ENOMEM; - goto out; - } + if (!sg) + return ERR_PTR(-ENOMEM); if (dev) max_segment = dma_max_mapping_size(dev->dev); if (max_segment == 0 || max_segment > SCATTERLIST_MAX_SEGMENT) max_segment = SCATTERLIST_MAX_SEGMENT; - ret = __sg_alloc_table_from_pages(sg, pages, nr_pages, 0, + sge = __sg_alloc_table_from_pages(sg, pages, nr_pages, 0, nr_pages << PAGE_SHIFT, - max_segment, GFP_KERNEL); - if (ret) - goto out; - + max_segment, + NULL, 0, GFP_KERNEL); + if (IS_ERR(sge)) { + kfree(sg); + sg = ERR_CAST(sge); + } return sg; -out: - kfree(sg); - return ERR_PTR(ret); } EXPORT_SYMBOL(drm_prime_pages_to_sg); diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig index 9afa5c4a6bf0..1e1cb245fca7 100644 --- a/drivers/gpu/drm/i915/Kconfig +++ b/drivers/gpu/drm/i915/Kconfig @@ -25,6 +25,7 @@ config DRM_I915 select CRC32 select SND_HDA_I915 if SND_HDA_CORE select CEC_CORE if CEC_NOTIFIER + select VMAP_PFN help Choose this option if you have a system that has "Intel Graphics Media Accelerator" or "HD Graphics" integrated graphics, diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug index 1cb28c20807c..25cd9788a4d5 100644 --- a/drivers/gpu/drm/i915/Kconfig.debug +++ b/drivers/gpu/drm/i915/Kconfig.debug @@ -153,6 +153,7 @@ config DRM_I915_SELFTEST select DRM_EXPORT_FOR_TESTS if m select FAULT_INJECTION select PRIME_NUMBERS + select CRC32 help Choose this option to allow the driver to perform selftests upon loading; also requires the i915.selftest=1 module parameter. To diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c index 4d06178cd76c..cdcb7b1034ae 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.c +++ b/drivers/gpu/drm/i915/display/intel_ddi.c @@ -2742,7 +2742,7 @@ tgl_dkl_phy_ddi_vswing_sequence(struct intel_encoder *encoder, int link_clock, u32 n_entries, val, ln, dpcnt_mask, dpcnt_val; int rate = 0; - if (type == INTEL_OUTPUT_HDMI) { + if (type != INTEL_OUTPUT_HDMI) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); rate = intel_dp->link_rate; diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index 631b4338224e..829b2a40a482 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -3434,6 +3434,14 @@ initial_plane_vma(struct drm_i915_private *i915, if (IS_ERR(obj)) return NULL; + /* + * Mark it WT ahead of time to avoid changing the + * cache_level during fbdev initialization. The + * unbind there would get stuck waiting for rcu. + */ + i915_gem_object_set_cache_coherency(obj, HAS_WT(i915) ? + I915_CACHE_WT : I915_CACHE_NONE); + switch (plane_config->tiling) { case I915_TILING_NONE: break; @@ -4093,8 +4101,7 @@ static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state) int skl_check_plane_surface(struct intel_plane_state *plane_state) { const struct drm_framebuffer *fb = plane_state->hw.fb; - int ret; - bool needs_aux = false; + int ret, i; ret = intel_plane_compute_gtt(plane_state); if (ret) @@ -4108,7 +4115,6 @@ int skl_check_plane_surface(struct intel_plane_state *plane_state) * it. */ if (is_ccs_modifier(fb->modifier)) { - needs_aux = true; ret = skl_check_ccs_aux_surface(plane_state); if (ret) return ret; @@ -4116,20 +4122,15 @@ int skl_check_plane_surface(struct intel_plane_state *plane_state) if (intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier)) { - needs_aux = true; ret = skl_check_nv12_aux_surface(plane_state); if (ret) return ret; } - if (!needs_aux) { - int i; - - for (i = 1; i < fb->format->num_planes; i++) { - plane_state->color_plane[i].offset = ~0xfff; - plane_state->color_plane[i].x = 0; - plane_state->color_plane[i].y = 0; - } + for (i = fb->format->num_planes; i < ARRAY_SIZE(plane_state->color_plane); i++) { + plane_state->color_plane[i].offset = ~0xfff; + plane_state->color_plane[i].x = 0; + plane_state->color_plane[i].y = 0; } ret = skl_check_main_surface(plane_state); diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c index acbd7eb66cbe..036f504ac7db 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c +++ b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c @@ -52,17 +52,11 @@ static void set_aux_backlight_enable(struct intel_dp *intel_dp, bool enable) } } -/* - * Read the current backlight value from DPCD register(s) based - * on if 8-bit(MSB) or 16-bit(MSB and LSB) values are supported - */ -static u32 intel_dp_aux_get_backlight(struct intel_connector *connector) +static bool intel_dp_aux_backlight_dpcd_mode(struct intel_connector *connector) { struct intel_dp *intel_dp = intel_attached_dp(connector); struct drm_i915_private *i915 = dp_to_i915(intel_dp); - u8 read_val[2] = { 0x0 }; u8 mode_reg; - u16 level = 0; if (drm_dp_dpcd_readb(&intel_dp->aux, DP_EDP_BACKLIGHT_MODE_SET_REGISTER, @@ -70,15 +64,29 @@ static u32 intel_dp_aux_get_backlight(struct intel_connector *connector) drm_dbg_kms(&i915->drm, "Failed to read the DPCD register 0x%x\n", DP_EDP_BACKLIGHT_MODE_SET_REGISTER); - return 0; + return false; } + return (mode_reg & DP_EDP_BACKLIGHT_CONTROL_MODE_MASK) == + DP_EDP_BACKLIGHT_CONTROL_MODE_DPCD; +} + +/* + * Read the current backlight value from DPCD register(s) based + * on if 8-bit(MSB) or 16-bit(MSB and LSB) values are supported + */ +static u32 intel_dp_aux_get_backlight(struct intel_connector *connector) +{ + struct intel_dp *intel_dp = intel_attached_dp(connector); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + u8 read_val[2] = { 0x0 }; + u16 level = 0; + /* * If we're not in DPCD control mode yet, the programmed brightness * value is meaningless and we should assume max brightness */ - if ((mode_reg & DP_EDP_BACKLIGHT_CONTROL_MODE_MASK) != - DP_EDP_BACKLIGHT_CONTROL_MODE_DPCD) + if (!intel_dp_aux_backlight_dpcd_mode(connector)) return connector->panel.backlight.max; if (drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_BACKLIGHT_BRIGHTNESS_MSB, @@ -319,7 +327,8 @@ static int intel_dp_aux_setup_backlight(struct intel_connector *connector, panel->backlight.min = 0; panel->backlight.level = intel_dp_aux_get_backlight(connector); - panel->backlight.enabled = panel->backlight.level != 0; + panel->backlight.enabled = intel_dp_aux_backlight_dpcd_mode(connector) && + panel->backlight.level != 0; return 0; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c index 4b09bcd70cf4..1904e6e5ea64 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c @@ -287,8 +287,8 @@ struct i915_execbuffer { u64 invalid_flags; /** Set of execobj.flags that are invalid */ u32 context_flags; /** Set of execobj.flags to insert from the ctx */ + u64 batch_len; /** Length of batch within object */ u32 batch_start_offset; /** Location within object of batch */ - u32 batch_len; /** Length of batch within object */ u32 batch_flags; /** Flags composed for emit_bb_start() */ struct intel_gt_buffer_pool_node *batch_pool; /** pool node for batch buffer */ @@ -871,6 +871,10 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb) if (eb->batch_len == 0) eb->batch_len = eb->batch->vma->size - eb->batch_start_offset; + if (unlikely(eb->batch_len == 0)) { /* impossible! */ + drm_dbg(&i915->drm, "Invalid batch length\n"); + return -EINVAL; + } return 0; @@ -2424,7 +2428,7 @@ static int eb_parse(struct i915_execbuffer *eb) struct drm_i915_private *i915 = eb->i915; struct intel_gt_buffer_pool_node *pool = eb->batch_pool; struct i915_vma *shadow, *trampoline, *batch; - unsigned int len; + unsigned long len; int err; if (!eb_use_cmdparser(eb)) { @@ -2449,6 +2453,8 @@ static int eb_parse(struct i915_execbuffer *eb) } else { len += I915_CMD_PARSER_TRAMPOLINE_SIZE; } + if (unlikely(len < eb->batch_len)) /* last paranoid check of overflow */ + return -EINVAL; if (!pool) { pool = intel_gt_get_buffer_pool(eb->engine->gt, len); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index d6eeefab3d01..f60ca6dc911f 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -162,8 +162,6 @@ static void unmap_object(struct drm_i915_gem_object *obj, void *ptr) { if (is_vmalloc_addr(ptr)) vunmap(ptr); - else - kunmap(kmap_to_page(ptr)); } struct sg_table * @@ -234,34 +232,21 @@ unlock: return err; } -static inline pte_t iomap_pte(resource_size_t base, - dma_addr_t offset, - pgprot_t prot) -{ - return pte_mkspecial(pfn_pte((base + offset) >> PAGE_SHIFT, prot)); -} - /* The 'mapping' part of i915_gem_object_pin_map() below */ -static void *i915_gem_object_map(struct drm_i915_gem_object *obj, - enum i915_map_type type) +static void *i915_gem_object_map_page(struct drm_i915_gem_object *obj, + enum i915_map_type type) { - unsigned long n_pte = obj->base.size >> PAGE_SHIFT; - struct sg_table *sgt = obj->mm.pages; - pte_t *stack[32], **mem; - struct vm_struct *area; + unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i; + struct page *stack[32], **pages = stack, *page; + struct sgt_iter iter; pgprot_t pgprot; + void *vaddr; - if (!i915_gem_object_has_struct_page(obj) && type != I915_MAP_WC) - return NULL; - - if (GEM_WARN_ON(type == I915_MAP_WC && - !static_cpu_has(X86_FEATURE_PAT))) - return NULL; - - /* A single page can always be kmapped */ - if (n_pte == 1 && type == I915_MAP_WB) { - struct page *page = sg_page(sgt->sgl); - + switch (type) { + default: + MISSING_CASE(type); + fallthrough; /* to use PAGE_KERNEL anyway */ + case I915_MAP_WB: /* * On 32b, highmem using a finite set of indirect PTE (i.e. * vmap) to provide virtual mappings of the high pages. @@ -277,33 +262,10 @@ static void *i915_gem_object_map(struct drm_i915_gem_object *obj, * forever. * * So if the page is beyond the 32b boundary, make an explicit - * vmap. On 64b, this check will be optimised away as we can - * directly kmap any page on the system. + * vmap. */ - if (!PageHighMem(page)) - return kmap(page); - } - - mem = stack; - if (n_pte > ARRAY_SIZE(stack)) { - /* Too big for stack -- allocate temporary array instead */ - mem = kvmalloc_array(n_pte, sizeof(*mem), GFP_KERNEL); - if (!mem) - return NULL; - } - - area = alloc_vm_area(obj->base.size, mem); - if (!area) { - if (mem != stack) - kvfree(mem); - return NULL; - } - - switch (type) { - default: - MISSING_CASE(type); - fallthrough; /* to use PAGE_KERNEL anyway */ - case I915_MAP_WB: + if (n_pages == 1 && !PageHighMem(sg_page(obj->mm.pages->sgl))) + return page_address(sg_page(obj->mm.pages->sgl)); pgprot = PAGE_KERNEL; break; case I915_MAP_WC: @@ -311,30 +273,50 @@ static void *i915_gem_object_map(struct drm_i915_gem_object *obj, break; } - if (i915_gem_object_has_struct_page(obj)) { - struct sgt_iter iter; - struct page *page; - pte_t **ptes = mem; + if (n_pages > ARRAY_SIZE(stack)) { + /* Too big for stack -- allocate temporary array instead */ + pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL); + if (!pages) + return NULL; + } - for_each_sgt_page(page, iter, sgt) - **ptes++ = mk_pte(page, pgprot); - } else { - resource_size_t iomap; - struct sgt_iter iter; - pte_t **ptes = mem; - dma_addr_t addr; + i = 0; + for_each_sgt_page(page, iter, obj->mm.pages) + pages[i++] = page; + vaddr = vmap(pages, n_pages, 0, pgprot); + if (pages != stack) + kvfree(pages); + return vaddr; +} - iomap = obj->mm.region->iomap.base; - iomap -= obj->mm.region->region.start; +static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj, + enum i915_map_type type) +{ + resource_size_t iomap = obj->mm.region->iomap.base - + obj->mm.region->region.start; + unsigned long n_pfn = obj->base.size >> PAGE_SHIFT; + unsigned long stack[32], *pfns = stack, i; + struct sgt_iter iter; + dma_addr_t addr; + void *vaddr; + + if (type != I915_MAP_WC) + return NULL; - for_each_sgt_daddr(addr, iter, sgt) - **ptes++ = iomap_pte(iomap, addr, pgprot); + if (n_pfn > ARRAY_SIZE(stack)) { + /* Too big for stack -- allocate temporary array instead */ + pfns = kvmalloc_array(n_pfn, sizeof(*pfns), GFP_KERNEL); + if (!pfns) + return NULL; } - if (mem != stack) - kvfree(mem); - - return area->addr; + i = 0; + for_each_sgt_daddr(addr, iter, obj->mm.pages) + pfns[i++] = (iomap + addr) >> PAGE_SHIFT; + vaddr = vmap_pfn(pfns, n_pfn, pgprot_writecombine(PAGE_KERNEL_IO)); + if (pfns != stack) + kvfree(pfns); + return vaddr; } /* get, pin, and map the pages of the object into kernel space */ @@ -386,7 +368,13 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, } if (!ptr) { - ptr = i915_gem_object_map(obj, type); + if (GEM_WARN_ON(type == I915_MAP_WC && + !static_cpu_has(X86_FEATURE_PAT))) + ptr = NULL; + else if (i915_gem_object_has_struct_page(obj)) + ptr = i915_gem_object_map_page(obj, type); + else + ptr = i915_gem_object_map_pfn(obj, type); if (!ptr) { err = -ENOMEM; goto err_unpin; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c index 0be5e8683337..84b2707d8b17 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c @@ -53,8 +53,10 @@ int i915_gem_stolen_insert_node(struct drm_i915_private *i915, struct drm_mm_node *node, u64 size, unsigned alignment) { - return i915_gem_stolen_insert_node_in_range(i915, node, size, - alignment, 0, U64_MAX); + return i915_gem_stolen_insert_node_in_range(i915, node, + size, alignment, + I915_GEM_STOLEN_BIAS, + U64_MAX); } void i915_gem_stolen_remove_node(struct drm_i915_private *i915, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.h b/drivers/gpu/drm/i915/gem/i915_gem_stolen.h index e15c0adad8af..61e028063f9f 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.h @@ -30,4 +30,6 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv resource_size_t stolen_offset, resource_size_t size); +#define I915_GEM_STOLEN_BIAS SZ_128K + #endif /* __I915_GEM_STOLEN_H__ */ diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c index 12b30075134a..f2eaed6aca3d 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c @@ -403,6 +403,7 @@ __i915_gem_userptr_alloc_pages(struct drm_i915_gem_object *obj, unsigned int max_segment = i915_sg_segment_size(); struct sg_table *st; unsigned int sg_page_sizes; + struct scatterlist *sg; int ret; st = kmalloc(sizeof(*st), GFP_KERNEL); @@ -410,13 +411,12 @@ __i915_gem_userptr_alloc_pages(struct drm_i915_gem_object *obj, return ERR_PTR(-ENOMEM); alloc_table: - ret = __sg_alloc_table_from_pages(st, pvec, num_pages, - 0, num_pages << PAGE_SHIFT, - max_segment, - GFP_KERNEL); - if (ret) { + sg = __sg_alloc_table_from_pages(st, pvec, num_pages, 0, + num_pages << PAGE_SHIFT, max_segment, + NULL, 0, GFP_KERNEL); + if (IS_ERR(sg)) { kfree(st); - return ERR_PTR(ret); + return ERR_CAST(sg); } ret = i915_gem_gtt_prepare_pages(obj, st); diff --git a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c index fd0d24d28763..c30adc05fa98 100644 --- a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c @@ -239,18 +239,24 @@ static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt) I915_CACHE_NONE, PTE_READ_ONLY); vm->scratch[1] = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); - if (IS_ERR(vm->scratch[1])) - return PTR_ERR(vm->scratch[1]); + if (IS_ERR(vm->scratch[1])) { + ret = PTR_ERR(vm->scratch[1]); + goto err_scratch0; + } ret = pin_pt_dma(vm, vm->scratch[1]); - if (ret) { - i915_gem_object_put(vm->scratch[1]); - return ret; - } + if (ret) + goto err_scratch1; fill32_px(vm->scratch[1], vm->scratch[0]->encode); return 0; + +err_scratch1: + i915_gem_object_put(vm->scratch[1]); +err_scratch0: + i915_gem_object_put(vm->scratch[0]); + return ret; } static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt) diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c index eb64f474a78c..38c7069b7749 100644 --- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c @@ -604,7 +604,8 @@ static int gen8_init_scratch(struct i915_address_space *vm) return 0; free_scratch: - free_scratch(vm); + while (i--) + i915_gem_object_put(vm->scratch[i]); return -ENOMEM; } diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index c400aaa2287b..ee6312601c56 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -278,7 +278,7 @@ struct intel_engine_execlists { * * Note these register may be either mmio or HWSP shadow. */ - u32 *csb_status; + u64 *csb_status; /** * @csb_size: context status buffer FIFO size diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index 0412a44f25f2..a32aabce7901 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -1140,9 +1140,8 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine) /* Check in case we rollback so far we wrap [size/2] */ if (intel_ring_direction(rq->ring, - intel_ring_wrap(rq->ring, - rq->tail), - rq->ring->tail) > 0) + rq->tail, + rq->ring->tail + 8) > 0) rq->context->lrc.desc |= CTX_DESC_FORCE_RESTORE; active = rq; @@ -2464,7 +2463,7 @@ cancel_port_requests(struct intel_engine_execlists * const execlists) } static inline void -invalidate_csb_entries(const u32 *first, const u32 *last) +invalidate_csb_entries(const u64 *first, const u64 *last) { clflush((void *)first); clflush((void *)last); @@ -2496,14 +2495,25 @@ invalidate_csb_entries(const u32 *first, const u32 *last) * bits 47-57: sw context id of the lrc the GT switched away from * bits 58-63: sw counter of the lrc the GT switched away from */ -static inline bool -gen12_csb_parse(const struct intel_engine_execlists *execlists, const u32 *csb) -{ - u32 lower_dw = csb[0]; - u32 upper_dw = csb[1]; - bool ctx_to_valid = GEN12_CSB_CTX_VALID(lower_dw); - bool ctx_away_valid = GEN12_CSB_CTX_VALID(upper_dw); - bool new_queue = lower_dw & GEN12_CTX_STATUS_SWITCHED_TO_NEW_QUEUE; +static inline bool gen12_csb_parse(const u64 *csb) +{ + bool ctx_away_valid; + bool new_queue; + u64 entry; + + /* HSD#22011248461 */ + entry = READ_ONCE(*csb); + if (unlikely(entry == -1)) { + preempt_disable(); + if (wait_for_atomic_us((entry = READ_ONCE(*csb)) != -1, 50)) + GEM_WARN_ON("50us CSB timeout"); + preempt_enable(); + } + WRITE_ONCE(*(u64 *)csb, -1); + + ctx_away_valid = GEN12_CSB_CTX_VALID(upper_32_bits(entry)); + new_queue = + lower_32_bits(entry) & GEN12_CTX_STATUS_SWITCHED_TO_NEW_QUEUE; /* * The context switch detail is not guaranteed to be 5 when a preemption @@ -2513,7 +2523,7 @@ gen12_csb_parse(const struct intel_engine_execlists *execlists, const u32 *csb) * would require some extra handling, but we don't support that. */ if (!ctx_away_valid || new_queue) { - GEM_BUG_ON(!ctx_to_valid); + GEM_BUG_ON(!GEN12_CSB_CTX_VALID(lower_32_bits(entry))); return true; } @@ -2522,12 +2532,11 @@ gen12_csb_parse(const struct intel_engine_execlists *execlists, const u32 *csb) * context switch on an unsuccessful wait instruction since we always * use polling mode. */ - GEM_BUG_ON(GEN12_CTX_SWITCH_DETAIL(upper_dw)); + GEM_BUG_ON(GEN12_CTX_SWITCH_DETAIL(upper_32_bits(entry))); return false; } -static inline bool -gen8_csb_parse(const struct intel_engine_execlists *execlists, const u32 *csb) +static inline bool gen8_csb_parse(const u64 *csb) { return *csb & (GEN8_CTX_STATUS_IDLE_ACTIVE | GEN8_CTX_STATUS_PREEMPTED); } @@ -2535,7 +2544,7 @@ gen8_csb_parse(const struct intel_engine_execlists *execlists, const u32 *csb) static void process_csb(struct intel_engine_cs *engine) { struct intel_engine_execlists * const execlists = &engine->execlists; - const u32 * const buf = execlists->csb_status; + const u64 * const buf = execlists->csb_status; const u8 num_entries = execlists->csb_size; u8 head, tail; @@ -2616,12 +2625,14 @@ static void process_csb(struct intel_engine_cs *engine) */ ENGINE_TRACE(engine, "csb[%d]: status=0x%08x:0x%08x\n", - head, buf[2 * head + 0], buf[2 * head + 1]); + head, + upper_32_bits(buf[head]), + lower_32_bits(buf[head])); if (INTEL_GEN(engine->i915) >= 12) - promote = gen12_csb_parse(execlists, buf + 2 * head); + promote = gen12_csb_parse(buf + head); else - promote = gen8_csb_parse(execlists, buf + 2 * head); + promote = gen8_csb_parse(buf + head); if (promote) { struct i915_request * const *old = execlists->active; @@ -2649,6 +2660,9 @@ static void process_csb(struct intel_engine_cs *engine) smp_wmb(); /* complete the seqlock */ WRITE_ONCE(execlists->active, execlists->inflight); + /* XXX Magic delay for tgl */ + ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR); + WRITE_ONCE(execlists->pending[0], NULL); } else { if (GEM_WARN_ON(!*execlists->active)) { @@ -4005,6 +4019,8 @@ static void reset_csb_pointers(struct intel_engine_cs *engine) WRITE_ONCE(*execlists->csb_write, reset_value); wmb(); /* Make sure this is visible to HW (paranoia?) */ + /* Check that the GPU does indeed update the CSB entries! */ + memset(execlists->csb_status, -1, (reset_value + 1) * sizeof(u64)); invalidate_csb_entries(&execlists->csb_status[0], &execlists->csb_status[reset_value]); @@ -5157,7 +5173,7 @@ int intel_execlists_submission_setup(struct intel_engine_cs *engine) } execlists->csb_status = - &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX]; + (u64 *)&engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX]; execlists->csb_write = &engine->status_page.addr[intel_hws_csb_write_index(i915)]; diff --git a/drivers/gpu/drm/i915/gt/intel_mocs.c b/drivers/gpu/drm/i915/gt/intel_mocs.c index 632e08a4592b..b8f56e62158e 100644 --- a/drivers/gpu/drm/i915/gt/intel_mocs.c +++ b/drivers/gpu/drm/i915/gt/intel_mocs.c @@ -234,11 +234,17 @@ static const struct drm_i915_mocs_entry broxton_mocs_table[] = { L3_1_UC) static const struct drm_i915_mocs_entry tgl_mocs_table[] = { - /* Base - Error (Reserved for Non-Use) */ - MOCS_ENTRY(0, 0x0, 0x0), - /* Base - Reserved */ - MOCS_ENTRY(1, 0x0, 0x0), - + /* + * NOTE: + * Reserved and unspecified MOCS indices have been set to (L3 + LCC). + * These reserved entries should never be used, they may be changed + * to low performant variants with better coherency in the future if + * more entries are needed. We are programming index I915_MOCS_PTE(1) + * only, __init_mocs_table() take care to program unused index with + * this entry. + */ + MOCS_ENTRY(1, LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_3_WB), GEN11_MOCS_ENTRIES, /* Implicitly enable L1 - HDC:L1 + L3 + LLC */ diff --git a/drivers/gpu/drm/i915/gt/selftest_reset.c b/drivers/gpu/drm/i915/gt/selftest_reset.c index 35406ecdf0b2..ef5aeebbeeb0 100644 --- a/drivers/gpu/drm/i915/gt/selftest_reset.c +++ b/drivers/gpu/drm/i915/gt/selftest_reset.c @@ -3,9 +3,203 @@ * Copyright © 2018 Intel Corporation */ +#include <linux/crc32.h> + +#include "gem/i915_gem_stolen.h" + +#include "i915_memcpy.h" #include "i915_selftest.h" #include "selftests/igt_reset.h" #include "selftests/igt_atomic.h" +#include "selftests/igt_spinner.h" + +static int +__igt_reset_stolen(struct intel_gt *gt, + intel_engine_mask_t mask, + const char *msg) +{ + struct i915_ggtt *ggtt = >->i915->ggtt; + const struct resource *dsm = >->i915->dsm; + resource_size_t num_pages, page; + struct intel_engine_cs *engine; + intel_wakeref_t wakeref; + enum intel_engine_id id; + struct igt_spinner spin; + long max, count; + void *tmp; + u32 *crc; + int err; + + if (!drm_mm_node_allocated(&ggtt->error_capture)) + return 0; + + num_pages = resource_size(dsm) >> PAGE_SHIFT; + if (!num_pages) + return 0; + + crc = kmalloc_array(num_pages, sizeof(u32), GFP_KERNEL); + if (!crc) + return -ENOMEM; + + tmp = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!tmp) { + err = -ENOMEM; + goto err_crc; + } + + igt_global_reset_lock(gt); + wakeref = intel_runtime_pm_get(gt->uncore->rpm); + + err = igt_spinner_init(&spin, gt); + if (err) + goto err_lock; + + for_each_engine(engine, gt, id) { + struct intel_context *ce; + struct i915_request *rq; + + if (!(mask & engine->mask)) + continue; + + if (!intel_engine_can_store_dword(engine)) + continue; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) { + err = PTR_ERR(ce); + goto err_spin; + } + rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK); + intel_context_put(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_spin; + } + i915_request_add(rq); + } + + for (page = 0; page < num_pages; page++) { + dma_addr_t dma = (dma_addr_t)dsm->start + (page << PAGE_SHIFT); + void __iomem *s; + void *in; + + ggtt->vm.insert_page(&ggtt->vm, dma, + ggtt->error_capture.start, + I915_CACHE_NONE, 0); + mb(); + + s = io_mapping_map_wc(&ggtt->iomap, + ggtt->error_capture.start, + PAGE_SIZE); + + if (!__drm_mm_interval_first(>->i915->mm.stolen, + page << PAGE_SHIFT, + ((page + 1) << PAGE_SHIFT) - 1)) + memset32(s, STACK_MAGIC, PAGE_SIZE / sizeof(u32)); + + in = s; + if (i915_memcpy_from_wc(tmp, s, PAGE_SIZE)) + in = tmp; + crc[page] = crc32_le(0, in, PAGE_SIZE); + + io_mapping_unmap(s); + } + mb(); + ggtt->vm.clear_range(&ggtt->vm, ggtt->error_capture.start, PAGE_SIZE); + + if (mask == ALL_ENGINES) { + intel_gt_reset(gt, mask, NULL); + } else { + for_each_engine(engine, gt, id) { + if (mask & engine->mask) + intel_engine_reset(engine, NULL); + } + } + + max = -1; + count = 0; + for (page = 0; page < num_pages; page++) { + dma_addr_t dma = (dma_addr_t)dsm->start + (page << PAGE_SHIFT); + void __iomem *s; + void *in; + u32 x; + + ggtt->vm.insert_page(&ggtt->vm, dma, + ggtt->error_capture.start, + I915_CACHE_NONE, 0); + mb(); + + s = io_mapping_map_wc(&ggtt->iomap, + ggtt->error_capture.start, + PAGE_SIZE); + + in = s; + if (i915_memcpy_from_wc(tmp, s, PAGE_SIZE)) + in = tmp; + x = crc32_le(0, in, PAGE_SIZE); + + if (x != crc[page] && + !__drm_mm_interval_first(>->i915->mm.stolen, + page << PAGE_SHIFT, + ((page + 1) << PAGE_SHIFT) - 1)) { + pr_debug("unused stolen page %pa modified by GPU reset\n", + &page); + if (count++ == 0) + igt_hexdump(in, PAGE_SIZE); + max = page; + } + + io_mapping_unmap(s); + } + mb(); + ggtt->vm.clear_range(&ggtt->vm, ggtt->error_capture.start, PAGE_SIZE); + + if (count > 0) { + pr_info("%s reset clobbered %ld pages of stolen, last clobber at page %ld\n", + msg, count, max); + } + if (max >= I915_GEM_STOLEN_BIAS >> PAGE_SHIFT) { + pr_err("%s reset clobbered unreserved area [above %x] of stolen; may cause severe faults\n", + msg, I915_GEM_STOLEN_BIAS); + err = -EINVAL; + } + +err_spin: + igt_spinner_fini(&spin); + +err_lock: + intel_runtime_pm_put(gt->uncore->rpm, wakeref); + igt_global_reset_unlock(gt); + + kfree(tmp); +err_crc: + kfree(crc); + return err; +} + +static int igt_reset_device_stolen(void *arg) +{ + return __igt_reset_stolen(arg, ALL_ENGINES, "device"); +} + +static int igt_reset_engines_stolen(void *arg) +{ + struct intel_gt *gt = arg; + struct intel_engine_cs *engine; + enum intel_engine_id id; + int err; + + if (!intel_has_reset_engine(gt)) + return 0; + + for_each_engine(engine, gt, id) { + err = __igt_reset_stolen(gt, engine->mask, engine->name); + if (err) + return err; + } + + return 0; +} static int igt_global_reset(void *arg) { @@ -164,6 +358,8 @@ int intel_reset_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { SUBTEST(igt_global_reset), /* attempt to recover GPU first */ + SUBTEST(igt_reset_device_stolen), + SUBTEST(igt_reset_engines_stolen), SUBTEST(igt_wedged_reset), SUBTEST(igt_atomic_reset), SUBTEST(igt_atomic_engine_reset), diff --git a/drivers/gpu/drm/i915/gt/shmem_utils.c b/drivers/gpu/drm/i915/gt/shmem_utils.c index 43c7acbdc79d..f011ea42487e 100644 --- a/drivers/gpu/drm/i915/gt/shmem_utils.c +++ b/drivers/gpu/drm/i915/gt/shmem_utils.c @@ -49,80 +49,40 @@ struct file *shmem_create_from_object(struct drm_i915_gem_object *obj) return file; } -static size_t shmem_npte(struct file *file) -{ - return file->f_mapping->host->i_size >> PAGE_SHIFT; -} - -static void __shmem_unpin_map(struct file *file, void *ptr, size_t n_pte) -{ - unsigned long pfn; - - vunmap(ptr); - - for (pfn = 0; pfn < n_pte; pfn++) { - struct page *page; - - page = shmem_read_mapping_page_gfp(file->f_mapping, pfn, - GFP_KERNEL); - if (!WARN_ON(IS_ERR(page))) { - put_page(page); - put_page(page); - } - } -} - void *shmem_pin_map(struct file *file) { - const size_t n_pte = shmem_npte(file); - pte_t *stack[32], **ptes, **mem; - struct vm_struct *area; - unsigned long pfn; - - mem = stack; - if (n_pte > ARRAY_SIZE(stack)) { - mem = kvmalloc_array(n_pte, sizeof(*mem), GFP_KERNEL); - if (!mem) - return NULL; - } + struct page **pages; + size_t n_pages, i; + void *vaddr; - area = alloc_vm_area(n_pte << PAGE_SHIFT, mem); - if (!area) { - if (mem != stack) - kvfree(mem); + n_pages = file->f_mapping->host->i_size >> PAGE_SHIFT; + pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL); + if (!pages) return NULL; - } - ptes = mem; - for (pfn = 0; pfn < n_pte; pfn++) { - struct page *page; - - page = shmem_read_mapping_page_gfp(file->f_mapping, pfn, - GFP_KERNEL); - if (IS_ERR(page)) + for (i = 0; i < n_pages; i++) { + pages[i] = shmem_read_mapping_page_gfp(file->f_mapping, i, + GFP_KERNEL); + if (IS_ERR(pages[i])) goto err_page; - - **ptes++ = mk_pte(page, PAGE_KERNEL); } - if (mem != stack) - kvfree(mem); - + vaddr = vmap(pages, n_pages, VM_MAP_PUT_PAGES, PAGE_KERNEL); + if (!vaddr) + goto err_page; mapping_set_unevictable(file->f_mapping); - return area->addr; - + return vaddr; err_page: - if (mem != stack) - kvfree(mem); - - __shmem_unpin_map(file, area->addr, pfn); + while (--i >= 0) + put_page(pages[i]); + kvfree(pages); return NULL; } void shmem_unpin_map(struct file *file, void *ptr) { mapping_clear_unevictable(file->f_mapping); - __shmem_unpin_map(file, ptr, shmem_npte(file)); + vfree(ptr); } static int __shmem_rw(struct file *file, loff_t off, diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index eef9a821c49c..8426d5974669 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -33,6 +33,8 @@ #include <uapi/drm/i915_drm.h> #include <uapi/drm/drm_fourcc.h> +#include <asm/hypervisor.h> + #include <linux/io-mapping.h> #include <linux/i2c.h> #include <linux/i2c-algo-bit.h> @@ -1760,7 +1762,9 @@ static inline bool intel_vtd_active(void) if (intel_iommu_gfx_mapped) return true; #endif - return false; + + /* Running as a guest, we assume the host is enforcing VT'd */ + return !hypervisor_is_type(X86_HYPER_NATIVE); } static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv) diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index a635ec8d0b94..cf6e47adfde6 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -1312,7 +1312,7 @@ capture_vma(struct intel_engine_capture_vma *next, } strcpy(c->name, name); - c->vma = i915_vma_get(vma); + c->vma = vma; /* reference held while active */ c->next = next; return c; @@ -1402,7 +1402,6 @@ intel_engine_coredump_add_vma(struct intel_engine_coredump *ee, compress)); i915_active_release(&vma->active); - i915_vma_put(vma); capture = this->next; kfree(this); diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index 263ffcb832b7..97ded2a59cf4 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -1209,6 +1209,18 @@ unclaimed_reg_debug(struct intel_uncore *uncore, spin_unlock(&uncore->debug->lock); } +#define __vgpu_read(x) \ +static u##x \ +vgpu_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \ + u##x val = __raw_uncore_read##x(uncore, reg); \ + trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ + return val; \ +} +__vgpu_read(8) +__vgpu_read(16) +__vgpu_read(32) +__vgpu_read(64) + #define GEN2_READ_HEADER(x) \ u##x val = 0; \ assert_rpm_wakelock_held(uncore->rpm); @@ -1414,6 +1426,16 @@ __gen_reg_write_funcs(gen8); #undef GEN6_WRITE_FOOTER #undef GEN6_WRITE_HEADER +#define __vgpu_write(x) \ +static void \ +vgpu_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \ + trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ + __raw_uncore_write##x(uncore, reg, val); \ +} +__vgpu_write(8) +__vgpu_write(16) +__vgpu_write(32) + #define ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, x) \ do { \ (uncore)->funcs.mmio_writeb = x##_write8; \ @@ -1735,7 +1757,10 @@ static void uncore_raw_init(struct intel_uncore *uncore) { GEM_BUG_ON(intel_uncore_has_forcewake(uncore)); - if (IS_GEN(uncore->i915, 5)) { + if (intel_vgpu_active(uncore->i915)) { + ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, vgpu); + ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, vgpu); + } else if (IS_GEN(uncore->i915, 5)) { ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen5); ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen5); } else { diff --git a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c index 4d29568be3f5..ac038572164d 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c @@ -481,7 +481,7 @@ static void mtk_drm_crtc_hw_config(struct mtk_drm_crtc *mtk_crtc) mbox_flush(mtk_crtc->cmdq_client->chan, 2000); cmdq_handle = cmdq_pkt_create(mtk_crtc->cmdq_client, PAGE_SIZE); cmdq_pkt_clear_event(cmdq_handle, mtk_crtc->cmdq_event); - cmdq_pkt_wfe(cmdq_handle, mtk_crtc->cmdq_event); + cmdq_pkt_wfe(cmdq_handle, mtk_crtc->cmdq_event, false); mtk_crtc_ddp_config(crtc, cmdq_handle); cmdq_pkt_finalize(cmdq_handle); cmdq_pkt_flush_async(cmdq_handle, ddp_cmdq_cb, cmdq_handle); diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 70b3bee27850..eb4b7df02ca0 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -647,7 +647,7 @@ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, /* Don't evict this BO if it's outside of the * requested placement range */ - if (place->fpfn >= (bo->mem.start + bo->mem.size) || + if (place->fpfn >= (bo->mem.start + bo->mem.num_pages) || (place->lpfn && place->lpfn <= bo->mem.start)) return false; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c index 7f0310441da1..73116ec70ba5 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c @@ -432,6 +432,7 @@ static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt) int ret = 0; static size_t sgl_size; static size_t sgt_size; + struct scatterlist *sg; if (vmw_tt->mapped) return 0; @@ -454,13 +455,15 @@ static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt) if (unlikely(ret != 0)) return ret; - ret = __sg_alloc_table_from_pages - (&vmw_tt->sgt, vsgt->pages, vsgt->num_pages, 0, - (unsigned long) vsgt->num_pages << PAGE_SHIFT, - dma_get_max_seg_size(dev_priv->dev->dev), - GFP_KERNEL); - if (unlikely(ret != 0)) + sg = __sg_alloc_table_from_pages(&vmw_tt->sgt, vsgt->pages, + vsgt->num_pages, 0, + (unsigned long) vsgt->num_pages << PAGE_SHIFT, + dma_get_max_seg_size(dev_priv->dev->dev), + NULL, 0, GFP_KERNEL); + if (IS_ERR(sg)) { + ret = PTR_ERR(sg); goto out_sg_alloc_fail; + } if (vsgt->num_pages > vmw_tt->sgt.orig_nents) { uint64_t over_alloc = diff --git a/drivers/hid/hid-rmi.c b/drivers/hid/hid-rmi.c index 7f41213d5ae3..311eee599ce9 100644 --- a/drivers/hid/hid-rmi.c +++ b/drivers/hid/hid-rmi.c @@ -720,7 +720,7 @@ static int rmi_probe(struct hid_device *hdev, const struct hid_device_id *id) } if (data->device_flags & RMI_DEVICE_HAS_PHYS_BUTTONS) - rmi_hid_pdata.f30_data.disable = true; + rmi_hid_pdata.gpio_data.disable = true; data->xport.dev = hdev->dev.parent; data->xport.pdata = rmi_hid_pdata; diff --git a/drivers/hid/intel-ish-hid/ipc/ipc.c b/drivers/hid/intel-ish-hid/ipc/ipc.c index 8f8dfdf64833..a45ac7fa417b 100644 --- a/drivers/hid/intel-ish-hid/ipc/ipc.c +++ b/drivers/hid/intel-ish-hid/ipc/ipc.c @@ -755,7 +755,7 @@ static int _ish_hw_reset(struct ishtp_device *dev) csr |= PCI_D3hot; pci_write_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, csr); - mdelay(pdev->d3_delay); + mdelay(pdev->d3hot_delay); csr &= ~PCI_PM_CTRL_STATE_MASK; csr |= PCI_D0; diff --git a/drivers/hv/hv_balloon.c b/drivers/hv/hv_balloon.c index 32e3bc0aa665..b64d2efbefe7 100644 --- a/drivers/hv/hv_balloon.c +++ b/drivers/hv/hv_balloon.c @@ -726,7 +726,7 @@ static void hv_mem_hot_add(unsigned long start, unsigned long size, nid = memory_add_physaddr_to_nid(PFN_PHYS(start_pfn)); ret = add_memory(nid, PFN_PHYS((start_pfn)), - (HA_CHUNK << PAGE_SHIFT)); + (HA_CHUNK << PAGE_SHIFT), MEMHP_MERGE_RESOURCE); if (ret) { pr_err("hot_add memory failed error is %d\n", ret); diff --git a/drivers/hwmon/s3c-hwmon.c b/drivers/hwmon/s3c-hwmon.c index b490fe3d2ee8..f2703c5460d0 100644 --- a/drivers/hwmon/s3c-hwmon.c +++ b/drivers/hwmon/s3c-hwmon.c @@ -20,7 +20,7 @@ #include <linux/hwmon.h> #include <linux/hwmon-sysfs.h> -#include <plat/adc.h> +#include <linux/soc/samsung/s3c-adc.h> #include <linux/platform_data/hwmon-s3c.h> struct s3c_hwmon_attr { diff --git a/drivers/i2c/Kconfig b/drivers/i2c/Kconfig index bae1dc08ec9a..438905e2a1d0 100644 --- a/drivers/i2c/Kconfig +++ b/drivers/i2c/Kconfig @@ -101,7 +101,6 @@ source "drivers/i2c/busses/Kconfig" config I2C_STUB tristate "I2C/SMBus Test Stub" depends on m - default 'n' help This module may be useful to developers of SMBus client drivers, especially for certain kinds of sensor chips. @@ -126,6 +125,14 @@ config I2C_SLAVE_EEPROM This backend makes Linux behave like an I2C EEPROM. Please read Documentation/i2c/slave-eeprom-backend.rst for further details. +config I2C_SLAVE_TESTUNIT + tristate "I2C eeprom testunit driver" + help + This backend can be used to trigger test cases for I2C bus masters + which require a remote device with certain capabilities, e.g. + multi-master, SMBus Host Notify, etc. Please read + Documentation/i2c/slave-testunit-backend.rst for further details. + endif config I2C_DEBUG_CORE diff --git a/drivers/i2c/Makefile b/drivers/i2c/Makefile index bed6ba63c983..c1d493dc9bac 100644 --- a/drivers/i2c/Makefile +++ b/drivers/i2c/Makefile @@ -16,5 +16,6 @@ obj-$(CONFIG_I2C_MUX) += i2c-mux.o obj-y += algos/ busses/ muxes/ obj-$(CONFIG_I2C_STUB) += i2c-stub.o obj-$(CONFIG_I2C_SLAVE_EEPROM) += i2c-slave-eeprom.o +obj-$(CONFIG_I2C_SLAVE_TESTUNIT) += i2c-slave-testunit.o ccflags-$(CONFIG_I2C_DEBUG_CORE) := -DDEBUG diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index 293e7a0760e7..a4f473ef4e5c 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -147,6 +147,7 @@ config I2C_I801 Tiger Lake (PCH) Jasper Lake (SOC) Emmitsburg (PCH) + Alder Lake (PCH) This driver can also be built as a module. If so, the module will be called i2c-i801. @@ -730,6 +731,19 @@ config I2C_LPC2K This driver can also be built as a module. If so, the module will be called i2c-lpc2k. +config I2C_MLXBF + tristate "Mellanox BlueField I2C controller" + depends on ARM64 + help + Enabling this option will add I2C SMBus support for Mellanox BlueField + system. + + This driver can also be built as a module. If so, the module will be + called i2c-mlxbf. + + This driver implements an I2C SMBus host controller and enables both + master and slave functions. + config I2C_MESON tristate "Amlogic Meson I2C controller" depends on ARCH_MESON || COMPILE_TEST @@ -840,7 +854,6 @@ config I2C_PASEMI config I2C_PCA_PLATFORM tristate "PCA9564/PCA9665 as platform device" select I2C_ALGOPCA - default n help This driver supports a memory mapped Philips PCA9564/PCA9665 parallel bus to I2C bus controller. @@ -1026,6 +1039,7 @@ config I2C_STM32F7 tristate "STMicroelectronics STM32F7 I2C support" depends on ARCH_STM32 || COMPILE_TEST select I2C_SLAVE + select I2C_SMBUS help Enable this option to add support for STM32 I2C controller embedded in STM32F7 SoCs. @@ -1181,6 +1195,8 @@ config I2C_RCAR tristate "Renesas R-Car I2C Controller" depends on ARCH_RENESAS || COMPILE_TEST select I2C_SLAVE + select I2C_SMBUS + select RESET_CONTROLLER if ARCH_RCAR_GEN3 help If you say yes to this option, support will be included for the R-Car I2C controller. @@ -1240,7 +1256,6 @@ config I2C_TAOS_EVM depends on TTY select SERIO select SERIO_SERPORT - default n help This supports TAOS evaluation modules on serial port. In order to use this driver, you will need the inputattach tool, which is part @@ -1324,7 +1339,6 @@ config I2C_PCA_ISA tristate "PCA9564/PCA9665 on an ISA bus" depends on ISA select I2C_ALGOPCA - default n help This driver supports ISA boards using the Philips PCA9564/PCA9665 parallel bus to I2C bus controller. diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile index 19aff0e45cb5..683c49faca05 100644 --- a/drivers/i2c/busses/Makefile +++ b/drivers/i2c/busses/Makefile @@ -140,6 +140,7 @@ obj-$(CONFIG_I2C_BRCMSTB) += i2c-brcmstb.o obj-$(CONFIG_I2C_CROS_EC_TUNNEL) += i2c-cros-ec-tunnel.o obj-$(CONFIG_I2C_ELEKTOR) += i2c-elektor.o obj-$(CONFIG_I2C_ICY) += i2c-icy.o +obj-$(CONFIG_I2C_MLXBF) += i2c-mlxbf.o obj-$(CONFIG_I2C_MLXCPLD) += i2c-mlxcpld.o obj-$(CONFIG_I2C_OPAL) += i2c-opal.o obj-$(CONFIG_I2C_PCA_ISA) += i2c-pca-isa.o diff --git a/drivers/i2c/busses/i2c-amd-mp2-plat.c b/drivers/i2c/busses/i2c-amd-mp2-plat.c index 17df9e8845b6..506433bc0ff2 100644 --- a/drivers/i2c/busses/i2c-amd-mp2-plat.c +++ b/drivers/i2c/busses/i2c-amd-mp2-plat.c @@ -155,7 +155,7 @@ static int i2c_amd_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num) struct amd_i2c_dev *i2c_dev = i2c_get_adapdata(adap); int i; struct i2c_msg *pmsg; - int err; + int err = 0; /* the adapter might have been deleted while waiting for the bus lock */ if (unlikely(!i2c_dev->common.mp2_dev)) diff --git a/drivers/i2c/busses/i2c-bcm2835.c b/drivers/i2c/busses/i2c-bcm2835.c index 5dc519516292..37443edbf754 100644 --- a/drivers/i2c/busses/i2c-bcm2835.c +++ b/drivers/i2c/busses/i2c-bcm2835.c @@ -421,11 +421,9 @@ static int bcm2835_i2c_probe(struct platform_device *pdev) return PTR_ERR(i2c_dev->regs); mclk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(mclk)) { - if (PTR_ERR(mclk) != -EPROBE_DEFER) - dev_err(&pdev->dev, "Could not get clock\n"); - return PTR_ERR(mclk); - } + if (IS_ERR(mclk)) + return dev_err_probe(&pdev->dev, PTR_ERR(mclk), + "Could not get clock\n"); i2c_dev->bus_clk = bcm2835_i2c_register_div(&pdev->dev, mclk, i2c_dev); diff --git a/drivers/i2c/busses/i2c-efm32.c b/drivers/i2c/busses/i2c-efm32.c index 838ce0947191..f6e13ceeb2b3 100644 --- a/drivers/i2c/busses/i2c-efm32.c +++ b/drivers/i2c/busses/i2c-efm32.c @@ -332,21 +332,15 @@ static int efm32_i2c_probe(struct platform_device *pdev) return ret; } - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) { - dev_err(&pdev->dev, "failed to determine base address\n"); - return -ENODEV; - } + ddata->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(ddata->base)) + return PTR_ERR(ddata->base); if (resource_size(res) < 0x42) { dev_err(&pdev->dev, "memory resource too small\n"); return -EINVAL; } - ddata->base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(ddata->base)) - return PTR_ERR(ddata->base); - ret = platform_get_irq(pdev, 0); if (ret <= 0) { if (!ret) diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index bffca729e1c7..ae90713443fa 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -71,6 +71,7 @@ * Tiger Lake-H (PCH) 0x43a3 32 hard yes yes yes * Jasper Lake (SOC) 0x4da3 32 hard yes yes yes * Comet Lake-V (PCH) 0xa3a3 32 hard yes yes yes + * Alder Lake-S (PCH) 0x7aa3 32 hard yes yes yes * * Features supported by this driver: * Software PEC no @@ -228,6 +229,7 @@ #define PCI_DEVICE_ID_INTEL_ELKHART_LAKE_SMBUS 0x4b23 #define PCI_DEVICE_ID_INTEL_JASPER_LAKE_SMBUS 0x4da3 #define PCI_DEVICE_ID_INTEL_BROXTON_SMBUS 0x5ad4 +#define PCI_DEVICE_ID_INTEL_ALDER_LAKE_S_SMBUS 0x7aa3 #define PCI_DEVICE_ID_INTEL_LYNXPOINT_SMBUS 0x8c22 #define PCI_DEVICE_ID_INTEL_WILDCATPOINT_SMBUS 0x8ca2 #define PCI_DEVICE_ID_INTEL_WELLSBURG_SMBUS 0x8d22 @@ -1081,6 +1083,7 @@ static const struct pci_device_id i801_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TIGERLAKE_LP_SMBUS) }, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TIGERLAKE_H_SMBUS) }, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_JASPER_LAKE_SMBUS) }, + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ALDER_LAKE_S_SMBUS) }, { 0, } }; @@ -1274,6 +1277,7 @@ static const struct { /* * Additional individual entries were added after verification. */ + { "Latitude 5480", 0x29 }, { "Vostro V131", 0x1d }, }; @@ -1767,6 +1771,7 @@ static int i801_probe(struct pci_dev *dev, const struct pci_device_id *id) case PCI_DEVICE_ID_INTEL_TIGERLAKE_H_SMBUS: case PCI_DEVICE_ID_INTEL_JASPER_LAKE_SMBUS: case PCI_DEVICE_ID_INTEL_EBG_SMBUS: + case PCI_DEVICE_ID_INTEL_ALDER_LAKE_S_SMBUS: priv->features |= FEATURE_BLOCK_PROC; priv->features |= FEATURE_I2C_BLOCK_READ; priv->features |= FEATURE_IRQ; diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c index 0ab5381aa012..c98529c76348 100644 --- a/drivers/i2c/busses/i2c-imx.c +++ b/drivers/i2c/busses/i2c-imx.c @@ -1159,11 +1159,9 @@ static int i2c_imx_probe(struct platform_device *pdev) /* Get I2C clock */ i2c_imx->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(i2c_imx->clk)) { - if (PTR_ERR(i2c_imx->clk) != -EPROBE_DEFER) - dev_err(&pdev->dev, "can't get I2C clock\n"); - return PTR_ERR(i2c_imx->clk); - } + if (IS_ERR(i2c_imx->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(i2c_imx->clk), + "can't get I2C clock\n"); ret = clk_prepare_enable(i2c_imx->clk); if (ret) { @@ -1171,14 +1169,6 @@ static int i2c_imx_probe(struct platform_device *pdev) return ret; } - /* Request IRQ */ - ret = devm_request_irq(&pdev->dev, irq, i2c_imx_isr, IRQF_SHARED, - pdev->name, i2c_imx); - if (ret) { - dev_err(&pdev->dev, "can't claim irq %d\n", irq); - goto clk_disable; - } - /* Init queue */ init_waitqueue_head(&i2c_imx->queue); @@ -1197,6 +1187,14 @@ static int i2c_imx_probe(struct platform_device *pdev) if (ret < 0) goto rpm_disable; + /* Request IRQ */ + ret = request_threaded_irq(irq, i2c_imx_isr, NULL, IRQF_SHARED, + pdev->name, i2c_imx); + if (ret) { + dev_err(&pdev->dev, "can't claim irq %d\n", irq); + goto rpm_disable; + } + /* Set up clock divider */ i2c_imx->bitrate = I2C_MAX_STANDARD_MODE_FREQ; ret = of_property_read_u32(pdev->dev.of_node, @@ -1239,13 +1237,12 @@ static int i2c_imx_probe(struct platform_device *pdev) clk_notifier_unregister: clk_notifier_unregister(i2c_imx->clk, &i2c_imx->clk_change_nb); + free_irq(irq, i2c_imx); rpm_disable: pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); pm_runtime_set_suspended(&pdev->dev); pm_runtime_dont_use_autosuspend(&pdev->dev); - -clk_disable: clk_disable_unprepare(i2c_imx->clk); return ret; } @@ -1253,7 +1250,7 @@ clk_disable: static int i2c_imx_remove(struct platform_device *pdev) { struct imx_i2c_struct *i2c_imx = platform_get_drvdata(pdev); - int ret; + int irq, ret; ret = pm_runtime_get_sync(&pdev->dev); if (ret < 0) @@ -1273,6 +1270,9 @@ static int i2c_imx_remove(struct platform_device *pdev) imx_i2c_write_reg(0, i2c_imx, IMX_I2C_I2SR); clk_notifier_unregister(i2c_imx->clk, &i2c_imx->clk_change_nb); + irq = platform_get_irq(pdev, 0); + if (irq >= 0) + free_irq(irq, i2c_imx); clk_disable_unprepare(i2c_imx->clk); pm_runtime_put_noidle(&pdev->dev); diff --git a/drivers/i2c/busses/i2c-ismt.c b/drivers/i2c/busses/i2c-ismt.c index 2f95e25a10f7..a35a27c320e7 100644 --- a/drivers/i2c/busses/i2c-ismt.c +++ b/drivers/i2c/busses/i2c-ismt.c @@ -77,6 +77,7 @@ #define PCI_DEVICE_ID_INTEL_S1200_SMT1 0x0c5a #define PCI_DEVICE_ID_INTEL_CDF_SMT 0x18ac #define PCI_DEVICE_ID_INTEL_DNV_SMT 0x19ac +#define PCI_DEVICE_ID_INTEL_EBG_SMT 0x1bff #define PCI_DEVICE_ID_INTEL_AVOTON_SMT 0x1f15 #define ISMT_DESC_ENTRIES 2 /* number of descriptor entries */ @@ -176,14 +177,12 @@ struct ismt_priv { u8 buffer[I2C_SMBUS_BLOCK_MAX + 16]; /* temp R/W data buffer */ }; -/** - * ismt_ids - PCI device IDs supported by this driver - */ static const struct pci_device_id ismt_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_S1200_SMT0) }, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_S1200_SMT1) }, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CDF_SMT) }, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_DNV_SMT) }, + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_EBG_SMT) }, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_AVOTON_SMT) }, { 0, } }; @@ -197,6 +196,8 @@ MODULE_PARM_DESC(bus_speed, "Bus Speed in kHz (0 = BIOS default)"); /** * __ismt_desc_dump() - dump the contents of a specific descriptor + * @dev: the iSMT device + * @desc: the iSMT hardware descriptor */ static void __ismt_desc_dump(struct device *dev, const struct ismt_desc *desc) { @@ -628,11 +629,6 @@ static u32 ismt_func(struct i2c_adapter *adap) I2C_FUNC_SMBUS_PEC; } -/** - * smbus_algorithm - the adapter algorithm and supported functionality - * @smbus_xfer: the adapter algorithm - * @functionality: functionality supported by the adapter - */ static const struct i2c_algorithm smbus_algorithm = { .smbus_xfer = ismt_access, .functionality = ismt_func, diff --git a/drivers/i2c/busses/i2c-jz4780.c b/drivers/i2c/busses/i2c-jz4780.c index ba831df6661e..cb4a25ebb890 100644 --- a/drivers/i2c/busses/i2c-jz4780.c +++ b/drivers/i2c/busses/i2c-jz4780.c @@ -752,6 +752,7 @@ static const struct ingenic_i2c_config x1000_i2c_config = { }; static const struct of_device_id jz4780_i2c_of_matches[] = { + { .compatible = "ingenic,jz4770-i2c", .data = &jz4780_i2c_config }, { .compatible = "ingenic,jz4780-i2c", .data = &jz4780_i2c_config }, { .compatible = "ingenic,x1000-i2c", .data = &x1000_i2c_config }, { /* sentinel */ } @@ -856,7 +857,7 @@ static struct platform_driver jz4780_i2c_driver = { .remove = jz4780_i2c_remove, .driver = { .name = "jz4780-i2c", - .of_match_table = of_match_ptr(jz4780_i2c_of_matches), + .of_match_table = jz4780_i2c_of_matches, }, }; diff --git a/drivers/i2c/busses/i2c-mlxbf.c b/drivers/i2c/busses/i2c-mlxbf.c new file mode 100644 index 000000000000..ee59e0da082d --- /dev/null +++ b/drivers/i2c/busses/i2c-mlxbf.c @@ -0,0 +1,2506 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Mellanox BlueField I2C bus driver + * + * Copyright (C) 2020 Mellanox Technologies, Ltd. + */ + +#include <linux/acpi.h> +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/i2c.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/string.h> + +/* Defines what functionality is present. */ +#define MLXBF_I2C_FUNC_SMBUS_BLOCK \ + (I2C_FUNC_SMBUS_BLOCK_DATA | I2C_FUNC_SMBUS_BLOCK_PROC_CALL) + +#define MLXBF_I2C_FUNC_SMBUS_DEFAULT \ + (I2C_FUNC_SMBUS_BYTE | I2C_FUNC_SMBUS_BYTE_DATA | \ + I2C_FUNC_SMBUS_WORD_DATA | I2C_FUNC_SMBUS_I2C_BLOCK | \ + I2C_FUNC_SMBUS_PROC_CALL) + +#define MLXBF_I2C_FUNC_ALL \ + (MLXBF_I2C_FUNC_SMBUS_DEFAULT | MLXBF_I2C_FUNC_SMBUS_BLOCK | \ + I2C_FUNC_SMBUS_QUICK | I2C_FUNC_SLAVE) + +#define MLXBF_I2C_SMBUS_MAX 3 + +/* Shared resources info in BlueField platforms. */ + +#define MLXBF_I2C_COALESCE_TYU_ADDR 0x02801300 +#define MLXBF_I2C_COALESCE_TYU_SIZE 0x010 + +#define MLXBF_I2C_GPIO_TYU_ADDR 0x02802000 +#define MLXBF_I2C_GPIO_TYU_SIZE 0x100 + +#define MLXBF_I2C_COREPLL_TYU_ADDR 0x02800358 +#define MLXBF_I2C_COREPLL_TYU_SIZE 0x008 + +#define MLXBF_I2C_COREPLL_YU_ADDR 0x02800c30 +#define MLXBF_I2C_COREPLL_YU_SIZE 0x00c + +#define MLXBF_I2C_SHARED_RES_MAX 3 + +/* + * Note that the following SMBus, CAUSE, GPIO and PLL register addresses + * refer to their respective offsets relative to the corresponding + * memory-mapped region whose addresses are specified in either the DT or + * the ACPI tables or above. + */ + +/* + * SMBus Master core clock frequency. Timing configurations are + * strongly dependent on the core clock frequency of the SMBus + * Master. Default value is set to 400MHz. + */ +#define MLXBF_I2C_TYU_PLL_OUT_FREQ (400 * 1000 * 1000) +/* Reference clock for Bluefield 1 - 156 MHz. */ +#define MLXBF_I2C_TYU_PLL_IN_FREQ (156 * 1000 * 1000) +/* Reference clock for BlueField 2 - 200 MHz. */ +#define MLXBF_I2C_YU_PLL_IN_FREQ (200 * 1000 * 1000) + +/* Constant used to determine the PLL frequency. */ +#define MLNXBF_I2C_COREPLL_CONST 16384 + +/* PLL registers. */ +#define MLXBF_I2C_CORE_PLL_REG0 0x0 +#define MLXBF_I2C_CORE_PLL_REG1 0x4 +#define MLXBF_I2C_CORE_PLL_REG2 0x8 + +/* OR cause register. */ +#define MLXBF_I2C_CAUSE_OR_EVTEN0 0x14 +#define MLXBF_I2C_CAUSE_OR_CLEAR 0x18 + +/* Arbiter Cause Register. */ +#define MLXBF_I2C_CAUSE_ARBITER 0x1c + +/* + * Cause Status flags. Note that those bits might be considered + * as interrupt enabled bits. + */ + +/* Transaction ended with STOP. */ +#define MLXBF_I2C_CAUSE_TRANSACTION_ENDED BIT(0) +/* Master arbitration lost. */ +#define MLXBF_I2C_CAUSE_M_ARBITRATION_LOST BIT(1) +/* Unexpected start detected. */ +#define MLXBF_I2C_CAUSE_UNEXPECTED_START BIT(2) +/* Unexpected stop detected. */ +#define MLXBF_I2C_CAUSE_UNEXPECTED_STOP BIT(3) +/* Wait for transfer continuation. */ +#define MLXBF_I2C_CAUSE_WAIT_FOR_FW_DATA BIT(4) +/* Failed to generate STOP. */ +#define MLXBF_I2C_CAUSE_PUT_STOP_FAILED BIT(5) +/* Failed to generate START. */ +#define MLXBF_I2C_CAUSE_PUT_START_FAILED BIT(6) +/* Clock toggle completed. */ +#define MLXBF_I2C_CAUSE_CLK_TOGGLE_DONE BIT(7) +/* Transfer timeout occurred. */ +#define MLXBF_I2C_CAUSE_M_FW_TIMEOUT BIT(8) +/* Master busy bit reset. */ +#define MLXBF_I2C_CAUSE_M_GW_BUSY_FALL BIT(9) + +#define MLXBF_I2C_CAUSE_MASTER_ARBITER_BITS_MASK GENMASK(9, 0) + +#define MLXBF_I2C_CAUSE_MASTER_STATUS_ERROR \ + (MLXBF_I2C_CAUSE_M_ARBITRATION_LOST | \ + MLXBF_I2C_CAUSE_UNEXPECTED_START | \ + MLXBF_I2C_CAUSE_UNEXPECTED_STOP | \ + MLXBF_I2C_CAUSE_PUT_STOP_FAILED | \ + MLXBF_I2C_CAUSE_PUT_START_FAILED | \ + MLXBF_I2C_CAUSE_CLK_TOGGLE_DONE | \ + MLXBF_I2C_CAUSE_M_FW_TIMEOUT) + +/* + * Slave cause status flags. Note that those bits might be considered + * as interrupt enabled bits. + */ + +/* Write transaction received successfully. */ +#define MLXBF_I2C_CAUSE_WRITE_SUCCESS BIT(0) +/* Read transaction received, waiting for response. */ +#define MLXBF_I2C_CAUSE_READ_WAIT_FW_RESPONSE BIT(13) +/* Slave busy bit reset. */ +#define MLXBF_I2C_CAUSE_S_GW_BUSY_FALL BIT(18) + +#define MLXBF_I2C_CAUSE_SLAVE_ARBITER_BITS_MASK GENMASK(20, 0) + +/* Cause coalesce registers. */ +#define MLXBF_I2C_CAUSE_COALESCE_0 0x00 +#define MLXBF_I2C_CAUSE_COALESCE_1 0x04 +#define MLXBF_I2C_CAUSE_COALESCE_2 0x08 + +#define MLXBF_I2C_CAUSE_TYU_SLAVE_BIT MLXBF_I2C_SMBUS_MAX +#define MLXBF_I2C_CAUSE_YU_SLAVE_BIT 1 + +/* Functional enable register. */ +#define MLXBF_I2C_GPIO_0_FUNC_EN_0 0x28 +/* Force OE enable register. */ +#define MLXBF_I2C_GPIO_0_FORCE_OE_EN 0x30 +/* + * Note that Smbus GWs are on GPIOs 30:25. Two pins are used to control + * SDA/SCL lines: + * + * SMBUS GW0 -> bits[26:25] + * SMBUS GW1 -> bits[28:27] + * SMBUS GW2 -> bits[30:29] + */ +#define MLXBF_I2C_GPIO_SMBUS_GW_PINS(num) (25 + ((num) << 1)) + +/* Note that gw_id can be 0,1 or 2. */ +#define MLXBF_I2C_GPIO_SMBUS_GW_MASK(num) \ + (0xffffffff & (~(0x3 << MLXBF_I2C_GPIO_SMBUS_GW_PINS(num)))) + +#define MLXBF_I2C_GPIO_SMBUS_GW_RESET_PINS(num, val) \ + ((val) & MLXBF_I2C_GPIO_SMBUS_GW_MASK(num)) + +#define MLXBF_I2C_GPIO_SMBUS_GW_ASSERT_PINS(num, val) \ + ((val) | (0x3 << MLXBF_I2C_GPIO_SMBUS_GW_PINS(num))) + +/* SMBus timing parameters. */ +#define MLXBF_I2C_SMBUS_TIMER_SCL_LOW_SCL_HIGH 0x00 +#define MLXBF_I2C_SMBUS_TIMER_FALL_RISE_SPIKE 0x04 +#define MLXBF_I2C_SMBUS_TIMER_THOLD 0x08 +#define MLXBF_I2C_SMBUS_TIMER_TSETUP_START_STOP 0x0c +#define MLXBF_I2C_SMBUS_TIMER_TSETUP_DATA 0x10 +#define MLXBF_I2C_SMBUS_THIGH_MAX_TBUF 0x14 +#define MLXBF_I2C_SMBUS_SCL_LOW_TIMEOUT 0x18 + +enum { + MLXBF_I2C_TIMING_100KHZ = 100000, + MLXBF_I2C_TIMING_400KHZ = 400000, + MLXBF_I2C_TIMING_1000KHZ = 1000000, +}; + +/* + * Defines SMBus operating frequency and core clock frequency. + * According to ADB files, default values are compliant to 100KHz SMBus + * @ 400MHz core clock. The driver should be able to calculate core + * frequency based on PLL parameters. + */ +#define MLXBF_I2C_COREPLL_FREQ MLXBF_I2C_TYU_PLL_OUT_FREQ + +/* Core PLL TYU configuration. */ +#define MLXBF_I2C_COREPLL_CORE_F_TYU_MASK GENMASK(12, 0) +#define MLXBF_I2C_COREPLL_CORE_OD_TYU_MASK GENMASK(3, 0) +#define MLXBF_I2C_COREPLL_CORE_R_TYU_MASK GENMASK(5, 0) + +#define MLXBF_I2C_COREPLL_CORE_F_TYU_SHIFT 3 +#define MLXBF_I2C_COREPLL_CORE_OD_TYU_SHIFT 16 +#define MLXBF_I2C_COREPLL_CORE_R_TYU_SHIFT 20 + +/* Core PLL YU configuration. */ +#define MLXBF_I2C_COREPLL_CORE_F_YU_MASK GENMASK(25, 0) +#define MLXBF_I2C_COREPLL_CORE_OD_YU_MASK GENMASK(3, 0) +#define MLXBF_I2C_COREPLL_CORE_R_YU_MASK GENMASK(5, 0) + +#define MLXBF_I2C_COREPLL_CORE_F_YU_SHIFT 0 +#define MLXBF_I2C_COREPLL_CORE_OD_YU_SHIFT 1 +#define MLXBF_I2C_COREPLL_CORE_R_YU_SHIFT 26 + +/* Core PLL frequency. */ +static u64 mlxbf_i2c_corepll_frequency; + +/* SMBus Master GW. */ +#define MLXBF_I2C_SMBUS_MASTER_GW 0x200 +/* Number of bytes received and sent. */ +#define MLXBF_I2C_SMBUS_RS_BYTES 0x300 +/* Packet error check (PEC) value. */ +#define MLXBF_I2C_SMBUS_MASTER_PEC 0x304 +/* Status bits (ACK/NACK/FW Timeout). */ +#define MLXBF_I2C_SMBUS_MASTER_STATUS 0x308 +/* SMbus Master Finite State Machine. */ +#define MLXBF_I2C_SMBUS_MASTER_FSM 0x310 + +/* + * When enabled, the master will issue a stop condition in case of + * timeout while waiting for FW response. + */ +#define MLXBF_I2C_SMBUS_EN_FW_TIMEOUT 0x31c + +/* SMBus master GW control bits offset in MLXBF_I2C_SMBUS_MASTER_GW[31:3]. */ +#define MLXBF_I2C_MASTER_LOCK_BIT BIT(31) /* Lock bit. */ +#define MLXBF_I2C_MASTER_BUSY_BIT BIT(30) /* Busy bit. */ +#define MLXBF_I2C_MASTER_START_BIT BIT(29) /* Control start. */ +#define MLXBF_I2C_MASTER_CTL_WRITE_BIT BIT(28) /* Control write phase. */ +#define MLXBF_I2C_MASTER_CTL_READ_BIT BIT(19) /* Control read phase. */ +#define MLXBF_I2C_MASTER_STOP_BIT BIT(3) /* Control stop. */ + +#define MLXBF_I2C_MASTER_ENABLE \ + (MLXBF_I2C_MASTER_LOCK_BIT | MLXBF_I2C_MASTER_BUSY_BIT | \ + MLXBF_I2C_MASTER_START_BIT | MLXBF_I2C_MASTER_STOP_BIT) + +#define MLXBF_I2C_MASTER_ENABLE_WRITE \ + (MLXBF_I2C_MASTER_ENABLE | MLXBF_I2C_MASTER_CTL_WRITE_BIT) + +#define MLXBF_I2C_MASTER_ENABLE_READ \ + (MLXBF_I2C_MASTER_ENABLE | MLXBF_I2C_MASTER_CTL_READ_BIT) + +#define MLXBF_I2C_MASTER_SLV_ADDR_SHIFT 12 /* Slave address shift. */ +#define MLXBF_I2C_MASTER_WRITE_SHIFT 21 /* Control write bytes shift. */ +#define MLXBF_I2C_MASTER_SEND_PEC_SHIFT 20 /* Send PEC byte shift. */ +#define MLXBF_I2C_MASTER_PARSE_EXP_SHIFT 11 /* Parse expected bytes shift. */ +#define MLXBF_I2C_MASTER_READ_SHIFT 4 /* Control read bytes shift. */ + +/* SMBus master GW Data descriptor. */ +#define MLXBF_I2C_MASTER_DATA_DESC_ADDR 0x280 +#define MLXBF_I2C_MASTER_DATA_DESC_SIZE 0x80 /* Size in bytes. */ + +/* Maximum bytes to read/write per SMBus transaction. */ +#define MLXBF_I2C_MASTER_DATA_R_LENGTH MLXBF_I2C_MASTER_DATA_DESC_SIZE +#define MLXBF_I2C_MASTER_DATA_W_LENGTH (MLXBF_I2C_MASTER_DATA_DESC_SIZE - 1) + +/* All bytes were transmitted. */ +#define MLXBF_I2C_SMBUS_STATUS_BYTE_CNT_DONE BIT(0) +/* NACK received. */ +#define MLXBF_I2C_SMBUS_STATUS_NACK_RCV BIT(1) +/* Slave's byte count >128 bytes. */ +#define MLXBF_I2C_SMBUS_STATUS_READ_ERR BIT(2) +/* Timeout occurred. */ +#define MLXBF_I2C_SMBUS_STATUS_FW_TIMEOUT BIT(3) + +#define MLXBF_I2C_SMBUS_MASTER_STATUS_MASK GENMASK(3, 0) + +#define MLXBF_I2C_SMBUS_MASTER_STATUS_ERROR \ + (MLXBF_I2C_SMBUS_STATUS_NACK_RCV | \ + MLXBF_I2C_SMBUS_STATUS_READ_ERR | \ + MLXBF_I2C_SMBUS_STATUS_FW_TIMEOUT) + +#define MLXBF_I2C_SMBUS_MASTER_FSM_STOP_MASK BIT(31) +#define MLXBF_I2C_SMBUS_MASTER_FSM_PS_STATE_MASK BIT(15) + +/* SMBus slave GW. */ +#define MLXBF_I2C_SMBUS_SLAVE_GW 0x400 +/* Number of bytes received and sent from/to master. */ +#define MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES 0x500 +/* Packet error check (PEC) value. */ +#define MLXBF_I2C_SMBUS_SLAVE_PEC 0x504 +/* SMBus slave Finite State Machine (FSM). */ +#define MLXBF_I2C_SMBUS_SLAVE_FSM 0x510 +/* + * Should be set when all raised causes handled, and cleared by HW on + * every new cause. + */ +#define MLXBF_I2C_SMBUS_SLAVE_READY 0x52c + +/* SMBus slave GW control bits offset in MLXBF_I2C_SMBUS_SLAVE_GW[31:19]. */ +#define MLXBF_I2C_SLAVE_BUSY_BIT BIT(30) /* Busy bit. */ +#define MLXBF_I2C_SLAVE_WRITE_BIT BIT(29) /* Control write enable. */ + +#define MLXBF_I2C_SLAVE_ENABLE \ + (MLXBF_I2C_SLAVE_BUSY_BIT | MLXBF_I2C_SLAVE_WRITE_BIT) + +#define MLXBF_I2C_SLAVE_WRITE_BYTES_SHIFT 22 /* Number of bytes to write. */ +#define MLXBF_I2C_SLAVE_SEND_PEC_SHIFT 21 /* Send PEC byte shift. */ + +/* SMBus slave GW Data descriptor. */ +#define MLXBF_I2C_SLAVE_DATA_DESC_ADDR 0x480 +#define MLXBF_I2C_SLAVE_DATA_DESC_SIZE 0x80 /* Size in bytes. */ + +/* SMbus slave configuration registers. */ +#define MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG 0x514 +#define MLXBF_I2C_SMBUS_SLAVE_ADDR_CNT 16 +#define MLXBF_I2C_SMBUS_SLAVE_ADDR_EN_BIT 7 +#define MLXBF_I2C_SMBUS_SLAVE_ADDR_MASK GENMASK(6, 0) + +#define MLXBF_I2C_SLAVE_ADDR_ENABLED(addr) \ + ((addr) & (1 << MLXBF_I2C_SMBUS_SLAVE_ADDR_EN_BIT)) + +/* + * Timeout is given in microsends. Note also that timeout handling is not + * exact. + */ +#define MLXBF_I2C_SMBUS_TIMEOUT (300 * 1000) /* 300ms */ + +/* Encapsulates timing parameters. */ +struct mlxbf_i2c_timings { + u16 scl_high; /* Clock high period. */ + u16 scl_low; /* Clock low period. */ + u8 sda_rise; /* Data rise time. */ + u8 sda_fall; /* Data fall time. */ + u8 scl_rise; /* Clock rise time. */ + u8 scl_fall; /* Clock fall time. */ + u16 hold_start; /* Hold time after (REPEATED) START. */ + u16 hold_data; /* Data hold time. */ + u16 setup_start; /* REPEATED START condition setup time. */ + u16 setup_stop; /* STOP condition setup time. */ + u16 setup_data; /* Data setup time. */ + u16 pad; /* Padding. */ + u16 buf; /* Bus free time between STOP and START. */ + u16 thigh_max; /* Thigh max. */ + u32 timeout; /* Detect clock low timeout. */ +}; + +enum { + MLXBF_I2C_F_READ = BIT(0), + MLXBF_I2C_F_WRITE = BIT(1), + MLXBF_I2C_F_NORESTART = BIT(3), + MLXBF_I2C_F_SMBUS_OPERATION = BIT(4), + MLXBF_I2C_F_SMBUS_BLOCK = BIT(5), + MLXBF_I2C_F_SMBUS_PEC = BIT(6), + MLXBF_I2C_F_SMBUS_PROCESS_CALL = BIT(7), +}; + +struct mlxbf_i2c_smbus_operation { + u32 flags; + u32 length; /* Buffer length in bytes. */ + u8 *buffer; +}; + +#define MLXBF_I2C_SMBUS_OP_CNT_1 1 +#define MLXBF_I2C_SMBUS_OP_CNT_2 2 +#define MLXBF_I2C_SMBUS_OP_CNT_3 3 +#define MLXBF_I2C_SMBUS_MAX_OP_CNT MLXBF_I2C_SMBUS_OP_CNT_3 + +struct mlxbf_i2c_smbus_request { + u8 slave; + u8 operation_cnt; + struct mlxbf_i2c_smbus_operation operation[MLXBF_I2C_SMBUS_MAX_OP_CNT]; +}; + +struct mlxbf_i2c_resource { + void __iomem *io; + struct resource *params; + struct mutex *lock; /* Mutex to protect mlxbf_i2c_resource. */ + u8 type; +}; + +/* List of chip resources that are being accessed by the driver. */ +enum { + MLXBF_I2C_SMBUS_RES, + MLXBF_I2C_MST_CAUSE_RES, + MLXBF_I2C_SLV_CAUSE_RES, + MLXBF_I2C_COALESCE_RES, + MLXBF_I2C_COREPLL_RES, + MLXBF_I2C_GPIO_RES, + MLXBF_I2C_END_RES, +}; + +/* Helper macro to define an I2C resource parameters. */ +#define MLXBF_I2C_RES_PARAMS(addr, size, str) \ + { \ + .start = (addr), \ + .end = (addr) + (size) - 1, \ + .name = (str) \ + } + +static struct resource mlxbf_i2c_coalesce_tyu_params = + MLXBF_I2C_RES_PARAMS(MLXBF_I2C_COALESCE_TYU_ADDR, + MLXBF_I2C_COALESCE_TYU_SIZE, + "COALESCE_MEM"); +static struct resource mlxbf_i2c_corepll_tyu_params = + MLXBF_I2C_RES_PARAMS(MLXBF_I2C_COREPLL_TYU_ADDR, + MLXBF_I2C_COREPLL_TYU_SIZE, + "COREPLL_MEM"); +static struct resource mlxbf_i2c_corepll_yu_params = + MLXBF_I2C_RES_PARAMS(MLXBF_I2C_COREPLL_YU_ADDR, + MLXBF_I2C_COREPLL_YU_SIZE, + "COREPLL_MEM"); +static struct resource mlxbf_i2c_gpio_tyu_params = + MLXBF_I2C_RES_PARAMS(MLXBF_I2C_GPIO_TYU_ADDR, + MLXBF_I2C_GPIO_TYU_SIZE, + "GPIO_MEM"); + +static struct mutex mlxbf_i2c_coalesce_lock; +static struct mutex mlxbf_i2c_corepll_lock; +static struct mutex mlxbf_i2c_gpio_lock; + +/* Mellanox BlueField chip type. */ +enum mlxbf_i2c_chip_type { + MLXBF_I2C_CHIP_TYPE_1, /* Mellanox BlueField-1 chip. */ + MLXBF_I2C_CHIP_TYPE_2, /* Mallanox BlueField-2 chip. */ +}; + +struct mlxbf_i2c_chip_info { + enum mlxbf_i2c_chip_type type; + /* Chip shared resources that are being used by the I2C controller. */ + struct mlxbf_i2c_resource *shared_res[MLXBF_I2C_SHARED_RES_MAX]; + + /* Callback to calculate the core PLL frequency. */ + u64 (*calculate_freq)(struct mlxbf_i2c_resource *corepll_res); +}; + +struct mlxbf_i2c_priv { + const struct mlxbf_i2c_chip_info *chip; + struct i2c_adapter adap; + struct mlxbf_i2c_resource *smbus; + struct mlxbf_i2c_resource *mst_cause; + struct mlxbf_i2c_resource *slv_cause; + struct mlxbf_i2c_resource *coalesce; + u64 frequency; /* Core frequency in Hz. */ + int bus; /* Physical bus identifier. */ + int irq; + struct i2c_client *slave; +}; + +static struct mlxbf_i2c_resource mlxbf_i2c_coalesce_res[] = { + [MLXBF_I2C_CHIP_TYPE_1] = { + .params = &mlxbf_i2c_coalesce_tyu_params, + .lock = &mlxbf_i2c_coalesce_lock, + .type = MLXBF_I2C_COALESCE_RES + }, + {} +}; + +static struct mlxbf_i2c_resource mlxbf_i2c_corepll_res[] = { + [MLXBF_I2C_CHIP_TYPE_1] = { + .params = &mlxbf_i2c_corepll_tyu_params, + .lock = &mlxbf_i2c_corepll_lock, + .type = MLXBF_I2C_COREPLL_RES + }, + [MLXBF_I2C_CHIP_TYPE_2] = { + .params = &mlxbf_i2c_corepll_yu_params, + .lock = &mlxbf_i2c_corepll_lock, + .type = MLXBF_I2C_COREPLL_RES, + } +}; + +static struct mlxbf_i2c_resource mlxbf_i2c_gpio_res[] = { + [MLXBF_I2C_CHIP_TYPE_1] = { + .params = &mlxbf_i2c_gpio_tyu_params, + .lock = &mlxbf_i2c_gpio_lock, + .type = MLXBF_I2C_GPIO_RES + }, + {} +}; + +static u8 mlxbf_i2c_bus_count; + +static struct mutex mlxbf_i2c_bus_lock; + +/* Polling frequency in microseconds. */ +#define MLXBF_I2C_POLL_FREQ_IN_USEC 200 + +#define MLXBF_I2C_SHIFT_0 0 +#define MLXBF_I2C_SHIFT_8 8 +#define MLXBF_I2C_SHIFT_16 16 +#define MLXBF_I2C_SHIFT_24 24 + +#define MLXBF_I2C_MASK_8 GENMASK(7, 0) +#define MLXBF_I2C_MASK_16 GENMASK(15, 0) + +#define MLXBF_I2C_FREQUENCY_1GHZ 1000000000 + +static void mlxbf_i2c_write(void __iomem *io, int reg, u32 val) +{ + writel(val, io + reg); +} + +static u32 mlxbf_i2c_read(void __iomem *io, int reg) +{ + return readl(io + reg); +} + +/* + * This function is used to read data from Master GW Data Descriptor. + * Data bytes in the Master GW Data Descriptor are shifted left so the + * data starts at the MSB of the descriptor registers as set by the + * underlying hardware. TYU_READ_DATA enables byte swapping while + * reading data bytes, and MUST be called by the SMBus read routines + * to copy data from the 32 * 32-bit HW Data registers a.k.a Master GW + * Data Descriptor. + */ +static u32 mlxbf_i2c_read_data(void __iomem *io, int reg) +{ + return (u32)be32_to_cpu(mlxbf_i2c_read(io, reg)); +} + +/* + * This function is used to write data to the Master GW Data Descriptor. + * Data copied to the Master GW Data Descriptor MUST be shifted left so + * the data starts at the MSB of the descriptor registers as required by + * the underlying hardware. TYU_WRITE_DATA enables byte swapping when + * writing data bytes, and MUST be called by the SMBus write routines to + * copy data to the 32 * 32-bit HW Data registers a.k.a Master GW Data + * Descriptor. + */ +static void mlxbf_i2c_write_data(void __iomem *io, int reg, u32 val) +{ + mlxbf_i2c_write(io, reg, (u32)cpu_to_be32(val)); +} + +/* + * Function to poll a set of bits at a specific address; it checks whether + * the bits are equal to zero when eq_zero is set to 'true', and not equal + * to zero when eq_zero is set to 'false'. + * Note that the timeout is given in microseconds. + */ +static u32 mlxbf_smbus_poll(void __iomem *io, u32 addr, u32 mask, + bool eq_zero, u32 timeout) +{ + u32 bits; + + timeout = (timeout / MLXBF_I2C_POLL_FREQ_IN_USEC) + 1; + + do { + bits = mlxbf_i2c_read(io, addr) & mask; + if (eq_zero ? bits == 0 : bits != 0) + return eq_zero ? 1 : bits; + udelay(MLXBF_I2C_POLL_FREQ_IN_USEC); + } while (timeout-- != 0); + + return 0; +} + +/* + * SW must make sure that the SMBus Master GW is idle before starting + * a transaction. Accordingly, this function polls the Master FSM stop + * bit; it returns false when the bit is asserted, true if not. + */ +static bool mlxbf_smbus_master_wait_for_idle(struct mlxbf_i2c_priv *priv) +{ + u32 mask = MLXBF_I2C_SMBUS_MASTER_FSM_STOP_MASK; + u32 addr = MLXBF_I2C_SMBUS_MASTER_FSM; + u32 timeout = MLXBF_I2C_SMBUS_TIMEOUT; + + if (mlxbf_smbus_poll(priv->smbus->io, addr, mask, true, timeout)) + return true; + + return false; +} + +static bool mlxbf_i2c_smbus_transaction_success(u32 master_status, + u32 cause_status) +{ + /* + * When transaction ended with STOP, all bytes were transmitted, + * and no NACK received, then the transaction ended successfully. + * On the other hand, when the GW is configured with the stop bit + * de-asserted then the SMBus expects the following GW configuration + * for transfer continuation. + */ + if ((cause_status & MLXBF_I2C_CAUSE_WAIT_FOR_FW_DATA) || + ((cause_status & MLXBF_I2C_CAUSE_TRANSACTION_ENDED) && + (master_status & MLXBF_I2C_SMBUS_STATUS_BYTE_CNT_DONE) && + !(master_status & MLXBF_I2C_SMBUS_STATUS_NACK_RCV))) + return true; + + return false; +} + +/* + * Poll SMBus master status and return transaction status, + * i.e. whether succeeded or failed. I2C and SMBus fault codes + * are returned as negative numbers from most calls, with zero + * or some positive number indicating a non-fault return. + */ +static int mlxbf_i2c_smbus_check_status(struct mlxbf_i2c_priv *priv) +{ + u32 master_status_bits; + u32 cause_status_bits; + + /* + * GW busy bit is raised by the driver and cleared by the HW + * when the transaction is completed. The busy bit is a good + * indicator of transaction status. So poll the busy bit, and + * then read the cause and master status bits to determine if + * errors occurred during the transaction. + */ + mlxbf_smbus_poll(priv->smbus->io, MLXBF_I2C_SMBUS_MASTER_GW, + MLXBF_I2C_MASTER_BUSY_BIT, true, + MLXBF_I2C_SMBUS_TIMEOUT); + + /* Read cause status bits. */ + cause_status_bits = mlxbf_i2c_read(priv->mst_cause->io, + MLXBF_I2C_CAUSE_ARBITER); + cause_status_bits &= MLXBF_I2C_CAUSE_MASTER_ARBITER_BITS_MASK; + + /* + * Parse both Cause and Master GW bits, then return transaction status. + */ + + master_status_bits = mlxbf_i2c_read(priv->smbus->io, + MLXBF_I2C_SMBUS_MASTER_STATUS); + master_status_bits &= MLXBF_I2C_SMBUS_MASTER_STATUS_MASK; + + if (mlxbf_i2c_smbus_transaction_success(master_status_bits, + cause_status_bits)) + return 0; + + /* + * In case of timeout on GW busy, the ISR will clear busy bit but + * transaction ended bits cause will not be set so the transaction + * fails. Then, we must check Master GW status bits. + */ + if ((master_status_bits & MLXBF_I2C_SMBUS_MASTER_STATUS_ERROR) && + (cause_status_bits & (MLXBF_I2C_CAUSE_TRANSACTION_ENDED | + MLXBF_I2C_CAUSE_M_GW_BUSY_FALL))) + return -EIO; + + if (cause_status_bits & MLXBF_I2C_CAUSE_MASTER_STATUS_ERROR) + return -EAGAIN; + + return -ETIMEDOUT; +} + +static void mlxbf_i2c_smbus_write_data(struct mlxbf_i2c_priv *priv, + const u8 *data, u8 length, u32 addr) +{ + u8 offset, aligned_length; + u32 data32; + + aligned_length = round_up(length, 4); + + /* Copy data bytes from 4-byte aligned source buffer. */ + for (offset = 0; offset < aligned_length; offset += sizeof(u32)) { + data32 = *((u32 *)(data + offset)); + mlxbf_i2c_write_data(priv->smbus->io, addr + offset, data32); + } +} + +static void mlxbf_i2c_smbus_read_data(struct mlxbf_i2c_priv *priv, + u8 *data, u8 length, u32 addr) +{ + u32 data32, mask; + u8 byte, offset; + + mask = sizeof(u32) - 1; + + for (offset = 0; offset < (length & ~mask); offset += sizeof(u32)) { + data32 = mlxbf_i2c_read_data(priv->smbus->io, addr + offset); + *((u32 *)(data + offset)) = data32; + } + + if (!(length & mask)) + return; + + data32 = mlxbf_i2c_read_data(priv->smbus->io, addr + offset); + + for (byte = 0; byte < (length & mask); byte++) { + data[offset + byte] = data32 & GENMASK(7, 0); + data32 = ror32(data32, MLXBF_I2C_SHIFT_8); + } +} + +static int mlxbf_i2c_smbus_enable(struct mlxbf_i2c_priv *priv, u8 slave, + u8 len, u8 block_en, u8 pec_en, bool read) +{ + u32 command; + + /* Set Master GW control word. */ + if (read) { + command = MLXBF_I2C_MASTER_ENABLE_READ; + command |= rol32(len, MLXBF_I2C_MASTER_READ_SHIFT); + } else { + command = MLXBF_I2C_MASTER_ENABLE_WRITE; + command |= rol32(len, MLXBF_I2C_MASTER_WRITE_SHIFT); + } + command |= rol32(slave, MLXBF_I2C_MASTER_SLV_ADDR_SHIFT); + command |= rol32(block_en, MLXBF_I2C_MASTER_PARSE_EXP_SHIFT); + command |= rol32(pec_en, MLXBF_I2C_MASTER_SEND_PEC_SHIFT); + + /* Clear status bits. */ + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_MASTER_STATUS, 0x0); + /* Set the cause data. */ + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_CAUSE_OR_CLEAR, ~0x0); + /* Zero PEC byte. */ + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_MASTER_PEC, 0x0); + /* Zero byte count. */ + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_RS_BYTES, 0x0); + + /* GW activation. */ + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_MASTER_GW, command); + + /* + * Poll master status and check status bits. An ACK is sent when + * completing writing data to the bus (Master 'byte_count_done' bit + * is set to 1). + */ + return mlxbf_i2c_smbus_check_status(priv); +} + +static int +mlxbf_i2c_smbus_start_transaction(struct mlxbf_i2c_priv *priv, + struct mlxbf_i2c_smbus_request *request) +{ + u8 data_desc[MLXBF_I2C_MASTER_DATA_DESC_SIZE] = { 0 }; + u8 op_idx, data_idx, data_len, write_len, read_len; + struct mlxbf_i2c_smbus_operation *operation; + u8 read_en, write_en, block_en, pec_en; + u8 slave, flags, addr; + u8 *read_buf; + int ret = 0; + + if (request->operation_cnt > MLXBF_I2C_SMBUS_MAX_OP_CNT) + return -EINVAL; + + read_buf = NULL; + data_idx = 0; + read_en = 0; + write_en = 0; + write_len = 0; + read_len = 0; + block_en = 0; + pec_en = 0; + slave = request->slave & GENMASK(6, 0); + addr = slave << 1; + + /* First of all, check whether the HW is idle. */ + if (WARN_ON(!mlxbf_smbus_master_wait_for_idle(priv))) + return -EBUSY; + + /* Set first byte. */ + data_desc[data_idx++] = addr; + + for (op_idx = 0; op_idx < request->operation_cnt; op_idx++) { + operation = &request->operation[op_idx]; + flags = operation->flags; + + /* + * Note that read and write operations might be handled by a + * single command. If the MLXBF_I2C_F_SMBUS_OPERATION is set + * then write command byte and set the optional SMBus specific + * bits such as block_en and pec_en. These bits MUST be + * submitted by the first operation only. + */ + if (op_idx == 0 && flags & MLXBF_I2C_F_SMBUS_OPERATION) { + block_en = flags & MLXBF_I2C_F_SMBUS_BLOCK; + pec_en = flags & MLXBF_I2C_F_SMBUS_PEC; + } + + if (flags & MLXBF_I2C_F_WRITE) { + write_en = 1; + write_len += operation->length; + memcpy(data_desc + data_idx, + operation->buffer, operation->length); + data_idx += operation->length; + } + /* + * We assume that read operations are performed only once per + * SMBus transaction. *TBD* protect this statement so it won't + * be executed twice? or return an error if we try to read more + * than once? + */ + if (flags & MLXBF_I2C_F_READ) { + read_en = 1; + /* Subtract 1 as required by HW. */ + read_len = operation->length - 1; + read_buf = operation->buffer; + } + } + + /* Set Master GW data descriptor. */ + data_len = write_len + 1; /* Add one byte of the slave address. */ + /* + * Note that data_len cannot be 0. Indeed, the slave address byte + * must be written to the data registers. + */ + mlxbf_i2c_smbus_write_data(priv, (const u8 *)data_desc, data_len, + MLXBF_I2C_MASTER_DATA_DESC_ADDR); + + if (write_en) { + ret = mlxbf_i2c_smbus_enable(priv, slave, write_len, block_en, + pec_en, 0); + if (ret) + return ret; + } + + if (read_en) { + /* Write slave address to Master GW data descriptor. */ + mlxbf_i2c_smbus_write_data(priv, (const u8 *)&addr, 1, + MLXBF_I2C_MASTER_DATA_DESC_ADDR); + ret = mlxbf_i2c_smbus_enable(priv, slave, read_len, block_en, + pec_en, 1); + if (!ret) { + /* Get Master GW data descriptor. */ + mlxbf_i2c_smbus_read_data(priv, data_desc, read_len + 1, + MLXBF_I2C_MASTER_DATA_DESC_ADDR); + + /* Get data from Master GW data descriptor. */ + memcpy(read_buf, data_desc, read_len + 1); + } + + /* + * After a read operation the SMBus FSM ps (present state) + * needs to be 'manually' reset. This should be removed in + * next tag integration. + */ + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_MASTER_FSM, + MLXBF_I2C_SMBUS_MASTER_FSM_PS_STATE_MASK); + } + + return ret; +} + +/* I2C SMBus protocols. */ + +static void +mlxbf_i2c_smbus_quick_command(struct mlxbf_i2c_smbus_request *request, + u8 read) +{ + request->operation_cnt = MLXBF_I2C_SMBUS_OP_CNT_1; + + request->operation[0].length = 0; + request->operation[0].flags = MLXBF_I2C_F_WRITE; + request->operation[0].flags |= read ? MLXBF_I2C_F_READ : 0; +} + +static void mlxbf_i2c_smbus_byte_func(struct mlxbf_i2c_smbus_request *request, + u8 *data, bool read, bool pec_check) +{ + request->operation_cnt = MLXBF_I2C_SMBUS_OP_CNT_1; + + request->operation[0].length = 1; + request->operation[0].length += pec_check; + + request->operation[0].flags = MLXBF_I2C_F_SMBUS_OPERATION; + request->operation[0].flags |= read ? + MLXBF_I2C_F_READ : MLXBF_I2C_F_WRITE; + request->operation[0].flags |= pec_check ? MLXBF_I2C_F_SMBUS_PEC : 0; + + request->operation[0].buffer = data; +} + +static void +mlxbf_i2c_smbus_data_byte_func(struct mlxbf_i2c_smbus_request *request, + u8 *command, u8 *data, bool read, bool pec_check) +{ + request->operation_cnt = MLXBF_I2C_SMBUS_OP_CNT_2; + + request->operation[0].length = 1; + request->operation[0].flags = + MLXBF_I2C_F_SMBUS_OPERATION | MLXBF_I2C_F_WRITE; + request->operation[0].flags |= pec_check ? MLXBF_I2C_F_SMBUS_PEC : 0; + request->operation[0].buffer = command; + + request->operation[1].length = 1; + request->operation[1].length += pec_check; + request->operation[1].flags = read ? + MLXBF_I2C_F_READ : MLXBF_I2C_F_WRITE; + request->operation[1].buffer = data; +} + +static void +mlxbf_i2c_smbus_data_word_func(struct mlxbf_i2c_smbus_request *request, + u8 *command, u8 *data, bool read, bool pec_check) +{ + request->operation_cnt = MLXBF_I2C_SMBUS_OP_CNT_2; + + request->operation[0].length = 1; + request->operation[0].flags = + MLXBF_I2C_F_SMBUS_OPERATION | MLXBF_I2C_F_WRITE; + request->operation[0].flags |= pec_check ? MLXBF_I2C_F_SMBUS_PEC : 0; + request->operation[0].buffer = command; + + request->operation[1].length = 2; + request->operation[1].length += pec_check; + request->operation[1].flags = read ? + MLXBF_I2C_F_READ : MLXBF_I2C_F_WRITE; + request->operation[1].buffer = data; +} + +static void +mlxbf_i2c_smbus_i2c_block_func(struct mlxbf_i2c_smbus_request *request, + u8 *command, u8 *data, u8 *data_len, bool read, + bool pec_check) +{ + request->operation_cnt = MLXBF_I2C_SMBUS_OP_CNT_2; + + request->operation[0].length = 1; + request->operation[0].flags = + MLXBF_I2C_F_SMBUS_OPERATION | MLXBF_I2C_F_WRITE; + request->operation[0].flags |= pec_check ? MLXBF_I2C_F_SMBUS_PEC : 0; + request->operation[0].buffer = command; + + /* + * As specified in the standard, the max number of bytes to read/write + * per block operation is 32 bytes. In Golan code, the controller can + * read up to 128 bytes and write up to 127 bytes. + */ + request->operation[1].length = + (*data_len + pec_check > I2C_SMBUS_BLOCK_MAX) ? + I2C_SMBUS_BLOCK_MAX : *data_len + pec_check; + request->operation[1].flags = read ? + MLXBF_I2C_F_READ : MLXBF_I2C_F_WRITE; + /* + * Skip the first data byte, which corresponds to the number of bytes + * to read/write. + */ + request->operation[1].buffer = data + 1; + + *data_len = request->operation[1].length; + + /* Set the number of byte to read. This will be used by userspace. */ + if (read) + data[0] = *data_len; +} + +static void mlxbf_i2c_smbus_block_func(struct mlxbf_i2c_smbus_request *request, + u8 *command, u8 *data, u8 *data_len, + bool read, bool pec_check) +{ + request->operation_cnt = MLXBF_I2C_SMBUS_OP_CNT_2; + + request->operation[0].length = 1; + request->operation[0].flags = + MLXBF_I2C_F_SMBUS_OPERATION | MLXBF_I2C_F_WRITE; + request->operation[0].flags |= MLXBF_I2C_F_SMBUS_BLOCK; + request->operation[0].flags |= pec_check ? MLXBF_I2C_F_SMBUS_PEC : 0; + request->operation[0].buffer = command; + + request->operation[1].length = + (*data_len + pec_check > I2C_SMBUS_BLOCK_MAX) ? + I2C_SMBUS_BLOCK_MAX : *data_len + pec_check; + request->operation[1].flags = read ? + MLXBF_I2C_F_READ : MLXBF_I2C_F_WRITE; + request->operation[1].buffer = data + 1; + + *data_len = request->operation[1].length; + + /* Set the number of bytes to read. This will be used by userspace. */ + if (read) + data[0] = *data_len; +} + +static void +mlxbf_i2c_smbus_process_call_func(struct mlxbf_i2c_smbus_request *request, + u8 *command, u8 *data, bool pec_check) +{ + request->operation_cnt = MLXBF_I2C_SMBUS_OP_CNT_3; + + request->operation[0].length = 1; + request->operation[0].flags = + MLXBF_I2C_F_SMBUS_OPERATION | MLXBF_I2C_F_WRITE; + request->operation[0].flags |= MLXBF_I2C_F_SMBUS_BLOCK; + request->operation[0].flags |= pec_check ? MLXBF_I2C_F_SMBUS_PEC : 0; + request->operation[0].buffer = command; + + request->operation[1].length = 2; + request->operation[1].flags = MLXBF_I2C_F_WRITE; + request->operation[1].buffer = data; + + request->operation[2].length = 3; + request->operation[2].flags = MLXBF_I2C_F_READ; + request->operation[2].buffer = data; +} + +static void +mlxbf_i2c_smbus_blk_process_call_func(struct mlxbf_i2c_smbus_request *request, + u8 *command, u8 *data, u8 *data_len, + bool pec_check) +{ + u32 length; + + request->operation_cnt = MLXBF_I2C_SMBUS_OP_CNT_3; + + request->operation[0].length = 1; + request->operation[0].flags = + MLXBF_I2C_F_SMBUS_OPERATION | MLXBF_I2C_F_WRITE; + request->operation[0].flags |= MLXBF_I2C_F_SMBUS_BLOCK; + request->operation[0].flags |= (pec_check) ? MLXBF_I2C_F_SMBUS_PEC : 0; + request->operation[0].buffer = command; + + length = (*data_len + pec_check > I2C_SMBUS_BLOCK_MAX) ? + I2C_SMBUS_BLOCK_MAX : *data_len + pec_check; + + request->operation[1].length = length - pec_check; + request->operation[1].flags = MLXBF_I2C_F_WRITE; + request->operation[1].buffer = data; + + request->operation[2].length = length; + request->operation[2].flags = MLXBF_I2C_F_READ; + request->operation[2].buffer = data; + + *data_len = length; /* including PEC byte. */ +} + +/* Initialization functions. */ + +static bool mlxbf_i2c_has_chip_type(struct mlxbf_i2c_priv *priv, u8 type) +{ + return priv->chip->type == type; +} + +static struct mlxbf_i2c_resource * +mlxbf_i2c_get_shared_resource(struct mlxbf_i2c_priv *priv, u8 type) +{ + const struct mlxbf_i2c_chip_info *chip = priv->chip; + struct mlxbf_i2c_resource *res; + u8 res_idx = 0; + + for (res_idx = 0; res_idx < MLXBF_I2C_SHARED_RES_MAX; res_idx++) { + res = chip->shared_res[res_idx]; + if (res && res->type == type) + return res; + } + + return NULL; +} + +static int mlxbf_i2c_init_resource(struct platform_device *pdev, + struct mlxbf_i2c_resource **res, + u8 type) +{ + struct mlxbf_i2c_resource *tmp_res; + struct device *dev = &pdev->dev; + + if (!res || *res || type >= MLXBF_I2C_END_RES) + return -EINVAL; + + tmp_res = devm_kzalloc(dev, sizeof(struct mlxbf_i2c_resource), + GFP_KERNEL); + if (!tmp_res) + return -ENOMEM; + + tmp_res->params = platform_get_resource(pdev, IORESOURCE_MEM, type); + if (!tmp_res->params) { + devm_kfree(dev, tmp_res); + return -EIO; + } + + tmp_res->io = devm_ioremap_resource(dev, tmp_res->params); + if (IS_ERR(tmp_res->io)) { + devm_kfree(dev, tmp_res); + return PTR_ERR(tmp_res->io); + } + + tmp_res->type = type; + + *res = tmp_res; + + return 0; +} + +static u32 mlxbf_i2c_get_ticks(struct mlxbf_i2c_priv *priv, u64 nanoseconds, + bool minimum) +{ + u64 frequency; + u32 ticks; + + /* + * Compute ticks as follow: + * + * Ticks + * Time = --------- x 10^9 => Ticks = Time x Frequency x 10^-9 + * Frequency + */ + frequency = priv->frequency; + ticks = (nanoseconds * frequency) / MLXBF_I2C_FREQUENCY_1GHZ; + /* + * The number of ticks is rounded down and if minimum is equal to 1 + * then add one tick. + */ + if (minimum) + ticks++; + + return ticks; +} + +static u32 mlxbf_i2c_set_timer(struct mlxbf_i2c_priv *priv, u64 nsec, bool opt, + u32 mask, u8 shift) +{ + u32 val = (mlxbf_i2c_get_ticks(priv, nsec, opt) & mask) << shift; + + return val; +} + +static void mlxbf_i2c_set_timings(struct mlxbf_i2c_priv *priv, + const struct mlxbf_i2c_timings *timings) +{ + u32 timer; + + timer = mlxbf_i2c_set_timer(priv, timings->scl_high, + false, MLXBF_I2C_MASK_16, + MLXBF_I2C_SHIFT_0); + timer |= mlxbf_i2c_set_timer(priv, timings->scl_low, + false, MLXBF_I2C_MASK_16, + MLXBF_I2C_SHIFT_16); + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_TIMER_SCL_LOW_SCL_HIGH, + timer); + + timer = mlxbf_i2c_set_timer(priv, timings->sda_rise, false, + MLXBF_I2C_MASK_8, MLXBF_I2C_SHIFT_0); + timer |= mlxbf_i2c_set_timer(priv, timings->sda_fall, false, + MLXBF_I2C_MASK_8, MLXBF_I2C_SHIFT_8); + timer |= mlxbf_i2c_set_timer(priv, timings->scl_rise, false, + MLXBF_I2C_MASK_8, MLXBF_I2C_SHIFT_16); + timer |= mlxbf_i2c_set_timer(priv, timings->scl_fall, false, + MLXBF_I2C_MASK_8, MLXBF_I2C_SHIFT_24); + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_TIMER_FALL_RISE_SPIKE, + timer); + + timer = mlxbf_i2c_set_timer(priv, timings->hold_start, true, + MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_0); + timer |= mlxbf_i2c_set_timer(priv, timings->hold_data, true, + MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_16); + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_TIMER_THOLD, timer); + + timer = mlxbf_i2c_set_timer(priv, timings->setup_start, true, + MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_0); + timer |= mlxbf_i2c_set_timer(priv, timings->setup_stop, true, + MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_16); + mlxbf_i2c_write(priv->smbus->io, + MLXBF_I2C_SMBUS_TIMER_TSETUP_START_STOP, timer); + + timer = mlxbf_i2c_set_timer(priv, timings->setup_data, true, + MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_0); + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_TIMER_TSETUP_DATA, + timer); + + timer = mlxbf_i2c_set_timer(priv, timings->buf, false, + MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_0); + timer |= mlxbf_i2c_set_timer(priv, timings->thigh_max, false, + MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_16); + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_THIGH_MAX_TBUF, + timer); + + timer = timings->timeout; + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_SCL_LOW_TIMEOUT, + timer); +} + +enum mlxbf_i2c_timings_config { + MLXBF_I2C_TIMING_CONFIG_100KHZ, + MLXBF_I2C_TIMING_CONFIG_400KHZ, + MLXBF_I2C_TIMING_CONFIG_1000KHZ, +}; + +/* + * Note that the mlxbf_i2c_timings->timeout value is not related to the + * bus frequency, it is impacted by the time it takes the driver to + * complete data transmission before transaction abort. + */ +static const struct mlxbf_i2c_timings mlxbf_i2c_timings[] = { + [MLXBF_I2C_TIMING_CONFIG_100KHZ] = { + .scl_high = 4810, + .scl_low = 5000, + .hold_start = 4000, + .setup_start = 4800, + .setup_stop = 4000, + .setup_data = 250, + .sda_rise = 50, + .sda_fall = 50, + .scl_rise = 50, + .scl_fall = 50, + .hold_data = 300, + .buf = 20000, + .thigh_max = 5000, + .timeout = 106500 + }, + [MLXBF_I2C_TIMING_CONFIG_400KHZ] = { + .scl_high = 1011, + .scl_low = 1300, + .hold_start = 600, + .setup_start = 700, + .setup_stop = 600, + .setup_data = 100, + .sda_rise = 50, + .sda_fall = 50, + .scl_rise = 50, + .scl_fall = 50, + .hold_data = 300, + .buf = 20000, + .thigh_max = 5000, + .timeout = 106500 + }, + [MLXBF_I2C_TIMING_CONFIG_1000KHZ] = { + .scl_high = 600, + .scl_low = 1300, + .hold_start = 600, + .setup_start = 600, + .setup_stop = 600, + .setup_data = 100, + .sda_rise = 50, + .sda_fall = 50, + .scl_rise = 50, + .scl_fall = 50, + .hold_data = 300, + .buf = 20000, + .thigh_max = 5000, + .timeout = 106500 + } +}; + +static int mlxbf_i2c_init_timings(struct platform_device *pdev, + struct mlxbf_i2c_priv *priv) +{ + enum mlxbf_i2c_timings_config config_idx; + struct device *dev = &pdev->dev; + u32 config_khz; + + int ret; + + ret = device_property_read_u32(dev, "clock-frequency", &config_khz); + if (ret < 0) + config_khz = MLXBF_I2C_TIMING_100KHZ; + + switch (config_khz) { + default: + /* Default settings is 100 KHz. */ + pr_warn("Illegal value %d: defaulting to 100 KHz\n", + config_khz); + fallthrough; + case MLXBF_I2C_TIMING_100KHZ: + config_idx = MLXBF_I2C_TIMING_CONFIG_100KHZ; + break; + + case MLXBF_I2C_TIMING_400KHZ: + config_idx = MLXBF_I2C_TIMING_CONFIG_400KHZ; + break; + + case MLXBF_I2C_TIMING_1000KHZ: + config_idx = MLXBF_I2C_TIMING_CONFIG_1000KHZ; + break; + } + + mlxbf_i2c_set_timings(priv, &mlxbf_i2c_timings[config_idx]); + + return 0; +} + +static int mlxbf_i2c_get_gpio(struct platform_device *pdev, + struct mlxbf_i2c_priv *priv) +{ + struct mlxbf_i2c_resource *gpio_res; + struct device *dev = &pdev->dev; + struct resource *params; + resource_size_t size; + + gpio_res = mlxbf_i2c_get_shared_resource(priv, MLXBF_I2C_GPIO_RES); + if (!gpio_res) + return -EPERM; + + /* + * The GPIO region in TYU space is shared among I2C busses. + * This function MUST be serialized to avoid racing when + * claiming the memory region and/or setting up the GPIO. + */ + lockdep_assert_held(gpio_res->lock); + + /* Check whether the memory map exist. */ + if (gpio_res->io) + return 0; + + params = gpio_res->params; + size = resource_size(params); + + if (!devm_request_mem_region(dev, params->start, size, params->name)) + return -EFAULT; + + gpio_res->io = devm_ioremap(dev, params->start, size); + if (IS_ERR(gpio_res->io)) { + devm_release_mem_region(dev, params->start, size); + return PTR_ERR(gpio_res->io); + } + + return 0; +} + +static int mlxbf_i2c_release_gpio(struct platform_device *pdev, + struct mlxbf_i2c_priv *priv) +{ + struct mlxbf_i2c_resource *gpio_res; + struct device *dev = &pdev->dev; + struct resource *params; + + gpio_res = mlxbf_i2c_get_shared_resource(priv, MLXBF_I2C_GPIO_RES); + if (!gpio_res) + return 0; + + mutex_lock(gpio_res->lock); + + if (gpio_res->io) { + /* Release the GPIO resource. */ + params = gpio_res->params; + devm_iounmap(dev, gpio_res->io); + devm_release_mem_region(dev, params->start, + resource_size(params)); + } + + mutex_unlock(gpio_res->lock); + + return 0; +} + +static int mlxbf_i2c_get_corepll(struct platform_device *pdev, + struct mlxbf_i2c_priv *priv) +{ + struct mlxbf_i2c_resource *corepll_res; + struct device *dev = &pdev->dev; + struct resource *params; + resource_size_t size; + + corepll_res = mlxbf_i2c_get_shared_resource(priv, + MLXBF_I2C_COREPLL_RES); + if (!corepll_res) + return -EPERM; + + /* + * The COREPLL region in TYU space is shared among I2C busses. + * This function MUST be serialized to avoid racing when + * claiming the memory region. + */ + lockdep_assert_held(corepll_res->lock); + + /* Check whether the memory map exist. */ + if (corepll_res->io) + return 0; + + params = corepll_res->params; + size = resource_size(params); + + if (!devm_request_mem_region(dev, params->start, size, params->name)) + return -EFAULT; + + corepll_res->io = devm_ioremap(dev, params->start, size); + if (IS_ERR(corepll_res->io)) { + devm_release_mem_region(dev, params->start, size); + return PTR_ERR(corepll_res->io); + } + + return 0; +} + +static int mlxbf_i2c_release_corepll(struct platform_device *pdev, + struct mlxbf_i2c_priv *priv) +{ + struct mlxbf_i2c_resource *corepll_res; + struct device *dev = &pdev->dev; + struct resource *params; + + corepll_res = mlxbf_i2c_get_shared_resource(priv, + MLXBF_I2C_COREPLL_RES); + + mutex_lock(corepll_res->lock); + + if (corepll_res->io) { + /* Release the CorePLL resource. */ + params = corepll_res->params; + devm_iounmap(dev, corepll_res->io); + devm_release_mem_region(dev, params->start, + resource_size(params)); + } + + mutex_unlock(corepll_res->lock); + + return 0; +} + +static int mlxbf_i2c_init_master(struct platform_device *pdev, + struct mlxbf_i2c_priv *priv) +{ + struct mlxbf_i2c_resource *gpio_res; + struct device *dev = &pdev->dev; + u32 config_reg; + int ret; + + /* This configuration is only needed for BlueField 1. */ + if (!mlxbf_i2c_has_chip_type(priv, MLXBF_I2C_CHIP_TYPE_1)) + return 0; + + gpio_res = mlxbf_i2c_get_shared_resource(priv, MLXBF_I2C_GPIO_RES); + if (!gpio_res) + return -EPERM; + + /* + * The GPIO region in TYU space is shared among I2C busses. + * This function MUST be serialized to avoid racing when + * claiming the memory region and/or setting up the GPIO. + */ + + mutex_lock(gpio_res->lock); + + ret = mlxbf_i2c_get_gpio(pdev, priv); + if (ret < 0) { + dev_err(dev, "Failed to get gpio resource"); + mutex_unlock(gpio_res->lock); + return ret; + } + + /* + * TYU - Configuration for GPIO pins. Those pins must be asserted in + * MLXBF_I2C_GPIO_0_FUNC_EN_0, i.e. GPIO 0 is controlled by HW, and must + * be reset in MLXBF_I2C_GPIO_0_FORCE_OE_EN, i.e. GPIO_OE will be driven + * instead of HW_OE. + * For now, we do not reset the GPIO state when the driver is removed. + * First, it is not necessary to disable the bus since we are using + * the same busses. Then, some busses might be shared among Linux and + * platform firmware; disabling the bus might compromise the system + * functionality. + */ + config_reg = mlxbf_i2c_read(gpio_res->io, + MLXBF_I2C_GPIO_0_FUNC_EN_0); + config_reg = MLXBF_I2C_GPIO_SMBUS_GW_ASSERT_PINS(priv->bus, + config_reg); + mlxbf_i2c_write(gpio_res->io, MLXBF_I2C_GPIO_0_FUNC_EN_0, + config_reg); + + config_reg = mlxbf_i2c_read(gpio_res->io, + MLXBF_I2C_GPIO_0_FORCE_OE_EN); + config_reg = MLXBF_I2C_GPIO_SMBUS_GW_RESET_PINS(priv->bus, + config_reg); + mlxbf_i2c_write(gpio_res->io, MLXBF_I2C_GPIO_0_FORCE_OE_EN, + config_reg); + + mutex_unlock(gpio_res->lock); + + return 0; +} + +static u64 mlxbf_calculate_freq_from_tyu(struct mlxbf_i2c_resource *corepll_res) +{ + u64 core_frequency, pad_frequency; + u8 core_od, core_r; + u32 corepll_val; + u16 core_f; + + pad_frequency = MLXBF_I2C_TYU_PLL_IN_FREQ; + + corepll_val = mlxbf_i2c_read(corepll_res->io, + MLXBF_I2C_CORE_PLL_REG1); + + /* Get Core PLL configuration bits. */ + core_f = rol32(corepll_val, MLXBF_I2C_COREPLL_CORE_F_TYU_SHIFT) & + MLXBF_I2C_COREPLL_CORE_F_TYU_MASK; + core_od = rol32(corepll_val, MLXBF_I2C_COREPLL_CORE_OD_TYU_SHIFT) & + MLXBF_I2C_COREPLL_CORE_OD_TYU_MASK; + core_r = rol32(corepll_val, MLXBF_I2C_COREPLL_CORE_R_TYU_SHIFT) & + MLXBF_I2C_COREPLL_CORE_R_TYU_MASK; + + /* + * Compute PLL output frequency as follow: + * + * CORE_F + 1 + * PLL_OUT_FREQ = PLL_IN_FREQ * ---------------------------- + * (CORE_R + 1) * (CORE_OD + 1) + * + * Where PLL_OUT_FREQ and PLL_IN_FREQ refer to CoreFrequency + * and PadFrequency, respectively. + */ + core_frequency = pad_frequency * (++core_f); + core_frequency /= (++core_r) * (++core_od); + + return core_frequency; +} + +static u64 mlxbf_calculate_freq_from_yu(struct mlxbf_i2c_resource *corepll_res) +{ + u32 corepll_reg1_val, corepll_reg2_val; + u64 corepll_frequency, pad_frequency; + u8 core_od, core_r; + u32 core_f; + + pad_frequency = MLXBF_I2C_YU_PLL_IN_FREQ; + + corepll_reg1_val = mlxbf_i2c_read(corepll_res->io, + MLXBF_I2C_CORE_PLL_REG1); + corepll_reg2_val = mlxbf_i2c_read(corepll_res->io, + MLXBF_I2C_CORE_PLL_REG2); + + /* Get Core PLL configuration bits */ + core_f = rol32(corepll_reg1_val, MLXBF_I2C_COREPLL_CORE_F_YU_SHIFT) & + MLXBF_I2C_COREPLL_CORE_F_YU_MASK; + core_r = rol32(corepll_reg1_val, MLXBF_I2C_COREPLL_CORE_R_YU_SHIFT) & + MLXBF_I2C_COREPLL_CORE_R_YU_MASK; + core_od = rol32(corepll_reg2_val, MLXBF_I2C_COREPLL_CORE_OD_YU_SHIFT) & + MLXBF_I2C_COREPLL_CORE_OD_YU_MASK; + + /* + * Compute PLL output frequency as follow: + * + * CORE_F / 16384 + * PLL_OUT_FREQ = PLL_IN_FREQ * ---------------------------- + * (CORE_R + 1) * (CORE_OD + 1) + * + * Where PLL_OUT_FREQ and PLL_IN_FREQ refer to CoreFrequency + * and PadFrequency, respectively. + */ + corepll_frequency = (pad_frequency * core_f) / MLNXBF_I2C_COREPLL_CONST; + corepll_frequency /= (++core_r) * (++core_od); + + return corepll_frequency; +} + +static int mlxbf_i2c_calculate_corepll_freq(struct platform_device *pdev, + struct mlxbf_i2c_priv *priv) +{ + const struct mlxbf_i2c_chip_info *chip = priv->chip; + struct mlxbf_i2c_resource *corepll_res; + struct device *dev = &pdev->dev; + u64 *freq = &priv->frequency; + int ret; + + corepll_res = mlxbf_i2c_get_shared_resource(priv, + MLXBF_I2C_COREPLL_RES); + if (!corepll_res) + return -EPERM; + + /* + * First, check whether the TYU core Clock frequency is set. + * The TYU core frequency is the same for all I2C busses; when + * the first device gets probed the frequency is determined and + * stored into a globally visible variable. So, first of all, + * check whether the frequency is already set. Here, we assume + * that the frequency is expected to be greater than 0. + */ + mutex_lock(corepll_res->lock); + if (!mlxbf_i2c_corepll_frequency) { + if (!chip->calculate_freq) { + mutex_unlock(corepll_res->lock); + return -EPERM; + } + + ret = mlxbf_i2c_get_corepll(pdev, priv); + if (ret < 0) { + dev_err(dev, "Failed to get corePLL resource"); + mutex_unlock(corepll_res->lock); + return ret; + } + + mlxbf_i2c_corepll_frequency = chip->calculate_freq(corepll_res); + } + mutex_unlock(corepll_res->lock); + + *freq = mlxbf_i2c_corepll_frequency; + + return 0; +} + +static int mlxbf_slave_enable(struct mlxbf_i2c_priv *priv, u8 addr) +{ + u32 slave_reg, slave_reg_tmp, slave_reg_avail, slave_addr_mask; + u8 reg, reg_cnt, byte, addr_tmp, reg_avail, byte_avail; + bool avail, disabled; + + disabled = false; + avail = false; + + if (!priv) + return -EPERM; + + reg_cnt = MLXBF_I2C_SMBUS_SLAVE_ADDR_CNT >> 2; + slave_addr_mask = MLXBF_I2C_SMBUS_SLAVE_ADDR_MASK; + + /* + * Read the slave registers. There are 4 * 32-bit slave registers. + * Each slave register can hold up to 4 * 8-bit slave configuration + * (7-bit address, 1 status bit (1 if enabled, 0 if not)). + */ + for (reg = 0; reg < reg_cnt; reg++) { + slave_reg = mlxbf_i2c_read(priv->smbus->io, + MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG + reg * 0x4); + /* + * Each register holds 4 slave addresses. So, we have to keep + * the byte order consistent with the value read in order to + * update the register correctly, if needed. + */ + slave_reg_tmp = slave_reg; + for (byte = 0; byte < 4; byte++) { + addr_tmp = slave_reg_tmp & GENMASK(7, 0); + + /* + * Mark the first available slave address slot, i.e. its + * enabled bit should be unset. This slot might be used + * later on to register our slave. + */ + if (!avail && !MLXBF_I2C_SLAVE_ADDR_ENABLED(addr_tmp)) { + avail = true; + reg_avail = reg; + byte_avail = byte; + slave_reg_avail = slave_reg; + } + + /* + * Parse slave address bytes and check whether the + * slave address already exists and it's enabled, + * i.e. most significant bit is set. + */ + if ((addr_tmp & slave_addr_mask) == addr) { + if (MLXBF_I2C_SLAVE_ADDR_ENABLED(addr_tmp)) + return 0; + disabled = true; + break; + } + + /* Parse next byte. */ + slave_reg_tmp >>= 8; + } + + /* Exit the loop if the slave address is found. */ + if (disabled) + break; + } + + if (!avail && !disabled) + return -EINVAL; /* No room for a new slave address. */ + + if (avail && !disabled) { + reg = reg_avail; + byte = byte_avail; + /* Set the slave address. */ + slave_reg_avail &= ~(slave_addr_mask << (byte * 8)); + slave_reg_avail |= addr << (byte * 8); + slave_reg = slave_reg_avail; + } + + /* Enable the slave address and update the register. */ + slave_reg |= (1 << MLXBF_I2C_SMBUS_SLAVE_ADDR_EN_BIT) << (byte * 8); + mlxbf_i2c_write(priv->smbus->io, + MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG + reg * 0x4, slave_reg); + + return 0; +} + +static int mlxbf_slave_disable(struct mlxbf_i2c_priv *priv) +{ + u32 slave_reg, slave_reg_tmp, slave_addr_mask; + u8 addr, addr_tmp, reg, reg_cnt, slave_byte; + struct i2c_client *client = priv->slave; + bool exist; + + exist = false; + + addr = client->addr; + reg_cnt = MLXBF_I2C_SMBUS_SLAVE_ADDR_CNT >> 2; + slave_addr_mask = MLXBF_I2C_SMBUS_SLAVE_ADDR_MASK; + + /* + * Read the slave registers. There are 4 * 32-bit slave registers. + * Each slave register can hold up to 4 * 8-bit slave configuration + * (7-bit address, 1 status bit (1 if enabled, 0 if not)). + */ + for (reg = 0; reg < reg_cnt; reg++) { + slave_reg = mlxbf_i2c_read(priv->smbus->io, + MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG + reg * 0x4); + + /* Check whether the address slots are empty. */ + if (slave_reg == 0) + continue; + + /* + * Each register holds 4 slave addresses. So, we have to keep + * the byte order consistent with the value read in order to + * update the register correctly, if needed. + */ + slave_reg_tmp = slave_reg; + slave_byte = 0; + while (slave_reg_tmp != 0) { + addr_tmp = slave_reg_tmp & slave_addr_mask; + /* + * Parse slave address bytes and check whether the + * slave address already exists. + */ + if (addr_tmp == addr) { + exist = true; + break; + } + + /* Parse next byte. */ + slave_reg_tmp >>= 8; + slave_byte += 1; + } + + /* Exit the loop if the slave address is found. */ + if (exist) + break; + } + + if (!exist) + return 0; /* Slave is not registered, nothing to do. */ + + /* Cleanup the slave address slot. */ + slave_reg &= ~(GENMASK(7, 0) << (slave_byte * 8)); + mlxbf_i2c_write(priv->smbus->io, + MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG + reg * 0x4, slave_reg); + + return 0; +} + +static int mlxbf_i2c_init_coalesce(struct platform_device *pdev, + struct mlxbf_i2c_priv *priv) +{ + struct mlxbf_i2c_resource *coalesce_res; + struct resource *params; + resource_size_t size; + int ret = 0; + + /* + * Unlike BlueField-1 platform, the coalesce registers is a dedicated + * resource in the next generations of BlueField. + */ + if (mlxbf_i2c_has_chip_type(priv, MLXBF_I2C_CHIP_TYPE_1)) { + coalesce_res = mlxbf_i2c_get_shared_resource(priv, + MLXBF_I2C_COALESCE_RES); + if (!coalesce_res) + return -EPERM; + + /* + * The Cause Coalesce group in TYU space is shared among + * I2C busses. This function MUST be serialized to avoid + * racing when claiming the memory region. + */ + lockdep_assert_held(mlxbf_i2c_gpio_res->lock); + + /* Check whether the memory map exist. */ + if (coalesce_res->io) { + priv->coalesce = coalesce_res; + return 0; + } + + params = coalesce_res->params; + size = resource_size(params); + + if (!request_mem_region(params->start, size, params->name)) + return -EFAULT; + + coalesce_res->io = ioremap(params->start, size); + if (IS_ERR(coalesce_res->io)) { + release_mem_region(params->start, size); + return PTR_ERR(coalesce_res->io); + } + + priv->coalesce = coalesce_res; + + } else { + ret = mlxbf_i2c_init_resource(pdev, &priv->coalesce, + MLXBF_I2C_COALESCE_RES); + } + + return ret; +} + +static int mlxbf_i2c_release_coalesce(struct platform_device *pdev, + struct mlxbf_i2c_priv *priv) +{ + struct mlxbf_i2c_resource *coalesce_res; + struct device *dev = &pdev->dev; + struct resource *params; + resource_size_t size; + + coalesce_res = priv->coalesce; + + if (coalesce_res->io) { + params = coalesce_res->params; + size = resource_size(params); + if (mlxbf_i2c_has_chip_type(priv, MLXBF_I2C_CHIP_TYPE_1)) { + mutex_lock(coalesce_res->lock); + iounmap(coalesce_res->io); + release_mem_region(params->start, size); + mutex_unlock(coalesce_res->lock); + } else { + devm_release_mem_region(dev, params->start, size); + } + } + + return 0; +} + +static int mlxbf_i2c_init_slave(struct platform_device *pdev, + struct mlxbf_i2c_priv *priv) +{ + struct device *dev = &pdev->dev; + u32 int_reg; + int ret; + + /* Reset FSM. */ + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_SLAVE_FSM, 0); + + /* + * Enable slave cause interrupt bits. Drive + * MLXBF_I2C_CAUSE_READ_WAIT_FW_RESPONSE and + * MLXBF_I2C_CAUSE_WRITE_SUCCESS, these are enabled when an external + * masters issue a Read and Write, respectively. But, clear all + * interrupts first. + */ + mlxbf_i2c_write(priv->slv_cause->io, + MLXBF_I2C_CAUSE_OR_CLEAR, ~0); + int_reg = MLXBF_I2C_CAUSE_READ_WAIT_FW_RESPONSE; + int_reg |= MLXBF_I2C_CAUSE_WRITE_SUCCESS; + mlxbf_i2c_write(priv->slv_cause->io, + MLXBF_I2C_CAUSE_OR_EVTEN0, int_reg); + + /* Finally, set the 'ready' bit to start handling transactions. */ + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_SLAVE_READY, 0x1); + + /* Initialize the cause coalesce resource. */ + ret = mlxbf_i2c_init_coalesce(pdev, priv); + if (ret < 0) { + dev_err(dev, "failed to initialize cause coalesce\n"); + return ret; + } + + return 0; +} + +static bool mlxbf_i2c_has_coalesce(struct mlxbf_i2c_priv *priv, bool *read, + bool *write) +{ + const struct mlxbf_i2c_chip_info *chip = priv->chip; + u32 coalesce0_reg, cause_reg; + u8 slave_shift, is_set; + + *write = false; + *read = false; + + slave_shift = chip->type != MLXBF_I2C_CHIP_TYPE_1 ? + MLXBF_I2C_CAUSE_YU_SLAVE_BIT : + priv->bus + MLXBF_I2C_CAUSE_TYU_SLAVE_BIT; + + coalesce0_reg = mlxbf_i2c_read(priv->coalesce->io, + MLXBF_I2C_CAUSE_COALESCE_0); + is_set = coalesce0_reg & (1 << slave_shift); + + if (!is_set) + return false; + + /* Check the source of the interrupt, i.e. whether a Read or Write. */ + cause_reg = mlxbf_i2c_read(priv->slv_cause->io, + MLXBF_I2C_CAUSE_ARBITER); + if (cause_reg & MLXBF_I2C_CAUSE_READ_WAIT_FW_RESPONSE) + *read = true; + else if (cause_reg & MLXBF_I2C_CAUSE_WRITE_SUCCESS) + *write = true; + + /* Clear cause bits. */ + mlxbf_i2c_write(priv->slv_cause->io, MLXBF_I2C_CAUSE_OR_CLEAR, ~0x0); + + return true; +} + +static bool mlxbf_smbus_slave_wait_for_idle(struct mlxbf_i2c_priv *priv, + u32 timeout) +{ + u32 mask = MLXBF_I2C_CAUSE_S_GW_BUSY_FALL; + u32 addr = MLXBF_I2C_CAUSE_ARBITER; + + if (mlxbf_smbus_poll(priv->slv_cause->io, addr, mask, false, timeout)) + return true; + + return false; +} + +/* Send byte to 'external' smbus master. */ +static int mlxbf_smbus_irq_send(struct mlxbf_i2c_priv *priv, u8 recv_bytes) +{ + u8 data_desc[MLXBF_I2C_SLAVE_DATA_DESC_SIZE] = { 0 }; + u8 write_size, pec_en, addr, byte, value, byte_cnt, desc_size; + struct i2c_client *slave = priv->slave; + u32 control32, data32; + int ret; + + if (!slave) + return -EINVAL; + + addr = 0; + byte = 0; + desc_size = MLXBF_I2C_SLAVE_DATA_DESC_SIZE; + + /* + * Read bytes received from the external master. These bytes should + * be located in the first data descriptor register of the slave GW. + * These bytes are the slave address byte and the internal register + * address, if supplied. + */ + if (recv_bytes > 0) { + data32 = mlxbf_i2c_read_data(priv->smbus->io, + MLXBF_I2C_SLAVE_DATA_DESC_ADDR); + + /* Parse the received bytes. */ + switch (recv_bytes) { + case 2: + byte = (data32 >> 8) & GENMASK(7, 0); + fallthrough; + case 1: + addr = (data32 & GENMASK(7, 0)) >> 1; + } + + /* Check whether it's our slave address. */ + if (slave->addr != addr) + return -EINVAL; + } + + /* + * I2C read transactions may start by a WRITE followed by a READ. + * Indeed, most slave devices would expect the internal address + * following the slave address byte. So, write that byte first, + * and then, send the requested data bytes to the master. + */ + if (recv_bytes > 1) { + i2c_slave_event(slave, I2C_SLAVE_WRITE_REQUESTED, &value); + value = byte; + ret = i2c_slave_event(slave, I2C_SLAVE_WRITE_RECEIVED, + &value); + i2c_slave_event(slave, I2C_SLAVE_STOP, &value); + + if (ret < 0) + return ret; + } + + /* + * Now, send data to the master; currently, the driver supports + * READ_BYTE, READ_WORD and BLOCK READ protocols. Note that the + * hardware can send up to 128 bytes per transfer. That is the + * size of its data registers. + */ + i2c_slave_event(slave, I2C_SLAVE_READ_REQUESTED, &value); + + for (byte_cnt = 0; byte_cnt < desc_size; byte_cnt++) { + data_desc[byte_cnt] = value; + i2c_slave_event(slave, I2C_SLAVE_READ_PROCESSED, &value); + } + + /* Send a stop condition to the backend. */ + i2c_slave_event(slave, I2C_SLAVE_STOP, &value); + + /* Handle the actual transfer. */ + + /* Set the number of bytes to write to master. */ + write_size = (byte_cnt - 1) & 0x7f; + + /* Write data to Slave GW data descriptor. */ + mlxbf_i2c_smbus_write_data(priv, data_desc, byte_cnt, + MLXBF_I2C_SLAVE_DATA_DESC_ADDR); + + pec_en = 0; /* Disable PEC since it is not supported. */ + + /* Prepare control word. */ + control32 = MLXBF_I2C_SLAVE_ENABLE; + control32 |= rol32(write_size, MLXBF_I2C_SLAVE_WRITE_BYTES_SHIFT); + control32 |= rol32(pec_en, MLXBF_I2C_SLAVE_SEND_PEC_SHIFT); + + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_SLAVE_GW, control32); + + /* + * Wait until the transfer is completed; the driver will wait + * until the GW is idle, a cause will rise on fall of GW busy. + */ + mlxbf_smbus_slave_wait_for_idle(priv, MLXBF_I2C_SMBUS_TIMEOUT); + + /* Release the Slave GW. */ + mlxbf_i2c_write(priv->smbus->io, + MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES, 0x0); + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_SLAVE_PEC, 0x0); + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_SLAVE_READY, 0x1); + + return 0; +} + +/* Receive bytes from 'external' smbus master. */ +static int mlxbf_smbus_irq_recv(struct mlxbf_i2c_priv *priv, u8 recv_bytes) +{ + u8 data_desc[MLXBF_I2C_SLAVE_DATA_DESC_SIZE] = { 0 }; + struct i2c_client *slave = priv->slave; + u8 value, byte, addr; + int ret = 0; + + if (!slave) + return -EINVAL; + + /* Read data from Slave GW data descriptor. */ + mlxbf_i2c_smbus_read_data(priv, data_desc, recv_bytes, + MLXBF_I2C_SLAVE_DATA_DESC_ADDR); + + /* Check whether its our slave address. */ + addr = data_desc[0] >> 1; + if (slave->addr != addr) + return -EINVAL; + + /* + * Notify the slave backend; another I2C master wants to write data + * to us. This event is sent once the slave address and the write bit + * is detected. + */ + i2c_slave_event(slave, I2C_SLAVE_WRITE_REQUESTED, &value); + + /* Send the received data to the slave backend. */ + for (byte = 1; byte < recv_bytes; byte++) { + value = data_desc[byte]; + ret = i2c_slave_event(slave, I2C_SLAVE_WRITE_RECEIVED, + &value); + if (ret < 0) + break; + } + + /* Send a stop condition to the backend. */ + i2c_slave_event(slave, I2C_SLAVE_STOP, &value); + + /* Release the Slave GW. */ + mlxbf_i2c_write(priv->smbus->io, + MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES, 0x0); + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_SLAVE_PEC, 0x0); + mlxbf_i2c_write(priv->smbus->io, MLXBF_I2C_SMBUS_SLAVE_READY, 0x1); + + return ret; +} + +static irqreturn_t mlxbf_smbus_irq(int irq, void *ptr) +{ + struct mlxbf_i2c_priv *priv = ptr; + bool read, write, irq_is_set; + u32 rw_bytes_reg; + u8 recv_bytes; + + /* + * Read TYU interrupt register and determine the source of the + * interrupt. Based on the source of the interrupt one of the + * following actions are performed: + * - Receive data and send response to master. + * - Send data and release slave GW. + * + * Handle read/write transaction only. CRmaster and Iarp requests + * are ignored for now. + */ + irq_is_set = mlxbf_i2c_has_coalesce(priv, &read, &write); + if (!irq_is_set || (!read && !write)) { + /* Nothing to do here, interrupt was not from this device. */ + return IRQ_NONE; + } + + /* + * The MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES includes the number of + * bytes from/to master. These are defined by 8-bits each. If the lower + * 8 bits are set, then the master expect to read N bytes from the + * slave, if the higher 8 bits are sent then the slave expect N bytes + * from the master. + */ + rw_bytes_reg = mlxbf_i2c_read(priv->smbus->io, + MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES); + recv_bytes = (rw_bytes_reg >> 8) & GENMASK(7, 0); + + /* + * For now, the slave supports 128 bytes transfer. Discard remaining + * data bytes if the master wrote more than + * MLXBF_I2C_SLAVE_DATA_DESC_SIZE, i.e, the actual size of the slave + * data descriptor. + * + * Note that we will never expect to transfer more than 128 bytes; as + * specified in the SMBus standard, block transactions cannot exceed + * 32 bytes. + */ + recv_bytes = recv_bytes > MLXBF_I2C_SLAVE_DATA_DESC_SIZE ? + MLXBF_I2C_SLAVE_DATA_DESC_SIZE : recv_bytes; + + if (read) + mlxbf_smbus_irq_send(priv, recv_bytes); + else + mlxbf_smbus_irq_recv(priv, recv_bytes); + + return IRQ_HANDLED; +} + +/* Return negative errno on error. */ +static s32 mlxbf_i2c_smbus_xfer(struct i2c_adapter *adap, u16 addr, + unsigned short flags, char read_write, + u8 command, int size, + union i2c_smbus_data *data) +{ + struct mlxbf_i2c_smbus_request request = { 0 }; + struct mlxbf_i2c_priv *priv; + bool read, pec; + u8 byte_cnt; + + request.slave = addr; + + read = (read_write == I2C_SMBUS_READ); + pec = flags & I2C_FUNC_SMBUS_PEC; + + switch (size) { + case I2C_SMBUS_QUICK: + mlxbf_i2c_smbus_quick_command(&request, read); + dev_dbg(&adap->dev, "smbus quick, slave 0x%02x\n", addr); + break; + + case I2C_SMBUS_BYTE: + mlxbf_i2c_smbus_byte_func(&request, + read ? &data->byte : &command, read, + pec); + dev_dbg(&adap->dev, "smbus %s byte, slave 0x%02x.\n", + read ? "read" : "write", addr); + break; + + case I2C_SMBUS_BYTE_DATA: + mlxbf_i2c_smbus_data_byte_func(&request, &command, &data->byte, + read, pec); + dev_dbg(&adap->dev, "smbus %s byte data at 0x%02x, slave 0x%02x.\n", + read ? "read" : "write", command, addr); + break; + + case I2C_SMBUS_WORD_DATA: + mlxbf_i2c_smbus_data_word_func(&request, &command, + (u8 *)&data->word, read, pec); + dev_dbg(&adap->dev, "smbus %s word data at 0x%02x, slave 0x%02x.\n", + read ? "read" : "write", command, addr); + break; + + case I2C_SMBUS_I2C_BLOCK_DATA: + byte_cnt = data->block[0]; + mlxbf_i2c_smbus_i2c_block_func(&request, &command, data->block, + &byte_cnt, read, pec); + dev_dbg(&adap->dev, "i2c %s block data, %d bytes at 0x%02x, slave 0x%02x.\n", + read ? "read" : "write", byte_cnt, command, addr); + break; + + case I2C_SMBUS_BLOCK_DATA: + byte_cnt = read ? I2C_SMBUS_BLOCK_MAX : data->block[0]; + mlxbf_i2c_smbus_block_func(&request, &command, data->block, + &byte_cnt, read, pec); + dev_dbg(&adap->dev, "smbus %s block data, %d bytes at 0x%02x, slave 0x%02x.\n", + read ? "read" : "write", byte_cnt, command, addr); + break; + + case I2C_FUNC_SMBUS_PROC_CALL: + mlxbf_i2c_smbus_process_call_func(&request, &command, + (u8 *)&data->word, pec); + dev_dbg(&adap->dev, "process call, wr/rd at 0x%02x, slave 0x%02x.\n", + command, addr); + break; + + case I2C_FUNC_SMBUS_BLOCK_PROC_CALL: + byte_cnt = data->block[0]; + mlxbf_i2c_smbus_blk_process_call_func(&request, &command, + data->block, &byte_cnt, + pec); + dev_dbg(&adap->dev, "block process call, wr/rd %d bytes, slave 0x%02x.\n", + byte_cnt, addr); + break; + + default: + dev_dbg(&adap->dev, "Unsupported I2C/SMBus command %d\n", + size); + return -EOPNOTSUPP; + } + + priv = i2c_get_adapdata(adap); + + return mlxbf_i2c_smbus_start_transaction(priv, &request); +} + +static int mlxbf_i2c_reg_slave(struct i2c_client *slave) +{ + struct mlxbf_i2c_priv *priv = i2c_get_adapdata(slave->adapter); + int ret; + + if (priv->slave) + return -EBUSY; + + /* + * Do not support ten bit chip address and do not use Packet Error + * Checking (PEC). + */ + if (slave->flags & (I2C_CLIENT_TEN | I2C_CLIENT_PEC)) + return -EAFNOSUPPORT; + + ret = mlxbf_slave_enable(priv, slave->addr); + if (ret < 0) + return ret; + + priv->slave = slave; + + return 0; +} + +static int mlxbf_i2c_unreg_slave(struct i2c_client *slave) +{ + struct mlxbf_i2c_priv *priv = i2c_get_adapdata(slave->adapter); + int ret; + + WARN_ON(!priv->slave); + + /* Unregister slave, i.e. disable the slave address in hardware. */ + ret = mlxbf_slave_disable(priv); + if (ret < 0) + return ret; + + priv->slave = NULL; + + return 0; +} + +static u32 mlxbf_i2c_functionality(struct i2c_adapter *adap) +{ + return MLXBF_I2C_FUNC_ALL; +} + +static struct mlxbf_i2c_chip_info mlxbf_i2c_chip[] = { + [MLXBF_I2C_CHIP_TYPE_1] = { + .type = MLXBF_I2C_CHIP_TYPE_1, + .shared_res = { + [0] = &mlxbf_i2c_coalesce_res[MLXBF_I2C_CHIP_TYPE_1], + [1] = &mlxbf_i2c_corepll_res[MLXBF_I2C_CHIP_TYPE_1], + [2] = &mlxbf_i2c_gpio_res[MLXBF_I2C_CHIP_TYPE_1] + }, + .calculate_freq = mlxbf_calculate_freq_from_tyu + }, + [MLXBF_I2C_CHIP_TYPE_2] = { + .type = MLXBF_I2C_CHIP_TYPE_2, + .shared_res = { + [0] = &mlxbf_i2c_corepll_res[MLXBF_I2C_CHIP_TYPE_2] + }, + .calculate_freq = mlxbf_calculate_freq_from_yu + } +}; + +static const struct i2c_algorithm mlxbf_i2c_algo = { + .smbus_xfer = mlxbf_i2c_smbus_xfer, + .functionality = mlxbf_i2c_functionality, + .reg_slave = mlxbf_i2c_reg_slave, + .unreg_slave = mlxbf_i2c_unreg_slave, +}; + +static struct i2c_adapter_quirks mlxbf_i2c_quirks = { + .max_read_len = MLXBF_I2C_MASTER_DATA_R_LENGTH, + .max_write_len = MLXBF_I2C_MASTER_DATA_W_LENGTH, +}; + +static const struct of_device_id mlxbf_i2c_dt_ids[] = { + { + .compatible = "mellanox,i2c-mlxbf1", + .data = &mlxbf_i2c_chip[MLXBF_I2C_CHIP_TYPE_1] + }, + { + .compatible = "mellanox,i2c-mlxbf2", + .data = &mlxbf_i2c_chip[MLXBF_I2C_CHIP_TYPE_2] + }, + {}, +}; + +MODULE_DEVICE_TABLE(of, mlxbf_i2c_dt_ids); + +static const struct acpi_device_id mlxbf_i2c_acpi_ids[] = { + { "MLNXBF03", (kernel_ulong_t)&mlxbf_i2c_chip[MLXBF_I2C_CHIP_TYPE_1] }, + { "MLNXBF23", (kernel_ulong_t)&mlxbf_i2c_chip[MLXBF_I2C_CHIP_TYPE_2] }, + {}, +}; + +MODULE_DEVICE_TABLE(acpi, mlxbf_i2c_acpi_ids); + +static int mlxbf_i2c_acpi_probe(struct device *dev, struct mlxbf_i2c_priv *priv) +{ + const struct acpi_device_id *aid; + struct acpi_device *adev; + unsigned long bus_id = 0; + const char *uid; + int ret; + + if (acpi_disabled) + return -ENOENT; + + adev = ACPI_COMPANION(dev); + if (!adev) + return -ENXIO; + + aid = acpi_match_device(mlxbf_i2c_acpi_ids, dev); + if (!aid) + return -ENODEV; + + priv->chip = (struct mlxbf_i2c_chip_info *)aid->driver_data; + + uid = acpi_device_uid(adev); + if (!uid || !(*uid)) { + dev_err(dev, "Cannot retrieve UID\n"); + return -ENODEV; + } + + ret = kstrtoul(uid, 0, &bus_id); + if (!ret) + priv->bus = bus_id; + + return ret; +} + +static int mlxbf_i2c_of_probe(struct device *dev, struct mlxbf_i2c_priv *priv) +{ + const struct of_device_id *oid; + int bus_id = -1; + + if (IS_ENABLED(CONFIG_OF) && dev->of_node) { + oid = of_match_node(mlxbf_i2c_dt_ids, dev->of_node); + if (!oid) + return -ENODEV; + + priv->chip = oid->data; + + bus_id = of_alias_get_id(dev->of_node, "i2c"); + if (bus_id >= 0) + priv->bus = bus_id; + } + + if (bus_id < 0) { + dev_err(dev, "Cannot get bus id"); + return bus_id; + } + + return 0; +} + +static int mlxbf_i2c_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct mlxbf_i2c_priv *priv; + struct i2c_adapter *adap; + int irq, ret; + + priv = devm_kzalloc(dev, sizeof(struct mlxbf_i2c_priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + ret = mlxbf_i2c_acpi_probe(dev, priv); + if (ret < 0 && ret != -ENOENT && ret != -ENXIO) + ret = mlxbf_i2c_of_probe(dev, priv); + + if (ret < 0) + return ret; + + ret = mlxbf_i2c_init_resource(pdev, &priv->smbus, + MLXBF_I2C_SMBUS_RES); + if (ret < 0) { + dev_err(dev, "Cannot fetch smbus resource info"); + return ret; + } + + ret = mlxbf_i2c_init_resource(pdev, &priv->mst_cause, + MLXBF_I2C_MST_CAUSE_RES); + if (ret < 0) { + dev_err(dev, "Cannot fetch cause master resource info"); + return ret; + } + + ret = mlxbf_i2c_init_resource(pdev, &priv->slv_cause, + MLXBF_I2C_SLV_CAUSE_RES); + if (ret < 0) { + dev_err(dev, "Cannot fetch cause slave resource info"); + return ret; + } + + adap = &priv->adap; + adap->owner = THIS_MODULE; + adap->class = I2C_CLASS_HWMON; + adap->algo = &mlxbf_i2c_algo; + adap->quirks = &mlxbf_i2c_quirks; + adap->dev.parent = dev; + adap->dev.of_node = dev->of_node; + adap->nr = priv->bus; + + snprintf(adap->name, sizeof(adap->name), "i2c%d", adap->nr); + i2c_set_adapdata(adap, priv); + + /* Read Core PLL frequency. */ + ret = mlxbf_i2c_calculate_corepll_freq(pdev, priv); + if (ret < 0) { + dev_err(dev, "cannot get core clock frequency\n"); + /* Set to default value. */ + priv->frequency = MLXBF_I2C_COREPLL_FREQ; + } + + /* + * Initialize master. + * Note that a physical bus might be shared among Linux and firmware + * (e.g., ATF). Thus, the bus should be initialized and ready and + * bus initialization would be unnecessary. This requires additional + * knowledge about physical busses. But, since an extra initialization + * does not really hurt, then keep the code as is. + */ + ret = mlxbf_i2c_init_master(pdev, priv); + if (ret < 0) { + dev_err(dev, "failed to initialize smbus master %d", + priv->bus); + return ret; + } + + mlxbf_i2c_init_timings(pdev, priv); + + mlxbf_i2c_init_slave(pdev, priv); + + irq = platform_get_irq(pdev, 0); + ret = devm_request_irq(dev, irq, mlxbf_smbus_irq, + IRQF_ONESHOT | IRQF_SHARED | IRQF_PROBE_SHARED, + dev_name(dev), priv); + if (ret < 0) { + dev_err(dev, "Cannot get irq %d\n", irq); + return ret; + } + + priv->irq = irq; + + platform_set_drvdata(pdev, priv); + + ret = i2c_add_numbered_adapter(adap); + if (ret < 0) + return ret; + + mutex_lock(&mlxbf_i2c_bus_lock); + mlxbf_i2c_bus_count++; + mutex_unlock(&mlxbf_i2c_bus_lock); + + return 0; +} + +static int mlxbf_i2c_remove(struct platform_device *pdev) +{ + struct mlxbf_i2c_priv *priv = platform_get_drvdata(pdev); + struct device *dev = &pdev->dev; + struct resource *params; + + params = priv->smbus->params; + devm_release_mem_region(dev, params->start, resource_size(params)); + + params = priv->mst_cause->params; + devm_release_mem_region(dev, params->start, resource_size(params)); + + params = priv->slv_cause->params; + devm_release_mem_region(dev, params->start, resource_size(params)); + + /* + * Release shared resources. This should be done when releasing + * the I2C controller. + */ + mutex_lock(&mlxbf_i2c_bus_lock); + if (--mlxbf_i2c_bus_count == 0) { + mlxbf_i2c_release_coalesce(pdev, priv); + mlxbf_i2c_release_corepll(pdev, priv); + mlxbf_i2c_release_gpio(pdev, priv); + } + mutex_unlock(&mlxbf_i2c_bus_lock); + + devm_free_irq(dev, priv->irq, priv); + + i2c_del_adapter(&priv->adap); + + return 0; +} + +static struct platform_driver mlxbf_i2c_driver = { + .probe = mlxbf_i2c_probe, + .remove = mlxbf_i2c_remove, + .driver = { + .name = "i2c-mlxbf", + .of_match_table = mlxbf_i2c_dt_ids, + .acpi_match_table = ACPI_PTR(mlxbf_i2c_acpi_ids), + }, +}; + +static int __init mlxbf_i2c_init(void) +{ + mutex_init(&mlxbf_i2c_coalesce_lock); + mutex_init(&mlxbf_i2c_corepll_lock); + mutex_init(&mlxbf_i2c_gpio_lock); + + mutex_init(&mlxbf_i2c_bus_lock); + + return platform_driver_register(&mlxbf_i2c_driver); +} +module_init(mlxbf_i2c_init); + +static void __exit mlxbf_i2c_exit(void) +{ + platform_driver_unregister(&mlxbf_i2c_driver); + + mutex_destroy(&mlxbf_i2c_bus_lock); + + mutex_destroy(&mlxbf_i2c_gpio_lock); + mutex_destroy(&mlxbf_i2c_corepll_lock); + mutex_destroy(&mlxbf_i2c_coalesce_lock); +} +module_exit(mlxbf_i2c_exit); + +MODULE_DESCRIPTION("Mellanox BlueField I2C bus driver"); +MODULE_AUTHOR("Khalil Blaiech <kblaiech@mellanox.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/i2c/busses/i2c-mv64xxx.c b/drivers/i2c/busses/i2c-mv64xxx.c index 8d9d4ffdcd24..e0e45fc19b8f 100644 --- a/drivers/i2c/busses/i2c-mv64xxx.c +++ b/drivers/i2c/busses/i2c-mv64xxx.c @@ -496,11 +496,10 @@ static irqreturn_t mv64xxx_i2c_intr(int irq, void *dev_id) { struct mv64xxx_i2c_data *drv_data = dev_id; - unsigned long flags; u32 status; irqreturn_t rc = IRQ_NONE; - spin_lock_irqsave(&drv_data->lock, flags); + spin_lock(&drv_data->lock); if (drv_data->offload_enabled) rc = mv64xxx_i2c_intr_offload(drv_data); @@ -517,7 +516,7 @@ mv64xxx_i2c_intr(int irq, void *dev_id) rc = IRQ_HANDLED; } - spin_unlock_irqrestore(&drv_data->lock, flags); + spin_unlock(&drv_data->lock); return rc; } diff --git a/drivers/i2c/busses/i2c-nvidia-gpu.c b/drivers/i2c/busses/i2c-nvidia-gpu.c index f480105000b8..f9a69b109e5c 100644 --- a/drivers/i2c/busses/i2c-nvidia-gpu.c +++ b/drivers/i2c/busses/i2c-nvidia-gpu.c @@ -125,8 +125,7 @@ static int gpu_i2c_read(struct gpu_i2c_dev *i2cd, u8 *data, u16 len) put_unaligned_be16(val, data); break; case 3: - put_unaligned_be16(val >> 8, data); - data[2] = val; + put_unaligned_be24(val, data); break; case 4: put_unaligned_be32(val, data); diff --git a/drivers/i2c/busses/i2c-owl.c b/drivers/i2c/busses/i2c-owl.c index a163b8f308c1..9918b2a0b909 100644 --- a/drivers/i2c/busses/i2c-owl.c +++ b/drivers/i2c/busses/i2c-owl.c @@ -165,10 +165,9 @@ static irqreturn_t owl_i2c_interrupt(int irq, void *_dev) { struct owl_i2c_dev *i2c_dev = _dev; struct i2c_msg *msg = i2c_dev->msg; - unsigned long flags; unsigned int stat, fifostat; - spin_lock_irqsave(&i2c_dev->lock, flags); + spin_lock(&i2c_dev->lock); i2c_dev->err = 0; @@ -214,7 +213,7 @@ stop: OWL_I2C_STAT_IRQP, true); complete_all(&i2c_dev->msg_complete); - spin_unlock_irqrestore(&i2c_dev->lock, flags); + spin_unlock(&i2c_dev->lock); return IRQ_HANDLED; } diff --git a/drivers/i2c/busses/i2c-qcom-geni.c b/drivers/i2c/busses/i2c-qcom-geni.c index dead5db3315a..8b4c35f47a70 100644 --- a/drivers/i2c/busses/i2c-qcom-geni.c +++ b/drivers/i2c/busses/i2c-qcom-geni.c @@ -210,9 +210,8 @@ static irqreturn_t geni_i2c_irq(int irq, void *dev) u32 dma; u32 val; struct i2c_msg *cur; - unsigned long flags; - spin_lock_irqsave(&gi2c->lock, flags); + spin_lock(&gi2c->lock); m_stat = readl_relaxed(base + SE_GENI_M_IRQ_STATUS); rx_st = readl_relaxed(base + SE_GENI_RX_FIFO_STATUS); dm_tx_st = readl_relaxed(base + SE_DMA_TX_IRQ_STAT); @@ -294,7 +293,7 @@ static irqreturn_t geni_i2c_irq(int irq, void *dev) dm_rx_st & RX_DMA_DONE || dm_rx_st & RX_RESET_DONE) complete(&gi2c->done); - spin_unlock_irqrestore(&gi2c->lock, flags); + spin_unlock(&gi2c->lock); return IRQ_HANDLED; } diff --git a/drivers/i2c/busses/i2c-rcar.c b/drivers/i2c/busses/i2c-rcar.c index c7c543483b08..217def2d7cb4 100644 --- a/drivers/i2c/busses/i2c-rcar.c +++ b/drivers/i2c/busses/i2c-rcar.c @@ -19,7 +19,9 @@ #include <linux/err.h> #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/iopoll.h> #include <linux/i2c.h> +#include <linux/i2c-smbus.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/of_device.h> @@ -105,10 +107,11 @@ #define ID_ARBLOST (1 << 3) #define ID_NACK (1 << 4) /* persistent flags */ +#define ID_P_HOST_NOTIFY BIT(28) #define ID_P_REP_AFTER_RD BIT(29) #define ID_P_NO_RXDMA BIT(30) /* HW forbids RXDMA sometimes */ #define ID_P_PM_BLOCKED BIT(31) -#define ID_P_MASK GENMASK(31, 29) +#define ID_P_MASK GENMASK(31, 28) enum rcar_i2c_type { I2C_RCAR_GEN1, @@ -140,14 +143,13 @@ struct rcar_i2c_priv { struct reset_control *rstc; int irq; + + struct i2c_client *host_notify_client; }; #define rcar_i2c_priv_to_dev(p) ((p)->adap.dev.parent) #define rcar_i2c_is_recv(p) ((p)->msg->flags & I2C_M_RD) -#define LOOP_TIMEOUT 1024 - - static void rcar_i2c_write(struct rcar_i2c_priv *priv, int reg, u32 val) { writel(val, priv->io + reg); @@ -221,18 +223,18 @@ static void rcar_i2c_init(struct rcar_i2c_priv *priv) static int rcar_i2c_bus_barrier(struct rcar_i2c_priv *priv) { - int i; + int ret; + u32 val; - for (i = 0; i < LOOP_TIMEOUT; i++) { - /* make sure that bus is not busy */ - if (!(rcar_i2c_read(priv, ICMCR) & FSDA)) - return 0; - udelay(1); + ret = readl_poll_timeout(priv->io + ICMCR, val, !(val & FSDA), 10, + priv->adap.timeout); + if (ret) { + /* Waiting did not help, try to recover */ + priv->recovery_icmcr = MDBS | OBPC | FSDA | FSCL; + ret = i2c_recover_bus(&priv->adap); } - /* Waiting did not help, try to recover */ - priv->recovery_icmcr = MDBS | OBPC | FSDA | FSCL; - return i2c_recover_bus(&priv->adap); + return ret; } static int rcar_i2c_clock_calculate(struct rcar_i2c_priv *priv) @@ -760,20 +762,14 @@ static void rcar_i2c_release_dma(struct rcar_i2c_priv *priv) /* I2C is a special case, we need to poll the status of a reset */ static int rcar_i2c_do_reset(struct rcar_i2c_priv *priv) { - int i, ret; + int ret; ret = reset_control_reset(priv->rstc); if (ret) return ret; - for (i = 0; i < LOOP_TIMEOUT; i++) { - ret = reset_control_status(priv->rstc); - if (ret == 0) - return 0; - udelay(1); - } - - return -ETIMEDOUT; + return read_poll_timeout_atomic(reset_control_status, ret, ret == 0, 1, + 100, false, priv->rstc); } static int rcar_i2c_master_xfer(struct i2c_adapter *adap, @@ -884,14 +880,21 @@ static int rcar_unreg_slave(struct i2c_client *slave) static u32 rcar_i2c_func(struct i2c_adapter *adap) { + struct rcar_i2c_priv *priv = i2c_get_adapdata(adap); + /* * This HW can't do: * I2C_SMBUS_QUICK (setting FSB during START didn't work) * I2C_M_NOSTART (automatically sends address after START) * I2C_M_IGNORE_NAK (automatically sends STOP after NAK) */ - return I2C_FUNC_I2C | I2C_FUNC_SLAVE | - (I2C_FUNC_SMBUS_EMUL & ~I2C_FUNC_SMBUS_QUICK); + u32 func = I2C_FUNC_I2C | I2C_FUNC_SLAVE | + (I2C_FUNC_SMBUS_EMUL & ~I2C_FUNC_SMBUS_QUICK); + + if (priv->flags & ID_P_HOST_NOTIFY) + func |= I2C_FUNC_SMBUS_HOST_NOTIFY; + + return func; } static const struct i2c_algorithm rcar_i2c_algo = { @@ -991,6 +994,8 @@ static int rcar_i2c_probe(struct platform_device *pdev) else pm_runtime_put(dev); + if (of_property_read_bool(dev->of_node, "smbus")) + priv->flags |= ID_P_HOST_NOTIFY; priv->irq = platform_get_irq(pdev, 0); ret = devm_request_irq(dev, priv->irq, rcar_i2c_irq, 0, dev_name(dev), priv); @@ -1005,10 +1010,20 @@ static int rcar_i2c_probe(struct platform_device *pdev) if (ret < 0) goto out_pm_disable; + if (priv->flags & ID_P_HOST_NOTIFY) { + priv->host_notify_client = i2c_new_slave_host_notify_device(adap); + if (IS_ERR(priv->host_notify_client)) { + ret = PTR_ERR(priv->host_notify_client); + goto out_del_device; + } + } + dev_info(dev, "probed\n"); return 0; + out_del_device: + i2c_del_adapter(&priv->adap); out_pm_put: pm_runtime_put(dev); out_pm_disable: @@ -1021,6 +1036,8 @@ static int rcar_i2c_remove(struct platform_device *pdev) struct rcar_i2c_priv *priv = platform_get_drvdata(pdev); struct device *dev = &pdev->dev; + if (priv->host_notify_client) + i2c_free_slave_host_notify_device(priv->host_notify_client); i2c_del_adapter(&priv->adap); rcar_i2c_release_dma(priv); if (priv->flags & ID_P_PM_BLOCKED) diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c index 8e3cc85d1921..819ab4ee517e 100644 --- a/drivers/i2c/busses/i2c-rk3x.c +++ b/drivers/i2c/busses/i2c-rk3x.c @@ -1312,18 +1312,13 @@ static int rk3x_i2c_probe(struct platform_device *pdev) i2c->pclk = devm_clk_get(&pdev->dev, "pclk"); } - if (IS_ERR(i2c->clk)) { - ret = PTR_ERR(i2c->clk); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "Can't get bus clk: %d\n", ret); - return ret; - } - if (IS_ERR(i2c->pclk)) { - ret = PTR_ERR(i2c->pclk); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "Can't get periph clk: %d\n", ret); - return ret; - } + if (IS_ERR(i2c->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(i2c->clk), + "Can't get bus clk\n"); + + if (IS_ERR(i2c->pclk)) + return dev_err_probe(&pdev->dev, PTR_ERR(i2c->pclk), + "Can't get periph clk\n"); ret = clk_prepare(i2c->clk); if (ret < 0) { diff --git a/drivers/i2c/busses/i2c-stm32.c b/drivers/i2c/busses/i2c-stm32.c index 3f69a3bb6119..157c64e27d0b 100644 --- a/drivers/i2c/busses/i2c-stm32.c +++ b/drivers/i2c/busses/i2c-stm32.c @@ -26,8 +26,9 @@ struct stm32_i2c_dma *stm32_i2c_dma_request(struct device *dev, dma->chan_tx = dma_request_chan(dev, "tx"); if (IS_ERR(dma->chan_tx)) { ret = PTR_ERR(dma->chan_tx); - if (ret != -EPROBE_DEFER) - dev_err(dev, "can't request DMA tx channel\n"); + if (ret != -ENODEV) + ret = dev_err_probe(dev, ret, + "can't request DMA tx channel\n"); goto fail_al; } @@ -46,8 +47,9 @@ struct stm32_i2c_dma *stm32_i2c_dma_request(struct device *dev, dma->chan_rx = dma_request_chan(dev, "rx"); if (IS_ERR(dma->chan_rx)) { ret = PTR_ERR(dma->chan_rx); - if (ret != -EPROBE_DEFER) - dev_err(dev, "can't request DMA rx channel\n"); + if (ret != -ENODEV) + ret = dev_err_probe(dev, ret, + "can't request DMA rx channel\n"); goto fail_tx; } @@ -76,8 +78,6 @@ fail_tx: dma_release_channel(dma->chan_tx); fail_al: devm_kfree(dev, dma); - if (ret != -EPROBE_DEFER) - dev_info(dev, "can't use DMA\n"); return ERR_PTR(ret); } diff --git a/drivers/i2c/busses/i2c-stm32f4.c b/drivers/i2c/busses/i2c-stm32f4.c index 48e269284369..937c2c8fd349 100644 --- a/drivers/i2c/busses/i2c-stm32f4.c +++ b/drivers/i2c/busses/i2c-stm32f4.c @@ -797,10 +797,8 @@ static int stm32f4_i2c_probe(struct platform_device *pdev) rst = devm_reset_control_get_exclusive(&pdev->dev, NULL); if (IS_ERR(rst)) { - ret = PTR_ERR(rst); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "Error: Missing reset ctrl\n"); - + ret = dev_err_probe(&pdev->dev, PTR_ERR(rst), + "Error: Missing reset ctrl\n"); goto clk_free; } reset_control_assert(rst); diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c index bff3479fe122..f41f51a176a1 100644 --- a/drivers/i2c/busses/i2c-stm32f7.c +++ b/drivers/i2c/busses/i2c-stm32f7.c @@ -18,6 +18,7 @@ #include <linux/delay.h> #include <linux/err.h> #include <linux/i2c.h> +#include <linux/i2c-smbus.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/iopoll.h> @@ -50,6 +51,7 @@ /* STM32F7 I2C control 1 */ #define STM32F7_I2C_CR1_PECEN BIT(23) +#define STM32F7_I2C_CR1_SMBHEN BIT(20) #define STM32F7_I2C_CR1_WUPEN BIT(18) #define STM32F7_I2C_CR1_SBC BIT(16) #define STM32F7_I2C_CR1_RXDMAEN BIT(15) @@ -150,7 +152,12 @@ #define STM32F7_I2C_MAX_LEN 0xff #define STM32F7_I2C_DMA_LEN_MIN 0x16 -#define STM32F7_I2C_MAX_SLAVE 0x2 +enum { + STM32F7_SLAVE_HOSTNOTIFY, + STM32F7_SLAVE_7_10_BITS_ADDR, + STM32F7_SLAVE_7_BITS_ADDR, + STM32F7_I2C_MAX_SLAVE +}; #define STM32F7_I2C_DNF_DEFAULT 0 #define STM32F7_I2C_DNF_MAX 16 @@ -301,6 +308,8 @@ struct stm32f7_i2c_msg { * @fmp_creg: register address for clearing Fast Mode Plus bits * @fmp_mask: mask for Fast Mode Plus bits in set register * @wakeup_src: boolean to know if the device is a wakeup source + * @smbus_mode: states that the controller is configured in SMBus mode + * @host_notify_client: SMBus host-notify client */ struct stm32f7_i2c_dev { struct i2c_adapter adap; @@ -327,6 +336,8 @@ struct stm32f7_i2c_dev { u32 fmp_creg; u32 fmp_mask; bool wakeup_src; + bool smbus_mode; + struct i2c_client *host_notify_client; }; /* @@ -1321,11 +1332,20 @@ static int stm32f7_i2c_get_free_slave_id(struct stm32f7_i2c_dev *i2c_dev, int i; /* - * slave[0] supports 7-bit and 10-bit slave address - * slave[1] supports 7-bit slave address only + * slave[STM32F7_SLAVE_HOSTNOTIFY] support only SMBus Host address (0x8) + * slave[STM32F7_SLAVE_7_10_BITS_ADDR] supports 7-bit and 10-bit slave address + * slave[STM32F7_SLAVE_7_BITS_ADDR] supports 7-bit slave address only */ - for (i = STM32F7_I2C_MAX_SLAVE - 1; i >= 0; i--) { - if (i == 1 && (slave->flags & I2C_CLIENT_TEN)) + if (i2c_dev->smbus_mode && (slave->addr == 0x08)) { + if (i2c_dev->slave[STM32F7_SLAVE_HOSTNOTIFY]) + goto fail; + *id = STM32F7_SLAVE_HOSTNOTIFY; + return 0; + } + + for (i = STM32F7_I2C_MAX_SLAVE - 1; i > STM32F7_SLAVE_HOSTNOTIFY; i--) { + if ((i == STM32F7_SLAVE_7_BITS_ADDR) && + (slave->flags & I2C_CLIENT_TEN)) continue; if (!i2c_dev->slave[i]) { *id = i; @@ -1333,6 +1353,7 @@ static int stm32f7_i2c_get_free_slave_id(struct stm32f7_i2c_dev *i2c_dev, } } +fail: dev_err(dev, "Slave 0x%x could not be registered\n", slave->addr); return -EINVAL; @@ -1776,7 +1797,13 @@ static int stm32f7_i2c_reg_slave(struct i2c_client *slave) if (!stm32f7_i2c_is_slave_registered(i2c_dev)) stm32f7_i2c_enable_wakeup(i2c_dev, true); - if (id == 0) { + switch (id) { + case 0: + /* Slave SMBus Host */ + i2c_dev->slave[id] = slave; + break; + + case 1: /* Configure Own Address 1 */ oar1 = readl_relaxed(i2c_dev->base + STM32F7_I2C_OAR1); oar1 &= ~STM32F7_I2C_OAR1_MASK; @@ -1789,7 +1816,9 @@ static int stm32f7_i2c_reg_slave(struct i2c_client *slave) oar1 |= STM32F7_I2C_OAR1_OA1EN; i2c_dev->slave[id] = slave; writel_relaxed(oar1, i2c_dev->base + STM32F7_I2C_OAR1); - } else if (id == 1) { + break; + + case 2: /* Configure Own Address 2 */ oar2 = readl_relaxed(i2c_dev->base + STM32F7_I2C_OAR2); oar2 &= ~STM32F7_I2C_OAR2_MASK; @@ -1802,7 +1831,10 @@ static int stm32f7_i2c_reg_slave(struct i2c_client *slave) oar2 |= STM32F7_I2C_OAR2_OA2EN; i2c_dev->slave[id] = slave; writel_relaxed(oar2, i2c_dev->base + STM32F7_I2C_OAR2); - } else { + break; + + default: + dev_err(dev, "I2C slave id not supported\n"); ret = -ENODEV; goto pm_free; } @@ -1843,10 +1875,10 @@ static int stm32f7_i2c_unreg_slave(struct i2c_client *slave) if (ret < 0) return ret; - if (id == 0) { + if (id == 1) { mask = STM32F7_I2C_OAR1_OA1EN; stm32f7_i2c_clr_bits(base + STM32F7_I2C_OAR1, mask); - } else { + } else if (id == 2) { mask = STM32F7_I2C_OAR2_OA2EN; stm32f7_i2c_clr_bits(base + STM32F7_I2C_OAR2, mask); } @@ -1911,14 +1943,51 @@ static int stm32f7_i2c_setup_fm_plus_bits(struct platform_device *pdev, &i2c_dev->fmp_mask); } +static int stm32f7_i2c_enable_smbus_host(struct stm32f7_i2c_dev *i2c_dev) +{ + struct i2c_adapter *adap = &i2c_dev->adap; + void __iomem *base = i2c_dev->base; + struct i2c_client *client; + + client = i2c_new_slave_host_notify_device(adap); + if (IS_ERR(client)) + return PTR_ERR(client); + + i2c_dev->host_notify_client = client; + + /* Enable SMBus Host address */ + stm32f7_i2c_set_bits(base + STM32F7_I2C_CR1, STM32F7_I2C_CR1_SMBHEN); + + return 0; +} + +static void stm32f7_i2c_disable_smbus_host(struct stm32f7_i2c_dev *i2c_dev) +{ + void __iomem *base = i2c_dev->base; + + if (i2c_dev->host_notify_client) { + /* Disable SMBus Host address */ + stm32f7_i2c_clr_bits(base + STM32F7_I2C_CR1, + STM32F7_I2C_CR1_SMBHEN); + i2c_free_slave_host_notify_device(i2c_dev->host_notify_client); + } +} + static u32 stm32f7_i2c_func(struct i2c_adapter *adap) { - return I2C_FUNC_I2C | I2C_FUNC_10BIT_ADDR | I2C_FUNC_SLAVE | - I2C_FUNC_SMBUS_QUICK | I2C_FUNC_SMBUS_BYTE | - I2C_FUNC_SMBUS_BYTE_DATA | I2C_FUNC_SMBUS_WORD_DATA | - I2C_FUNC_SMBUS_BLOCK_DATA | I2C_FUNC_SMBUS_BLOCK_PROC_CALL | - I2C_FUNC_SMBUS_PROC_CALL | I2C_FUNC_SMBUS_PEC | - I2C_FUNC_SMBUS_I2C_BLOCK; + struct stm32f7_i2c_dev *i2c_dev = i2c_get_adapdata(adap); + + u32 func = I2C_FUNC_I2C | I2C_FUNC_10BIT_ADDR | I2C_FUNC_SLAVE | + I2C_FUNC_SMBUS_QUICK | I2C_FUNC_SMBUS_BYTE | + I2C_FUNC_SMBUS_BYTE_DATA | I2C_FUNC_SMBUS_WORD_DATA | + I2C_FUNC_SMBUS_BLOCK_DATA | I2C_FUNC_SMBUS_BLOCK_PROC_CALL | + I2C_FUNC_SMBUS_PROC_CALL | I2C_FUNC_SMBUS_PEC | + I2C_FUNC_SMBUS_I2C_BLOCK; + + if (i2c_dev->smbus_mode) + func |= I2C_FUNC_SMBUS_HOST_NOTIFY; + + return func; } static const struct i2c_algorithm stm32f7_i2c_algo = { @@ -1968,11 +2037,9 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) "wakeup-source"); i2c_dev->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(i2c_dev->clk)) { - if (PTR_ERR(i2c_dev->clk) != -EPROBE_DEFER) - dev_err(&pdev->dev, "Failed to get controller clock\n"); - return PTR_ERR(i2c_dev->clk); - } + if (IS_ERR(i2c_dev->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(i2c_dev->clk), + "Failed to get controller clock\n"); ret = clk_prepare_enable(i2c_dev->clk); if (ret) { @@ -1982,10 +2049,8 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) rst = devm_reset_control_get(&pdev->dev, NULL); if (IS_ERR(rst)) { - ret = PTR_ERR(rst); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "Error: Missing reset ctrl\n"); - + ret = dev_err_probe(&pdev->dev, PTR_ERR(rst), + "Error: Missing reset ctrl\n"); goto clk_free; } reset_control_assert(rst); @@ -2052,14 +2117,13 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) i2c_dev->dma = stm32_i2c_dma_request(i2c_dev->dev, phy_addr, STM32F7_I2C_TXDR, STM32F7_I2C_RXDR); - if (PTR_ERR(i2c_dev->dma) == -ENODEV) - i2c_dev->dma = NULL; - else if (IS_ERR(i2c_dev->dma)) { + if (IS_ERR(i2c_dev->dma)) { ret = PTR_ERR(i2c_dev->dma); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, - "Failed to request dma error %i\n", ret); - goto fmp_clear; + /* DMA support is optional, only report other errors */ + if (ret != -ENODEV) + goto fmp_clear; + dev_dbg(i2c_dev->dev, "No DMA option: fallback using interrupts\n"); + i2c_dev->dma = NULL; } if (i2c_dev->wakeup_src) { @@ -2084,10 +2148,22 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) stm32f7_i2c_hw_config(i2c_dev); + i2c_dev->smbus_mode = of_property_read_bool(pdev->dev.of_node, "smbus"); + ret = i2c_add_adapter(adap); if (ret) goto pm_disable; + if (i2c_dev->smbus_mode) { + ret = stm32f7_i2c_enable_smbus_host(i2c_dev); + if (ret) { + dev_err(i2c_dev->dev, + "failed to enable SMBus Host-Notify protocol (%d)\n", + ret); + goto i2c_adapter_remove; + } + } + dev_info(i2c_dev->dev, "STM32F7 I2C-%d bus adapter\n", adap->nr); pm_runtime_mark_last_busy(i2c_dev->dev); @@ -2095,6 +2171,9 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) return 0; +i2c_adapter_remove: + i2c_del_adapter(adap); + pm_disable: pm_runtime_put_noidle(i2c_dev->dev); pm_runtime_disable(i2c_dev->dev); @@ -2126,6 +2205,8 @@ static int stm32f7_i2c_remove(struct platform_device *pdev) { struct stm32f7_i2c_dev *i2c_dev = platform_get_drvdata(pdev); + stm32f7_i2c_disable_smbus_host(i2c_dev); + i2c_del_adapter(&i2c_dev->adap); pm_runtime_get_sync(i2c_dev->dev); diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c index 00d3e4d7a01e..6f08c0c3238d 100644 --- a/drivers/i2c/busses/i2c-tegra.c +++ b/drivers/i2c/busses/i2c-tegra.c @@ -136,7 +136,7 @@ /* configuration load timeout in microseconds */ #define I2C_CONFIG_LOAD_TIMEOUT 1000000 -/* Packet header size in bytes */ +/* packet header size in bytes */ #define I2C_PACKET_HEADER_SIZE 12 /* @@ -148,11 +148,10 @@ #define I2C_PIO_MODE_PREFERRED_LEN 32 /* - * msg_end_type: The bus control which need to be send at end of transfer. - * @MSG_END_STOP: Send stop pulse at end of transfer. - * @MSG_END_REPEAT_START: Send repeat start at end of transfer. - * @MSG_END_CONTINUE: The following on message is coming and so do not send - * stop or repeat start. + * msg_end_type: The bus control which needs to be sent at end of transfer. + * @MSG_END_STOP: Send stop pulse. + * @MSG_END_REPEAT_START: Send repeat-start. + * @MSG_END_CONTINUE: Don't send stop or repeat-start. */ enum msg_end_type { MSG_END_STOP, @@ -161,13 +160,10 @@ enum msg_end_type { }; /** - * struct tegra_i2c_hw_feature : Different HW support on Tegra - * @has_continue_xfer_support: Continue transfer supports. + * struct tegra_i2c_hw_feature : per hardware generation features + * @has_continue_xfer_support: continue-transfer supported * @has_per_pkt_xfer_complete_irq: Has enable/disable capability for transfer - * complete interrupt per packet basis. - * @has_single_clk_source: The I2C controller has single clock source. Tegra30 - * and earlier SoCs have two clock sources i.e. div-clk and - * fast-clk. + * completion interrupt on per packet basis. * @has_config_load_reg: Has the config load register to load the new * configuration. * @clk_divisor_hs_mode: Clock divisor in HS mode. @@ -187,7 +183,7 @@ enum msg_end_type { * @has_mst_fifo: The I2C controller contains the new MST FIFO interface that * provides additional features and allows for longer messages to * be transferred in one go. - * @quirks: i2c adapter quirks for limiting write/read transfer size and not + * @quirks: I2C adapter quirks for limiting write/read transfer size and not * allowing 0 length transfers. * @supports_bus_clear: Bus Clear support to recover from bus hang during * SDA stuck low from device for some unknown reasons. @@ -208,22 +204,21 @@ enum msg_end_type { struct tegra_i2c_hw_feature { bool has_continue_xfer_support; bool has_per_pkt_xfer_complete_irq; - bool has_single_clk_source; bool has_config_load_reg; - int clk_divisor_hs_mode; - int clk_divisor_std_mode; - int clk_divisor_fast_mode; - u16 clk_divisor_fast_plus_mode; + u32 clk_divisor_hs_mode; + u32 clk_divisor_std_mode; + u32 clk_divisor_fast_mode; + u32 clk_divisor_fast_plus_mode; bool has_multi_master_mode; bool has_slcg_override_reg; bool has_mst_fifo; const struct i2c_adapter_quirks *quirks; bool supports_bus_clear; bool has_apb_dma; - u8 tlow_std_mode; - u8 thigh_std_mode; - u8 tlow_fast_fastplus_mode; - u8 thigh_fast_fastplus_mode; + u32 tlow_std_mode; + u32 thigh_std_mode; + u32 tlow_fast_fastplus_mode; + u32 thigh_fast_fastplus_mode; u32 setup_hold_time_std_mode; u32 setup_hold_time_fast_fast_plus_mode; u32 setup_hold_time_hs_mode; @@ -236,7 +231,8 @@ struct tegra_i2c_hw_feature { * @hw: Tegra I2C HW feature * @adapter: core I2C layer adapter information * @div_clk: clock reference for div clock of I2C controller - * @fast_clk: clock reference for fast clock of I2C controller + * @clocks: array of I2C controller clocks + * @nclocks: number of clocks in the array * @rst: reset control for the I2C controller * @base: ioremapped registers cookie * @base_phys: physical base address of the I2C controller @@ -248,101 +244,103 @@ struct tegra_i2c_hw_feature { * @msg_err: error code for completed message * @msg_buf: pointer to current message data * @msg_buf_remaining: size of unsent data in the message buffer - * @msg_read: identifies read transfers + * @msg_read: indicates that the transfer is a read access * @bus_clk_rate: current I2C bus clock rate - * @clk_divisor_non_hs_mode: clock divider for non-high-speed modes - * @is_multimaster_mode: track if I2C controller is in multi-master mode + * @multimaster_mode: indicates that I2C controller is in multi-master mode * @tx_dma_chan: DMA transmit channel * @rx_dma_chan: DMA receive channel * @dma_phys: handle to DMA resources * @dma_buf: pointer to allocated DMA buffer * @dma_buf_size: DMA buffer size - * @is_curr_dma_xfer: indicates active DMA transfer + * @dma_mode: indicates active DMA transfer * @dma_complete: DMA completion notifier - * @is_curr_atomic_xfer: indicates active atomic transfer + * @atomic_mode: indicates active atomic transfer */ struct tegra_i2c_dev { struct device *dev; - const struct tegra_i2c_hw_feature *hw; struct i2c_adapter adapter; - struct clk *div_clk; - struct clk *fast_clk; - struct clk *slow_clk; + + const struct tegra_i2c_hw_feature *hw; struct reset_control *rst; - void __iomem *base; + unsigned int cont_id; + unsigned int irq; + phys_addr_t base_phys; - int cont_id; - int irq; - int is_dvc; - bool is_vi; + void __iomem *base; + + struct clk_bulk_data clocks[2]; + unsigned int nclocks; + + struct clk *div_clk; + u32 bus_clk_rate; + struct completion msg_complete; + size_t msg_buf_remaining; int msg_err; u8 *msg_buf; - size_t msg_buf_remaining; - int msg_read; - u32 bus_clk_rate; - u16 clk_divisor_non_hs_mode; - bool is_multimaster_mode; + + struct completion dma_complete; struct dma_chan *tx_dma_chan; struct dma_chan *rx_dma_chan; - dma_addr_t dma_phys; - u32 *dma_buf; unsigned int dma_buf_size; - bool is_curr_dma_xfer; - struct completion dma_complete; - bool is_curr_atomic_xfer; -}; + dma_addr_t dma_phys; + void *dma_buf; -static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit); + bool multimaster_mode; + bool atomic_mode; + bool dma_mode; + bool msg_read; + bool is_dvc; + bool is_vi; +}; static void dvc_writel(struct tegra_i2c_dev *i2c_dev, u32 val, - unsigned long reg) + unsigned int reg) { writel_relaxed(val, i2c_dev->base + reg); } -static u32 dvc_readl(struct tegra_i2c_dev *i2c_dev, unsigned long reg) +static u32 dvc_readl(struct tegra_i2c_dev *i2c_dev, unsigned int reg) { return readl_relaxed(i2c_dev->base + reg); } /* - * i2c_writel and i2c_readl will offset the register if necessary to talk - * to the I2C block inside the DVC block + * If necessary, i2c_writel() and i2c_readl() will offset the register + * in order to talk to the I2C block inside the DVC block. */ -static unsigned long tegra_i2c_reg_addr(struct tegra_i2c_dev *i2c_dev, - unsigned long reg) +static u32 tegra_i2c_reg_addr(struct tegra_i2c_dev *i2c_dev, unsigned int reg) { if (i2c_dev->is_dvc) reg += (reg >= I2C_TX_FIFO) ? 0x10 : 0x40; else if (i2c_dev->is_vi) reg = 0xc00 + (reg << 2); + return reg; } -static void i2c_writel(struct tegra_i2c_dev *i2c_dev, u32 val, - unsigned long reg) +static void i2c_writel(struct tegra_i2c_dev *i2c_dev, u32 val, unsigned int reg) { writel_relaxed(val, i2c_dev->base + tegra_i2c_reg_addr(i2c_dev, reg)); - /* Read back register to make sure that register writes completed */ + /* read back register to make sure that register writes completed */ if (reg != I2C_TX_FIFO) readl_relaxed(i2c_dev->base + tegra_i2c_reg_addr(i2c_dev, reg)); } -static u32 i2c_readl(struct tegra_i2c_dev *i2c_dev, unsigned long reg) +static u32 i2c_readl(struct tegra_i2c_dev *i2c_dev, unsigned int reg) { return readl_relaxed(i2c_dev->base + tegra_i2c_reg_addr(i2c_dev, reg)); } static void i2c_writesl(struct tegra_i2c_dev *i2c_dev, void *data, - unsigned long reg, int len) + unsigned int reg, unsigned int len) { writesl(i2c_dev->base + tegra_i2c_reg_addr(i2c_dev, reg), data, len); } static void i2c_readsl(struct tegra_i2c_dev *i2c_dev, void *data, - unsigned long reg, int len) + unsigned int reg, unsigned int len) { readsl(i2c_dev->base + tegra_i2c_reg_addr(i2c_dev, reg), data, len); } @@ -377,21 +375,27 @@ static int tegra_i2c_dma_submit(struct tegra_i2c_dev *i2c_dev, size_t len) struct dma_chan *chan; dev_dbg(i2c_dev->dev, "starting DMA for length: %zu\n", len); + reinit_completion(&i2c_dev->dma_complete); + dir = i2c_dev->msg_read ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV; chan = i2c_dev->msg_read ? i2c_dev->rx_dma_chan : i2c_dev->tx_dma_chan; + dma_desc = dmaengine_prep_slave_single(chan, i2c_dev->dma_phys, len, dir, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); if (!dma_desc) { - dev_err(i2c_dev->dev, "failed to get DMA descriptor\n"); + dev_err(i2c_dev->dev, "failed to get %s DMA descriptor\n", + i2c_dev->msg_read ? "RX" : "TX"); return -EINVAL; } dma_desc->callback = tegra_i2c_dma_complete; dma_desc->callback_param = i2c_dev; + dmaengine_submit(dma_desc); dma_async_issue_pending(chan); + return 0; } @@ -417,15 +421,15 @@ static void tegra_i2c_release_dma(struct tegra_i2c_dev *i2c_dev) static int tegra_i2c_init_dma(struct tegra_i2c_dev *i2c_dev) { struct dma_chan *chan; - u32 *dma_buf; dma_addr_t dma_phys; + u32 *dma_buf; int err; if (!i2c_dev->hw->has_apb_dma || i2c_dev->is_vi) return 0; if (!IS_ENABLED(CONFIG_TEGRA20_APB_DMA)) { - dev_dbg(i2c_dev->dev, "Support for APB DMA not enabled!\n"); + dev_dbg(i2c_dev->dev, "DMA support not enabled\n"); return 0; } @@ -445,16 +449,20 @@ static int tegra_i2c_init_dma(struct tegra_i2c_dev *i2c_dev) i2c_dev->tx_dma_chan = chan; + i2c_dev->dma_buf_size = i2c_dev->hw->quirks->max_write_len + + I2C_PACKET_HEADER_SIZE; + dma_buf = dma_alloc_coherent(i2c_dev->dev, i2c_dev->dma_buf_size, &dma_phys, GFP_KERNEL | __GFP_NOWARN); if (!dma_buf) { - dev_err(i2c_dev->dev, "failed to allocate the DMA buffer\n"); + dev_err(i2c_dev->dev, "failed to allocate DMA buffer\n"); err = -ENOMEM; goto err_out; } i2c_dev->dma_buf = dma_buf; i2c_dev->dma_phys = dma_phys; + return 0; err_out: @@ -468,171 +476,12 @@ err_out: return err; } -static int tegra_i2c_flush_fifos(struct tegra_i2c_dev *i2c_dev) -{ - unsigned long timeout = jiffies + HZ; - unsigned int offset; - u32 mask, val; - - if (i2c_dev->hw->has_mst_fifo) { - mask = I2C_MST_FIFO_CONTROL_TX_FLUSH | - I2C_MST_FIFO_CONTROL_RX_FLUSH; - offset = I2C_MST_FIFO_CONTROL; - } else { - mask = I2C_FIFO_CONTROL_TX_FLUSH | - I2C_FIFO_CONTROL_RX_FLUSH; - offset = I2C_FIFO_CONTROL; - } - - val = i2c_readl(i2c_dev, offset); - val |= mask; - i2c_writel(i2c_dev, val, offset); - - while (i2c_readl(i2c_dev, offset) & mask) { - if (time_after(jiffies, timeout)) { - dev_warn(i2c_dev->dev, "timeout waiting for fifo flush\n"); - return -ETIMEDOUT; - } - usleep_range(1000, 2000); - } - return 0; -} - -static int tegra_i2c_empty_rx_fifo(struct tegra_i2c_dev *i2c_dev) -{ - u32 val; - int rx_fifo_avail; - u8 *buf = i2c_dev->msg_buf; - size_t buf_remaining = i2c_dev->msg_buf_remaining; - int words_to_transfer; - - /* - * Catch overflow due to message fully sent - * before the check for RX FIFO availability. - */ - if (WARN_ON_ONCE(!(i2c_dev->msg_buf_remaining))) - return -EINVAL; - - if (i2c_dev->hw->has_mst_fifo) { - val = i2c_readl(i2c_dev, I2C_MST_FIFO_STATUS); - rx_fifo_avail = FIELD_GET(I2C_MST_FIFO_STATUS_RX, val); - } else { - val = i2c_readl(i2c_dev, I2C_FIFO_STATUS); - rx_fifo_avail = FIELD_GET(I2C_FIFO_STATUS_RX, val); - } - - /* Rounds down to not include partial word at the end of buf */ - words_to_transfer = buf_remaining / BYTES_PER_FIFO_WORD; - if (words_to_transfer > rx_fifo_avail) - words_to_transfer = rx_fifo_avail; - - i2c_readsl(i2c_dev, buf, I2C_RX_FIFO, words_to_transfer); - - buf += words_to_transfer * BYTES_PER_FIFO_WORD; - buf_remaining -= words_to_transfer * BYTES_PER_FIFO_WORD; - rx_fifo_avail -= words_to_transfer; - - /* - * If there is a partial word at the end of buf, handle it manually to - * prevent overwriting past the end of buf - */ - if (rx_fifo_avail > 0 && buf_remaining > 0) { - /* - * buf_remaining > 3 check not needed as rx_fifo_avail == 0 - * when (words_to_transfer was > rx_fifo_avail) earlier - * in this function. - */ - val = i2c_readl(i2c_dev, I2C_RX_FIFO); - val = cpu_to_le32(val); - memcpy(buf, &val, buf_remaining); - buf_remaining = 0; - rx_fifo_avail--; - } - - /* RX FIFO must be drained, otherwise it's an Overflow case. */ - if (WARN_ON_ONCE(rx_fifo_avail)) - return -EINVAL; - - i2c_dev->msg_buf_remaining = buf_remaining; - i2c_dev->msg_buf = buf; - - return 0; -} - -static int tegra_i2c_fill_tx_fifo(struct tegra_i2c_dev *i2c_dev) -{ - u32 val; - int tx_fifo_avail; - u8 *buf = i2c_dev->msg_buf; - size_t buf_remaining = i2c_dev->msg_buf_remaining; - int words_to_transfer; - - if (i2c_dev->hw->has_mst_fifo) { - val = i2c_readl(i2c_dev, I2C_MST_FIFO_STATUS); - tx_fifo_avail = FIELD_GET(I2C_MST_FIFO_STATUS_TX, val); - } else { - val = i2c_readl(i2c_dev, I2C_FIFO_STATUS); - tx_fifo_avail = FIELD_GET(I2C_FIFO_STATUS_TX, val); - } - - /* Rounds down to not include partial word at the end of buf */ - words_to_transfer = buf_remaining / BYTES_PER_FIFO_WORD; - - /* It's very common to have < 4 bytes, so optimize that case. */ - if (words_to_transfer) { - if (words_to_transfer > tx_fifo_avail) - words_to_transfer = tx_fifo_avail; - - /* - * Update state before writing to FIFO. If this casues us - * to finish writing all bytes (AKA buf_remaining goes to 0) we - * have a potential for an interrupt (PACKET_XFER_COMPLETE is - * not maskable). We need to make sure that the isr sees - * buf_remaining as 0 and doesn't call us back re-entrantly. - */ - buf_remaining -= words_to_transfer * BYTES_PER_FIFO_WORD; - tx_fifo_avail -= words_to_transfer; - i2c_dev->msg_buf_remaining = buf_remaining; - i2c_dev->msg_buf = buf + - words_to_transfer * BYTES_PER_FIFO_WORD; - barrier(); - - i2c_writesl(i2c_dev, buf, I2C_TX_FIFO, words_to_transfer); - - buf += words_to_transfer * BYTES_PER_FIFO_WORD; - } - - /* - * If there is a partial word at the end of buf, handle it manually to - * prevent reading past the end of buf, which could cross a page - * boundary and fault. - */ - if (tx_fifo_avail > 0 && buf_remaining > 0) { - /* - * buf_remaining > 3 check not needed as tx_fifo_avail == 0 - * when (words_to_transfer was > tx_fifo_avail) earlier - * in this function for non-zero words_to_transfer. - */ - memcpy(&val, buf, buf_remaining); - val = le32_to_cpu(val); - - /* Again update before writing to FIFO to make sure isr sees. */ - i2c_dev->msg_buf_remaining = 0; - i2c_dev->msg_buf = NULL; - barrier(); - - i2c_writel(i2c_dev, val, I2C_TX_FIFO); - } - - return 0; -} - /* * One of the Tegra I2C blocks is inside the DVC (Digital Voltage Controller) * block. This block is identical to the rest of the I2C blocks, except that * it only supports master mode, it has registers moved around, and it needs * some extra init to get it into I2C mode. The register moves are handled - * by i2c_readl and i2c_writel + * by i2c_readl() and i2c_writel(). */ static void tegra_dvc_init(struct tegra_i2c_dev *i2c_dev) { @@ -648,140 +497,112 @@ static void tegra_dvc_init(struct tegra_i2c_dev *i2c_dev) dvc_writel(i2c_dev, val, DVC_CTRL_REG1); } -static int __maybe_unused tegra_i2c_runtime_resume(struct device *dev) +static void tegra_i2c_vi_init(struct tegra_i2c_dev *i2c_dev) { - struct tegra_i2c_dev *i2c_dev = dev_get_drvdata(dev); - int ret; - - ret = pinctrl_pm_select_default_state(i2c_dev->dev); - if (ret) - return ret; + u32 value; - ret = clk_enable(i2c_dev->fast_clk); - if (ret < 0) { - dev_err(i2c_dev->dev, - "Enabling fast clk failed, err %d\n", ret); - return ret; - } + value = FIELD_PREP(I2C_INTERFACE_TIMING_THIGH, 2) | + FIELD_PREP(I2C_INTERFACE_TIMING_TLOW, 4); + i2c_writel(i2c_dev, value, I2C_INTERFACE_TIMING_0); - ret = clk_enable(i2c_dev->slow_clk); - if (ret < 0) { - dev_err(dev, "failed to enable slow clock: %d\n", ret); - goto disable_fast_clk; - } + value = FIELD_PREP(I2C_INTERFACE_TIMING_TBUF, 4) | + FIELD_PREP(I2C_INTERFACE_TIMING_TSU_STO, 7) | + FIELD_PREP(I2C_INTERFACE_TIMING_THD_STA, 4) | + FIELD_PREP(I2C_INTERFACE_TIMING_TSU_STA, 4); + i2c_writel(i2c_dev, value, I2C_INTERFACE_TIMING_1); - ret = clk_enable(i2c_dev->div_clk); - if (ret < 0) { - dev_err(i2c_dev->dev, - "Enabling div clk failed, err %d\n", ret); - goto disable_slow_clk; - } + value = FIELD_PREP(I2C_HS_INTERFACE_TIMING_THIGH, 3) | + FIELD_PREP(I2C_HS_INTERFACE_TIMING_TLOW, 8); + i2c_writel(i2c_dev, value, I2C_HS_INTERFACE_TIMING_0); - /* - * VI I2C device is attached to VE power domain which goes through - * power ON/OFF during PM runtime resume/suspend. So, controller - * should go through reset and need to re-initialize after power - * domain ON. - */ - if (i2c_dev->is_vi) { - ret = tegra_i2c_init(i2c_dev, true); - if (ret) - goto disable_div_clk; - } + value = FIELD_PREP(I2C_HS_INTERFACE_TIMING_TSU_STO, 11) | + FIELD_PREP(I2C_HS_INTERFACE_TIMING_THD_STA, 11) | + FIELD_PREP(I2C_HS_INTERFACE_TIMING_TSU_STA, 11); + i2c_writel(i2c_dev, value, I2C_HS_INTERFACE_TIMING_1); - return 0; + value = FIELD_PREP(I2C_BC_SCLK_THRESHOLD, 9) | I2C_BC_STOP_COND; + i2c_writel(i2c_dev, value, I2C_BUS_CLEAR_CNFG); -disable_div_clk: - clk_disable(i2c_dev->div_clk); -disable_slow_clk: - clk_disable(i2c_dev->slow_clk); -disable_fast_clk: - clk_disable(i2c_dev->fast_clk); - return ret; + i2c_writel(i2c_dev, 0x0, I2C_TLOW_SEXT); } -static int __maybe_unused tegra_i2c_runtime_suspend(struct device *dev) +static int tegra_i2c_poll_register(struct tegra_i2c_dev *i2c_dev, + u32 reg, u32 mask, u32 delay_us, + u32 timeout_us) { - struct tegra_i2c_dev *i2c_dev = dev_get_drvdata(dev); + void __iomem *addr = i2c_dev->base + tegra_i2c_reg_addr(i2c_dev, reg); + u32 val; - clk_disable(i2c_dev->div_clk); - clk_disable(i2c_dev->slow_clk); - clk_disable(i2c_dev->fast_clk); + if (!i2c_dev->atomic_mode) + return readl_relaxed_poll_timeout(addr, val, !(val & mask), + delay_us, timeout_us); - return pinctrl_pm_select_idle_state(i2c_dev->dev); + return readl_relaxed_poll_timeout_atomic(addr, val, !(val & mask), + delay_us, timeout_us); } -static int tegra_i2c_wait_for_config_load(struct tegra_i2c_dev *i2c_dev) +static int tegra_i2c_flush_fifos(struct tegra_i2c_dev *i2c_dev) { - unsigned long reg_offset; - void __iomem *addr; - u32 val; + u32 mask, val, offset; int err; - if (i2c_dev->hw->has_config_load_reg) { - reg_offset = tegra_i2c_reg_addr(i2c_dev, I2C_CONFIG_LOAD); - addr = i2c_dev->base + reg_offset; - i2c_writel(i2c_dev, I2C_MSTR_CONFIG_LOAD, I2C_CONFIG_LOAD); + if (i2c_dev->hw->has_mst_fifo) { + mask = I2C_MST_FIFO_CONTROL_TX_FLUSH | + I2C_MST_FIFO_CONTROL_RX_FLUSH; + offset = I2C_MST_FIFO_CONTROL; + } else { + mask = I2C_FIFO_CONTROL_TX_FLUSH | + I2C_FIFO_CONTROL_RX_FLUSH; + offset = I2C_FIFO_CONTROL; + } - if (i2c_dev->is_curr_atomic_xfer) - err = readl_relaxed_poll_timeout_atomic( - addr, val, val == 0, 1000, - I2C_CONFIG_LOAD_TIMEOUT); - else - err = readl_relaxed_poll_timeout( - addr, val, val == 0, 1000, - I2C_CONFIG_LOAD_TIMEOUT); + val = i2c_readl(i2c_dev, offset); + val |= mask; + i2c_writel(i2c_dev, val, offset); - if (err) { - dev_warn(i2c_dev->dev, - "timeout waiting for config load\n"); - return err; - } + err = tegra_i2c_poll_register(i2c_dev, offset, mask, 1000, 1000000); + if (err) { + dev_err(i2c_dev->dev, "failed to flush FIFO\n"); + return err; } return 0; } -static void tegra_i2c_vi_init(struct tegra_i2c_dev *i2c_dev) +static int tegra_i2c_wait_for_config_load(struct tegra_i2c_dev *i2c_dev) { - u32 value; - - value = FIELD_PREP(I2C_INTERFACE_TIMING_THIGH, 2) | - FIELD_PREP(I2C_INTERFACE_TIMING_TLOW, 4); - i2c_writel(i2c_dev, value, I2C_INTERFACE_TIMING_0); - - value = FIELD_PREP(I2C_INTERFACE_TIMING_TBUF, 4) | - FIELD_PREP(I2C_INTERFACE_TIMING_TSU_STO, 7) | - FIELD_PREP(I2C_INTERFACE_TIMING_THD_STA, 4) | - FIELD_PREP(I2C_INTERFACE_TIMING_TSU_STA, 4); - i2c_writel(i2c_dev, value, I2C_INTERFACE_TIMING_1); + int err; - value = FIELD_PREP(I2C_HS_INTERFACE_TIMING_THIGH, 3) | - FIELD_PREP(I2C_HS_INTERFACE_TIMING_TLOW, 8); - i2c_writel(i2c_dev, value, I2C_HS_INTERFACE_TIMING_0); + if (!i2c_dev->hw->has_config_load_reg) + return 0; - value = FIELD_PREP(I2C_HS_INTERFACE_TIMING_TSU_STO, 11) | - FIELD_PREP(I2C_HS_INTERFACE_TIMING_THD_STA, 11) | - FIELD_PREP(I2C_HS_INTERFACE_TIMING_TSU_STA, 11); - i2c_writel(i2c_dev, value, I2C_HS_INTERFACE_TIMING_1); + i2c_writel(i2c_dev, I2C_MSTR_CONFIG_LOAD, I2C_CONFIG_LOAD); - value = FIELD_PREP(I2C_BC_SCLK_THRESHOLD, 9) | I2C_BC_STOP_COND; - i2c_writel(i2c_dev, value, I2C_BUS_CLEAR_CNFG); + err = tegra_i2c_poll_register(i2c_dev, I2C_CONFIG_LOAD, 0xffffffff, + 1000, I2C_CONFIG_LOAD_TIMEOUT); + if (err) { + dev_err(i2c_dev->dev, "failed to load config\n"); + return err; + } - i2c_writel(i2c_dev, 0x0, I2C_TLOW_SEXT); + return 0; } -static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit) +static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev) { - u32 val; + u32 val, clk_divisor, clk_multiplier, tsu_thd, tlow, thigh, non_hs_mode; int err; - u32 clk_divisor, clk_multiplier; - u32 tsu_thd; - u8 tlow, thigh; - reset_control_assert(i2c_dev->rst); - udelay(2); - reset_control_deassert(i2c_dev->rst); + /* + * The reset shouldn't ever fail in practice. The failure will be a + * sign of a severe problem that needs to be resolved. Still we don't + * want to fail the initialization completely because this may break + * kernel boot up since voltage regulators use I2C. Hence, we will + * emit a noisy warning on error, which won't stay unnoticed and + * won't hose machine entirely. + */ + err = reset_control_reset(i2c_dev->rst); + WARN_ON_ONCE(err); if (i2c_dev->is_dvc) tegra_dvc_init(i2c_dev); @@ -798,24 +619,33 @@ static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit) if (i2c_dev->is_vi) tegra_i2c_vi_init(i2c_dev); - /* Make sure clock divisor programmed correctly */ - clk_divisor = FIELD_PREP(I2C_CLK_DIVISOR_HSMODE, - i2c_dev->hw->clk_divisor_hs_mode) | - FIELD_PREP(I2C_CLK_DIVISOR_STD_FAST_MODE, - i2c_dev->clk_divisor_non_hs_mode); - i2c_writel(i2c_dev, clk_divisor, I2C_CLK_DIVISOR); - - if (i2c_dev->bus_clk_rate > I2C_MAX_STANDARD_MODE_FREQ && - i2c_dev->bus_clk_rate <= I2C_MAX_FAST_MODE_PLUS_FREQ) { + switch (i2c_dev->bus_clk_rate) { + case I2C_MAX_STANDARD_MODE_FREQ + 1 ... I2C_MAX_FAST_MODE_PLUS_FREQ: + default: tlow = i2c_dev->hw->tlow_fast_fastplus_mode; thigh = i2c_dev->hw->thigh_fast_fastplus_mode; tsu_thd = i2c_dev->hw->setup_hold_time_fast_fast_plus_mode; - } else { + + if (i2c_dev->bus_clk_rate > I2C_MAX_FAST_MODE_FREQ) + non_hs_mode = i2c_dev->hw->clk_divisor_fast_plus_mode; + else + non_hs_mode = i2c_dev->hw->clk_divisor_fast_mode; + break; + + case 0 ... I2C_MAX_STANDARD_MODE_FREQ: tlow = i2c_dev->hw->tlow_std_mode; thigh = i2c_dev->hw->thigh_std_mode; tsu_thd = i2c_dev->hw->setup_hold_time_std_mode; + non_hs_mode = i2c_dev->hw->clk_divisor_std_mode; + break; } + /* make sure clock divisor programmed correctly */ + clk_divisor = FIELD_PREP(I2C_CLK_DIVISOR_HSMODE, + i2c_dev->hw->clk_divisor_hs_mode) | + FIELD_PREP(I2C_CLK_DIVISOR_STD_FAST_MODE, non_hs_mode); + i2c_writel(i2c_dev, clk_divisor, I2C_CLK_DIVISOR); + if (i2c_dev->hw->has_interface_timing_reg) { val = FIELD_PREP(I2C_INTERFACE_TIMING_THIGH, thigh) | FIELD_PREP(I2C_INTERFACE_TIMING_TLOW, tlow); @@ -823,22 +653,19 @@ static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit) } /* - * configure setup and hold times only when tsu_thd is non-zero. - * otherwise, preserve the chip default values + * Configure setup and hold times only when tsu_thd is non-zero. + * Otherwise, preserve the chip default values. */ if (i2c_dev->hw->has_interface_timing_reg && tsu_thd) i2c_writel(i2c_dev, tsu_thd, I2C_INTERFACE_TIMING_1); - if (!clk_reinit) { - clk_multiplier = (tlow + thigh + 2); - clk_multiplier *= (i2c_dev->clk_divisor_non_hs_mode + 1); - err = clk_set_rate(i2c_dev->div_clk, - i2c_dev->bus_clk_rate * clk_multiplier); - if (err) { - dev_err(i2c_dev->dev, - "failed changing clock rate: %d\n", err); - return err; - } + clk_multiplier = (tlow + thigh + 2) * (non_hs_mode + 1); + + err = clk_set_rate(i2c_dev->div_clk, + i2c_dev->bus_clk_rate * clk_multiplier); + if (err) { + dev_err(i2c_dev->dev, "failed to set div-clk rate: %d\n", err); + return err; } if (!i2c_dev->is_dvc && !i2c_dev->is_vi) { @@ -854,7 +681,7 @@ static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit) if (err) return err; - if (i2c_dev->is_multimaster_mode && i2c_dev->hw->has_slcg_override_reg) + if (i2c_dev->multimaster_mode && i2c_dev->hw->has_slcg_override_reg) i2c_writel(i2c_dev, I2C_MST_CORE_CLKEN_OVR, I2C_CLKEN_OVERRIDE); err = tegra_i2c_wait_for_config_load(i2c_dev); @@ -870,7 +697,7 @@ static int tegra_i2c_disable_packet_mode(struct tegra_i2c_dev *i2c_dev) /* * NACK interrupt is generated before the I2C controller generates - * the STOP condition on the bus. So wait for 2 clock periods + * the STOP condition on the bus. So, wait for 2 clock periods * before disabling the controller so that the STOP condition has * been delivered properly. */ @@ -883,16 +710,145 @@ static int tegra_i2c_disable_packet_mode(struct tegra_i2c_dev *i2c_dev) return tegra_i2c_wait_for_config_load(i2c_dev); } +static int tegra_i2c_empty_rx_fifo(struct tegra_i2c_dev *i2c_dev) +{ + size_t buf_remaining = i2c_dev->msg_buf_remaining; + unsigned int words_to_transfer, rx_fifo_avail; + u8 *buf = i2c_dev->msg_buf; + u32 val; + + /* + * Catch overflow due to message fully sent before the check for + * RX FIFO availability. + */ + if (WARN_ON_ONCE(!(i2c_dev->msg_buf_remaining))) + return -EINVAL; + + if (i2c_dev->hw->has_mst_fifo) { + val = i2c_readl(i2c_dev, I2C_MST_FIFO_STATUS); + rx_fifo_avail = FIELD_GET(I2C_MST_FIFO_STATUS_RX, val); + } else { + val = i2c_readl(i2c_dev, I2C_FIFO_STATUS); + rx_fifo_avail = FIELD_GET(I2C_FIFO_STATUS_RX, val); + } + + /* round down to exclude partial word at the end of buffer */ + words_to_transfer = buf_remaining / BYTES_PER_FIFO_WORD; + if (words_to_transfer > rx_fifo_avail) + words_to_transfer = rx_fifo_avail; + + i2c_readsl(i2c_dev, buf, I2C_RX_FIFO, words_to_transfer); + + buf += words_to_transfer * BYTES_PER_FIFO_WORD; + buf_remaining -= words_to_transfer * BYTES_PER_FIFO_WORD; + rx_fifo_avail -= words_to_transfer; + + /* + * If there is a partial word at the end of buffer, handle it + * manually to prevent overwriting past the end of buffer. + */ + if (rx_fifo_avail > 0 && buf_remaining > 0) { + /* + * buf_remaining > 3 check not needed as rx_fifo_avail == 0 + * when (words_to_transfer was > rx_fifo_avail) earlier + * in this function. + */ + val = i2c_readl(i2c_dev, I2C_RX_FIFO); + val = cpu_to_le32(val); + memcpy(buf, &val, buf_remaining); + buf_remaining = 0; + rx_fifo_avail--; + } + + /* RX FIFO must be drained, otherwise it's an Overflow case. */ + if (WARN_ON_ONCE(rx_fifo_avail)) + return -EINVAL; + + i2c_dev->msg_buf_remaining = buf_remaining; + i2c_dev->msg_buf = buf; + + return 0; +} + +static int tegra_i2c_fill_tx_fifo(struct tegra_i2c_dev *i2c_dev) +{ + size_t buf_remaining = i2c_dev->msg_buf_remaining; + unsigned int words_to_transfer, tx_fifo_avail; + u8 *buf = i2c_dev->msg_buf; + u32 val; + + if (i2c_dev->hw->has_mst_fifo) { + val = i2c_readl(i2c_dev, I2C_MST_FIFO_STATUS); + tx_fifo_avail = FIELD_GET(I2C_MST_FIFO_STATUS_TX, val); + } else { + val = i2c_readl(i2c_dev, I2C_FIFO_STATUS); + tx_fifo_avail = FIELD_GET(I2C_FIFO_STATUS_TX, val); + } + + /* round down to exclude partial word at the end of buffer */ + words_to_transfer = buf_remaining / BYTES_PER_FIFO_WORD; + + /* + * This hunk pushes 4 bytes at a time into the TX FIFO. + * + * It's very common to have < 4 bytes, hence there is no word + * to push if we have less than 4 bytes to transfer. + */ + if (words_to_transfer) { + if (words_to_transfer > tx_fifo_avail) + words_to_transfer = tx_fifo_avail; + + /* + * Update state before writing to FIFO. Note that this may + * cause us to finish writing all bytes (AKA buf_remaining + * goes to 0), hence we have a potential for an interrupt + * (PACKET_XFER_COMPLETE is not maskable), but GIC interrupt + * is disabled at this point. + */ + buf_remaining -= words_to_transfer * BYTES_PER_FIFO_WORD; + tx_fifo_avail -= words_to_transfer; + + i2c_dev->msg_buf_remaining = buf_remaining; + i2c_dev->msg_buf = buf + words_to_transfer * BYTES_PER_FIFO_WORD; + + i2c_writesl(i2c_dev, buf, I2C_TX_FIFO, words_to_transfer); + + buf += words_to_transfer * BYTES_PER_FIFO_WORD; + } + + /* + * If there is a partial word at the end of buffer, handle it manually + * to prevent reading past the end of buffer, which could cross a page + * boundary and fault. + */ + if (tx_fifo_avail > 0 && buf_remaining > 0) { + /* + * buf_remaining > 3 check not needed as tx_fifo_avail == 0 + * when (words_to_transfer was > tx_fifo_avail) earlier + * in this function for non-zero words_to_transfer. + */ + memcpy(&val, buf, buf_remaining); + val = le32_to_cpu(val); + + i2c_dev->msg_buf_remaining = 0; + i2c_dev->msg_buf = NULL; + + i2c_writel(i2c_dev, val, I2C_TX_FIFO); + } + + return 0; +} + static irqreturn_t tegra_i2c_isr(int irq, void *dev_id) { - u32 status; const u32 status_err = I2C_INT_NO_ACK | I2C_INT_ARBITRATION_LOST; struct tegra_i2c_dev *i2c_dev = dev_id; + u32 status; status = i2c_readl(i2c_dev, I2C_INT_STATUS); if (status == 0) { - dev_warn(i2c_dev->dev, "irq status 0 %08x %08x %08x\n", + dev_warn(i2c_dev->dev, "IRQ status 0 %08x %08x %08x\n", i2c_readl(i2c_dev, I2C_PACKET_TRANSFER_STATUS), i2c_readl(i2c_dev, I2C_STATUS), i2c_readl(i2c_dev, I2C_CNFG)); @@ -900,7 +856,7 @@ static irqreturn_t tegra_i2c_isr(int irq, void *dev_id) goto err; } - if (unlikely(status & status_err)) { + if (status & status_err) { tegra_i2c_disable_packet_mode(i2c_dev); if (status & I2C_INT_NO_ACK) i2c_dev->msg_err |= I2C_ERR_NO_ACK; @@ -910,13 +866,13 @@ static irqreturn_t tegra_i2c_isr(int irq, void *dev_id) } /* - * I2C transfer is terminated during the bus clear so skip + * I2C transfer is terminated during the bus clear, so skip * processing the other interrupts. */ if (i2c_dev->hw->supports_bus_clear && (status & I2C_INT_BUS_CLR_DONE)) goto err; - if (!i2c_dev->is_curr_dma_xfer) { + if (!i2c_dev->dma_mode) { if (i2c_dev->msg_read && (status & I2C_INT_RX_FIFO_DATA_REQ)) { if (tegra_i2c_empty_rx_fifo(i2c_dev)) { /* @@ -946,11 +902,12 @@ static irqreturn_t tegra_i2c_isr(int irq, void *dev_id) * During message read XFER_COMPLETE interrupt is triggered prior to * DMA completion and during message write XFER_COMPLETE interrupt is * triggered after DMA completion. - * PACKETS_XFER_COMPLETE indicates completion of all bytes of transfer. + * + * PACKETS_XFER_COMPLETE indicates completion of all bytes of transfer, * so forcing msg_buf_remaining to 0 in DMA mode. */ if (status & I2C_INT_PACKET_XFER_COMPLETE) { - if (i2c_dev->is_curr_dma_xfer) + if (i2c_dev->dma_mode) i2c_dev->msg_buf_remaining = 0; /* * Underflow error condition: XFER_COMPLETE before message @@ -964,17 +921,23 @@ static irqreturn_t tegra_i2c_isr(int irq, void *dev_id) } goto done; err: - /* An error occurred, mask all interrupts */ - tegra_i2c_mask_irq(i2c_dev, I2C_INT_NO_ACK | I2C_INT_ARBITRATION_LOST | - I2C_INT_PACKET_XFER_COMPLETE | I2C_INT_TX_FIFO_DATA_REQ | - I2C_INT_RX_FIFO_DATA_REQ); + /* mask all interrupts on error */ + tegra_i2c_mask_irq(i2c_dev, + I2C_INT_NO_ACK | + I2C_INT_ARBITRATION_LOST | + I2C_INT_PACKET_XFER_COMPLETE | + I2C_INT_TX_FIFO_DATA_REQ | + I2C_INT_RX_FIFO_DATA_REQ); + if (i2c_dev->hw->supports_bus_clear) tegra_i2c_mask_irq(i2c_dev, I2C_INT_BUS_CLR_DONE); + i2c_writel(i2c_dev, status, I2C_INT_STATUS); + if (i2c_dev->is_dvc) dvc_writel(i2c_dev, DVC_STATUS_I2C_DONE_INTR, DVC_STATUS); - if (i2c_dev->is_curr_dma_xfer) { + if (i2c_dev->dma_mode) { if (i2c_dev->msg_read) dmaengine_terminate_async(i2c_dev->rx_dma_chan); else @@ -991,19 +954,17 @@ done: static void tegra_i2c_config_fifo_trig(struct tegra_i2c_dev *i2c_dev, size_t len) { - u32 val, reg; - u8 dma_burst; struct dma_slave_config slv_config = {0}; + u32 val, reg, dma_burst, reg_offset; struct dma_chan *chan; - int ret; - unsigned long reg_offset; + int err; if (i2c_dev->hw->has_mst_fifo) reg = I2C_MST_FIFO_CONTROL; else reg = I2C_FIFO_CONTROL; - if (i2c_dev->is_curr_dma_xfer) { + if (i2c_dev->dma_mode) { if (len & 0xF) dma_burst = 1; else if (len & 0x10) @@ -1014,6 +975,7 @@ static void tegra_i2c_config_fifo_trig(struct tegra_i2c_dev *i2c_dev, if (i2c_dev->msg_read) { chan = i2c_dev->rx_dma_chan; reg_offset = tegra_i2c_reg_addr(i2c_dev, I2C_RX_FIFO); + slv_config.src_addr = i2c_dev->base_phys + reg_offset; slv_config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; slv_config.src_maxburst = dma_burst; @@ -1025,6 +987,7 @@ static void tegra_i2c_config_fifo_trig(struct tegra_i2c_dev *i2c_dev, } else { chan = i2c_dev->tx_dma_chan; reg_offset = tegra_i2c_reg_addr(i2c_dev, I2C_TX_FIFO); + slv_config.dst_addr = i2c_dev->base_phys + reg_offset; slv_config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; slv_config.dst_maxburst = dma_burst; @@ -1036,13 +999,13 @@ static void tegra_i2c_config_fifo_trig(struct tegra_i2c_dev *i2c_dev, } slv_config.device_fc = true; - ret = dmaengine_slave_config(chan, &slv_config); - if (ret < 0) { - dev_err(i2c_dev->dev, "DMA slave config failed: %d\n", - ret); + err = dmaengine_slave_config(chan, &slv_config); + if (err) { + dev_err(i2c_dev->dev, "DMA config failed: %d\n", err); dev_err(i2c_dev->dev, "falling back to PIO\n"); + tegra_i2c_release_dma(i2c_dev); - i2c_dev->is_curr_dma_xfer = false; + i2c_dev->dma_mode = false; } else { goto out; } @@ -1058,10 +1021,9 @@ out: i2c_writel(i2c_dev, val, reg); } -static unsigned long -tegra_i2c_poll_completion_timeout(struct tegra_i2c_dev *i2c_dev, - struct completion *complete, - unsigned int timeout_ms) +static unsigned long tegra_i2c_poll_completion(struct tegra_i2c_dev *i2c_dev, + struct completion *complete, + unsigned int timeout_ms) { ktime_t ktime = ktime_get(); ktime_t ktimeout = ktime_add_ms(ktime, timeout_ms); @@ -1085,16 +1047,14 @@ tegra_i2c_poll_completion_timeout(struct tegra_i2c_dev *i2c_dev, return 0; } -static unsigned long -tegra_i2c_wait_completion_timeout(struct tegra_i2c_dev *i2c_dev, - struct completion *complete, - unsigned int timeout_ms) +static unsigned long tegra_i2c_wait_completion(struct tegra_i2c_dev *i2c_dev, + struct completion *complete, + unsigned int timeout_ms) { unsigned long ret; - if (i2c_dev->is_curr_atomic_xfer) { - ret = tegra_i2c_poll_completion_timeout(i2c_dev, complete, - timeout_ms); + if (i2c_dev->atomic_mode) { + ret = tegra_i2c_poll_completion(i2c_dev, complete, timeout_ms); } else { enable_irq(i2c_dev->irq); ret = wait_for_completion_timeout(complete, @@ -1112,8 +1072,7 @@ tegra_i2c_wait_completion_timeout(struct tegra_i2c_dev *i2c_dev, * needs to be checked after timeout. */ if (ret == 0) - ret = tegra_i2c_poll_completion_timeout(i2c_dev, - complete, 0); + ret = tegra_i2c_poll_completion(i2c_dev, complete, 0); } return ret; @@ -1122,60 +1081,134 @@ tegra_i2c_wait_completion_timeout(struct tegra_i2c_dev *i2c_dev, static int tegra_i2c_issue_bus_clear(struct i2c_adapter *adap) { struct tegra_i2c_dev *i2c_dev = i2c_get_adapdata(adap); + u32 val, time_left; int err; - unsigned long time_left; - u32 reg; reinit_completion(&i2c_dev->msg_complete); - reg = FIELD_PREP(I2C_BC_SCLK_THRESHOLD, 9) | I2C_BC_STOP_COND | + + val = FIELD_PREP(I2C_BC_SCLK_THRESHOLD, 9) | I2C_BC_STOP_COND | I2C_BC_TERMINATE; - i2c_writel(i2c_dev, reg, I2C_BUS_CLEAR_CNFG); - if (i2c_dev->hw->has_config_load_reg) { - err = tegra_i2c_wait_for_config_load(i2c_dev); - if (err) - return err; - } + i2c_writel(i2c_dev, val, I2C_BUS_CLEAR_CNFG); + + err = tegra_i2c_wait_for_config_load(i2c_dev); + if (err) + return err; - reg |= I2C_BC_ENABLE; - i2c_writel(i2c_dev, reg, I2C_BUS_CLEAR_CNFG); + val |= I2C_BC_ENABLE; + i2c_writel(i2c_dev, val, I2C_BUS_CLEAR_CNFG); tegra_i2c_unmask_irq(i2c_dev, I2C_INT_BUS_CLR_DONE); - time_left = tegra_i2c_wait_completion_timeout( - i2c_dev, &i2c_dev->msg_complete, 50); + time_left = tegra_i2c_wait_completion(i2c_dev, &i2c_dev->msg_complete, 50); + tegra_i2c_mask_irq(i2c_dev, I2C_INT_BUS_CLR_DONE); + if (time_left == 0) { - dev_err(i2c_dev->dev, "timed out for bus clear\n"); + dev_err(i2c_dev->dev, "failed to clear bus\n"); return -ETIMEDOUT; } - reg = i2c_readl(i2c_dev, I2C_BUS_CLEAR_STATUS); - if (!(reg & I2C_BC_STATUS)) { - dev_err(i2c_dev->dev, - "un-recovered arbitration lost\n"); + val = i2c_readl(i2c_dev, I2C_BUS_CLEAR_STATUS); + if (!(val & I2C_BC_STATUS)) { + dev_err(i2c_dev->dev, "un-recovered arbitration lost\n"); return -EIO; } return -EAGAIN; } +static void tegra_i2c_push_packet_header(struct tegra_i2c_dev *i2c_dev, + struct i2c_msg *msg, + enum msg_end_type end_state) +{ + u32 *dma_buf = i2c_dev->dma_buf; + u32 packet_header; + + packet_header = FIELD_PREP(PACKET_HEADER0_HEADER_SIZE, 0) | + FIELD_PREP(PACKET_HEADER0_PROTOCOL, + PACKET_HEADER0_PROTOCOL_I2C) | + FIELD_PREP(PACKET_HEADER0_CONT_ID, i2c_dev->cont_id) | + FIELD_PREP(PACKET_HEADER0_PACKET_ID, 1); + + if (i2c_dev->dma_mode && !i2c_dev->msg_read) + *dma_buf++ = packet_header; + else + i2c_writel(i2c_dev, packet_header, I2C_TX_FIFO); + + packet_header = msg->len - 1; + + if (i2c_dev->dma_mode && !i2c_dev->msg_read) + *dma_buf++ = packet_header; + else + i2c_writel(i2c_dev, packet_header, I2C_TX_FIFO); + + packet_header = I2C_HEADER_IE_ENABLE; + + if (end_state == MSG_END_CONTINUE) + packet_header |= I2C_HEADER_CONTINUE_XFER; + else if (end_state == MSG_END_REPEAT_START) + packet_header |= I2C_HEADER_REPEAT_START; + + if (msg->flags & I2C_M_TEN) { + packet_header |= msg->addr; + packet_header |= I2C_HEADER_10BIT_ADDR; + } else { + packet_header |= msg->addr << I2C_HEADER_SLAVE_ADDR_SHIFT; + } + + if (msg->flags & I2C_M_IGNORE_NAK) + packet_header |= I2C_HEADER_CONT_ON_NAK; + + if (msg->flags & I2C_M_RD) + packet_header |= I2C_HEADER_READ; + + if (i2c_dev->dma_mode && !i2c_dev->msg_read) + *dma_buf++ = packet_header; + else + i2c_writel(i2c_dev, packet_header, I2C_TX_FIFO); +} + +static int tegra_i2c_error_recover(struct tegra_i2c_dev *i2c_dev, + struct i2c_msg *msg) +{ + if (i2c_dev->msg_err == I2C_ERR_NONE) + return 0; + + tegra_i2c_init(i2c_dev); + + /* start recovery upon arbitration loss in single master mode */ + if (i2c_dev->msg_err == I2C_ERR_ARBITRATION_LOST) { + if (!i2c_dev->multimaster_mode) + return i2c_recover_bus(&i2c_dev->adapter); + + return -EAGAIN; + } + + if (i2c_dev->msg_err == I2C_ERR_NO_ACK) { + if (msg->flags & I2C_M_IGNORE_NAK) + return 0; + + return -EREMOTEIO; + } + + return -EIO; +} + static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, struct i2c_msg *msg, enum msg_end_type end_state) { - u32 packet_header; - u32 int_mask; - unsigned long time_left; + unsigned long time_left, xfer_time = 100; size_t xfer_size; - u32 *buffer = NULL; - int err = 0; - bool dma; - u16 xfer_time = 100; + u32 int_mask; + int err; - tegra_i2c_flush_fifos(i2c_dev); + err = tegra_i2c_flush_fifos(i2c_dev); + if (err) + return err; i2c_dev->msg_buf = msg->buf; i2c_dev->msg_buf_remaining = msg->len; i2c_dev->msg_err = I2C_ERR_NONE; - i2c_dev->msg_read = (msg->flags & I2C_M_RD); + i2c_dev->msg_read = !!(msg->flags & I2C_M_RD); reinit_completion(&i2c_dev->msg_complete); if (i2c_dev->msg_read) @@ -1184,93 +1217,52 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, xfer_size = msg->len + I2C_PACKET_HEADER_SIZE; xfer_size = ALIGN(xfer_size, BYTES_PER_FIFO_WORD); - i2c_dev->is_curr_dma_xfer = (xfer_size > I2C_PIO_MODE_PREFERRED_LEN) && - i2c_dev->dma_buf && - !i2c_dev->is_curr_atomic_xfer; + + i2c_dev->dma_mode = xfer_size > I2C_PIO_MODE_PREFERRED_LEN && + i2c_dev->dma_buf && !i2c_dev->atomic_mode; + tegra_i2c_config_fifo_trig(i2c_dev, xfer_size); - dma = i2c_dev->is_curr_dma_xfer; + /* * Transfer time in mSec = Total bits / transfer rate * Total bits = 9 bits per byte (including ACK bit) + Start & stop bits */ xfer_time += DIV_ROUND_CLOSEST(((xfer_size * 9) + 2) * MSEC_PER_SEC, - i2c_dev->bus_clk_rate); + i2c_dev->bus_clk_rate); int_mask = I2C_INT_NO_ACK | I2C_INT_ARBITRATION_LOST; tegra_i2c_unmask_irq(i2c_dev, int_mask); - if (dma) { + + if (i2c_dev->dma_mode) { if (i2c_dev->msg_read) { dma_sync_single_for_device(i2c_dev->dev, i2c_dev->dma_phys, - xfer_size, - DMA_FROM_DEVICE); + xfer_size, DMA_FROM_DEVICE); + err = tegra_i2c_dma_submit(i2c_dev, xfer_size); - if (err < 0) { - dev_err(i2c_dev->dev, - "starting RX DMA failed, err %d\n", - err); + if (err) return err; - } - } else { dma_sync_single_for_cpu(i2c_dev->dev, i2c_dev->dma_phys, - xfer_size, - DMA_TO_DEVICE); - buffer = i2c_dev->dma_buf; + xfer_size, DMA_TO_DEVICE); } } - packet_header = FIELD_PREP(PACKET_HEADER0_HEADER_SIZE, 0) | - FIELD_PREP(PACKET_HEADER0_PROTOCOL, - PACKET_HEADER0_PROTOCOL_I2C) | - FIELD_PREP(PACKET_HEADER0_CONT_ID, i2c_dev->cont_id) | - FIELD_PREP(PACKET_HEADER0_PACKET_ID, 1); - if (dma && !i2c_dev->msg_read) - *buffer++ = packet_header; - else - i2c_writel(i2c_dev, packet_header, I2C_TX_FIFO); - - packet_header = msg->len - 1; - if (dma && !i2c_dev->msg_read) - *buffer++ = packet_header; - else - i2c_writel(i2c_dev, packet_header, I2C_TX_FIFO); - - packet_header = I2C_HEADER_IE_ENABLE; - if (end_state == MSG_END_CONTINUE) - packet_header |= I2C_HEADER_CONTINUE_XFER; - else if (end_state == MSG_END_REPEAT_START) - packet_header |= I2C_HEADER_REPEAT_START; - if (msg->flags & I2C_M_TEN) { - packet_header |= msg->addr; - packet_header |= I2C_HEADER_10BIT_ADDR; - } else { - packet_header |= msg->addr << I2C_HEADER_SLAVE_ADDR_SHIFT; - } - if (msg->flags & I2C_M_IGNORE_NAK) - packet_header |= I2C_HEADER_CONT_ON_NAK; - if (msg->flags & I2C_M_RD) - packet_header |= I2C_HEADER_READ; - if (dma && !i2c_dev->msg_read) - *buffer++ = packet_header; - else - i2c_writel(i2c_dev, packet_header, I2C_TX_FIFO); + tegra_i2c_push_packet_header(i2c_dev, msg, end_state); if (!i2c_dev->msg_read) { - if (dma) { - memcpy(buffer, msg->buf, msg->len); + if (i2c_dev->dma_mode) { + memcpy(i2c_dev->dma_buf + I2C_PACKET_HEADER_SIZE, + msg->buf, msg->len); + dma_sync_single_for_device(i2c_dev->dev, i2c_dev->dma_phys, - xfer_size, - DMA_TO_DEVICE); + xfer_size, DMA_TO_DEVICE); + err = tegra_i2c_dma_submit(i2c_dev, xfer_size); - if (err < 0) { - dev_err(i2c_dev->dev, - "starting TX DMA failed, err %d\n", - err); + if (err) return err; - } } else { tegra_i2c_fill_tx_fifo(i2c_dev); } @@ -1278,7 +1270,8 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, if (i2c_dev->hw->has_per_pkt_xfer_complete_irq) int_mask |= I2C_INT_PACKET_XFER_COMPLETE; - if (!dma) { + + if (!i2c_dev->dma_mode) { if (msg->flags & I2C_M_RD) int_mask |= I2C_INT_RX_FIFO_DATA_REQ; else if (i2c_dev->msg_buf_remaining) @@ -1286,12 +1279,13 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, } tegra_i2c_unmask_irq(i2c_dev, int_mask); - dev_dbg(i2c_dev->dev, "unmasked irq: %02x\n", + dev_dbg(i2c_dev->dev, "unmasked IRQ: %02x\n", i2c_readl(i2c_dev, I2C_INT_MASK)); - if (dma) { - time_left = tegra_i2c_wait_completion_timeout( - i2c_dev, &i2c_dev->dma_complete, xfer_time); + if (i2c_dev->dma_mode) { + time_left = tegra_i2c_wait_completion(i2c_dev, + &i2c_dev->dma_complete, + xfer_time); /* * Synchronize DMA first, since dmaengine_terminate_sync() @@ -1307,29 +1301,28 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, i2c_dev->tx_dma_chan); if (!time_left && !completion_done(&i2c_dev->dma_complete)) { - dev_err(i2c_dev->dev, "DMA transfer timeout\n"); - tegra_i2c_init(i2c_dev, true); + dev_err(i2c_dev->dev, "DMA transfer timed out\n"); + tegra_i2c_init(i2c_dev); return -ETIMEDOUT; } if (i2c_dev->msg_read && i2c_dev->msg_err == I2C_ERR_NONE) { dma_sync_single_for_cpu(i2c_dev->dev, i2c_dev->dma_phys, - xfer_size, - DMA_FROM_DEVICE); - memcpy(i2c_dev->msg_buf, i2c_dev->dma_buf, - msg->len); + xfer_size, DMA_FROM_DEVICE); + + memcpy(i2c_dev->msg_buf, i2c_dev->dma_buf, msg->len); } } - time_left = tegra_i2c_wait_completion_timeout( - i2c_dev, &i2c_dev->msg_complete, xfer_time); + time_left = tegra_i2c_wait_completion(i2c_dev, &i2c_dev->msg_complete, + xfer_time); tegra_i2c_mask_irq(i2c_dev, int_mask); if (time_left == 0) { - dev_err(i2c_dev->dev, "i2c transfer timed out\n"); - tegra_i2c_init(i2c_dev, true); + dev_err(i2c_dev->dev, "I2C transfer timed out\n"); + tegra_i2c_init(i2c_dev); return -ETIMEDOUT; } @@ -1337,37 +1330,25 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, time_left, completion_done(&i2c_dev->msg_complete), i2c_dev->msg_err); - i2c_dev->is_curr_dma_xfer = false; - if (likely(i2c_dev->msg_err == I2C_ERR_NONE)) - return 0; + i2c_dev->dma_mode = false; - tegra_i2c_init(i2c_dev, true); - /* start recovery upon arbitration loss in single master mode */ - if (i2c_dev->msg_err == I2C_ERR_ARBITRATION_LOST) { - if (!i2c_dev->is_multimaster_mode) - return i2c_recover_bus(&i2c_dev->adapter); - return -EAGAIN; - } - - if (i2c_dev->msg_err == I2C_ERR_NO_ACK) { - if (msg->flags & I2C_M_IGNORE_NAK) - return 0; - return -EREMOTEIO; - } + err = tegra_i2c_error_recover(i2c_dev, msg); + if (err) + return err; - return -EIO; + return 0; } static int tegra_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) { struct tegra_i2c_dev *i2c_dev = i2c_get_adapdata(adap); - int i; - int ret; + int i, ret; ret = pm_runtime_get_sync(i2c_dev->dev); if (ret < 0) { dev_err(i2c_dev->dev, "runtime resume failed %d\n", ret); + pm_runtime_put_noidle(i2c_dev->dev); return ret; } @@ -1375,6 +1356,7 @@ static int tegra_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], enum msg_end_type end_type = MSG_END_STOP; if (i < (num - 1)) { + /* check whether follow up message is coming */ if (msgs[i + 1].flags & I2C_M_NOSTART) end_type = MSG_END_CONTINUE; else @@ -1396,9 +1378,9 @@ static int tegra_i2c_xfer_atomic(struct i2c_adapter *adap, struct tegra_i2c_dev *i2c_dev = i2c_get_adapdata(adap); int ret; - i2c_dev->is_curr_atomic_xfer = true; + i2c_dev->atomic_mode = true; ret = tegra_i2c_xfer(adap, msgs, num); - i2c_dev->is_curr_atomic_xfer = false; + i2c_dev->atomic_mode = false; return ret; } @@ -1411,22 +1393,8 @@ static u32 tegra_i2c_func(struct i2c_adapter *adap) if (i2c_dev->hw->has_continue_xfer_support) ret |= I2C_FUNC_NOSTART; - return ret; -} -static void tegra_i2c_parse_dt(struct tegra_i2c_dev *i2c_dev) -{ - struct device_node *np = i2c_dev->dev->of_node; - int ret; - bool multi_mode; - - ret = of_property_read_u32(np, "clock-frequency", - &i2c_dev->bus_clk_rate); - if (ret) - i2c_dev->bus_clk_rate = I2C_MAX_STANDARD_MODE_FREQ; /* default clock rate */ - - multi_mode = of_property_read_bool(np, "multi-master"); - i2c_dev->is_multimaster_mode = multi_mode; + return ret; } static const struct i2c_algorithm tegra_i2c_algo = { @@ -1454,7 +1422,6 @@ static struct i2c_bus_recovery_info tegra_i2c_recovery_info = { static const struct tegra_i2c_hw_feature tegra20_i2c_hw = { .has_continue_xfer_support = false, .has_per_pkt_xfer_complete_irq = false, - .has_single_clk_source = false, .clk_divisor_hs_mode = 3, .clk_divisor_std_mode = 0, .clk_divisor_fast_mode = 0, @@ -1479,7 +1446,6 @@ static const struct tegra_i2c_hw_feature tegra20_i2c_hw = { static const struct tegra_i2c_hw_feature tegra30_i2c_hw = { .has_continue_xfer_support = true, .has_per_pkt_xfer_complete_irq = false, - .has_single_clk_source = false, .clk_divisor_hs_mode = 3, .clk_divisor_std_mode = 0, .clk_divisor_fast_mode = 0, @@ -1504,7 +1470,6 @@ static const struct tegra_i2c_hw_feature tegra30_i2c_hw = { static const struct tegra_i2c_hw_feature tegra114_i2c_hw = { .has_continue_xfer_support = true, .has_per_pkt_xfer_complete_irq = true, - .has_single_clk_source = true, .clk_divisor_hs_mode = 1, .clk_divisor_std_mode = 0x19, .clk_divisor_fast_mode = 0x19, @@ -1529,7 +1494,6 @@ static const struct tegra_i2c_hw_feature tegra114_i2c_hw = { static const struct tegra_i2c_hw_feature tegra124_i2c_hw = { .has_continue_xfer_support = true, .has_per_pkt_xfer_complete_irq = true, - .has_single_clk_source = true, .clk_divisor_hs_mode = 1, .clk_divisor_std_mode = 0x19, .clk_divisor_fast_mode = 0x19, @@ -1554,7 +1518,6 @@ static const struct tegra_i2c_hw_feature tegra124_i2c_hw = { static const struct tegra_i2c_hw_feature tegra210_i2c_hw = { .has_continue_xfer_support = true, .has_per_pkt_xfer_complete_irq = true, - .has_single_clk_source = true, .clk_divisor_hs_mode = 1, .clk_divisor_std_mode = 0x19, .clk_divisor_fast_mode = 0x19, @@ -1579,7 +1542,6 @@ static const struct tegra_i2c_hw_feature tegra210_i2c_hw = { static const struct tegra_i2c_hw_feature tegra186_i2c_hw = { .has_continue_xfer_support = true, .has_per_pkt_xfer_complete_irq = true, - .has_single_clk_source = true, .clk_divisor_hs_mode = 1, .clk_divisor_std_mode = 0x16, .clk_divisor_fast_mode = 0x19, @@ -1604,7 +1566,6 @@ static const struct tegra_i2c_hw_feature tegra186_i2c_hw = { static const struct tegra_i2c_hw_feature tegra194_i2c_hw = { .has_continue_xfer_support = true, .has_per_pkt_xfer_complete_irq = true, - .has_single_clk_source = true, .clk_divisor_hs_mode = 1, .clk_divisor_std_mode = 0x4f, .clk_divisor_fast_mode = 0x3c, @@ -1626,7 +1587,6 @@ static const struct tegra_i2c_hw_feature tegra194_i2c_hw = { .has_interface_timing_reg = true, }; -/* Match table for of_platform binding */ static const struct of_device_id tegra_i2c_of_match[] = { { .compatible = "nvidia,tegra194-i2c", .data = &tegra194_i2c_hw, }, { .compatible = "nvidia,tegra186-i2c", .data = &tegra186_i2c_hw, }, @@ -1641,223 +1601,196 @@ static const struct of_device_id tegra_i2c_of_match[] = { }; MODULE_DEVICE_TABLE(of, tegra_i2c_of_match); -static int tegra_i2c_probe(struct platform_device *pdev) +static void tegra_i2c_parse_dt(struct tegra_i2c_dev *i2c_dev) { - struct device *dev = &pdev->dev; - struct tegra_i2c_dev *i2c_dev; - struct resource *res; - struct clk *div_clk; - struct clk *fast_clk; - void __iomem *base; - phys_addr_t base_phys; - int irq; - int ret; + struct device_node *np = i2c_dev->dev->of_node; + bool multi_mode; + int err; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - base_phys = res->start; - base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(base)) - return PTR_ERR(base); + err = of_property_read_u32(np, "clock-frequency", + &i2c_dev->bus_clk_rate); + if (err) + i2c_dev->bus_clk_rate = I2C_MAX_STANDARD_MODE_FREQ; - res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!res) { - dev_err(&pdev->dev, "no irq resource\n"); - return -EINVAL; - } - irq = res->start; + multi_mode = of_property_read_bool(np, "multi-master"); + i2c_dev->multimaster_mode = multi_mode; + + if (of_device_is_compatible(np, "nvidia,tegra20-i2c-dvc")) + i2c_dev->is_dvc = true; + + if (of_device_is_compatible(np, "nvidia,tegra210-i2c-vi")) + i2c_dev->is_vi = true; +} + +static int tegra_i2c_init_clocks(struct tegra_i2c_dev *i2c_dev) +{ + int err; + + i2c_dev->clocks[i2c_dev->nclocks++].id = "div-clk"; + + if (i2c_dev->hw == &tegra20_i2c_hw || i2c_dev->hw == &tegra30_i2c_hw) + i2c_dev->clocks[i2c_dev->nclocks++].id = "fast-clk"; - div_clk = devm_clk_get(&pdev->dev, "div-clk"); - if (IS_ERR(div_clk)) { - if (PTR_ERR(div_clk) != -EPROBE_DEFER) - dev_err(&pdev->dev, "missing controller clock\n"); + if (i2c_dev->is_vi) + i2c_dev->clocks[i2c_dev->nclocks++].id = "slow"; + + err = devm_clk_bulk_get(i2c_dev->dev, i2c_dev->nclocks, + i2c_dev->clocks); + if (err) + return err; + + err = clk_bulk_prepare(i2c_dev->nclocks, i2c_dev->clocks); + if (err) + return err; + + i2c_dev->div_clk = i2c_dev->clocks[0].clk; + + if (!i2c_dev->multimaster_mode) + return 0; - return PTR_ERR(div_clk); + err = clk_enable(i2c_dev->div_clk); + if (err) { + dev_err(i2c_dev->dev, "failed to enable div-clk: %d\n", err); + goto unprepare_clocks; } + return 0; + +unprepare_clocks: + clk_bulk_unprepare(i2c_dev->nclocks, i2c_dev->clocks); + + return err; +} + +static void tegra_i2c_release_clocks(struct tegra_i2c_dev *i2c_dev) +{ + if (i2c_dev->multimaster_mode) + clk_disable(i2c_dev->div_clk); + + clk_bulk_unprepare(i2c_dev->nclocks, i2c_dev->clocks); +} + +static int tegra_i2c_init_hardware(struct tegra_i2c_dev *i2c_dev) +{ + int ret; + + ret = pm_runtime_get_sync(i2c_dev->dev); + if (ret < 0) + dev_err(i2c_dev->dev, "runtime resume failed: %d\n", ret); + else + ret = tegra_i2c_init(i2c_dev); + + pm_runtime_put(i2c_dev->dev); + + return ret; +} + +static int tegra_i2c_probe(struct platform_device *pdev) +{ + struct tegra_i2c_dev *i2c_dev; + struct resource *res; + int err; + i2c_dev = devm_kzalloc(&pdev->dev, sizeof(*i2c_dev), GFP_KERNEL); if (!i2c_dev) return -ENOMEM; - i2c_dev->base = base; - i2c_dev->base_phys = base_phys; - i2c_dev->div_clk = div_clk; - i2c_dev->adapter.algo = &tegra_i2c_algo; - i2c_dev->adapter.retries = 1; - i2c_dev->adapter.timeout = 6 * HZ; - i2c_dev->irq = irq; + platform_set_drvdata(pdev, i2c_dev); + + init_completion(&i2c_dev->msg_complete); + init_completion(&i2c_dev->dma_complete); + + i2c_dev->hw = of_device_get_match_data(&pdev->dev); i2c_dev->cont_id = pdev->id; i2c_dev->dev = &pdev->dev; - i2c_dev->rst = devm_reset_control_get_exclusive(&pdev->dev, "i2c"); - if (IS_ERR(i2c_dev->rst)) { - dev_err(&pdev->dev, "missing controller reset\n"); - return PTR_ERR(i2c_dev->rst); - } + i2c_dev->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(i2c_dev->base)) + return PTR_ERR(i2c_dev->base); - tegra_i2c_parse_dt(i2c_dev); - - i2c_dev->hw = of_device_get_match_data(&pdev->dev); - i2c_dev->is_dvc = of_device_is_compatible(pdev->dev.of_node, - "nvidia,tegra20-i2c-dvc"); - i2c_dev->is_vi = of_device_is_compatible(dev->of_node, - "nvidia,tegra210-i2c-vi"); - i2c_dev->adapter.quirks = i2c_dev->hw->quirks; - i2c_dev->dma_buf_size = i2c_dev->adapter.quirks->max_write_len + - I2C_PACKET_HEADER_SIZE; - init_completion(&i2c_dev->msg_complete); - init_completion(&i2c_dev->dma_complete); + i2c_dev->base_phys = res->start; - if (!i2c_dev->hw->has_single_clk_source) { - fast_clk = devm_clk_get(&pdev->dev, "fast-clk"); - if (IS_ERR(fast_clk)) { - dev_err(&pdev->dev, "missing fast clock\n"); - return PTR_ERR(fast_clk); - } - i2c_dev->fast_clk = fast_clk; - } + err = platform_get_irq(pdev, 0); + if (err < 0) + return err; - if (i2c_dev->is_vi) { - i2c_dev->slow_clk = devm_clk_get(dev, "slow"); - if (IS_ERR(i2c_dev->slow_clk)) { - if (PTR_ERR(i2c_dev->slow_clk) != -EPROBE_DEFER) - dev_err(dev, "failed to get slow clock: %ld\n", - PTR_ERR(i2c_dev->slow_clk)); + i2c_dev->irq = err; - return PTR_ERR(i2c_dev->slow_clk); - } - } + /* interrupt will be enabled during of transfer time */ + irq_set_status_flags(i2c_dev->irq, IRQ_NOAUTOEN); - platform_set_drvdata(pdev, i2c_dev); + err = devm_request_irq(i2c_dev->dev, i2c_dev->irq, tegra_i2c_isr, + IRQF_NO_SUSPEND, dev_name(i2c_dev->dev), + i2c_dev); + if (err) + return err; - ret = clk_prepare(i2c_dev->fast_clk); - if (ret < 0) { - dev_err(i2c_dev->dev, "Clock prepare failed %d\n", ret); - return ret; + i2c_dev->rst = devm_reset_control_get_exclusive(i2c_dev->dev, "i2c"); + if (IS_ERR(i2c_dev->rst)) { + dev_err_probe(i2c_dev->dev, PTR_ERR(i2c_dev->rst), + "failed to get reset control\n"); + return PTR_ERR(i2c_dev->rst); } - ret = clk_prepare(i2c_dev->slow_clk); - if (ret < 0) { - dev_err(dev, "failed to prepare slow clock: %d\n", ret); - goto unprepare_fast_clk; - } + tegra_i2c_parse_dt(i2c_dev); - if (i2c_dev->bus_clk_rate > I2C_MAX_FAST_MODE_FREQ && - i2c_dev->bus_clk_rate <= I2C_MAX_FAST_MODE_PLUS_FREQ) - i2c_dev->clk_divisor_non_hs_mode = - i2c_dev->hw->clk_divisor_fast_plus_mode; - else if (i2c_dev->bus_clk_rate > I2C_MAX_STANDARD_MODE_FREQ && - i2c_dev->bus_clk_rate <= I2C_MAX_FAST_MODE_FREQ) - i2c_dev->clk_divisor_non_hs_mode = - i2c_dev->hw->clk_divisor_fast_mode; - else - i2c_dev->clk_divisor_non_hs_mode = - i2c_dev->hw->clk_divisor_std_mode; + err = tegra_i2c_init_clocks(i2c_dev); + if (err) + return err; - ret = clk_prepare(i2c_dev->div_clk); - if (ret < 0) { - dev_err(i2c_dev->dev, "Clock prepare failed %d\n", ret); - goto unprepare_slow_clk; - } + err = tegra_i2c_init_dma(i2c_dev); + if (err) + goto release_clocks; /* - * VI I2C is in VE power domain which is not always on and not - * an IRQ safe. So, IRQ safe device can't be attached to a non-IRQ - * safe domain as it prevents powering off the PM domain. - * Also, VI I2C device don't need to use runtime IRQ safe as it will - * not be used for atomic transfers. + * VI I2C is in VE power domain which is not always ON and not + * IRQ-safe. Thus, IRQ-safe device shouldn't be attached to a + * non IRQ-safe domain because this prevents powering off the power + * domain. + * + * VI I2C device shouldn't be marked as IRQ-safe because VI I2C won't + * be used for atomic transfers. */ if (!i2c_dev->is_vi) - pm_runtime_irq_safe(&pdev->dev); - pm_runtime_enable(&pdev->dev); - if (!pm_runtime_enabled(&pdev->dev)) { - ret = tegra_i2c_runtime_resume(&pdev->dev); - if (ret < 0) { - dev_err(&pdev->dev, "runtime resume failed\n"); - goto unprepare_div_clk; - } - } else { - ret = pm_runtime_get_sync(i2c_dev->dev); - if (ret < 0) { - dev_err(&pdev->dev, "runtime resume failed\n"); - goto disable_rpm; - } - } + pm_runtime_irq_safe(i2c_dev->dev); - if (i2c_dev->is_multimaster_mode) { - ret = clk_enable(i2c_dev->div_clk); - if (ret < 0) { - dev_err(i2c_dev->dev, "div_clk enable failed %d\n", - ret); - goto put_rpm; - } - } + pm_runtime_enable(i2c_dev->dev); - if (i2c_dev->hw->supports_bus_clear) - i2c_dev->adapter.bus_recovery_info = &tegra_i2c_recovery_info; - - ret = tegra_i2c_init_dma(i2c_dev); - if (ret < 0) - goto disable_div_clk; - - ret = tegra_i2c_init(i2c_dev, false); - if (ret) { - dev_err(&pdev->dev, "Failed to initialize i2c controller\n"); - goto release_dma; - } - - irq_set_status_flags(i2c_dev->irq, IRQ_NOAUTOEN); - - ret = devm_request_irq(&pdev->dev, i2c_dev->irq, tegra_i2c_isr, - IRQF_NO_SUSPEND, dev_name(&pdev->dev), i2c_dev); - if (ret) { - dev_err(&pdev->dev, "Failed to request irq %i\n", i2c_dev->irq); - goto release_dma; - } + err = tegra_i2c_init_hardware(i2c_dev); + if (err) + goto release_rpm; i2c_set_adapdata(&i2c_dev->adapter, i2c_dev); + i2c_dev->adapter.dev.of_node = i2c_dev->dev->of_node; + i2c_dev->adapter.dev.parent = i2c_dev->dev; + i2c_dev->adapter.retries = 1; + i2c_dev->adapter.timeout = 6 * HZ; + i2c_dev->adapter.quirks = i2c_dev->hw->quirks; i2c_dev->adapter.owner = THIS_MODULE; i2c_dev->adapter.class = I2C_CLASS_DEPRECATED; - strlcpy(i2c_dev->adapter.name, dev_name(&pdev->dev), - sizeof(i2c_dev->adapter.name)); - i2c_dev->adapter.dev.parent = &pdev->dev; + i2c_dev->adapter.algo = &tegra_i2c_algo; i2c_dev->adapter.nr = pdev->id; - i2c_dev->adapter.dev.of_node = pdev->dev.of_node; - - ret = i2c_add_numbered_adapter(&i2c_dev->adapter); - if (ret) - goto release_dma; - - pm_runtime_put(&pdev->dev); - return 0; - -release_dma: - tegra_i2c_release_dma(i2c_dev); - -disable_div_clk: - if (i2c_dev->is_multimaster_mode) - clk_disable(i2c_dev->div_clk); + if (i2c_dev->hw->supports_bus_clear) + i2c_dev->adapter.bus_recovery_info = &tegra_i2c_recovery_info; -put_rpm: - if (pm_runtime_enabled(&pdev->dev)) - pm_runtime_put_sync(&pdev->dev); - else - tegra_i2c_runtime_suspend(&pdev->dev); + strlcpy(i2c_dev->adapter.name, dev_name(i2c_dev->dev), + sizeof(i2c_dev->adapter.name)); -disable_rpm: - if (pm_runtime_enabled(&pdev->dev)) - pm_runtime_disable(&pdev->dev); + err = i2c_add_numbered_adapter(&i2c_dev->adapter); + if (err) + goto release_rpm; -unprepare_div_clk: - clk_unprepare(i2c_dev->div_clk); + return 0; -unprepare_slow_clk: - clk_unprepare(i2c_dev->slow_clk); +release_rpm: + pm_runtime_disable(i2c_dev->dev); -unprepare_fast_clk: - clk_unprepare(i2c_dev->fast_clk); + tegra_i2c_release_dma(i2c_dev); +release_clocks: + tegra_i2c_release_clocks(i2c_dev); - return ret; + return err; } static int tegra_i2c_remove(struct platform_device *pdev) @@ -1865,33 +1798,69 @@ static int tegra_i2c_remove(struct platform_device *pdev) struct tegra_i2c_dev *i2c_dev = platform_get_drvdata(pdev); i2c_del_adapter(&i2c_dev->adapter); + pm_runtime_disable(i2c_dev->dev); - if (i2c_dev->is_multimaster_mode) - clk_disable(i2c_dev->div_clk); + tegra_i2c_release_dma(i2c_dev); + tegra_i2c_release_clocks(i2c_dev); - pm_runtime_disable(&pdev->dev); - if (!pm_runtime_status_suspended(&pdev->dev)) - tegra_i2c_runtime_suspend(&pdev->dev); + return 0; +} - clk_unprepare(i2c_dev->div_clk); - clk_unprepare(i2c_dev->slow_clk); - clk_unprepare(i2c_dev->fast_clk); +static int __maybe_unused tegra_i2c_runtime_resume(struct device *dev) +{ + struct tegra_i2c_dev *i2c_dev = dev_get_drvdata(dev); + int err; + + err = pinctrl_pm_select_default_state(dev); + if (err) + return err; + + err = clk_bulk_enable(i2c_dev->nclocks, i2c_dev->clocks); + if (err) + return err; + + /* + * VI I2C device is attached to VE power domain which goes through + * power ON/OFF during runtime PM resume/suspend, meaning that + * controller needs to be re-initialized after power ON. + */ + if (i2c_dev->is_vi) { + err = tegra_i2c_init(i2c_dev); + if (err) + goto disable_clocks; + } - tegra_i2c_release_dma(i2c_dev); return 0; + +disable_clocks: + clk_bulk_disable(i2c_dev->nclocks, i2c_dev->clocks); + + return err; +} + +static int __maybe_unused tegra_i2c_runtime_suspend(struct device *dev) +{ + struct tegra_i2c_dev *i2c_dev = dev_get_drvdata(dev); + + clk_bulk_disable(i2c_dev->nclocks, i2c_dev->clocks); + + return pinctrl_pm_select_idle_state(dev); } static int __maybe_unused tegra_i2c_suspend(struct device *dev) { struct tegra_i2c_dev *i2c_dev = dev_get_drvdata(dev); - int err = 0; + int err; i2c_mark_adapter_suspended(&i2c_dev->adapter); - if (!pm_runtime_status_suspended(dev)) + if (!pm_runtime_status_suspended(dev)) { err = tegra_i2c_runtime_suspend(dev); + if (err) + return err; + } - return err; + return 0; } static int __maybe_unused tegra_i2c_resume(struct device *dev) @@ -1907,7 +1876,7 @@ static int __maybe_unused tegra_i2c_resume(struct device *dev) if (err) return err; - err = tegra_i2c_init(i2c_dev, false); + err = tegra_i2c_init(i2c_dev); if (err) return err; @@ -1934,17 +1903,16 @@ static const struct dev_pm_ops tegra_i2c_pm = { }; static struct platform_driver tegra_i2c_driver = { - .probe = tegra_i2c_probe, - .remove = tegra_i2c_remove, - .driver = { - .name = "tegra-i2c", + .probe = tegra_i2c_probe, + .remove = tegra_i2c_remove, + .driver = { + .name = "tegra-i2c", .of_match_table = tegra_i2c_of_match, - .pm = &tegra_i2c_pm, + .pm = &tegra_i2c_pm, }, }; - module_platform_driver(tegra_i2c_driver); -MODULE_DESCRIPTION("nVidia Tegra2 I2C Bus Controller driver"); +MODULE_DESCRIPTION("NVIDIA Tegra I2C Bus Controller driver"); MODULE_AUTHOR("Colin Cross"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/i2c/busses/i2c-xiic.c b/drivers/i2c/busses/i2c-xiic.c index 90c1c362394d..087b2951942e 100644 --- a/drivers/i2c/busses/i2c-xiic.c +++ b/drivers/i2c/busses/i2c-xiic.c @@ -46,34 +46,36 @@ enum xiic_endian { /** * struct xiic_i2c - Internal representation of the XIIC I2C bus - * @dev: Pointer to device structure - * @base: Memory base of the HW registers - * @wait: Wait queue for callers - * @adap: Kernel adapter representation - * @tx_msg: Messages from above to be sent - * @lock: Mutual exclusion - * @tx_pos: Current pos in TX message - * @nmsgs: Number of messages in tx_msg - * @state: See STATE_ - * @rx_msg: Current RX message - * @rx_pos: Position within current RX message + * @dev: Pointer to device structure + * @base: Memory base of the HW registers + * @wait: Wait queue for callers + * @adap: Kernel adapter representation + * @tx_msg: Messages from above to be sent + * @lock: Mutual exclusion + * @tx_pos: Current pos in TX message + * @nmsgs: Number of messages in tx_msg + * @rx_msg: Current RX message + * @rx_pos: Position within current RX message * @endianness: big/little-endian byte order - * @clk: Pointer to AXI4-lite input clock + * @clk: Pointer to AXI4-lite input clock + * @state: See STATE_ + * @singlemaster: Indicates bus is single master */ struct xiic_i2c { - struct device *dev; - void __iomem *base; - wait_queue_head_t wait; - struct i2c_adapter adap; - struct i2c_msg *tx_msg; - struct mutex lock; - unsigned int tx_pos; - unsigned int nmsgs; - enum xilinx_i2c_state state; - struct i2c_msg *rx_msg; - int rx_pos; - enum xiic_endian endianness; + struct device *dev; + void __iomem *base; + wait_queue_head_t wait; + struct i2c_adapter adap; + struct i2c_msg *tx_msg; + struct mutex lock; + unsigned int tx_pos; + unsigned int nmsgs; + struct i2c_msg *rx_msg; + int rx_pos; + enum xiic_endian endianness; struct clk *clk; + enum xilinx_i2c_state state; + bool singlemaster; }; @@ -526,6 +528,15 @@ static int xiic_busy(struct xiic_i2c *i2c) if (i2c->tx_msg) return -EBUSY; + /* In single master mode bus can only be busy, when in use by this + * driver. If the register indicates bus being busy for some reason we + * should ignore it, since bus will never be released and i2c will be + * stuck forever. + */ + if (i2c->singlemaster) { + return 0; + } + /* for instance if previous transfer was terminated due to TX error * it might be that the bus is on it's way to become available * give it at most 3 ms to wake @@ -811,6 +822,9 @@ static int xiic_i2c_probe(struct platform_device *pdev) goto err_clk_dis; } + i2c->singlemaster = + of_property_read_bool(pdev->dev.of_node, "single-master"); + /* * Detect endianness * Try to reset the TX FIFO. Then check the EMPTY flag. If it is not diff --git a/drivers/i2c/i2c-slave-testunit.c b/drivers/i2c/i2c-slave-testunit.c new file mode 100644 index 000000000000..c288102de324 --- /dev/null +++ b/drivers/i2c/i2c-slave-testunit.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * I2C slave mode testunit + * + * Copyright (C) 2020 by Wolfram Sang, Sang Engineering <wsa@sang-engineering.com> + * Copyright (C) 2020 by Renesas Electronics Corporation + */ + +#include <linux/bitops.h> +#include <linux/i2c.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/slab.h> +#include <linux/workqueue.h> /* FIXME: is system_long_wq the best choice? */ + +#define TU_CUR_VERSION 0x01 + +enum testunit_cmds { + TU_CMD_READ_BYTES = 1, /* save 0 for ABORT, RESET or similar */ + TU_CMD_HOST_NOTIFY, + TU_NUM_CMDS +}; + +enum testunit_regs { + TU_REG_CMD, + TU_REG_DATAL, + TU_REG_DATAH, + TU_REG_DELAY, + TU_NUM_REGS +}; + +enum testunit_flags { + TU_FLAG_IN_PROCESS, +}; + +struct testunit_data { + unsigned long flags; + u8 regs[TU_NUM_REGS]; + u8 reg_idx; + struct i2c_client *client; + struct delayed_work worker; +}; + +static void i2c_slave_testunit_work(struct work_struct *work) +{ + struct testunit_data *tu = container_of(work, struct testunit_data, worker.work); + struct i2c_msg msg; + u8 msgbuf[256]; + int ret = 0; + + msg.addr = I2C_CLIENT_END; + msg.buf = msgbuf; + + switch (tu->regs[TU_REG_CMD]) { + case TU_CMD_READ_BYTES: + msg.addr = tu->regs[TU_REG_DATAL]; + msg.flags = I2C_M_RD; + msg.len = tu->regs[TU_REG_DATAH]; + break; + + case TU_CMD_HOST_NOTIFY: + msg.addr = 0x08; + msg.flags = 0; + msg.len = 3; + msgbuf[0] = tu->client->addr; + msgbuf[1] = tu->regs[TU_REG_DATAL]; + msgbuf[2] = tu->regs[TU_REG_DATAH]; + break; + + default: + break; + } + + if (msg.addr != I2C_CLIENT_END) { + ret = i2c_transfer(tu->client->adapter, &msg, 1); + /* convert '0 msgs transferred' to errno */ + ret = (ret == 0) ? -EIO : ret; + } + + if (ret < 0) + dev_err(&tu->client->dev, "CMD%02X failed (%d)\n", tu->regs[TU_REG_CMD], ret); + + clear_bit(TU_FLAG_IN_PROCESS, &tu->flags); +} + +static int i2c_slave_testunit_slave_cb(struct i2c_client *client, + enum i2c_slave_event event, u8 *val) +{ + struct testunit_data *tu = i2c_get_clientdata(client); + int ret = 0; + + switch (event) { + case I2C_SLAVE_WRITE_RECEIVED: + if (test_bit(TU_FLAG_IN_PROCESS, &tu->flags)) + return -EBUSY; + + if (tu->reg_idx < TU_NUM_REGS) + tu->regs[tu->reg_idx] = *val; + else + ret = -EMSGSIZE; + + if (tu->reg_idx <= TU_NUM_REGS) + tu->reg_idx++; + + /* TU_REG_CMD always written at this point */ + if (tu->regs[TU_REG_CMD] >= TU_NUM_CMDS) + ret = -EINVAL; + + break; + + case I2C_SLAVE_STOP: + if (tu->reg_idx == TU_NUM_REGS) { + set_bit(TU_FLAG_IN_PROCESS, &tu->flags); + queue_delayed_work(system_long_wq, &tu->worker, + msecs_to_jiffies(10 * tu->regs[TU_REG_DELAY])); + } + fallthrough; + + case I2C_SLAVE_WRITE_REQUESTED: + tu->reg_idx = 0; + break; + + case I2C_SLAVE_READ_REQUESTED: + case I2C_SLAVE_READ_PROCESSED: + *val = TU_CUR_VERSION; + break; + } + + return ret; +} + +static int i2c_slave_testunit_probe(struct i2c_client *client) +{ + struct testunit_data *tu; + + tu = devm_kzalloc(&client->dev, sizeof(struct testunit_data), GFP_KERNEL); + if (!tu) + return -ENOMEM; + + tu->client = client; + i2c_set_clientdata(client, tu); + INIT_DELAYED_WORK(&tu->worker, i2c_slave_testunit_work); + + return i2c_slave_register(client, i2c_slave_testunit_slave_cb); +}; + +static int i2c_slave_testunit_remove(struct i2c_client *client) +{ + struct testunit_data *tu = i2c_get_clientdata(client); + + cancel_delayed_work_sync(&tu->worker); + i2c_slave_unregister(client); + return 0; +} + +static const struct i2c_device_id i2c_slave_testunit_id[] = { + { "slave-testunit", 0 }, + { } +}; +MODULE_DEVICE_TABLE(i2c, i2c_slave_testunit_id); + +static struct i2c_driver i2c_slave_testunit_driver = { + .driver = { + .name = "i2c-slave-testunit", + }, + .probe_new = i2c_slave_testunit_probe, + .remove = i2c_slave_testunit_remove, + .id_table = i2c_slave_testunit_id, +}; +module_i2c_driver(i2c_slave_testunit_driver); + +MODULE_AUTHOR("Wolfram Sang <wsa@sang-engineering.com>"); +MODULE_DESCRIPTION("I2C slave mode test unit"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/i2c/i2c-smbus.c b/drivers/i2c/i2c-smbus.c index dc0108287ccf..d3d06e3b4f3b 100644 --- a/drivers/i2c/i2c-smbus.c +++ b/drivers/i2c/i2c-smbus.c @@ -197,6 +197,113 @@ EXPORT_SYMBOL_GPL(i2c_handle_smbus_alert); module_i2c_driver(smbalert_driver); +#if IS_ENABLED(CONFIG_I2C_SLAVE) +#define SMBUS_HOST_NOTIFY_LEN 3 +struct i2c_slave_host_notify_status { + u8 index; + u8 addr; +}; + +static int i2c_slave_host_notify_cb(struct i2c_client *client, + enum i2c_slave_event event, u8 *val) +{ + struct i2c_slave_host_notify_status *status = client->dev.platform_data; + + switch (event) { + case I2C_SLAVE_WRITE_RECEIVED: + /* We only retrieve the first byte received (addr) + * since there is currently no support to retrieve the data + * parameter from the client. + */ + if (status->index == 0) + status->addr = *val; + if (status->index < U8_MAX) + status->index++; + break; + case I2C_SLAVE_STOP: + if (status->index == SMBUS_HOST_NOTIFY_LEN) + i2c_handle_smbus_host_notify(client->adapter, + status->addr); + fallthrough; + case I2C_SLAVE_WRITE_REQUESTED: + status->index = 0; + break; + case I2C_SLAVE_READ_REQUESTED: + case I2C_SLAVE_READ_PROCESSED: + *val = 0xff; + break; + } + + return 0; +} + +/** + * i2c_new_slave_host_notify_device - get a client for SMBus host-notify support + * @adapter: the target adapter + * Context: can sleep + * + * Setup handling of the SMBus host-notify protocol on a given I2C bus segment. + * + * Handling is done by creating a device and its callback and handling data + * received via the SMBus host-notify address (0x8) + * + * This returns the client, which should be ultimately freed using + * i2c_free_slave_host_notify_device(); or an ERRPTR to indicate an error. + */ +struct i2c_client *i2c_new_slave_host_notify_device(struct i2c_adapter *adapter) +{ + struct i2c_board_info host_notify_board_info = { + I2C_BOARD_INFO("smbus_host_notify", 0x08), + .flags = I2C_CLIENT_SLAVE, + }; + struct i2c_slave_host_notify_status *status; + struct i2c_client *client; + int ret; + + status = kzalloc(sizeof(struct i2c_slave_host_notify_status), + GFP_KERNEL); + if (!status) + return ERR_PTR(-ENOMEM); + + host_notify_board_info.platform_data = status; + + client = i2c_new_client_device(adapter, &host_notify_board_info); + if (IS_ERR(client)) { + kfree(status); + return client; + } + + ret = i2c_slave_register(client, i2c_slave_host_notify_cb); + if (ret) { + i2c_unregister_device(client); + kfree(status); + return ERR_PTR(ret); + } + + return client; +} +EXPORT_SYMBOL_GPL(i2c_new_slave_host_notify_device); + +/** + * i2c_free_slave_host_notify_device - free the client for SMBus host-notify + * support + * @client: the client to free + * Context: can sleep + * + * Free the i2c_client allocated via i2c_new_slave_host_notify_device + */ +void i2c_free_slave_host_notify_device(struct i2c_client *client) +{ + if (IS_ERR_OR_NULL(client)) + return; + + i2c_slave_unregister(client); + kfree(client->dev.platform_data); + i2c_unregister_device(client); +} +EXPORT_SYMBOL_GPL(i2c_free_slave_host_notify_device); +#endif + /* * SPD is not part of SMBus but we include it here for convenience as the * target systems are the same. diff --git a/drivers/i2c/muxes/i2c-mux-gpmux.c b/drivers/i2c/muxes/i2c-mux-gpmux.c index f830535cff12..d3acd8d66c32 100644 --- a/drivers/i2c/muxes/i2c-mux-gpmux.c +++ b/drivers/i2c/muxes/i2c-mux-gpmux.c @@ -85,18 +85,14 @@ static int i2c_mux_probe(struct platform_device *pdev) return -ENOMEM; mux->control = devm_mux_control_get(dev, NULL); - if (IS_ERR(mux->control)) { - if (PTR_ERR(mux->control) != -EPROBE_DEFER) - dev_err(dev, "failed to get control-mux\n"); - return PTR_ERR(mux->control); - } + if (IS_ERR(mux->control)) + return dev_err_probe(dev, PTR_ERR(mux->control), + "failed to get control-mux\n"); parent = mux_parent_adapter(dev); - if (IS_ERR(parent)) { - if (PTR_ERR(parent) != -EPROBE_DEFER) - dev_err(dev, "failed to get i2c-parent adapter\n"); - return PTR_ERR(parent); - } + if (IS_ERR(parent)) + return dev_err_probe(dev, PTR_ERR(parent), + "failed to get i2c-parent adapter\n"); children = of_get_child_count(np); diff --git a/drivers/i2c/muxes/i2c-mux-reg.c b/drivers/i2c/muxes/i2c-mux-reg.c index b59a62f8d7a6..0e0679f65cf7 100644 --- a/drivers/i2c/muxes/i2c-mux-reg.c +++ b/drivers/i2c/muxes/i2c-mux-reg.c @@ -171,13 +171,9 @@ static int i2c_mux_reg_probe(struct platform_device *pdev) sizeof(mux->data)); } else { ret = i2c_mux_reg_probe_dt(mux, pdev); - if (ret == -EPROBE_DEFER) - return ret; - - if (ret < 0) { - dev_err(&pdev->dev, "Error parsing device tree"); - return ret; - } + if (ret < 0) + return dev_err_probe(&pdev->dev, ret, + "Error parsing device tree"); } parent = i2c_get_adapter(mux->data.parent); diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c index 97f2e29265da..1c6b78ad5ade 100644 --- a/drivers/i3c/master.c +++ b/drivers/i3c/master.c @@ -1373,7 +1373,9 @@ static int i3c_master_reattach_i3c_dev(struct i3c_dev_desc *dev, enum i3c_addr_slot_status status; int ret; - if (dev->info.dyn_addr != old_dyn_addr) { + if (dev->info.dyn_addr != old_dyn_addr && + (!dev->boardinfo || + dev->info.dyn_addr != dev->boardinfo->init_dyn_addr)) { status = i3c_bus_get_addr_slot_status(&master->bus, dev->info.dyn_addr); if (status != I3C_ADDR_SLOT_FREE) @@ -1432,33 +1434,49 @@ static void i3c_master_detach_i2c_dev(struct i2c_dev_desc *dev) master->ops->detach_i2c_dev(dev); } -static void i3c_master_pre_assign_dyn_addr(struct i3c_dev_desc *dev) +static int i3c_master_early_i3c_dev_add(struct i3c_master_controller *master, + struct i3c_dev_boardinfo *boardinfo) { - struct i3c_master_controller *master = i3c_dev_get_master(dev); + struct i3c_device_info info = { + .static_addr = boardinfo->static_addr, + }; + struct i3c_dev_desc *i3cdev; int ret; - if (!dev->boardinfo || !dev->boardinfo->init_dyn_addr || - !dev->boardinfo->static_addr) - return; + i3cdev = i3c_master_alloc_i3c_dev(master, &info); + if (IS_ERR(i3cdev)) + return -ENOMEM; + + i3cdev->boardinfo = boardinfo; + + ret = i3c_master_attach_i3c_dev(master, i3cdev); + if (ret) + goto err_free_dev; - ret = i3c_master_setdasa_locked(master, dev->info.static_addr, - dev->boardinfo->init_dyn_addr); + ret = i3c_master_setdasa_locked(master, i3cdev->info.static_addr, + i3cdev->boardinfo->init_dyn_addr); if (ret) - return; + goto err_detach_dev; - dev->info.dyn_addr = dev->boardinfo->init_dyn_addr; - ret = i3c_master_reattach_i3c_dev(dev, 0); + i3cdev->info.dyn_addr = i3cdev->boardinfo->init_dyn_addr; + ret = i3c_master_reattach_i3c_dev(i3cdev, 0); if (ret) goto err_rstdaa; - ret = i3c_master_retrieve_dev_info(dev); + ret = i3c_master_retrieve_dev_info(i3cdev); if (ret) goto err_rstdaa; - return; + return 0; err_rstdaa: - i3c_master_rstdaa_locked(master, dev->boardinfo->init_dyn_addr); + i3c_master_rstdaa_locked(master, i3cdev->boardinfo->init_dyn_addr); +err_detach_dev: + i3c_master_detach_i3c_dev(i3cdev); +err_free_dev: + i3c_master_free_i3c_dev(i3cdev); + + return ret; } static void @@ -1625,8 +1643,8 @@ static void i3c_master_detach_free_devs(struct i3c_master_controller *master) * This function is following all initialisation steps described in the I3C * specification: * - * 1. Attach I2C and statically defined I3C devs to the master so that the - * master can fill its internal device table appropriately + * 1. Attach I2C devs to the master so that the master can fill its internal + * device table appropriately * * 2. Call &i3c_master_controller_ops->bus_init() method to initialize * the master controller. That's usually where the bus mode is selected @@ -1638,8 +1656,10 @@ static void i3c_master_detach_free_devs(struct i3c_master_controller *master) * * 4. Disable all slave events. * - * 5. Pre-assign dynamic addresses requested by the FW with SETDASA for I3C - * devices that have a static address + * 5. Reserve address slots for I3C devices with init_dyn_addr. And if devices + * also have static_addr, try to pre-assign dynamic addresses requested by + * the FW with SETDASA and attach corresponding statically defined I3C + * devices to the master. * * 6. Do a DAA (Dynamic Address Assignment) to assign dynamic addresses to all * remaining I3C devices @@ -1653,7 +1673,6 @@ static int i3c_master_bus_init(struct i3c_master_controller *master) enum i3c_addr_slot_status status; struct i2c_dev_boardinfo *i2cboardinfo; struct i3c_dev_boardinfo *i3cboardinfo; - struct i3c_dev_desc *i3cdev; struct i2c_dev_desc *i2cdev; int ret; @@ -1685,34 +1704,6 @@ static int i3c_master_bus_init(struct i3c_master_controller *master) goto err_detach_devs; } } - list_for_each_entry(i3cboardinfo, &master->boardinfo.i3c, node) { - struct i3c_device_info info = { - .static_addr = i3cboardinfo->static_addr, - }; - - if (i3cboardinfo->init_dyn_addr) { - status = i3c_bus_get_addr_slot_status(&master->bus, - i3cboardinfo->init_dyn_addr); - if (status != I3C_ADDR_SLOT_FREE) { - ret = -EBUSY; - goto err_detach_devs; - } - } - - i3cdev = i3c_master_alloc_i3c_dev(master, &info); - if (IS_ERR(i3cdev)) { - ret = PTR_ERR(i3cdev); - goto err_detach_devs; - } - - i3cdev->boardinfo = i3cboardinfo; - - ret = i3c_master_attach_i3c_dev(master, i3cdev); - if (ret) { - i3c_master_free_i3c_dev(i3cdev); - goto err_detach_devs; - } - } /* * Now execute the controller specific ->bus_init() routine, which @@ -1749,11 +1740,43 @@ static int i3c_master_bus_init(struct i3c_master_controller *master) goto err_bus_cleanup; /* - * Pre-assign dynamic address and retrieve device information if - * needed. + * Reserve init_dyn_addr first, and then try to pre-assign dynamic + * address and retrieve device information if needed. + * In case pre-assign dynamic address fails, setting dynamic address to + * the requested init_dyn_addr is retried after DAA is done in + * i3c_master_add_i3c_dev_locked(). */ - i3c_bus_for_each_i3cdev(&master->bus, i3cdev) - i3c_master_pre_assign_dyn_addr(i3cdev); + list_for_each_entry(i3cboardinfo, &master->boardinfo.i3c, node) { + + /* + * We don't reserve a dynamic address for devices that + * don't explicitly request one. + */ + if (!i3cboardinfo->init_dyn_addr) + continue; + + ret = i3c_bus_get_addr_slot_status(&master->bus, + i3cboardinfo->init_dyn_addr); + if (ret != I3C_ADDR_SLOT_FREE) { + ret = -EBUSY; + goto err_rstdaa; + } + + i3c_bus_set_addr_slot_status(&master->bus, + i3cboardinfo->init_dyn_addr, + I3C_ADDR_SLOT_I3C_DEV); + + /* + * Only try to create/attach devices that have a static + * address. Other devices will be created/attached when + * DAA happens, and the requested dynamic address will + * be set using SETNEWDA once those devices become + * addressable. + */ + + if (i3cboardinfo->static_addr) + i3c_master_early_i3c_dev_add(master, i3cboardinfo); + } ret = i3c_master_do_daa(master); if (ret) @@ -1782,6 +1805,21 @@ static void i3c_master_bus_cleanup(struct i3c_master_controller *master) i3c_master_detach_free_devs(master); } +static void i3c_master_attach_boardinfo(struct i3c_dev_desc *i3cdev) +{ + struct i3c_master_controller *master = i3cdev->common.master; + struct i3c_dev_boardinfo *i3cboardinfo; + + list_for_each_entry(i3cboardinfo, &master->boardinfo.i3c, node) { + if (i3cdev->info.pid != i3cboardinfo->pid) + continue; + + i3cdev->boardinfo = i3cboardinfo; + i3cdev->info.static_addr = i3cboardinfo->static_addr; + return; + } +} + static struct i3c_dev_desc * i3c_master_search_i3c_dev_duplicate(struct i3c_dev_desc *refdev) { @@ -1837,10 +1875,10 @@ int i3c_master_add_i3c_dev_locked(struct i3c_master_controller *master, if (ret) goto err_detach_dev; + i3c_master_attach_boardinfo(newdev); + olddev = i3c_master_search_i3c_dev_duplicate(newdev); if (olddev) { - newdev->boardinfo = olddev->boardinfo; - newdev->info.static_addr = olddev->info.static_addr; newdev->dev = olddev->dev; if (newdev->dev) newdev->dev->desc = newdev; diff --git a/drivers/i3c/master/i3c-master-cdns.c b/drivers/i3c/master/i3c-master-cdns.c index 3fee8bd7fe20..3f2226928fe0 100644 --- a/drivers/i3c/master/i3c-master-cdns.c +++ b/drivers/i3c/master/i3c-master-cdns.c @@ -1635,8 +1635,10 @@ static int cdns_i3c_master_probe(struct platform_device *pdev) master->ibi.slots = devm_kcalloc(&pdev->dev, master->ibi.num_slots, sizeof(*master->ibi.slots), GFP_KERNEL); - if (!master->ibi.slots) + if (!master->ibi.slots) { + ret = -ENOMEM; goto err_disable_sysclk; + } writel(IBIR_THR(1), master->regs + CMD_IBI_THR_CTRL); writel(MST_INT_IBIR_THR, master->regs + MST_IER); diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index 9a810e4a7946..56f5b8077cba 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -8,7 +8,7 @@ */ /* - * intel_idle is a cpuidle driver that loads on specific Intel processors + * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT * in lieu of the legacy ACPI processor_idle driver. The intent is to * make Linux more efficient on these processors, as intel_idle knows * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. @@ -20,7 +20,11 @@ * All CPUs have same idle states as boot CPU * * Chipset BM_STS (bus master status) bit is a NOP - * for preventing entry into deep C-stats + * for preventing entry into deep C-states + * + * CPU will flush caches as needed when entering a C-state via MWAIT + * (in contrast to entering ACPI C3, in which case the WBINVD + * instruction needs to be executed to flush the caches) */ /* @@ -1212,14 +1216,13 @@ static bool __init intel_idle_acpi_cst_extract(void) if (!intel_idle_cst_usable()) continue; - if (!acpi_processor_claim_cst_control()) { - acpi_state_table.count = 0; - return false; - } + if (!acpi_processor_claim_cst_control()) + break; return true; } + acpi_state_table.count = 0; pr_debug("ACPI _CST not found or not usable\n"); return false; } diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c index 57038ca48d93..dee1191de752 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c @@ -73,7 +73,7 @@ static int cros_ec_sensors_read(struct iio_dev *indio_dev, st->core.param.sensor_offset.flags = 0; ret = cros_ec_motion_send_host_cmd(&st->core, 0); - if (ret == -EPROTO) { + if (ret == -EPROTO || ret == -EOPNOTSUPP) { /* Reading calibscale is not supported on older EC. */ *val = 1; *val2 = 0; diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig index 91b023341b77..32a51432ec4f 100644 --- a/drivers/infiniband/Kconfig +++ b/drivers/infiniband/Kconfig @@ -48,6 +48,7 @@ config INFINIBAND_ON_DEMAND_PAGING depends on INFINIBAND_USER_MEM select MMU_NOTIFIER select INTERVAL_TREE + select HMM_MIRROR default y help On demand paging support for the InfiniBand subsystem. diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile index 24cb71a16a28..ccf2670ef45e 100644 --- a/drivers/infiniband/core/Makefile +++ b/drivers/infiniband/core/Makefile @@ -17,7 +17,7 @@ ib_core-y := packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \ ib_core-$(CONFIG_SECURITY_INFINIBAND) += security.o ib_core-$(CONFIG_CGROUP_RDMA) += cgroup.o -ib_cm-y := cm.o +ib_cm-y := cm.o cm_trace.o iw_cm-y := iwcm.o iwpm_util.o iwpm_msg.o diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index 3a98439bba83..0abce004a959 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -647,13 +647,12 @@ static void process_one_req(struct work_struct *_work) req->callback = NULL; spin_lock_bh(&lock); + /* + * Although the work will normally have been canceled by the workqueue, + * it can still be requeued as long as it is on the req_list. + */ + cancel_delayed_work(&req->work); if (!list_empty(&req->list)) { - /* - * Although the work will normally have been canceled by the - * workqueue, it can still be requeued as long as it is on the - * req_list. - */ - cancel_delayed_work(&req->work); list_del_init(&req->list); kfree(req); } diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index 5a76611e684a..8017c40dd110 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -133,7 +133,11 @@ static void dispatch_gid_change_event(struct ib_device *ib_dev, u8 port) } static const char * const gid_type_str[] = { + /* IB/RoCE v1 value is set for IB_GID_TYPE_IB and IB_GID_TYPE_ROCE for + * user space compatibility reasons. + */ [IB_GID_TYPE_IB] = "IB/RoCE v1", + [IB_GID_TYPE_ROCE] = "IB/RoCE v1", [IB_GID_TYPE_ROCE_UDP_ENCAP] = "RoCE v2", }; @@ -1220,7 +1224,7 @@ EXPORT_SYMBOL(ib_get_cached_port_state); const struct ib_gid_attr * rdma_get_gid_attr(struct ib_device *device, u8 port_num, int index) { - const struct ib_gid_attr *attr = ERR_PTR(-EINVAL); + const struct ib_gid_attr *attr = ERR_PTR(-ENODATA); struct ib_gid_table *table; unsigned long flags; @@ -1244,6 +1248,67 @@ done: EXPORT_SYMBOL(rdma_get_gid_attr); /** + * rdma_query_gid_table - Reads GID table entries of all the ports of a device up to max_entries. + * @device: The device to query. + * @entries: Entries where GID entries are returned. + * @max_entries: Maximum number of entries that can be returned. + * Entries array must be allocated to hold max_entries number of entries. + * @num_entries: Updated to the number of entries that were successfully read. + * + * Returns number of entries on success or appropriate error code. + */ +ssize_t rdma_query_gid_table(struct ib_device *device, + struct ib_uverbs_gid_entry *entries, + size_t max_entries) +{ + const struct ib_gid_attr *gid_attr; + ssize_t num_entries = 0, ret; + struct ib_gid_table *table; + unsigned int port_num, i; + struct net_device *ndev; + unsigned long flags; + + rdma_for_each_port(device, port_num) { + if (!rdma_ib_or_roce(device, port_num)) + continue; + + table = rdma_gid_table(device, port_num); + read_lock_irqsave(&table->rwlock, flags); + for (i = 0; i < table->sz; i++) { + if (!is_gid_entry_valid(table->data_vec[i])) + continue; + if (num_entries >= max_entries) { + ret = -EINVAL; + goto err; + } + + gid_attr = &table->data_vec[i]->attr; + + memcpy(&entries->gid, &gid_attr->gid, + sizeof(gid_attr->gid)); + entries->gid_index = gid_attr->index; + entries->port_num = gid_attr->port_num; + entries->gid_type = gid_attr->gid_type; + ndev = rcu_dereference_protected( + gid_attr->ndev, + lockdep_is_held(&table->rwlock)); + if (ndev) + entries->netdev_ifindex = ndev->ifindex; + + num_entries++; + entries++; + } + read_unlock_irqrestore(&table->rwlock, flags); + } + + return num_entries; +err: + read_unlock_irqrestore(&table->rwlock, flags); + return ret; +} +EXPORT_SYMBOL(rdma_query_gid_table); + +/** * rdma_put_gid_attr - Release reference to the GID attribute * @attr: Pointer to the GID attribute whose reference * needs to be released. @@ -1299,7 +1364,7 @@ struct net_device *rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr) struct ib_gid_table_entry *entry = container_of(attr, struct ib_gid_table_entry, attr); struct ib_device *device = entry->attr.device; - struct net_device *ndev = ERR_PTR(-ENODEV); + struct net_device *ndev = ERR_PTR(-EINVAL); u8 port_num = entry->attr.port_num; struct ib_gid_table *table; unsigned long flags; @@ -1311,8 +1376,7 @@ struct net_device *rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr) valid = is_gid_entry_valid(table->data_vec[attr->index]); if (valid) { ndev = rcu_dereference(attr->ndev); - if (!ndev || - (ndev && ((READ_ONCE(ndev->flags) & IFF_UP) == 0))) + if (!ndev) ndev = ERR_PTR(-ENODEV); } read_unlock_irqrestore(&table->rwlock, flags); diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index fbc28f1a8b92..5740d1ba3568 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -27,6 +27,7 @@ #include <rdma/ib_cm.h> #include "cm_msgs.h" #include "core_priv.h" +#include "cm_trace.h" MODULE_AUTHOR("Sean Hefty"); MODULE_DESCRIPTION("InfiniBand CM"); @@ -201,7 +202,6 @@ static struct attribute *cm_counter_default_attrs[] = { struct cm_port { struct cm_device *cm_dev; struct ib_mad_agent *mad_agent; - struct kobject port_obj; u8 port_num; struct list_head cm_priv_prim_list; struct list_head cm_priv_altr_list; @@ -1563,6 +1563,7 @@ int ib_send_cm_req(struct ib_cm_id *cm_id, cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg)); cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg)); + trace_icm_send_req(&cm_id_priv->id); spin_lock_irqsave(&cm_id_priv->lock, flags); ret = ib_post_send_mad(cm_id_priv->msg, NULL); if (ret) { @@ -1610,6 +1611,9 @@ static int cm_issue_rej(struct cm_port *port, IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length); } + trace_icm_issue_rej( + IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg), + IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg)); ret = ib_post_send_mad(msg, NULL); if (ret) cm_free_msg(msg); @@ -1961,6 +1965,7 @@ static void cm_dup_req_handler(struct cm_work *work, } spin_unlock_irq(&cm_id_priv->lock); + trace_icm_send_dup_req(&cm_id_priv->id); ret = ib_post_send_mad(msg, NULL); if (ret) goto free; @@ -2124,8 +2129,7 @@ static int cm_req_handler(struct cm_work *work) listen_cm_id_priv = cm_match_req(work, cm_id_priv); if (!listen_cm_id_priv) { - pr_debug("%s: local_id %d, no listen_cm_id_priv\n", __func__, - be32_to_cpu(cm_id_priv->id.local_id)); + trace_icm_no_listener_err(&cm_id_priv->id); cm_id_priv->id.state = IB_CM_IDLE; ret = -EINVAL; goto destroy; @@ -2274,8 +2278,7 @@ int ib_send_cm_rep(struct ib_cm_id *cm_id, spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id->state != IB_CM_REQ_RCVD && cm_id->state != IB_CM_MRA_REQ_SENT) { - pr_debug("%s: local_comm_id %d, cm_id->state: %d\n", __func__, - be32_to_cpu(cm_id_priv->id.local_id), cm_id->state); + trace_icm_send_rep_err(cm_id_priv->id.local_id, cm_id->state); ret = -EINVAL; goto out; } @@ -2289,6 +2292,7 @@ int ib_send_cm_rep(struct ib_cm_id *cm_id, msg->timeout_ms = cm_id_priv->timeout_ms; msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT; + trace_icm_send_rep(cm_id); ret = ib_post_send_mad(msg, NULL); if (ret) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); @@ -2348,8 +2352,7 @@ int ib_send_cm_rtu(struct ib_cm_id *cm_id, spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id->state != IB_CM_REP_RCVD && cm_id->state != IB_CM_MRA_REP_SENT) { - pr_debug("%s: local_id %d, cm_id->state %d\n", __func__, - be32_to_cpu(cm_id->local_id), cm_id->state); + trace_icm_send_cm_rtu_err(cm_id); ret = -EINVAL; goto error; } @@ -2361,6 +2364,7 @@ int ib_send_cm_rtu(struct ib_cm_id *cm_id, cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv, private_data, private_data_len); + trace_icm_send_rtu(cm_id); ret = ib_post_send_mad(msg, NULL); if (ret) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); @@ -2442,6 +2446,7 @@ static void cm_dup_rep_handler(struct cm_work *work) goto unlock; spin_unlock_irq(&cm_id_priv->lock); + trace_icm_send_dup_rep(&cm_id_priv->id); ret = ib_post_send_mad(msg, NULL); if (ret) goto free; @@ -2465,7 +2470,7 @@ static int cm_rep_handler(struct cm_work *work) cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0); if (!cm_id_priv) { cm_dup_rep_handler(work); - pr_debug("%s: remote_comm_id %d, no cm_id_priv\n", __func__, + trace_icm_remote_no_priv_err( IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); return -EINVAL; } @@ -2479,11 +2484,10 @@ static int cm_rep_handler(struct cm_work *work) break; default: ret = -EINVAL; - pr_debug( - "%s: cm_id_priv->id.state: %d, local_comm_id %d, remote_comm_id %d\n", - __func__, cm_id_priv->id.state, + trace_icm_rep_unknown_err( IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg), - IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); + IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg), + cm_id_priv->id.state); spin_unlock_irq(&cm_id_priv->lock); goto error; } @@ -2500,7 +2504,7 @@ static int cm_rep_handler(struct cm_work *work) spin_unlock(&cm.lock); spin_unlock_irq(&cm_id_priv->lock); ret = -EINVAL; - pr_debug("%s: Failed to insert remote id %d\n", __func__, + trace_icm_insert_failed_err( IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); goto error; } @@ -2517,9 +2521,8 @@ static int cm_rep_handler(struct cm_work *work) IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP, NULL, 0); ret = -EINVAL; - pr_debug( - "%s: Stale connection. local_comm_id %d, remote_comm_id %d\n", - __func__, IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg), + trace_icm_staleconn_err( + IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg), IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); if (cur_cm_id_priv) { @@ -2646,9 +2649,7 @@ static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv, return -EINVAL; if (cm_id_priv->id.state != IB_CM_ESTABLISHED) { - pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__, - be32_to_cpu(cm_id_priv->id.local_id), - cm_id_priv->id.state); + trace_icm_dreq_skipped(&cm_id_priv->id); return -EINVAL; } @@ -2667,6 +2668,7 @@ static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv, msg->timeout_ms = cm_id_priv->timeout_ms; msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT; + trace_icm_send_dreq(&cm_id_priv->id); ret = ib_post_send_mad(msg, NULL); if (ret) { cm_enter_timewait(cm_id_priv); @@ -2722,10 +2724,7 @@ static int cm_send_drep_locked(struct cm_id_private *cm_id_priv, return -EINVAL; if (cm_id_priv->id.state != IB_CM_DREQ_RCVD) { - pr_debug( - "%s: local_id %d, cm_idcm_id->state(%d) != IB_CM_DREQ_RCVD\n", - __func__, be32_to_cpu(cm_id_priv->id.local_id), - cm_id_priv->id.state); + trace_icm_send_drep_err(&cm_id_priv->id); kfree(private_data); return -EINVAL; } @@ -2740,6 +2739,7 @@ static int cm_send_drep_locked(struct cm_id_private *cm_id_priv, cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv, private_data, private_data_len); + trace_icm_send_drep(&cm_id_priv->id); ret = ib_post_send_mad(msg, NULL); if (ret) { cm_free_msg(msg); @@ -2789,6 +2789,9 @@ static int cm_issue_drep(struct cm_port *port, IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg, IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)); + trace_icm_issue_drep( + IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg), + IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)); ret = ib_post_send_mad(msg, NULL); if (ret) cm_free_msg(msg); @@ -2810,9 +2813,8 @@ static int cm_dreq_handler(struct cm_work *work) atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. counter[CM_DREQ_COUNTER]); cm_issue_drep(work->port, work->mad_recv_wc); - pr_debug( - "%s: no cm_id_priv, local_comm_id %d, remote_comm_id %d\n", - __func__, IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg), + trace_icm_no_priv_err( + IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg), IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)); return -EINVAL; } @@ -2858,9 +2860,7 @@ static int cm_dreq_handler(struct cm_work *work) counter[CM_DREQ_COUNTER]); goto unlock; default: - pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", - __func__, be32_to_cpu(cm_id_priv->id.local_id), - cm_id_priv->id.state); + trace_icm_dreq_unknown_err(&cm_id_priv->id); goto unlock; } cm_id_priv->id.state = IB_CM_DREQ_RCVD; @@ -2945,12 +2945,11 @@ static int cm_send_rej_locked(struct cm_id_private *cm_id_priv, state); break; default: - pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__, - be32_to_cpu(cm_id_priv->id.local_id), - cm_id_priv->id.state); + trace_icm_send_unknown_rej_err(&cm_id_priv->id); return -EINVAL; } + trace_icm_send_rej(&cm_id_priv->id, reason); ret = ib_post_send_mad(msg, NULL); if (ret) { cm_free_msg(msg); @@ -3060,9 +3059,7 @@ static int cm_rej_handler(struct cm_work *work) } fallthrough; default: - pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", - __func__, be32_to_cpu(cm_id_priv->id.local_id), - cm_id_priv->id.state); + trace_icm_rej_unknown_err(&cm_id_priv->id); spin_unlock_irq(&cm_id_priv->lock); goto out; } @@ -3118,9 +3115,7 @@ int ib_send_cm_mra(struct ib_cm_id *cm_id, } fallthrough; default: - pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", - __func__, be32_to_cpu(cm_id_priv->id.local_id), - cm_id_priv->id.state); + trace_icm_send_mra_unknown_err(&cm_id_priv->id); ret = -EINVAL; goto error1; } @@ -3133,6 +3128,7 @@ int ib_send_cm_mra(struct ib_cm_id *cm_id, cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, msg_response, service_timeout, private_data, private_data_len); + trace_icm_send_mra(cm_id); ret = ib_post_send_mad(msg, NULL); if (ret) goto error2; @@ -3229,9 +3225,7 @@ static int cm_mra_handler(struct cm_work *work) counter[CM_MRA_COUNTER]); fallthrough; default: - pr_debug("%s local_id %d, cm_id_priv->id.state: %d\n", - __func__, be32_to_cpu(cm_id_priv->id.local_id), - cm_id_priv->id.state); + trace_icm_mra_unknown_err(&cm_id_priv->id); goto out; } @@ -3505,10 +3499,12 @@ int ib_send_cm_sidr_req(struct ib_cm_id *cm_id, msg->context[1] = (void *) (unsigned long) IB_CM_SIDR_REQ_SENT; spin_lock_irqsave(&cm_id_priv->lock, flags); - if (cm_id->state == IB_CM_IDLE) + if (cm_id->state == IB_CM_IDLE) { + trace_icm_send_sidr_req(&cm_id_priv->id); ret = ib_post_send_mad(msg, NULL); - else + } else { ret = -EINVAL; + } if (ret) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); @@ -3670,6 +3666,7 @@ static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv, cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv, param); + trace_icm_send_sidr_rep(&cm_id_priv->id); ret = ib_post_send_mad(msg, NULL); if (ret) { cm_free_msg(msg); @@ -3767,8 +3764,7 @@ static void cm_process_send_error(struct ib_mad_send_buf *msg, if (msg != cm_id_priv->msg || state != cm_id_priv->id.state) goto discard; - pr_debug_ratelimited("CM: failed sending MAD in state %d. (%s)\n", - state, ib_wc_status_msg(wc_status)); + trace_icm_mad_send_err(state, wc_status); switch (state) { case IB_CM_REQ_SENT: case IB_CM_MRA_REQ_RCVD: @@ -3891,7 +3887,7 @@ static void cm_work_handler(struct work_struct *_work) ret = cm_timewait_handler(work); break; default: - pr_debug("cm_event.event: 0x%x\n", work->cm_event.event); + trace_icm_handler_err(work->cm_event.event); ret = -EINVAL; break; } @@ -3927,8 +3923,7 @@ static int cm_establish(struct ib_cm_id *cm_id) ret = -EISCONN; break; default: - pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__, - be32_to_cpu(cm_id->local_id), cm_id->state); + trace_icm_establish_err(cm_id); ret = -EINVAL; break; } @@ -4125,9 +4120,7 @@ static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv, ret = 0; break; default: - pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", - __func__, be32_to_cpu(cm_id_priv->id.local_id), - cm_id_priv->id.state); + trace_icm_qp_init_err(&cm_id_priv->id); ret = -EINVAL; break; } @@ -4175,9 +4168,7 @@ static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv, ret = 0; break; default: - pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", - __func__, be32_to_cpu(cm_id_priv->id.local_id), - cm_id_priv->id.state); + trace_icm_qp_rtr_err(&cm_id_priv->id); ret = -EINVAL; break; } @@ -4237,9 +4228,7 @@ static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv, ret = 0; break; default: - pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", - __func__, be32_to_cpu(cm_id_priv->id.local_id), - cm_id_priv->id.state); + trace_icm_qp_rts_err(&cm_id_priv->id); ret = -EINVAL; break; } @@ -4295,20 +4284,6 @@ static struct kobj_type cm_counter_obj_type = { .default_attrs = cm_counter_default_attrs }; -static char *cm_devnode(struct device *dev, umode_t *mode) -{ - if (mode) - *mode = 0666; - return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev)); -} - -struct class cm_class = { - .owner = THIS_MODULE, - .name = "infiniband_cm", - .devnode = cm_devnode, -}; -EXPORT_SYMBOL(cm_class); - static int cm_create_port_fs(struct cm_port *port) { int i, ret; @@ -4511,12 +4486,6 @@ static int __init ib_cm_init(void) get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand); INIT_LIST_HEAD(&cm.timewait_list); - ret = class_register(&cm_class); - if (ret) { - ret = -ENOMEM; - goto error1; - } - cm.wq = alloc_workqueue("ib_cm", 0, 1); if (!cm.wq) { ret = -ENOMEM; @@ -4531,8 +4500,6 @@ static int __init ib_cm_init(void) error3: destroy_workqueue(cm.wq); error2: - class_unregister(&cm_class); -error1: return ret; } @@ -4553,7 +4520,6 @@ static void __exit ib_cm_cleanup(void) kfree(timewait_info); } - class_unregister(&cm_class); WARN_ON(!xa_empty(&cm.local_id_table)); } diff --git a/drivers/infiniband/core/cm_trace.c b/drivers/infiniband/core/cm_trace.c new file mode 100644 index 000000000000..8f3482f66338 --- /dev/null +++ b/drivers/infiniband/core/cm_trace.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Trace points for the IB Connection Manager. + * + * Author: Chuck Lever <chuck.lever@oracle.com> + * + * Copyright (c) 2020, Oracle and/or its affiliates. + */ + +#include <rdma/rdma_cm.h> +#include "cma_priv.h" + +#define CREATE_TRACE_POINTS + +#include "cm_trace.h" diff --git a/drivers/infiniband/core/cm_trace.h b/drivers/infiniband/core/cm_trace.h new file mode 100644 index 000000000000..e9d282679ef1 --- /dev/null +++ b/drivers/infiniband/core/cm_trace.h @@ -0,0 +1,414 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Trace point definitions for the RDMA Connect Manager. + * + * Author: Chuck Lever <chuck.lever@oracle.com> + * + * Copyright (c) 2020 Oracle and/or its affiliates. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM ib_cma + +#if !defined(_TRACE_IB_CMA_H) || defined(TRACE_HEADER_MULTI_READ) + +#define _TRACE_IB_CMA_H + +#include <linux/tracepoint.h> +#include <rdma/ib_cm.h> +#include <trace/events/rdma.h> + +/* + * enum ib_cm_state, from include/rdma/ib_cm.h + */ +#define IB_CM_STATE_LIST \ + ib_cm_state(IDLE) \ + ib_cm_state(LISTEN) \ + ib_cm_state(REQ_SENT) \ + ib_cm_state(REQ_RCVD) \ + ib_cm_state(MRA_REQ_SENT) \ + ib_cm_state(MRA_REQ_RCVD) \ + ib_cm_state(REP_SENT) \ + ib_cm_state(REP_RCVD) \ + ib_cm_state(MRA_REP_SENT) \ + ib_cm_state(MRA_REP_RCVD) \ + ib_cm_state(ESTABLISHED) \ + ib_cm_state(DREQ_SENT) \ + ib_cm_state(DREQ_RCVD) \ + ib_cm_state(TIMEWAIT) \ + ib_cm_state(SIDR_REQ_SENT) \ + ib_cm_state_end(SIDR_REQ_RCVD) + +#undef ib_cm_state +#undef ib_cm_state_end +#define ib_cm_state(x) TRACE_DEFINE_ENUM(IB_CM_##x); +#define ib_cm_state_end(x) TRACE_DEFINE_ENUM(IB_CM_##x); + +IB_CM_STATE_LIST + +#undef ib_cm_state +#undef ib_cm_state_end +#define ib_cm_state(x) { IB_CM_##x, #x }, +#define ib_cm_state_end(x) { IB_CM_##x, #x } + +#define show_ib_cm_state(x) \ + __print_symbolic(x, IB_CM_STATE_LIST) + +/* + * enum ib_cm_lap_state, from include/rdma/ib_cm.h + */ +#define IB_CM_LAP_STATE_LIST \ + ib_cm_lap_state(LAP_UNINIT) \ + ib_cm_lap_state(LAP_IDLE) \ + ib_cm_lap_state(LAP_SENT) \ + ib_cm_lap_state(LAP_RCVD) \ + ib_cm_lap_state(MRA_LAP_SENT) \ + ib_cm_lap_state_end(MRA_LAP_RCVD) + +#undef ib_cm_lap_state +#undef ib_cm_lap_state_end +#define ib_cm_lap_state(x) TRACE_DEFINE_ENUM(IB_CM_##x); +#define ib_cm_lap_state_end(x) TRACE_DEFINE_ENUM(IB_CM_##x); + +IB_CM_LAP_STATE_LIST + +#undef ib_cm_lap_state +#undef ib_cm_lap_state_end +#define ib_cm_lap_state(x) { IB_CM_##x, #x }, +#define ib_cm_lap_state_end(x) { IB_CM_##x, #x } + +#define show_ib_cm_lap_state(x) \ + __print_symbolic(x, IB_CM_LAP_STATE_LIST) + +/* + * enum ib_cm_rej_reason, from include/rdma/ib_cm.h + */ +#define IB_CM_REJ_REASON_LIST \ + ib_cm_rej_reason(REJ_NO_QP) \ + ib_cm_rej_reason(REJ_NO_EEC) \ + ib_cm_rej_reason(REJ_NO_RESOURCES) \ + ib_cm_rej_reason(REJ_TIMEOUT) \ + ib_cm_rej_reason(REJ_UNSUPPORTED) \ + ib_cm_rej_reason(REJ_INVALID_COMM_ID) \ + ib_cm_rej_reason(REJ_INVALID_COMM_INSTANCE) \ + ib_cm_rej_reason(REJ_INVALID_SERVICE_ID) \ + ib_cm_rej_reason(REJ_INVALID_TRANSPORT_TYPE) \ + ib_cm_rej_reason(REJ_STALE_CONN) \ + ib_cm_rej_reason(REJ_RDC_NOT_EXIST) \ + ib_cm_rej_reason(REJ_INVALID_GID) \ + ib_cm_rej_reason(REJ_INVALID_LID) \ + ib_cm_rej_reason(REJ_INVALID_SL) \ + ib_cm_rej_reason(REJ_INVALID_TRAFFIC_CLASS) \ + ib_cm_rej_reason(REJ_INVALID_HOP_LIMIT) \ + ib_cm_rej_reason(REJ_INVALID_PACKET_RATE) \ + ib_cm_rej_reason(REJ_INVALID_ALT_GID) \ + ib_cm_rej_reason(REJ_INVALID_ALT_LID) \ + ib_cm_rej_reason(REJ_INVALID_ALT_SL) \ + ib_cm_rej_reason(REJ_INVALID_ALT_TRAFFIC_CLASS) \ + ib_cm_rej_reason(REJ_INVALID_ALT_HOP_LIMIT) \ + ib_cm_rej_reason(REJ_INVALID_ALT_PACKET_RATE) \ + ib_cm_rej_reason(REJ_PORT_CM_REDIRECT) \ + ib_cm_rej_reason(REJ_PORT_REDIRECT) \ + ib_cm_rej_reason(REJ_INVALID_MTU) \ + ib_cm_rej_reason(REJ_INSUFFICIENT_RESP_RESOURCES) \ + ib_cm_rej_reason(REJ_CONSUMER_DEFINED) \ + ib_cm_rej_reason(REJ_INVALID_RNR_RETRY) \ + ib_cm_rej_reason(REJ_DUPLICATE_LOCAL_COMM_ID) \ + ib_cm_rej_reason(REJ_INVALID_CLASS_VERSION) \ + ib_cm_rej_reason(REJ_INVALID_FLOW_LABEL) \ + ib_cm_rej_reason(REJ_INVALID_ALT_FLOW_LABEL) \ + ib_cm_rej_reason_end(REJ_VENDOR_OPTION_NOT_SUPPORTED) + +#undef ib_cm_rej_reason +#undef ib_cm_rej_reason_end +#define ib_cm_rej_reason(x) TRACE_DEFINE_ENUM(IB_CM_##x); +#define ib_cm_rej_reason_end(x) TRACE_DEFINE_ENUM(IB_CM_##x); + +IB_CM_REJ_REASON_LIST + +#undef ib_cm_rej_reason +#undef ib_cm_rej_reason_end +#define ib_cm_rej_reason(x) { IB_CM_##x, #x }, +#define ib_cm_rej_reason_end(x) { IB_CM_##x, #x } + +#define show_ib_cm_rej_reason(x) \ + __print_symbolic(x, IB_CM_REJ_REASON_LIST) + +DECLARE_EVENT_CLASS(icm_id_class, + TP_PROTO( + const struct ib_cm_id *cm_id + ), + + TP_ARGS(cm_id), + + TP_STRUCT__entry( + __field(const void *, cm_id) /* for eBPF scripts */ + __field(unsigned int, local_id) + __field(unsigned int, remote_id) + __field(unsigned long, state) + __field(unsigned long, lap_state) + ), + + TP_fast_assign( + __entry->cm_id = cm_id; + __entry->local_id = be32_to_cpu(cm_id->local_id); + __entry->remote_id = be32_to_cpu(cm_id->remote_id); + __entry->state = cm_id->state; + __entry->lap_state = cm_id->lap_state; + ), + + TP_printk("local_id=%u remote_id=%u state=%s lap_state=%s", + __entry->local_id, __entry->remote_id, + show_ib_cm_state(__entry->state), + show_ib_cm_lap_state(__entry->lap_state) + ) +); + +#define DEFINE_CM_SEND_EVENT(name) \ + DEFINE_EVENT(icm_id_class, \ + icm_send_##name, \ + TP_PROTO( \ + const struct ib_cm_id *cm_id \ + ), \ + TP_ARGS(cm_id)) + +DEFINE_CM_SEND_EVENT(req); +DEFINE_CM_SEND_EVENT(rep); +DEFINE_CM_SEND_EVENT(dup_req); +DEFINE_CM_SEND_EVENT(dup_rep); +DEFINE_CM_SEND_EVENT(rtu); +DEFINE_CM_SEND_EVENT(mra); +DEFINE_CM_SEND_EVENT(sidr_req); +DEFINE_CM_SEND_EVENT(sidr_rep); +DEFINE_CM_SEND_EVENT(dreq); +DEFINE_CM_SEND_EVENT(drep); + +TRACE_EVENT(icm_send_rej, + TP_PROTO( + const struct ib_cm_id *cm_id, + enum ib_cm_rej_reason reason + ), + + TP_ARGS(cm_id, reason), + + TP_STRUCT__entry( + __field(const void *, cm_id) + __field(u32, local_id) + __field(u32, remote_id) + __field(unsigned long, state) + __field(unsigned long, reason) + ), + + TP_fast_assign( + __entry->cm_id = cm_id; + __entry->local_id = be32_to_cpu(cm_id->local_id); + __entry->remote_id = be32_to_cpu(cm_id->remote_id); + __entry->state = cm_id->state; + __entry->reason = reason; + ), + + TP_printk("local_id=%u remote_id=%u state=%s reason=%s", + __entry->local_id, __entry->remote_id, + show_ib_cm_state(__entry->state), + show_ib_cm_rej_reason(__entry->reason) + ) +); + +#define DEFINE_CM_ERR_EVENT(name) \ + DEFINE_EVENT(icm_id_class, \ + icm_##name##_err, \ + TP_PROTO( \ + const struct ib_cm_id *cm_id \ + ), \ + TP_ARGS(cm_id)) + +DEFINE_CM_ERR_EVENT(send_cm_rtu); +DEFINE_CM_ERR_EVENT(establish); +DEFINE_CM_ERR_EVENT(no_listener); +DEFINE_CM_ERR_EVENT(send_drep); +DEFINE_CM_ERR_EVENT(dreq_unknown); +DEFINE_CM_ERR_EVENT(send_unknown_rej); +DEFINE_CM_ERR_EVENT(rej_unknown); +DEFINE_CM_ERR_EVENT(send_mra_unknown); +DEFINE_CM_ERR_EVENT(mra_unknown); +DEFINE_CM_ERR_EVENT(qp_init); +DEFINE_CM_ERR_EVENT(qp_rtr); +DEFINE_CM_ERR_EVENT(qp_rts); + +DEFINE_EVENT(icm_id_class, \ + icm_dreq_skipped, \ + TP_PROTO( \ + const struct ib_cm_id *cm_id \ + ), \ + TP_ARGS(cm_id) \ +); + +DECLARE_EVENT_CLASS(icm_local_class, + TP_PROTO( + unsigned int local_id, + unsigned int remote_id + ), + + TP_ARGS(local_id, remote_id), + + TP_STRUCT__entry( + __field(unsigned int, local_id) + __field(unsigned int, remote_id) + ), + + TP_fast_assign( + __entry->local_id = local_id; + __entry->remote_id = remote_id; + ), + + TP_printk("local_id=%u remote_id=%u", + __entry->local_id, __entry->remote_id + ) +); + +#define DEFINE_CM_LOCAL_EVENT(name) \ + DEFINE_EVENT(icm_local_class, \ + icm_##name, \ + TP_PROTO( \ + unsigned int local_id, \ + unsigned int remote_id \ + ), \ + TP_ARGS(local_id, remote_id)) + +DEFINE_CM_LOCAL_EVENT(issue_rej); +DEFINE_CM_LOCAL_EVENT(issue_drep); +DEFINE_CM_LOCAL_EVENT(staleconn_err); +DEFINE_CM_LOCAL_EVENT(no_priv_err); + +DECLARE_EVENT_CLASS(icm_remote_class, + TP_PROTO( + u32 remote_id + ), + + TP_ARGS(remote_id), + + TP_STRUCT__entry( + __field(u32, remote_id) + ), + + TP_fast_assign( + __entry->remote_id = remote_id; + ), + + TP_printk("remote_id=%u", + __entry->remote_id + ) +); + +#define DEFINE_CM_REMOTE_EVENT(name) \ + DEFINE_EVENT(icm_remote_class, \ + icm_##name, \ + TP_PROTO( \ + u32 remote_id \ + ), \ + TP_ARGS(remote_id)) + +DEFINE_CM_REMOTE_EVENT(remote_no_priv_err); +DEFINE_CM_REMOTE_EVENT(insert_failed_err); + +TRACE_EVENT(icm_send_rep_err, + TP_PROTO( + __be32 local_id, + enum ib_cm_state state + ), + + TP_ARGS(local_id, state), + + TP_STRUCT__entry( + __field(unsigned int, local_id) + __field(unsigned long, state) + ), + + TP_fast_assign( + __entry->local_id = be32_to_cpu(local_id); + __entry->state = state; + ), + + TP_printk("local_id=%u state=%s", + __entry->local_id, show_ib_cm_state(__entry->state) + ) +); + +TRACE_EVENT(icm_rep_unknown_err, + TP_PROTO( + unsigned int local_id, + unsigned int remote_id, + enum ib_cm_state state + ), + + TP_ARGS(local_id, remote_id, state), + + TP_STRUCT__entry( + __field(unsigned int, local_id) + __field(unsigned int, remote_id) + __field(unsigned long, state) + ), + + TP_fast_assign( + __entry->local_id = local_id; + __entry->remote_id = remote_id; + __entry->state = state; + ), + + TP_printk("local_id=%u remote_id=%u state=%s", + __entry->local_id, __entry->remote_id, + show_ib_cm_state(__entry->state) + ) +); + +TRACE_EVENT(icm_handler_err, + TP_PROTO( + enum ib_cm_event_type event + ), + + TP_ARGS(event), + + TP_STRUCT__entry( + __field(unsigned long, event) + ), + + TP_fast_assign( + __entry->event = event; + ), + + TP_printk("unhandled event=%s", + rdma_show_ib_cm_event(__entry->event) + ) +); + +TRACE_EVENT(icm_mad_send_err, + TP_PROTO( + enum ib_cm_state state, + enum ib_wc_status wc_status + ), + + TP_ARGS(state, wc_status), + + TP_STRUCT__entry( + __field(unsigned long, state) + __field(unsigned long, wc_status) + ), + + TP_fast_assign( + __entry->state = state; + __entry->wc_status = wc_status; + ), + + TP_printk("state=%s completion status=%s", + show_ib_cm_state(__entry->state), + rdma_show_wc_status(__entry->wc_status) + ) +); + +#endif /* _TRACE_IB_CMA_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../drivers/infiniband/core +#define TRACE_INCLUDE_FILE cm_trace + +#include <trace/define_trace.h> diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 5888311b2119..7c2ab1f2fbea 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -68,6 +68,9 @@ static const char * const cma_events[] = { [RDMA_CM_EVENT_TIMEWAIT_EXIT] = "timewait exit", }; +static void cma_set_mgid(struct rdma_id_private *id_priv, struct sockaddr *addr, + union ib_gid *mgid); + const char *__attribute_const__ rdma_event_msg(enum rdma_cm_event_type event) { size_t index = event; @@ -301,6 +304,10 @@ int cma_set_default_gid_type(struct cma_device *cma_dev, if (!rdma_is_port_valid(cma_dev->device, port)) return -EINVAL; + if (default_gid_type == IB_GID_TYPE_IB && + rdma_protocol_roce_eth_encap(cma_dev->device, port)) + default_gid_type = IB_GID_TYPE_ROCE; + supported_gids = roce_gid_type_mask_support(cma_dev->device, port); if (!(supported_gids & 1 << default_gid_type)) @@ -345,13 +352,10 @@ struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev) struct cma_multicast { struct rdma_id_private *id_priv; - union { - struct ib_sa_multicast *ib; - } multicast; + struct ib_sa_multicast *sa_mc; struct list_head list; void *context; struct sockaddr_storage addr; - struct kref mcref; u8 join_state; }; @@ -363,18 +367,6 @@ struct cma_work { struct rdma_cm_event event; }; -struct cma_ndev_work { - struct work_struct work; - struct rdma_id_private *id; - struct rdma_cm_event event; -}; - -struct iboe_mcast_work { - struct work_struct work; - struct rdma_id_private *id; - struct cma_multicast *mc; -}; - union cma_ip_addr { struct in6_addr ip6; struct { @@ -404,23 +396,21 @@ struct cma_req_info { u16 pkey; }; -static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp) -{ - unsigned long flags; - int ret; - - spin_lock_irqsave(&id_priv->lock, flags); - ret = (id_priv->state == comp); - spin_unlock_irqrestore(&id_priv->lock, flags); - return ret; -} - static int cma_comp_exch(struct rdma_id_private *id_priv, enum rdma_cm_state comp, enum rdma_cm_state exch) { unsigned long flags; int ret; + /* + * The FSM uses a funny double locking where state is protected by both + * the handler_mutex and the spinlock. State is not allowed to change + * away from a handler_mutex protected value without also holding + * handler_mutex. + */ + if (comp == RDMA_CM_CONNECT) + lockdep_assert_held(&id_priv->handler_mutex); + spin_lock_irqsave(&id_priv->lock, flags); if ((ret = (id_priv->state == comp))) id_priv->state = exch; @@ -467,10 +457,8 @@ static void _cma_attach_to_dev(struct rdma_id_private *id_priv, id_priv->id.route.addr.dev_addr.transport = rdma_node_get_transport(cma_dev->device->node_type); list_add_tail(&id_priv->list, &cma_dev->id_list); - if (id_priv->res.kern_name) - rdma_restrack_kadd(&id_priv->res); - else - rdma_restrack_uadd(&id_priv->res); + rdma_restrack_add(&id_priv->res); + trace_cm_id_attach(id_priv, cma_dev->device); } @@ -483,14 +471,6 @@ static void cma_attach_to_dev(struct rdma_id_private *id_priv, rdma_start_port(cma_dev->device)]; } -static inline void release_mc(struct kref *kref) -{ - struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref); - - kfree(mc->multicast.ib); - kfree(mc); -} - static void cma_release_dev(struct rdma_id_private *id_priv) { mutex_lock(&lock); @@ -844,10 +824,10 @@ static void cma_id_put(struct rdma_id_private *id_priv) complete(&id_priv->comp); } -struct rdma_cm_id *__rdma_create_id(struct net *net, - rdma_cm_event_handler event_handler, - void *context, enum rdma_ucm_port_space ps, - enum ib_qp_type qp_type, const char *caller) +static struct rdma_id_private * +__rdma_create_id(struct net *net, rdma_cm_event_handler event_handler, + void *context, enum rdma_ucm_port_space ps, + enum ib_qp_type qp_type, const struct rdma_id_private *parent) { struct rdma_id_private *id_priv; @@ -855,8 +835,6 @@ struct rdma_cm_id *__rdma_create_id(struct net *net, if (!id_priv) return ERR_PTR(-ENOMEM); - rdma_restrack_set_task(&id_priv->res, caller); - id_priv->res.type = RDMA_RESTRACK_CM_ID; id_priv->state = RDMA_CM_IDLE; id_priv->id.context = context; id_priv->id.event_handler = event_handler; @@ -876,9 +854,45 @@ struct rdma_cm_id *__rdma_create_id(struct net *net, id_priv->id.route.addr.dev_addr.net = get_net(net); id_priv->seq_num &= 0x00ffffff; - return &id_priv->id; + rdma_restrack_new(&id_priv->res, RDMA_RESTRACK_CM_ID); + if (parent) + rdma_restrack_parent_name(&id_priv->res, &parent->res); + + return id_priv; +} + +struct rdma_cm_id * +__rdma_create_kernel_id(struct net *net, rdma_cm_event_handler event_handler, + void *context, enum rdma_ucm_port_space ps, + enum ib_qp_type qp_type, const char *caller) +{ + struct rdma_id_private *ret; + + ret = __rdma_create_id(net, event_handler, context, ps, qp_type, NULL); + if (IS_ERR(ret)) + return ERR_CAST(ret); + + rdma_restrack_set_name(&ret->res, caller); + return &ret->id; } -EXPORT_SYMBOL(__rdma_create_id); +EXPORT_SYMBOL(__rdma_create_kernel_id); + +struct rdma_cm_id *rdma_create_user_id(rdma_cm_event_handler event_handler, + void *context, + enum rdma_ucm_port_space ps, + enum ib_qp_type qp_type) +{ + struct rdma_id_private *ret; + + ret = __rdma_create_id(current->nsproxy->net_ns, event_handler, context, + ps, qp_type, NULL); + if (IS_ERR(ret)) + return ERR_CAST(ret); + + rdma_restrack_set_name(&ret->res, NULL); + return &ret->id; +} +EXPORT_SYMBOL(rdma_create_user_id); static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) { @@ -1783,19 +1797,30 @@ static void cma_release_port(struct rdma_id_private *id_priv) mutex_unlock(&lock); } -static void cma_leave_roce_mc_group(struct rdma_id_private *id_priv, - struct cma_multicast *mc) +static void destroy_mc(struct rdma_id_private *id_priv, + struct cma_multicast *mc) { - struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; - struct net_device *ndev = NULL; + if (rdma_cap_ib_mcast(id_priv->id.device, id_priv->id.port_num)) + ib_sa_free_multicast(mc->sa_mc); - if (dev_addr->bound_dev_if) - ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if); - if (ndev) { - cma_igmp_send(ndev, &mc->multicast.ib->rec.mgid, false); - dev_put(ndev); + if (rdma_protocol_roce(id_priv->id.device, id_priv->id.port_num)) { + struct rdma_dev_addr *dev_addr = + &id_priv->id.route.addr.dev_addr; + struct net_device *ndev = NULL; + + if (dev_addr->bound_dev_if) + ndev = dev_get_by_index(dev_addr->net, + dev_addr->bound_dev_if); + if (ndev) { + union ib_gid mgid; + + cma_set_mgid(id_priv, (struct sockaddr *)&mc->addr, + &mgid); + cma_igmp_send(ndev, &mgid, false); + dev_put(ndev); + } } - kref_put(&mc->mcref, release_mc); + kfree(mc); } static void cma_leave_mc_groups(struct rdma_id_private *id_priv) @@ -1803,16 +1828,10 @@ static void cma_leave_mc_groups(struct rdma_id_private *id_priv) struct cma_multicast *mc; while (!list_empty(&id_priv->mc_list)) { - mc = container_of(id_priv->mc_list.next, - struct cma_multicast, list); + mc = list_first_entry(&id_priv->mc_list, struct cma_multicast, + list); list_del(&mc->list); - if (rdma_cap_ib_mcast(id_priv->cma_dev->device, - id_priv->id.port_num)) { - ib_sa_free_multicast(mc->multicast.ib); - kfree(mc); - } else { - cma_leave_roce_mc_group(id_priv, mc); - } + destroy_mc(id_priv, mc); } } @@ -1821,7 +1840,6 @@ static void _destroy_id(struct rdma_id_private *id_priv, { cma_cancel_operation(id_priv, state); - rdma_restrack_del(&id_priv->res); if (id_priv->cma_dev) { if (rdma_cap_ib_cm(id_priv->id.device, 1)) { if (id_priv->cm_id.ib) @@ -1847,6 +1865,7 @@ static void _destroy_id(struct rdma_id_private *id_priv, rdma_put_gid_attr(id_priv->id.route.addr.dev_addr.sgid_attr); put_net(id_priv->id.route.addr.dev_addr.net); + rdma_restrack_del(&id_priv->res); kfree(id_priv); } @@ -1949,13 +1968,15 @@ static int cma_ib_handler(struct ib_cm_id *cm_id, { struct rdma_id_private *id_priv = cm_id->context; struct rdma_cm_event event = {}; + enum rdma_cm_state state; int ret; mutex_lock(&id_priv->handler_mutex); + state = READ_ONCE(id_priv->state); if ((ib_event->event != IB_CM_TIMEWAIT_EXIT && - id_priv->state != RDMA_CM_CONNECT) || + state != RDMA_CM_CONNECT) || (ib_event->event == IB_CM_TIMEWAIT_EXIT && - id_priv->state != RDMA_CM_DISCONNECT)) + state != RDMA_CM_DISCONNECT)) goto out; switch (ib_event->event) { @@ -1965,7 +1986,7 @@ static int cma_ib_handler(struct ib_cm_id *cm_id, event.status = -ETIMEDOUT; break; case IB_CM_REP_RECEIVED: - if (cma_comp(id_priv, RDMA_CM_CONNECT) && + if (state == RDMA_CM_CONNECT && (id_priv->id.qp_type != IB_QPT_UD)) { trace_cm_send_mra(id_priv); ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); @@ -2043,14 +2064,15 @@ cma_ib_new_conn_id(const struct rdma_cm_id *listen_id, int ret; listen_id_priv = container_of(listen_id, struct rdma_id_private, id); - id = __rdma_create_id(listen_id->route.addr.dev_addr.net, - listen_id->event_handler, listen_id->context, - listen_id->ps, ib_event->param.req_rcvd.qp_type, - listen_id_priv->res.kern_name); - if (IS_ERR(id)) + id_priv = __rdma_create_id(listen_id->route.addr.dev_addr.net, + listen_id->event_handler, listen_id->context, + listen_id->ps, + ib_event->param.req_rcvd.qp_type, + listen_id_priv); + if (IS_ERR(id_priv)) return NULL; - id_priv = container_of(id, struct rdma_id_private, id); + id = &id_priv->id; if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr, (struct sockaddr *)&id->route.addr.dst_addr, listen_id, ib_event, ss_family, service_id)) @@ -2104,13 +2126,13 @@ cma_ib_new_udp_id(const struct rdma_cm_id *listen_id, int ret; listen_id_priv = container_of(listen_id, struct rdma_id_private, id); - id = __rdma_create_id(net, listen_id->event_handler, listen_id->context, - listen_id->ps, IB_QPT_UD, - listen_id_priv->res.kern_name); - if (IS_ERR(id)) + id_priv = __rdma_create_id(net, listen_id->event_handler, + listen_id->context, listen_id->ps, IB_QPT_UD, + listen_id_priv); + if (IS_ERR(id_priv)) return NULL; - id_priv = container_of(id, struct rdma_id_private, id); + id = &id_priv->id; if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr, (struct sockaddr *)&id->route.addr.dst_addr, listen_id, ib_event, ss_family, @@ -2184,7 +2206,7 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id, } mutex_lock(&listen_id->handler_mutex); - if (listen_id->state != RDMA_CM_LISTEN) { + if (READ_ONCE(listen_id->state) != RDMA_CM_LISTEN) { ret = -ECONNABORTED; goto err_unlock; } @@ -2226,8 +2248,8 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id, goto net_dev_put; } - if (cma_comp(conn_id, RDMA_CM_CONNECT) && - (conn_id->id.qp_type != IB_QPT_UD)) { + if (READ_ONCE(conn_id->state) == RDMA_CM_CONNECT && + conn_id->id.qp_type != IB_QPT_UD) { trace_cm_send_mra(cm_id->context); ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); } @@ -2288,7 +2310,7 @@ static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event) struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr; mutex_lock(&id_priv->handler_mutex); - if (id_priv->state != RDMA_CM_CONNECT) + if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT) goto out; switch (iw_event->event) { @@ -2346,7 +2368,6 @@ out: static int iw_conn_req_handler(struct iw_cm_id *cm_id, struct iw_cm_event *iw_event) { - struct rdma_cm_id *new_cm_id; struct rdma_id_private *listen_id, *conn_id; struct rdma_cm_event event = {}; int ret = -ECONNABORTED; @@ -2362,20 +2383,18 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id, listen_id = cm_id->context; mutex_lock(&listen_id->handler_mutex); - if (listen_id->state != RDMA_CM_LISTEN) + if (READ_ONCE(listen_id->state) != RDMA_CM_LISTEN) goto out; /* Create a new RDMA id for the new IW CM ID */ - new_cm_id = __rdma_create_id(listen_id->id.route.addr.dev_addr.net, - listen_id->id.event_handler, - listen_id->id.context, - RDMA_PS_TCP, IB_QPT_RC, - listen_id->res.kern_name); - if (IS_ERR(new_cm_id)) { + conn_id = __rdma_create_id(listen_id->id.route.addr.dev_addr.net, + listen_id->id.event_handler, + listen_id->id.context, RDMA_PS_TCP, + IB_QPT_RC, listen_id); + if (IS_ERR(conn_id)) { ret = -ENOMEM; goto out; } - conn_id = container_of(new_cm_id, struct rdma_id_private, id); mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); conn_id->state = RDMA_CM_CONNECT; @@ -2480,7 +2499,6 @@ static void cma_listen_on_dev(struct rdma_id_private *id_priv, struct cma_device *cma_dev) { struct rdma_id_private *dev_id_priv; - struct rdma_cm_id *id; struct net *net = id_priv->id.route.addr.dev_addr.net; int ret; @@ -2489,13 +2507,12 @@ static void cma_listen_on_dev(struct rdma_id_private *id_priv, if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev->device, 1)) return; - id = __rdma_create_id(net, cma_listen_handler, id_priv, id_priv->id.ps, - id_priv->id.qp_type, id_priv->res.kern_name); - if (IS_ERR(id)) + dev_id_priv = + __rdma_create_id(net, cma_listen_handler, id_priv, + id_priv->id.ps, id_priv->id.qp_type, id_priv); + if (IS_ERR(dev_id_priv)) return; - dev_id_priv = container_of(id, struct rdma_id_private, id); - dev_id_priv->state = RDMA_CM_ADDR_BOUND; memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv), rdma_addr_size(cma_src_addr(id_priv))); @@ -2508,7 +2525,7 @@ static void cma_listen_on_dev(struct rdma_id_private *id_priv, dev_id_priv->tos_set = id_priv->tos_set; dev_id_priv->tos = id_priv->tos; - ret = rdma_listen(id, id_priv->backlog); + ret = rdma_listen(&dev_id_priv->id, id_priv->backlog); if (ret) dev_warn(&cma_dev->device->dev, "RDMA CMA: cma_listen_on_dev, error %d\n", ret); @@ -2647,32 +2664,14 @@ static void cma_work_handler(struct work_struct *_work) struct rdma_id_private *id_priv = work->id; mutex_lock(&id_priv->handler_mutex); - if (!cma_comp_exch(id_priv, work->old_state, work->new_state)) + if (READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING || + READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL) goto out_unlock; - - if (cma_cm_event_handler(id_priv, &work->event)) { - cma_id_put(id_priv); - destroy_id_handler_unlock(id_priv); - goto out_free; + if (work->old_state != 0 || work->new_state != 0) { + if (!cma_comp_exch(id_priv, work->old_state, work->new_state)) + goto out_unlock; } -out_unlock: - mutex_unlock(&id_priv->handler_mutex); - cma_id_put(id_priv); -out_free: - kfree(work); -} - -static void cma_ndev_work_handler(struct work_struct *_work) -{ - struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work); - struct rdma_id_private *id_priv = work->id; - - mutex_lock(&id_priv->handler_mutex); - if (id_priv->state == RDMA_CM_DESTROYING || - id_priv->state == RDMA_CM_DEVICE_REMOVAL) - goto out_unlock; - if (cma_cm_event_handler(id_priv, &work->event)) { cma_id_put(id_priv); destroy_id_handler_unlock(id_priv); @@ -2683,6 +2682,8 @@ out_unlock: mutex_unlock(&id_priv->handler_mutex); cma_id_put(id_priv); out_free: + if (work->event.event == RDMA_CM_EVENT_MULTICAST_JOIN) + rdma_destroy_ah_attr(&work->event.param.ud.ah_attr); kfree(work); } @@ -3240,32 +3241,54 @@ static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, return rdma_bind_addr(id, src_addr); } -int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, - const struct sockaddr *dst_addr, unsigned long timeout_ms) +/* + * If required, resolve the source address for bind and leave the id_priv in + * state RDMA_CM_ADDR_BOUND. This oddly uses the state to determine the prior + * calls made by ULP, a previously bound ID will not be re-bound and src_addr is + * ignored. + */ +static int resolve_prepare_src(struct rdma_id_private *id_priv, + struct sockaddr *src_addr, + const struct sockaddr *dst_addr) { - struct rdma_id_private *id_priv; int ret; - id_priv = container_of(id, struct rdma_id_private, id); memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr)); - if (id_priv->state == RDMA_CM_IDLE) { - ret = cma_bind_addr(id, src_addr, dst_addr); - if (ret) { - memset(cma_dst_addr(id_priv), 0, - rdma_addr_size(dst_addr)); - return ret; + if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) { + /* For a well behaved ULP state will be RDMA_CM_IDLE */ + ret = cma_bind_addr(&id_priv->id, src_addr, dst_addr); + if (ret) + goto err_dst; + if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, + RDMA_CM_ADDR_QUERY))) { + ret = -EINVAL; + goto err_dst; } } if (cma_family(id_priv) != dst_addr->sa_family) { - memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr)); - return -EINVAL; + ret = -EINVAL; + goto err_state; } + return 0; - if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) { - memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr)); - return -EINVAL; - } +err_state: + cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND); +err_dst: + memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr)); + return ret; +} + +int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, + const struct sockaddr *dst_addr, unsigned long timeout_ms) +{ + struct rdma_id_private *id_priv = + container_of(id, struct rdma_id_private, id); + int ret; + + ret = resolve_prepare_src(id_priv, src_addr, dst_addr); + if (ret) + return ret; if (cma_any_addr(dst_addr)) { ret = cma_resolve_loopback(id_priv); @@ -3297,7 +3320,8 @@ int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse) id_priv = container_of(id, struct rdma_id_private, id); spin_lock_irqsave(&id_priv->lock, flags); - if (reuse || id_priv->state == RDMA_CM_IDLE) { + if ((reuse && id_priv->state != RDMA_CM_LISTEN) || + id_priv->state == RDMA_CM_IDLE) { id_priv->reuseaddr = reuse; ret = 0; } else { @@ -3491,8 +3515,7 @@ static int cma_check_port(struct rdma_bind_list *bind_list, if (id_priv == cur_id) continue; - if ((cur_id->state != RDMA_CM_LISTEN) && reuseaddr && - cur_id->reuseaddr) + if (reuseaddr && cur_id->reuseaddr) continue; cur_addr = cma_src_addr(cur_id); @@ -3533,18 +3556,6 @@ static int cma_use_port(enum rdma_ucm_port_space ps, return ret; } -static int cma_bind_listen(struct rdma_id_private *id_priv) -{ - struct rdma_bind_list *bind_list = id_priv->bind_list; - int ret = 0; - - mutex_lock(&lock); - if (bind_list->owners.first->next) - ret = cma_check_port(bind_list, id_priv, 0); - mutex_unlock(&lock); - return ret; -} - static enum rdma_ucm_port_space cma_select_inet_ps(struct rdma_id_private *id_priv) { @@ -3638,22 +3649,31 @@ static int cma_check_linklocal(struct rdma_dev_addr *dev_addr, int rdma_listen(struct rdma_cm_id *id, int backlog) { - struct rdma_id_private *id_priv; + struct rdma_id_private *id_priv = + container_of(id, struct rdma_id_private, id); int ret; - id_priv = container_of(id, struct rdma_id_private, id); - if (id_priv->state == RDMA_CM_IDLE) { + if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN)) { + /* For a well behaved ULP state will be RDMA_CM_IDLE */ id->route.addr.src_addr.ss_family = AF_INET; ret = rdma_bind_addr(id, cma_src_addr(id_priv)); if (ret) return ret; + if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, + RDMA_CM_LISTEN))) + return -EINVAL; } - if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN)) - return -EINVAL; - + /* + * Once the ID reaches RDMA_CM_LISTEN it is not allowed to be reusable + * any more, and has to be unique in the bind list. + */ if (id_priv->reuseaddr) { - ret = cma_bind_listen(id_priv); + mutex_lock(&lock); + ret = cma_check_port(id_priv->bind_list, id_priv, 0); + if (!ret) + id_priv->reuseaddr = 0; + mutex_unlock(&lock); if (ret) goto err; } @@ -3678,6 +3698,10 @@ int rdma_listen(struct rdma_cm_id *id, int backlog) return 0; err: id_priv->backlog = 0; + /* + * All the failure paths that lead here will not allow the req_handler's + * to have run. + */ cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND); return ret; } @@ -3732,7 +3756,6 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) return 0; err2: - rdma_restrack_del(&id_priv->res); if (id_priv->cma_dev) cma_release_dev(id_priv); err1: @@ -3781,7 +3804,7 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, int ret; mutex_lock(&id_priv->handler_mutex); - if (id_priv->state != RDMA_CM_CONNECT) + if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT) goto out; switch (ib_event->event) { @@ -4017,12 +4040,15 @@ out: int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) { - struct rdma_id_private *id_priv; + struct rdma_id_private *id_priv = + container_of(id, struct rdma_id_private, id); int ret; - id_priv = container_of(id, struct rdma_id_private, id); - if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT)) - return -EINVAL; + mutex_lock(&id_priv->handler_mutex); + if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT)) { + ret = -EINVAL; + goto err_unlock; + } if (!id->qp) { id_priv->qp_num = conn_param->qp_num; @@ -4039,11 +4065,13 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) else ret = -ENOSYS; if (ret) - goto err; - + goto err_state; + mutex_unlock(&id_priv->handler_mutex); return 0; -err: +err_state: cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED); +err_unlock: + mutex_unlock(&id_priv->handler_mutex); return ret; } EXPORT_SYMBOL(rdma_connect); @@ -4155,17 +4183,33 @@ static int cma_send_sidr_rep(struct rdma_id_private *id_priv, return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep); } -int __rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param, - const char *caller) +/** + * rdma_accept - Called to accept a connection request or response. + * @id: Connection identifier associated with the request. + * @conn_param: Information needed to establish the connection. This must be + * provided if accepting a connection request. If accepting a connection + * response, this parameter must be NULL. + * + * Typically, this routine is only called by the listener to accept a connection + * request. It must also be called on the active side of a connection if the + * user is performing their own QP transitions. + * + * In the case of error, a reject message is sent to the remote side and the + * state of the qp associated with the id is modified to error, such that any + * previously posted receive buffers would be flushed. + * + * This function is for use by kernel ULPs and must be called from under the + * handler callback. + */ +int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) { - struct rdma_id_private *id_priv; + struct rdma_id_private *id_priv = + container_of(id, struct rdma_id_private, id); int ret; - id_priv = container_of(id, struct rdma_id_private, id); - - rdma_restrack_set_task(&id_priv->res, caller); + lockdep_assert_held(&id_priv->handler_mutex); - if (!cma_comp(id_priv, RDMA_CM_CONNECT)) + if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT) return -EINVAL; if (!id->qp && conn_param) { @@ -4203,10 +4247,10 @@ reject: rdma_reject(id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); return ret; } -EXPORT_SYMBOL(__rdma_accept); +EXPORT_SYMBOL(rdma_accept); -int __rdma_accept_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param, - const char *caller, struct rdma_ucm_ece *ece) +int rdma_accept_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param, + struct rdma_ucm_ece *ece) { struct rdma_id_private *id_priv = container_of(id, struct rdma_id_private, id); @@ -4214,9 +4258,27 @@ int __rdma_accept_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param, id_priv->ece.vendor_id = ece->vendor_id; id_priv->ece.attr_mod = ece->attr_mod; - return __rdma_accept(id, conn_param, caller); + return rdma_accept(id, conn_param); +} +EXPORT_SYMBOL(rdma_accept_ece); + +void rdma_lock_handler(struct rdma_cm_id *id) +{ + struct rdma_id_private *id_priv = + container_of(id, struct rdma_id_private, id); + + mutex_lock(&id_priv->handler_mutex); } -EXPORT_SYMBOL(__rdma_accept_ece); +EXPORT_SYMBOL(rdma_lock_handler); + +void rdma_unlock_handler(struct rdma_cm_id *id) +{ + struct rdma_id_private *id_priv = + container_of(id, struct rdma_id_private, id); + + mutex_unlock(&id_priv->handler_mutex); +} +EXPORT_SYMBOL(rdma_unlock_handler); int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event) { @@ -4299,63 +4361,66 @@ out: } EXPORT_SYMBOL(rdma_disconnect); -static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) +static void cma_make_mc_event(int status, struct rdma_id_private *id_priv, + struct ib_sa_multicast *multicast, + struct rdma_cm_event *event, + struct cma_multicast *mc) { - struct rdma_id_private *id_priv; - struct cma_multicast *mc = multicast->context; - struct rdma_cm_event event = {}; - int ret = 0; - - id_priv = mc->id_priv; - mutex_lock(&id_priv->handler_mutex); - if (id_priv->state != RDMA_CM_ADDR_BOUND && - id_priv->state != RDMA_CM_ADDR_RESOLVED) - goto out; + struct rdma_dev_addr *dev_addr; + enum ib_gid_type gid_type; + struct net_device *ndev; if (!status) status = cma_set_qkey(id_priv, be32_to_cpu(multicast->rec.qkey)); else pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to join multicast. status %d\n", status); - mutex_lock(&id_priv->qp_mutex); - if (!status && id_priv->id.qp) { - status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid, - be16_to_cpu(multicast->rec.mlid)); - if (status) - pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to attach QP. status %d\n", - status); + + event->status = status; + event->param.ud.private_data = mc->context; + if (status) { + event->event = RDMA_CM_EVENT_MULTICAST_ERROR; + return; } - mutex_unlock(&id_priv->qp_mutex); - event.status = status; - event.param.ud.private_data = mc->context; - if (!status) { - struct rdma_dev_addr *dev_addr = - &id_priv->id.route.addr.dev_addr; - struct net_device *ndev = - dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if); - enum ib_gid_type gid_type = - id_priv->cma_dev->default_gid_type[id_priv->id.port_num - - rdma_start_port(id_priv->cma_dev->device)]; - - event.event = RDMA_CM_EVENT_MULTICAST_JOIN; - ret = ib_init_ah_from_mcmember(id_priv->id.device, - id_priv->id.port_num, - &multicast->rec, - ndev, gid_type, - &event.param.ud.ah_attr); - if (ret) - event.event = RDMA_CM_EVENT_MULTICAST_ERROR; + dev_addr = &id_priv->id.route.addr.dev_addr; + ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if); + gid_type = + id_priv->cma_dev + ->default_gid_type[id_priv->id.port_num - + rdma_start_port( + id_priv->cma_dev->device)]; + + event->event = RDMA_CM_EVENT_MULTICAST_JOIN; + if (ib_init_ah_from_mcmember(id_priv->id.device, id_priv->id.port_num, + &multicast->rec, ndev, gid_type, + &event->param.ud.ah_attr)) { + event->event = RDMA_CM_EVENT_MULTICAST_ERROR; + goto out; + } - event.param.ud.qp_num = 0xFFFFFF; - event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey); - if (ndev) - dev_put(ndev); - } else - event.event = RDMA_CM_EVENT_MULTICAST_ERROR; + event->param.ud.qp_num = 0xFFFFFF; + event->param.ud.qkey = be32_to_cpu(multicast->rec.qkey); - ret = cma_cm_event_handler(id_priv, &event); +out: + if (ndev) + dev_put(ndev); +} +static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) +{ + struct cma_multicast *mc = multicast->context; + struct rdma_id_private *id_priv = mc->id_priv; + struct rdma_cm_event event = {}; + int ret = 0; + + mutex_lock(&id_priv->handler_mutex); + if (READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL || + READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING) + goto out; + + cma_make_mc_event(status, id_priv, multicast, &event, mc); + ret = cma_cm_event_handler(id_priv, &event); rdma_destroy_ah_attr(&event.param.ud.ah_attr); if (ret) { destroy_id_handler_unlock(id_priv); @@ -4445,23 +4510,10 @@ static int cma_join_ib_multicast(struct rdma_id_private *id_priv, IB_SA_MCMEMBER_REC_MTU | IB_SA_MCMEMBER_REC_HOP_LIMIT; - mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device, - id_priv->id.port_num, &rec, - comp_mask, GFP_KERNEL, - cma_ib_mc_handler, mc); - return PTR_ERR_OR_ZERO(mc->multicast.ib); -} - -static void iboe_mcast_work_handler(struct work_struct *work) -{ - struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work); - struct cma_multicast *mc = mw->mc; - struct ib_sa_multicast *m = mc->multicast.ib; - - mc->multicast.ib->context = mc; - cma_ib_mc_handler(0, m); - kref_put(&mc->mcref, release_mc); - kfree(mw); + mc->sa_mc = ib_sa_join_multicast(&sa_client, id_priv->id.device, + id_priv->id.port_num, &rec, comp_mask, + GFP_KERNEL, cma_ib_mc_handler, mc); + return PTR_ERR_OR_ZERO(mc->sa_mc); } static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid, @@ -4496,52 +4548,47 @@ static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid, static int cma_iboe_join_multicast(struct rdma_id_private *id_priv, struct cma_multicast *mc) { - struct iboe_mcast_work *work; + struct cma_work *work; struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; int err = 0; struct sockaddr *addr = (struct sockaddr *)&mc->addr; struct net_device *ndev = NULL; + struct ib_sa_multicast ib; enum ib_gid_type gid_type; bool send_only; send_only = mc->join_state == BIT(SENDONLY_FULLMEMBER_JOIN); - if (cma_zero_addr((struct sockaddr *)&mc->addr)) + if (cma_zero_addr(addr)) return -EINVAL; work = kzalloc(sizeof *work, GFP_KERNEL); if (!work) return -ENOMEM; - mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL); - if (!mc->multicast.ib) { - err = -ENOMEM; - goto out1; - } - gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num - rdma_start_port(id_priv->cma_dev->device)]; - cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid, gid_type); + cma_iboe_set_mgid(addr, &ib.rec.mgid, gid_type); - mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff); + ib.rec.pkey = cpu_to_be16(0xffff); if (id_priv->id.ps == RDMA_PS_UDP) - mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY); + ib.rec.qkey = cpu_to_be32(RDMA_UDP_QKEY); if (dev_addr->bound_dev_if) ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if); if (!ndev) { err = -ENODEV; - goto out2; + goto err_free; } - mc->multicast.ib->rec.rate = iboe_get_rate(ndev); - mc->multicast.ib->rec.hop_limit = 1; - mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu); + ib.rec.rate = iboe_get_rate(ndev); + ib.rec.hop_limit = 1; + ib.rec.mtu = iboe_get_mtu(ndev->mtu); if (addr->sa_family == AF_INET) { if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { - mc->multicast.ib->rec.hop_limit = IPV6_DEFAULT_HOPLIMIT; + ib.rec.hop_limit = IPV6_DEFAULT_HOPLIMIT; if (!send_only) { - err = cma_igmp_send(ndev, &mc->multicast.ib->rec.mgid, + err = cma_igmp_send(ndev, &ib.rec.mgid, true); } } @@ -4550,24 +4597,22 @@ static int cma_iboe_join_multicast(struct rdma_id_private *id_priv, err = -ENOTSUPP; } dev_put(ndev); - if (err || !mc->multicast.ib->rec.mtu) { + if (err || !ib.rec.mtu) { if (!err) err = -EINVAL; - goto out2; + goto err_free; } rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, - &mc->multicast.ib->rec.port_gid); + &ib.rec.port_gid); work->id = id_priv; - work->mc = mc; - INIT_WORK(&work->work, iboe_mcast_work_handler); - kref_get(&mc->mcref); + INIT_WORK(&work->work, cma_work_handler); + cma_make_mc_event(0, id_priv, &ib, &work->event, mc); + /* Balances with cma_id_put() in cma_work_handler */ + cma_id_get(id_priv); queue_work(cma_wq, &work->work); - return 0; -out2: - kfree(mc->multicast.ib); -out1: +err_free: kfree(work); return err; } @@ -4575,19 +4620,21 @@ out1: int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, u8 join_state, void *context) { - struct rdma_id_private *id_priv; + struct rdma_id_private *id_priv = + container_of(id, struct rdma_id_private, id); struct cma_multicast *mc; int ret; - if (!id->device) + /* Not supported for kernel QPs */ + if (WARN_ON(id->qp)) return -EINVAL; - id_priv = container_of(id, struct rdma_id_private, id); - if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) && - !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED)) + /* ULP is calling this wrong. */ + if (!id->device || (READ_ONCE(id_priv->state) != RDMA_CM_ADDR_BOUND && + READ_ONCE(id_priv->state) != RDMA_CM_ADDR_RESOLVED)) return -EINVAL; - mc = kmalloc(sizeof *mc, GFP_KERNEL); + mc = kzalloc(sizeof(*mc), GFP_KERNEL); if (!mc) return -ENOMEM; @@ -4597,7 +4644,6 @@ int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, mc->join_state = join_state; if (rdma_protocol_roce(id->device, id->port_num)) { - kref_init(&mc->mcref); ret = cma_iboe_join_multicast(id_priv, mc); if (ret) goto out_err; @@ -4629,25 +4675,14 @@ void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr) id_priv = container_of(id, struct rdma_id_private, id); spin_lock_irq(&id_priv->lock); list_for_each_entry(mc, &id_priv->mc_list, list) { - if (!memcmp(&mc->addr, addr, rdma_addr_size(addr))) { - list_del(&mc->list); - spin_unlock_irq(&id_priv->lock); - - if (id->qp) - ib_detach_mcast(id->qp, - &mc->multicast.ib->rec.mgid, - be16_to_cpu(mc->multicast.ib->rec.mlid)); - - BUG_ON(id_priv->cma_dev->device != id->device); - - if (rdma_cap_ib_mcast(id->device, id->port_num)) { - ib_sa_free_multicast(mc->multicast.ib); - kfree(mc); - } else if (rdma_protocol_roce(id->device, id->port_num)) { - cma_leave_roce_mc_group(id_priv, mc); - } - return; - } + if (memcmp(&mc->addr, addr, rdma_addr_size(addr)) != 0) + continue; + list_del(&mc->list); + spin_unlock_irq(&id_priv->lock); + + WARN_ON(id_priv->cma_dev->device != id->device); + destroy_mc(id_priv, mc); + return; } spin_unlock_irq(&id_priv->lock); } @@ -4656,7 +4691,7 @@ EXPORT_SYMBOL(rdma_leave_multicast); static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv) { struct rdma_dev_addr *dev_addr; - struct cma_ndev_work *work; + struct cma_work *work; dev_addr = &id_priv->id.route.addr.dev_addr; @@ -4669,7 +4704,7 @@ static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id if (!work) return -ENOMEM; - INIT_WORK(&work->work, cma_ndev_work_handler); + INIT_WORK(&work->work, cma_work_handler); work->id = id_priv; work->event.event = RDMA_CM_EVENT_ADDR_CHANGE; cma_id_get(id_priv); diff --git a/drivers/infiniband/core/cma_configfs.c b/drivers/infiniband/core/cma_configfs.c index 3c1e2ca564fe..7ec4af2ed87a 100644 --- a/drivers/infiniband/core/cma_configfs.c +++ b/drivers/infiniband/core/cma_configfs.c @@ -123,16 +123,17 @@ static ssize_t default_roce_mode_store(struct config_item *item, { struct cma_device *cma_dev; struct cma_dev_port_group *group; - int gid_type = ib_cache_gid_parse_type_str(buf); + int gid_type; ssize_t ret; - if (gid_type < 0) - return -EINVAL; - ret = cma_configfs_params_get(item, &cma_dev, &group); if (ret) return ret; + gid_type = ib_cache_gid_parse_type_str(buf); + if (gid_type < 0) + return -EINVAL; + ret = cma_set_default_gid_type(cma_dev, group->port_num, gid_type); cma_configfs_params_put(cma_dev); diff --git a/drivers/infiniband/core/cma_trace.h b/drivers/infiniband/core/cma_trace.h index e6e20c36c538..e45264267bcc 100644 --- a/drivers/infiniband/core/cma_trace.h +++ b/drivers/infiniband/core/cma_trace.h @@ -17,46 +17,6 @@ #include <linux/tracepoint.h> #include <trace/events/rdma.h> -/* - * enum ib_cm_event_type, from include/rdma/ib_cm.h - */ -#define IB_CM_EVENT_LIST \ - ib_cm_event(REQ_ERROR) \ - ib_cm_event(REQ_RECEIVED) \ - ib_cm_event(REP_ERROR) \ - ib_cm_event(REP_RECEIVED) \ - ib_cm_event(RTU_RECEIVED) \ - ib_cm_event(USER_ESTABLISHED) \ - ib_cm_event(DREQ_ERROR) \ - ib_cm_event(DREQ_RECEIVED) \ - ib_cm_event(DREP_RECEIVED) \ - ib_cm_event(TIMEWAIT_EXIT) \ - ib_cm_event(MRA_RECEIVED) \ - ib_cm_event(REJ_RECEIVED) \ - ib_cm_event(LAP_ERROR) \ - ib_cm_event(LAP_RECEIVED) \ - ib_cm_event(APR_RECEIVED) \ - ib_cm_event(SIDR_REQ_ERROR) \ - ib_cm_event(SIDR_REQ_RECEIVED) \ - ib_cm_event_end(SIDR_REP_RECEIVED) - -#undef ib_cm_event -#undef ib_cm_event_end - -#define ib_cm_event(x) TRACE_DEFINE_ENUM(IB_CM_##x); -#define ib_cm_event_end(x) TRACE_DEFINE_ENUM(IB_CM_##x); - -IB_CM_EVENT_LIST - -#undef ib_cm_event -#undef ib_cm_event_end - -#define ib_cm_event(x) { IB_CM_##x, #x }, -#define ib_cm_event_end(x) { IB_CM_##x, #x } - -#define rdma_show_ib_cm_event(x) \ - __print_symbolic(x, IB_CM_EVENT_LIST) - DECLARE_EVENT_CLASS(cma_fsm_class, TP_PROTO( diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index a1e6a67b2c4a..e84b0fedaacb 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -44,6 +44,7 @@ #include <rdma/ib_mad.h> #include <rdma/restrack.h> #include "mad_priv.h" +#include "restrack.h" /* Total number of ports combined across all struct ib_devices's */ #define RDMA_MAX_PORTS 8192 @@ -352,6 +353,7 @@ static inline struct ib_qp *_ib_create_qp(struct ib_device *dev, INIT_LIST_HEAD(&qp->rdma_mrs); INIT_LIST_HEAD(&qp->sig_mrs); + rdma_restrack_new(&qp->res, RDMA_RESTRACK_QP); /* * We don't track XRC QPs for now, because they don't have PD * and more importantly they are created internaly by driver, @@ -359,14 +361,9 @@ static inline struct ib_qp *_ib_create_qp(struct ib_device *dev, */ is_xrc = qp_type == IB_QPT_XRC_INI || qp_type == IB_QPT_XRC_TGT; if ((qp_type < IB_QPT_MAX && !is_xrc) || qp_type == IB_QPT_DRIVER) { - qp->res.type = RDMA_RESTRACK_QP; - if (uobj) - rdma_restrack_uadd(&qp->res); - else - rdma_restrack_kadd(&qp->res); - } else - qp->res.valid = false; - + rdma_restrack_parent_name(&qp->res, &pd->res); + rdma_restrack_add(&qp->res); + } return qp; } diff --git a/drivers/infiniband/core/counters.c b/drivers/infiniband/core/counters.c index 636166880442..e4ff0d3328b6 100644 --- a/drivers/infiniband/core/counters.c +++ b/drivers/infiniband/core/counters.c @@ -80,8 +80,9 @@ static struct rdma_counter *rdma_counter_alloc(struct ib_device *dev, u8 port, counter->device = dev; counter->port = port; - counter->res.type = RDMA_RESTRACK_COUNTER; - counter->stats = dev->ops.counter_alloc_stats(counter); + + rdma_restrack_new(&counter->res, RDMA_RESTRACK_COUNTER); + counter->stats = dev->ops.counter_alloc_stats(counter); if (!counter->stats) goto err_stats; @@ -107,6 +108,7 @@ err_mode: mutex_unlock(&port_counter->lock); kfree(counter->stats); err_stats: + rdma_restrack_put(&counter->res); kfree(counter); return NULL; } @@ -248,13 +250,8 @@ next: static void rdma_counter_res_add(struct rdma_counter *counter, struct ib_qp *qp) { - if (rdma_is_kernel_res(&qp->res)) { - rdma_restrack_set_task(&counter->res, qp->res.kern_name); - rdma_restrack_kadd(&counter->res); - } else { - rdma_restrack_attach_task(&counter->res, qp->res.task); - rdma_restrack_uadd(&counter->res); - } + rdma_restrack_parent_name(&counter->res, &qp->res); + rdma_restrack_add(&counter->res); } static void counter_release(struct kref *kref) diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c index a92fc3f90bb5..12ebacf52958 100644 --- a/drivers/infiniband/core/cq.c +++ b/drivers/infiniband/core/cq.c @@ -197,24 +197,22 @@ static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private) } /** - * __ib_alloc_cq_user - allocate a completion queue + * __ib_alloc_cq allocate a completion queue * @dev: device to allocate the CQ for * @private: driver private data, accessible from cq->cq_context * @nr_cqe: number of CQEs to allocate * @comp_vector: HCA completion vectors for this CQ * @poll_ctx: context to poll the CQ from. * @caller: module owner name. - * @udata: Valid user data or NULL for kernel object * * This is the proper interface to allocate a CQ for in-kernel users. A * CQ allocated with this interface will automatically be polled from the * specified context. The ULP must use wr->wr_cqe instead of wr->wr_id * to use this CQ abstraction. */ -struct ib_cq *__ib_alloc_cq_user(struct ib_device *dev, void *private, - int nr_cqe, int comp_vector, - enum ib_poll_context poll_ctx, - const char *caller, struct ib_udata *udata) +struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, int nr_cqe, + int comp_vector, enum ib_poll_context poll_ctx, + const char *caller) { struct ib_cq_init_attr cq_attr = { .cqe = nr_cqe, @@ -237,15 +235,13 @@ struct ib_cq *__ib_alloc_cq_user(struct ib_device *dev, void *private, if (!cq->wc) goto out_free_cq; - cq->res.type = RDMA_RESTRACK_CQ; - rdma_restrack_set_task(&cq->res, caller); + rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ); + rdma_restrack_set_name(&cq->res, caller); ret = dev->ops.create_cq(cq, &cq_attr, NULL); if (ret) goto out_free_wc; - rdma_restrack_kadd(&cq->res); - rdma_dim_init(cq); switch (cq->poll_ctx) { @@ -271,21 +267,22 @@ struct ib_cq *__ib_alloc_cq_user(struct ib_device *dev, void *private, goto out_destroy_cq; } + rdma_restrack_add(&cq->res); trace_cq_alloc(cq, nr_cqe, comp_vector, poll_ctx); return cq; out_destroy_cq: rdma_dim_destroy(cq); - rdma_restrack_del(&cq->res); - cq->device->ops.destroy_cq(cq, udata); + cq->device->ops.destroy_cq(cq, NULL); out_free_wc: + rdma_restrack_put(&cq->res); kfree(cq->wc); out_free_cq: kfree(cq); trace_cq_alloc_error(nr_cqe, comp_vector, poll_ctx, ret); return ERR_PTR(ret); } -EXPORT_SYMBOL(__ib_alloc_cq_user); +EXPORT_SYMBOL(__ib_alloc_cq); /** * __ib_alloc_cq_any - allocate a completion queue @@ -310,18 +307,19 @@ struct ib_cq *__ib_alloc_cq_any(struct ib_device *dev, void *private, atomic_inc_return(&counter) % min_t(int, dev->num_comp_vectors, num_online_cpus()); - return __ib_alloc_cq_user(dev, private, nr_cqe, comp_vector, poll_ctx, - caller, NULL); + return __ib_alloc_cq(dev, private, nr_cqe, comp_vector, poll_ctx, + caller); } EXPORT_SYMBOL(__ib_alloc_cq_any); /** - * ib_free_cq_user - free a completion queue + * ib_free_cq - free a completion queue * @cq: completion queue to free. - * @udata: User data or NULL for kernel object */ -void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata) +void ib_free_cq(struct ib_cq *cq) { + int ret; + if (WARN_ON_ONCE(atomic_read(&cq->usecnt))) return; if (WARN_ON_ONCE(cq->cqe_used)) @@ -343,12 +341,13 @@ void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata) rdma_dim_destroy(cq); trace_cq_free(cq); + ret = cq->device->ops.destroy_cq(cq, NULL); + WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail"); rdma_restrack_del(&cq->res); - cq->device->ops.destroy_cq(cq, udata); kfree(cq->wc); kfree(cq); } -EXPORT_SYMBOL(ib_free_cq_user); +EXPORT_SYMBOL(ib_free_cq); void ib_cq_pool_init(struct ib_device *dev) { diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index 23ee65a9185f..a3b1fc84cdca 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -1177,58 +1177,23 @@ out: return ret; } -static void setup_dma_device(struct ib_device *device) +static void setup_dma_device(struct ib_device *device, + struct device *dma_device) { - struct device *parent = device->dev.parent; - - WARN_ON_ONCE(device->dma_device); - -#ifdef CONFIG_DMA_OPS - if (device->dev.dma_ops) { - /* - * The caller provided custom DMA operations. Copy the - * DMA-related fields that are used by e.g. dma_alloc_coherent() - * into device->dev. - */ - device->dma_device = &device->dev; - if (!device->dev.dma_mask) { - if (parent) - device->dev.dma_mask = parent->dma_mask; - else - WARN_ON_ONCE(true); - } - if (!device->dev.coherent_dma_mask) { - if (parent) - device->dev.coherent_dma_mask = - parent->coherent_dma_mask; - else - WARN_ON_ONCE(true); - } - } else -#endif /* CONFIG_DMA_OPS */ - { - /* - * The caller did not provide custom DMA operations. Use the - * DMA mapping operations of the parent device. - */ - WARN_ON_ONCE(!parent); - device->dma_device = parent; - } - - if (!device->dev.dma_parms) { - if (parent) { - /* - * The caller did not provide DMA parameters, so - * 'parent' probably represents a PCI device. The PCI - * core sets the maximum segment size to 64 - * KB. Increase this parameter to 2 GB. - */ - device->dev.dma_parms = parent->dma_parms; - dma_set_max_seg_size(device->dma_device, SZ_2G); - } else { - WARN_ON_ONCE(true); - } + /* + * If the caller does not provide a DMA capable device then the IB + * device will be used. In this case the caller should fully setup the + * ibdev for DMA. This usually means using dma_virt_ops. + */ +#ifdef CONFIG_DMA_VIRT_OPS + if (!dma_device) { + device->dev.dma_ops = &dma_virt_ops; + dma_device = &device->dev; } +#endif + WARN_ON(!dma_device); + device->dma_device = dma_device; + WARN_ON(!device->dma_device->dma_parms); } /* @@ -1241,7 +1206,6 @@ static int setup_device(struct ib_device *device) struct ib_udata uhw = {.outlen = 0, .inlen = 0}; int ret; - setup_dma_device(device); ib_device_check_mandatory(device); ret = setup_port_data(device); @@ -1354,7 +1318,10 @@ static void prevent_dealloc_device(struct ib_device *ib_dev) * ib_register_device - Register an IB device with IB core * @device: Device to register * @name: unique string device name. This may include a '%' which will - * cause a unique index to be added to the passed device name. + * cause a unique index to be added to the passed device name. + * @dma_device: pointer to a DMA-capable device. If %NULL, then the IB + * device will be used. In this case the caller should fully + * setup the ibdev for DMA. This usually means using dma_virt_ops. * * Low-level drivers use ib_register_device() to register their * devices with the IB core. All registered clients will receive a @@ -1365,7 +1332,8 @@ static void prevent_dealloc_device(struct ib_device *ib_dev) * asynchronously then the device pointer may become freed as soon as this * function returns. */ -int ib_register_device(struct ib_device *device, const char *name) +int ib_register_device(struct ib_device *device, const char *name, + struct device *dma_device) { int ret; @@ -1373,6 +1341,7 @@ int ib_register_device(struct ib_device *device, const char *name) if (ret) return ret; + setup_dma_device(device, dma_device); ret = setup_device(device); if (ret) return ret; @@ -2697,7 +2666,9 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_OBJ_SIZE(dev_ops, ib_ah); SET_OBJ_SIZE(dev_ops, ib_counters); SET_OBJ_SIZE(dev_ops, ib_cq); + SET_OBJ_SIZE(dev_ops, ib_mw); SET_OBJ_SIZE(dev_ops, ib_pd); + SET_OBJ_SIZE(dev_ops, ib_rwq_ind_table); SET_OBJ_SIZE(dev_ops, ib_srq); SET_OBJ_SIZE(dev_ops, ib_ucontext); SET_OBJ_SIZE(dev_ops, ib_xrcd); diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c index 6d3ed7c6e19e..ffe11b03724c 100644 --- a/drivers/infiniband/core/rdma_core.c +++ b/drivers/infiniband/core/rdma_core.c @@ -130,17 +130,6 @@ static int uverbs_destroy_uobject(struct ib_uobject *uobj, lockdep_assert_held(&ufile->hw_destroy_rwsem); assert_uverbs_usecnt(uobj, UVERBS_LOOKUP_WRITE); - if (reason == RDMA_REMOVE_ABORT_HWOBJ) { - reason = RDMA_REMOVE_ABORT; - ret = uobj->uapi_object->type_class->destroy_hw(uobj, reason, - attrs); - /* - * Drivers are not permitted to ignore RDMA_REMOVE_ABORT, see - * ib_is_destroy_retryable, cleanup_retryable == false here. - */ - WARN_ON(ret); - } - if (reason == RDMA_REMOVE_ABORT) { WARN_ON(!list_empty(&uobj->list)); WARN_ON(!uobj->context); @@ -674,11 +663,22 @@ void rdma_alloc_abort_uobject(struct ib_uobject *uobj, bool hw_obj_valid) { struct ib_uverbs_file *ufile = uobj->ufile; + int ret; + + if (hw_obj_valid) { + ret = uobj->uapi_object->type_class->destroy_hw( + uobj, RDMA_REMOVE_ABORT, attrs); + /* + * If the driver couldn't destroy the object then go ahead and + * commit it. Leaking objects that can't be destroyed is only + * done during FD close after the driver has a few more tries to + * destroy it. + */ + if (WARN_ON(ret)) + return rdma_alloc_commit_uobject(uobj, attrs); + } - uverbs_destroy_uobject(uobj, - hw_obj_valid ? RDMA_REMOVE_ABORT_HWOBJ : - RDMA_REMOVE_ABORT, - attrs); + uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT, attrs); /* Matches the down_read in rdma_alloc_begin_uobject */ up_read(&ufile->hw_destroy_rwsem); @@ -889,14 +889,14 @@ void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile, if (!ufile->ucontext) goto done; - ufile->ucontext->closing = true; ufile->ucontext->cleanup_retryable = true; while (!list_empty(&ufile->uobjects)) if (__uverbs_cleanup_ufile(ufile, reason)) { /* * No entry was cleaned-up successfully during this - * iteration + * iteration. It is a driver bug to fail destruction. */ + WARN_ON(!list_empty(&ufile->uobjects)); break; } diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c index 62fbb0ae9cb4..4aeeaaed0f17 100644 --- a/drivers/infiniband/core/restrack.c +++ b/drivers/infiniband/core/restrack.c @@ -123,32 +123,6 @@ int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type) } EXPORT_SYMBOL(rdma_restrack_count); -static void set_kern_name(struct rdma_restrack_entry *res) -{ - struct ib_pd *pd; - - switch (res->type) { - case RDMA_RESTRACK_QP: - pd = container_of(res, struct ib_qp, res)->pd; - if (!pd) { - WARN_ONCE(true, "XRC QPs are not supported\n"); - /* Survive, despite the programmer's error */ - res->kern_name = " "; - } - break; - case RDMA_RESTRACK_MR: - pd = container_of(res, struct ib_mr, res)->pd; - break; - default: - /* Other types set kern_name directly */ - pd = NULL; - break; - } - - if (pd) - res->kern_name = pd->res.kern_name; -} - static struct ib_device *res_to_dev(struct rdma_restrack_entry *res) { switch (res->type) { @@ -173,36 +147,77 @@ static struct ib_device *res_to_dev(struct rdma_restrack_entry *res) } } -void rdma_restrack_set_task(struct rdma_restrack_entry *res, - const char *caller) +/** + * rdma_restrack_attach_task() - attach the task onto this resource, + * valid for user space restrack entries. + * @res: resource entry + * @task: the task to attach + */ +static void rdma_restrack_attach_task(struct rdma_restrack_entry *res, + struct task_struct *task) { - if (caller) { - res->kern_name = caller; + if (WARN_ON_ONCE(!task)) return; - } if (res->task) put_task_struct(res->task); - get_task_struct(current); - res->task = current; + get_task_struct(task); + res->task = task; + res->user = true; } -EXPORT_SYMBOL(rdma_restrack_set_task); /** - * rdma_restrack_attach_task() - attach the task onto this resource + * rdma_restrack_set_name() - set the task for this resource * @res: resource entry - * @task: the task to attach, the current task will be used if it is NULL. + * @caller: kernel name, the current task will be used if the caller is NULL. */ -void rdma_restrack_attach_task(struct rdma_restrack_entry *res, - struct task_struct *task) +void rdma_restrack_set_name(struct rdma_restrack_entry *res, const char *caller) { - if (res->task) - put_task_struct(res->task); - get_task_struct(task); - res->task = task; + if (caller) { + res->kern_name = caller; + return; + } + + rdma_restrack_attach_task(res, current); +} +EXPORT_SYMBOL(rdma_restrack_set_name); + +/** + * rdma_restrack_parent_name() - set the restrack name properties based + * on parent restrack + * @dst: destination resource entry + * @parent: parent resource entry + */ +void rdma_restrack_parent_name(struct rdma_restrack_entry *dst, + const struct rdma_restrack_entry *parent) +{ + if (rdma_is_kernel_res(parent)) + dst->kern_name = parent->kern_name; + else + rdma_restrack_attach_task(dst, parent->task); +} +EXPORT_SYMBOL(rdma_restrack_parent_name); + +/** + * rdma_restrack_new() - Initializes new restrack entry to allow _put() interface + * to release memory in fully automatic way. + * @res - Entry to initialize + * @type - REstrack type + */ +void rdma_restrack_new(struct rdma_restrack_entry *res, + enum rdma_restrack_type type) +{ + kref_init(&res->kref); + init_completion(&res->comp); + res->type = type; } +EXPORT_SYMBOL(rdma_restrack_new); -static void rdma_restrack_add(struct rdma_restrack_entry *res) +/** + * rdma_restrack_add() - add object to the reource tracking database + * @res: resource entry + */ +void rdma_restrack_add(struct rdma_restrack_entry *res) { struct ib_device *dev = res_to_dev(res); struct rdma_restrack_root *rt; @@ -213,8 +228,6 @@ static void rdma_restrack_add(struct rdma_restrack_entry *res) rt = &dev->res[res->type]; - kref_init(&res->kref); - init_completion(&res->comp); if (res->type == RDMA_RESTRACK_QP) { /* Special case to ensure that LQPN points to right QP */ struct ib_qp *qp = container_of(res, struct ib_qp, res); @@ -236,38 +249,7 @@ static void rdma_restrack_add(struct rdma_restrack_entry *res) if (!ret) res->valid = true; } - -/** - * rdma_restrack_kadd() - add kernel object to the reource tracking database - * @res: resource entry - */ -void rdma_restrack_kadd(struct rdma_restrack_entry *res) -{ - res->task = NULL; - set_kern_name(res); - res->user = false; - rdma_restrack_add(res); -} -EXPORT_SYMBOL(rdma_restrack_kadd); - -/** - * rdma_restrack_uadd() - add user object to the reource tracking database - * @res: resource entry - */ -void rdma_restrack_uadd(struct rdma_restrack_entry *res) -{ - if ((res->type != RDMA_RESTRACK_CM_ID) && - (res->type != RDMA_RESTRACK_COUNTER)) - res->task = NULL; - - if (!res->task) - rdma_restrack_set_task(res, NULL); - res->kern_name = NULL; - - res->user = true; - rdma_restrack_add(res); -} -EXPORT_SYMBOL(rdma_restrack_uadd); +EXPORT_SYMBOL(rdma_restrack_add); int __must_check rdma_restrack_get(struct rdma_restrack_entry *res) { @@ -305,6 +287,10 @@ static void restrack_release(struct kref *kref) struct rdma_restrack_entry *res; res = container_of(kref, struct rdma_restrack_entry, kref); + if (res->task) { + put_task_struct(res->task); + res->task = NULL; + } complete(&res->comp); } @@ -314,14 +300,23 @@ int rdma_restrack_put(struct rdma_restrack_entry *res) } EXPORT_SYMBOL(rdma_restrack_put); +/** + * rdma_restrack_del() - delete object from the reource tracking database + * @res: resource entry + */ void rdma_restrack_del(struct rdma_restrack_entry *res) { struct rdma_restrack_entry *old; struct rdma_restrack_root *rt; struct ib_device *dev; - if (!res->valid) - goto out; + if (!res->valid) { + if (res->task) { + put_task_struct(res->task); + res->task = NULL; + } + return; + } dev = res_to_dev(res); if (WARN_ON(!dev)) @@ -330,16 +325,12 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) rt = &dev->res[res->type]; old = xa_erase(&rt->xa, res->id); + if (res->type == RDMA_RESTRACK_MR || res->type == RDMA_RESTRACK_QP) + return; WARN_ON(old != res); res->valid = false; rdma_restrack_put(res); wait_for_completion(&res->comp); - -out: - if (res->task) { - put_task_struct(res->task); - res->task = NULL; - } } EXPORT_SYMBOL(rdma_restrack_del); diff --git a/drivers/infiniband/core/restrack.h b/drivers/infiniband/core/restrack.h index d084e5f89849..6a04fc41f738 100644 --- a/drivers/infiniband/core/restrack.h +++ b/drivers/infiniband/core/restrack.h @@ -25,6 +25,12 @@ struct rdma_restrack_root { int rdma_restrack_init(struct ib_device *dev); void rdma_restrack_clean(struct ib_device *dev); -void rdma_restrack_attach_task(struct rdma_restrack_entry *res, - struct task_struct *task); +void rdma_restrack_add(struct rdma_restrack_entry *res); +void rdma_restrack_del(struct rdma_restrack_entry *res); +void rdma_restrack_new(struct rdma_restrack_entry *res, + enum rdma_restrack_type type); +void rdma_restrack_set_name(struct rdma_restrack_entry *res, + const char *caller); +void rdma_restrack_parent_name(struct rdma_restrack_entry *dst, + const struct rdma_restrack_entry *parent); #endif /* _RDMA_CORE_RESTRACK_H_ */ diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c index c11e50510e49..914cddea525d 100644 --- a/drivers/infiniband/core/sysfs.c +++ b/drivers/infiniband/core/sysfs.c @@ -59,7 +59,7 @@ struct ib_port { struct gid_attr_group *gid_attr_group; struct attribute_group gid_group; struct attribute_group *pkey_group; - struct attribute_group *pma_table; + const struct attribute_group *pma_table; struct attribute_group *hw_stats_ag; struct rdma_hw_stats *hw_stats; u8 port_num; @@ -387,7 +387,8 @@ static ssize_t _show_port_gid_attr( gid_attr = rdma_get_gid_attr(p->ibdev, p->port_num, tab_attr->index); if (IS_ERR(gid_attr)) - return PTR_ERR(gid_attr); + /* -EINVAL is returned for user space compatibility reasons. */ + return -EINVAL; ret = print(gid_attr, buf); rdma_put_gid_attr(gid_attr); @@ -653,17 +654,17 @@ static struct attribute *pma_attrs_noietf[] = { NULL }; -static struct attribute_group pma_group = { +static const struct attribute_group pma_group = { .name = "counters", .attrs = pma_attrs }; -static struct attribute_group pma_group_ext = { +static const struct attribute_group pma_group_ext = { .name = "counters", .attrs = pma_attrs_ext }; -static struct attribute_group pma_group_noietf = { +static const struct attribute_group pma_group_noietf = { .name = "counters", .attrs = pma_attrs_noietf }; @@ -778,8 +779,8 @@ err: * Figure out which counter table to use depending on * the device capabilities. */ -static struct attribute_group *get_counter_table(struct ib_device *dev, - int port_num) +static const struct attribute_group *get_counter_table(struct ib_device *dev, + int port_num) { struct ib_class_port_info cpi; diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c index 1d184ea05eba..ffe2563ad345 100644 --- a/drivers/infiniband/core/ucma.c +++ b/drivers/infiniband/core/ucma.c @@ -80,7 +80,6 @@ struct ucma_file { struct list_head ctx_list; struct list_head event_list; wait_queue_head_t poll_wait; - struct workqueue_struct *close_wq; }; struct ucma_context { @@ -88,7 +87,7 @@ struct ucma_context { struct completion comp; refcount_t ref; int events_reported; - int backlog; + atomic_t backlog; struct ucma_file *file; struct rdma_cm_id *cm_id; @@ -96,11 +95,6 @@ struct ucma_context { u64 uid; struct list_head list; - struct list_head mc_list; - /* mark that device is in process of destroying the internal HW - * resources, protected by the ctx_table lock - */ - int closing; /* sync between removal event and id destroy, protected by file mut */ int destroying; struct work_struct close_work; @@ -113,23 +107,22 @@ struct ucma_multicast { u64 uid; u8 join_state; - struct list_head list; struct sockaddr_storage addr; }; struct ucma_event { struct ucma_context *ctx; + struct ucma_context *conn_req_ctx; struct ucma_multicast *mc; struct list_head list; - struct rdma_cm_id *cm_id; struct rdma_ucm_event_resp resp; - struct work_struct close_work; }; static DEFINE_XARRAY_ALLOC(ctx_table); static DEFINE_XARRAY_ALLOC(multicast_table); static const struct file_operations ucma_fops; +static int __destroy_id(struct ucma_context *ctx); static inline struct ucma_context *_ucma_find_context(int id, struct ucma_file *file) @@ -139,7 +132,7 @@ static inline struct ucma_context *_ucma_find_context(int id, ctx = xa_load(&ctx_table, id); if (!ctx) ctx = ERR_PTR(-ENOENT); - else if (ctx->file != file || !ctx->cm_id) + else if (ctx->file != file) ctx = ERR_PTR(-EINVAL); return ctx; } @@ -150,12 +143,9 @@ static struct ucma_context *ucma_get_ctx(struct ucma_file *file, int id) xa_lock(&ctx_table); ctx = _ucma_find_context(id, file); - if (!IS_ERR(ctx)) { - if (ctx->closing) - ctx = ERR_PTR(-EIO); - else - refcount_inc(&ctx->ref); - } + if (!IS_ERR(ctx)) + if (!refcount_inc_not_zero(&ctx->ref)) + ctx = ERR_PTR(-ENXIO); xa_unlock(&ctx_table); return ctx; } @@ -183,14 +173,6 @@ static struct ucma_context *ucma_get_ctx_dev(struct ucma_file *file, int id) return ctx; } -static void ucma_close_event_id(struct work_struct *work) -{ - struct ucma_event *uevent_close = container_of(work, struct ucma_event, close_work); - - rdma_destroy_id(uevent_close->cm_id); - kfree(uevent_close); -} - static void ucma_close_id(struct work_struct *work) { struct ucma_context *ctx = container_of(work, struct ucma_context, close_work); @@ -203,6 +185,14 @@ static void ucma_close_id(struct work_struct *work) wait_for_completion(&ctx->comp); /* No new events will be generated after destroying the id. */ rdma_destroy_id(ctx->cm_id); + + /* + * At this point ctx->ref is zero so the only place the ctx can be is in + * a uevent or in __destroy_id(). Since the former doesn't touch + * ctx->cm_id and the latter sync cancels this, there is no races with + * this store. + */ + ctx->cm_id = NULL; } static struct ucma_context *ucma_alloc_ctx(struct ucma_file *file) @@ -216,39 +206,23 @@ static struct ucma_context *ucma_alloc_ctx(struct ucma_file *file) INIT_WORK(&ctx->close_work, ucma_close_id); refcount_set(&ctx->ref, 1); init_completion(&ctx->comp); - INIT_LIST_HEAD(&ctx->mc_list); + /* So list_del() will work if we don't do ucma_finish_ctx() */ + INIT_LIST_HEAD(&ctx->list); ctx->file = file; mutex_init(&ctx->mutex); - if (xa_alloc(&ctx_table, &ctx->id, ctx, xa_limit_32b, GFP_KERNEL)) - goto error; - - list_add_tail(&ctx->list, &file->ctx_list); + if (xa_alloc(&ctx_table, &ctx->id, NULL, xa_limit_32b, GFP_KERNEL)) { + kfree(ctx); + return NULL; + } return ctx; - -error: - kfree(ctx); - return NULL; } -static struct ucma_multicast* ucma_alloc_multicast(struct ucma_context *ctx) +static void ucma_finish_ctx(struct ucma_context *ctx) { - struct ucma_multicast *mc; - - mc = kzalloc(sizeof(*mc), GFP_KERNEL); - if (!mc) - return NULL; - - mc->ctx = ctx; - if (xa_alloc(&multicast_table, &mc->id, NULL, xa_limit_32b, GFP_KERNEL)) - goto error; - - list_add_tail(&mc->list, &ctx->mc_list); - return mc; - -error: - kfree(mc); - return NULL; + lockdep_assert_held(&ctx->file->mut); + list_add_tail(&ctx->list, &ctx->file->ctx_list); + xa_store(&ctx_table, ctx->id, ctx, GFP_KERNEL); } static void ucma_copy_conn_event(struct rdma_ucm_conn_param *dst, @@ -280,10 +254,15 @@ static void ucma_copy_ud_event(struct ib_device *device, dst->qkey = src->qkey; } -static void ucma_set_event_context(struct ucma_context *ctx, - struct rdma_cm_event *event, - struct ucma_event *uevent) +static struct ucma_event *ucma_create_uevent(struct ucma_context *ctx, + struct rdma_cm_event *event) { + struct ucma_event *uevent; + + uevent = kzalloc(sizeof(*uevent), GFP_KERNEL); + if (!uevent) + return NULL; + uevent->ctx = ctx; switch (event->event) { case RDMA_CM_EVENT_MULTICAST_JOIN: @@ -298,44 +277,56 @@ static void ucma_set_event_context(struct ucma_context *ctx, uevent->resp.id = ctx->id; break; } + uevent->resp.event = event->event; + uevent->resp.status = event->status; + if (ctx->cm_id->qp_type == IB_QPT_UD) + ucma_copy_ud_event(ctx->cm_id->device, &uevent->resp.param.ud, + &event->param.ud); + else + ucma_copy_conn_event(&uevent->resp.param.conn, + &event->param.conn); + + uevent->resp.ece.vendor_id = event->ece.vendor_id; + uevent->resp.ece.attr_mod = event->ece.attr_mod; + return uevent; } -/* Called with file->mut locked for the relevant context. */ -static void ucma_removal_event_handler(struct rdma_cm_id *cm_id) +static int ucma_connect_event_handler(struct rdma_cm_id *cm_id, + struct rdma_cm_event *event) { - struct ucma_context *ctx = cm_id->context; - struct ucma_event *con_req_eve; - int event_found = 0; + struct ucma_context *listen_ctx = cm_id->context; + struct ucma_context *ctx; + struct ucma_event *uevent; - if (ctx->destroying) - return; + if (!atomic_add_unless(&listen_ctx->backlog, -1, 0)) + return -ENOMEM; + ctx = ucma_alloc_ctx(listen_ctx->file); + if (!ctx) + goto err_backlog; + ctx->cm_id = cm_id; - /* only if context is pointing to cm_id that it owns it and can be - * queued to be closed, otherwise that cm_id is an inflight one that - * is part of that context event list pending to be detached and - * reattached to its new context as part of ucma_get_event, - * handled separately below. - */ - if (ctx->cm_id == cm_id) { - xa_lock(&ctx_table); - ctx->closing = 1; - xa_unlock(&ctx_table); - queue_work(ctx->file->close_wq, &ctx->close_work); - return; - } + uevent = ucma_create_uevent(listen_ctx, event); + if (!uevent) + goto err_alloc; + uevent->conn_req_ctx = ctx; + uevent->resp.id = ctx->id; - list_for_each_entry(con_req_eve, &ctx->file->event_list, list) { - if (con_req_eve->cm_id == cm_id && - con_req_eve->resp.event == RDMA_CM_EVENT_CONNECT_REQUEST) { - list_del(&con_req_eve->list); - INIT_WORK(&con_req_eve->close_work, ucma_close_event_id); - queue_work(ctx->file->close_wq, &con_req_eve->close_work); - event_found = 1; - break; - } - } - if (!event_found) - pr_err("ucma_removal_event_handler: warning: connect request event wasn't found\n"); + ctx->cm_id->context = ctx; + + mutex_lock(&ctx->file->mut); + ucma_finish_ctx(ctx); + list_add_tail(&uevent->list, &ctx->file->event_list); + mutex_unlock(&ctx->file->mut); + wake_up_interruptible(&ctx->file->poll_wait); + return 0; + +err_alloc: + xa_erase(&ctx_table, ctx->id); + kfree(ctx); +err_backlog: + atomic_inc(&listen_ctx->backlog); + /* Returning error causes the new ID to be destroyed */ + return -ENOMEM; } static int ucma_event_handler(struct rdma_cm_id *cm_id, @@ -343,66 +334,38 @@ static int ucma_event_handler(struct rdma_cm_id *cm_id, { struct ucma_event *uevent; struct ucma_context *ctx = cm_id->context; - int ret = 0; - - uevent = kzalloc(sizeof(*uevent), GFP_KERNEL); - if (!uevent) - return event->event == RDMA_CM_EVENT_CONNECT_REQUEST; - mutex_lock(&ctx->file->mut); - uevent->cm_id = cm_id; - ucma_set_event_context(ctx, event, uevent); - uevent->resp.event = event->event; - uevent->resp.status = event->status; - if (cm_id->qp_type == IB_QPT_UD) - ucma_copy_ud_event(cm_id->device, &uevent->resp.param.ud, - &event->param.ud); - else - ucma_copy_conn_event(&uevent->resp.param.conn, - &event->param.conn); - - uevent->resp.ece.vendor_id = event->ece.vendor_id; - uevent->resp.ece.attr_mod = event->ece.attr_mod; - - if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) { - if (!ctx->backlog) { - ret = -ENOMEM; - kfree(uevent); - goto out; - } - ctx->backlog--; - } else if (!ctx->uid || ctx->cm_id != cm_id) { - /* - * We ignore events for new connections until userspace has set - * their context. This can only happen if an error occurs on a - * new connection before the user accepts it. This is okay, - * since the accept will just fail later. However, we do need - * to release the underlying HW resources in case of a device - * removal event. - */ - if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) - ucma_removal_event_handler(cm_id); + if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) + return ucma_connect_event_handler(cm_id, event); - kfree(uevent); - goto out; + /* + * We ignore events for new connections until userspace has set their + * context. This can only happen if an error occurs on a new connection + * before the user accepts it. This is okay, since the accept will just + * fail later. However, we do need to release the underlying HW + * resources in case of a device removal event. + */ + if (ctx->uid) { + uevent = ucma_create_uevent(ctx, event); + if (!uevent) + return 0; + + mutex_lock(&ctx->file->mut); + list_add_tail(&uevent->list, &ctx->file->event_list); + mutex_unlock(&ctx->file->mut); + wake_up_interruptible(&ctx->file->poll_wait); } - list_add_tail(&uevent->list, &ctx->file->event_list); - wake_up_interruptible(&ctx->file->poll_wait); - if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) - ucma_removal_event_handler(cm_id); -out: - mutex_unlock(&ctx->file->mut); - return ret; + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL && !ctx->destroying) + queue_work(system_unbound_wq, &ctx->close_work); + return 0; } static ssize_t ucma_get_event(struct ucma_file *file, const char __user *inbuf, int in_len, int out_len) { - struct ucma_context *ctx; struct rdma_ucm_get_event cmd; struct ucma_event *uevent; - int ret = 0; /* * Old 32 bit user space does not send the 4 byte padding in the @@ -429,35 +392,25 @@ static ssize_t ucma_get_event(struct ucma_file *file, const char __user *inbuf, mutex_lock(&file->mut); } - uevent = list_entry(file->event_list.next, struct ucma_event, list); - - if (uevent->resp.event == RDMA_CM_EVENT_CONNECT_REQUEST) { - ctx = ucma_alloc_ctx(file); - if (!ctx) { - ret = -ENOMEM; - goto done; - } - uevent->ctx->backlog++; - ctx->cm_id = uevent->cm_id; - ctx->cm_id->context = ctx; - uevent->resp.id = ctx->id; - } + uevent = list_first_entry(&file->event_list, struct ucma_event, list); if (copy_to_user(u64_to_user_ptr(cmd.response), &uevent->resp, min_t(size_t, out_len, sizeof(uevent->resp)))) { - ret = -EFAULT; - goto done; + mutex_unlock(&file->mut); + return -EFAULT; } list_del(&uevent->list); uevent->ctx->events_reported++; if (uevent->mc) uevent->mc->events_reported++; - kfree(uevent); -done: + if (uevent->resp.event == RDMA_CM_EVENT_CONNECT_REQUEST) + atomic_inc(&uevent->ctx->backlog); mutex_unlock(&file->mut); - return ret; + + kfree(uevent); + return 0; } static int ucma_get_qp_type(struct rdma_ucm_create_id *cmd, enum ib_qp_type *qp_type) @@ -498,58 +451,60 @@ static ssize_t ucma_create_id(struct ucma_file *file, const char __user *inbuf, if (ret) return ret; - mutex_lock(&file->mut); ctx = ucma_alloc_ctx(file); - mutex_unlock(&file->mut); if (!ctx) return -ENOMEM; ctx->uid = cmd.uid; - cm_id = __rdma_create_id(current->nsproxy->net_ns, - ucma_event_handler, ctx, cmd.ps, qp_type, NULL); + cm_id = rdma_create_user_id(ucma_event_handler, ctx, cmd.ps, qp_type); if (IS_ERR(cm_id)) { ret = PTR_ERR(cm_id); goto err1; } + ctx->cm_id = cm_id; resp.id = ctx->id; if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp))) { - ret = -EFAULT; - goto err2; + xa_erase(&ctx_table, ctx->id); + __destroy_id(ctx); + return -EFAULT; } - ctx->cm_id = cm_id; + mutex_lock(&file->mut); + ucma_finish_ctx(ctx); + mutex_unlock(&file->mut); return 0; -err2: - rdma_destroy_id(cm_id); err1: xa_erase(&ctx_table, ctx->id); - mutex_lock(&file->mut); - list_del(&ctx->list); - mutex_unlock(&file->mut); kfree(ctx); return ret; } static void ucma_cleanup_multicast(struct ucma_context *ctx) { - struct ucma_multicast *mc, *tmp; + struct ucma_multicast *mc; + unsigned long index; - mutex_lock(&ctx->file->mut); - list_for_each_entry_safe(mc, tmp, &ctx->mc_list, list) { - list_del(&mc->list); - xa_erase(&multicast_table, mc->id); + xa_for_each(&multicast_table, index, mc) { + if (mc->ctx != ctx) + continue; + /* + * At this point mc->ctx->ref is 0 so the mc cannot leave the + * lock on the reader and this is enough serialization + */ + xa_erase(&multicast_table, index); kfree(mc); } - mutex_unlock(&ctx->file->mut); } static void ucma_cleanup_mc_events(struct ucma_multicast *mc) { struct ucma_event *uevent, *tmp; + rdma_lock_handler(mc->ctx->cm_id); + mutex_lock(&mc->ctx->file->mut); list_for_each_entry_safe(uevent, tmp, &mc->ctx->file->event_list, list) { if (uevent->mc != mc) continue; @@ -557,6 +512,8 @@ static void ucma_cleanup_mc_events(struct ucma_multicast *mc) list_del(&uevent->list); kfree(uevent); } + mutex_unlock(&mc->ctx->file->mut); + rdma_unlock_handler(mc->ctx->cm_id); } /* @@ -564,10 +521,6 @@ static void ucma_cleanup_mc_events(struct ucma_multicast *mc) * this point, no new events will be reported from the hardware. However, we * still need to cleanup the UCMA context for this ID. Specifically, there * might be events that have not yet been consumed by the user space software. - * These might include pending connect requests which we have not completed - * processing. We cannot call rdma_destroy_id while holding the lock of the - * context (file->mut), as it might cause a deadlock. We therefore extract all - * relevant events from the context pending events list while holding the * mutex. After that we release them as needed. */ static int ucma_free_ctx(struct ucma_context *ctx) @@ -576,31 +529,57 @@ static int ucma_free_ctx(struct ucma_context *ctx) struct ucma_event *uevent, *tmp; LIST_HEAD(list); - ucma_cleanup_multicast(ctx); /* Cleanup events not yet reported to the user. */ mutex_lock(&ctx->file->mut); list_for_each_entry_safe(uevent, tmp, &ctx->file->event_list, list) { - if (uevent->ctx == ctx) + if (uevent->ctx == ctx || uevent->conn_req_ctx == ctx) list_move_tail(&uevent->list, &list); } list_del(&ctx->list); + events_reported = ctx->events_reported; mutex_unlock(&ctx->file->mut); + /* + * If this was a listening ID then any connections spawned from it + * that have not been delivered to userspace are cleaned up too. + * Must be done outside any locks. + */ list_for_each_entry_safe(uevent, tmp, &list, list) { list_del(&uevent->list); - if (uevent->resp.event == RDMA_CM_EVENT_CONNECT_REQUEST) - rdma_destroy_id(uevent->cm_id); + if (uevent->resp.event == RDMA_CM_EVENT_CONNECT_REQUEST && + uevent->conn_req_ctx != ctx) + __destroy_id(uevent->conn_req_ctx); kfree(uevent); } - events_reported = ctx->events_reported; mutex_destroy(&ctx->mutex); kfree(ctx); return events_reported; } +static int __destroy_id(struct ucma_context *ctx) +{ + /* + * If the refcount is already 0 then ucma_close_id() has already + * destroyed the cm_id, otherwise holding the refcount keeps cm_id + * valid. Prevent queue_work() from being called. + */ + if (refcount_inc_not_zero(&ctx->ref)) { + rdma_lock_handler(ctx->cm_id); + ctx->destroying = 1; + rdma_unlock_handler(ctx->cm_id); + ucma_put_ctx(ctx); + } + + cancel_work_sync(&ctx->close_work); + /* At this point it's guaranteed that there is no inflight closing task */ + if (ctx->cm_id) + ucma_close_id(&ctx->close_work); + return ucma_free_ctx(ctx); +} + static ssize_t ucma_destroy_id(struct ucma_file *file, const char __user *inbuf, int in_len, int out_len) { @@ -624,24 +603,7 @@ static ssize_t ucma_destroy_id(struct ucma_file *file, const char __user *inbuf, if (IS_ERR(ctx)) return PTR_ERR(ctx); - mutex_lock(&ctx->file->mut); - ctx->destroying = 1; - mutex_unlock(&ctx->file->mut); - - flush_workqueue(ctx->file->close_wq); - /* At this point it's guaranteed that there is no inflight - * closing task */ - xa_lock(&ctx_table); - if (!ctx->closing) { - xa_unlock(&ctx_table); - ucma_put_ctx(ctx); - wait_for_completion(&ctx->comp); - rdma_destroy_id(ctx->cm_id); - } else { - xa_unlock(&ctx_table); - } - - resp.events_reported = ucma_free_ctx(ctx); + resp.events_reported = __destroy_id(ctx); if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp))) ret = -EFAULT; @@ -1124,10 +1086,12 @@ static ssize_t ucma_listen(struct ucma_file *file, const char __user *inbuf, if (IS_ERR(ctx)) return PTR_ERR(ctx); - ctx->backlog = cmd.backlog > 0 && cmd.backlog < max_backlog ? - cmd.backlog : max_backlog; + if (cmd.backlog <= 0 || cmd.backlog > max_backlog) + cmd.backlog = max_backlog; + atomic_set(&ctx->backlog, cmd.backlog); + mutex_lock(&ctx->mutex); - ret = rdma_listen(ctx->cm_id, ctx->backlog); + ret = rdma_listen(ctx->cm_id, cmd.backlog); mutex_unlock(&ctx->mutex); ucma_put_ctx(ctx); return ret; @@ -1160,16 +1124,20 @@ static ssize_t ucma_accept(struct ucma_file *file, const char __user *inbuf, if (cmd.conn_param.valid) { ucma_copy_conn_param(ctx->cm_id, &conn_param, &cmd.conn_param); - mutex_lock(&file->mut); mutex_lock(&ctx->mutex); - ret = __rdma_accept_ece(ctx->cm_id, &conn_param, NULL, &ece); - mutex_unlock(&ctx->mutex); - if (!ret) + rdma_lock_handler(ctx->cm_id); + ret = rdma_accept_ece(ctx->cm_id, &conn_param, &ece); + if (!ret) { + /* The uid must be set atomically with the handler */ ctx->uid = cmd.uid; - mutex_unlock(&file->mut); + } + rdma_unlock_handler(ctx->cm_id); + mutex_unlock(&ctx->mutex); } else { mutex_lock(&ctx->mutex); - ret = __rdma_accept_ece(ctx->cm_id, NULL, NULL, &ece); + rdma_lock_handler(ctx->cm_id); + ret = rdma_accept_ece(ctx->cm_id, NULL, &ece); + rdma_unlock_handler(ctx->cm_id); mutex_unlock(&ctx->mutex); } ucma_put_ctx(ctx); @@ -1482,44 +1450,52 @@ static ssize_t ucma_process_join(struct ucma_file *file, if (IS_ERR(ctx)) return PTR_ERR(ctx); - mutex_lock(&file->mut); - mc = ucma_alloc_multicast(ctx); + mc = kzalloc(sizeof(*mc), GFP_KERNEL); if (!mc) { ret = -ENOMEM; - goto err1; + goto err_put_ctx; } + + mc->ctx = ctx; mc->join_state = join_state; mc->uid = cmd->uid; memcpy(&mc->addr, addr, cmd->addr_size); + + if (xa_alloc(&multicast_table, &mc->id, NULL, xa_limit_32b, + GFP_KERNEL)) { + ret = -ENOMEM; + goto err_free_mc; + } + mutex_lock(&ctx->mutex); ret = rdma_join_multicast(ctx->cm_id, (struct sockaddr *)&mc->addr, join_state, mc); mutex_unlock(&ctx->mutex); if (ret) - goto err2; + goto err_xa_erase; resp.id = mc->id; if (copy_to_user(u64_to_user_ptr(cmd->response), &resp, sizeof(resp))) { ret = -EFAULT; - goto err3; + goto err_leave_multicast; } xa_store(&multicast_table, mc->id, mc, 0); - mutex_unlock(&file->mut); ucma_put_ctx(ctx); return 0; -err3: +err_leave_multicast: + mutex_lock(&ctx->mutex); rdma_leave_multicast(ctx->cm_id, (struct sockaddr *) &mc->addr); + mutex_unlock(&ctx->mutex); ucma_cleanup_mc_events(mc); -err2: +err_xa_erase: xa_erase(&multicast_table, mc->id); - list_del(&mc->list); +err_free_mc: kfree(mc); -err1: - mutex_unlock(&file->mut); +err_put_ctx: ucma_put_ctx(ctx); return ret; } @@ -1581,7 +1557,7 @@ static ssize_t ucma_leave_multicast(struct ucma_file *file, mc = xa_load(&multicast_table, cmd.id); if (!mc) mc = ERR_PTR(-ENOENT); - else if (mc->ctx->file != file) + else if (READ_ONCE(mc->ctx->file) != file) mc = ERR_PTR(-EINVAL); else if (!refcount_inc_not_zero(&mc->ctx->ref)) mc = ERR_PTR(-ENXIO); @@ -1598,10 +1574,7 @@ static ssize_t ucma_leave_multicast(struct ucma_file *file, rdma_leave_multicast(mc->ctx->cm_id, (struct sockaddr *) &mc->addr); mutex_unlock(&mc->ctx->mutex); - mutex_lock(&mc->ctx->file->mut); ucma_cleanup_mc_events(mc); - list_del(&mc->list); - mutex_unlock(&mc->ctx->file->mut); ucma_put_ctx(mc->ctx); resp.events_reported = mc->events_reported; @@ -1614,45 +1587,15 @@ out: return ret; } -static void ucma_lock_files(struct ucma_file *file1, struct ucma_file *file2) -{ - /* Acquire mutex's based on pointer comparison to prevent deadlock. */ - if (file1 < file2) { - mutex_lock(&file1->mut); - mutex_lock_nested(&file2->mut, SINGLE_DEPTH_NESTING); - } else { - mutex_lock(&file2->mut); - mutex_lock_nested(&file1->mut, SINGLE_DEPTH_NESTING); - } -} - -static void ucma_unlock_files(struct ucma_file *file1, struct ucma_file *file2) -{ - if (file1 < file2) { - mutex_unlock(&file2->mut); - mutex_unlock(&file1->mut); - } else { - mutex_unlock(&file1->mut); - mutex_unlock(&file2->mut); - } -} - -static void ucma_move_events(struct ucma_context *ctx, struct ucma_file *file) -{ - struct ucma_event *uevent, *tmp; - - list_for_each_entry_safe(uevent, tmp, &ctx->file->event_list, list) - if (uevent->ctx == ctx) - list_move_tail(&uevent->list, &file->event_list); -} - static ssize_t ucma_migrate_id(struct ucma_file *new_file, const char __user *inbuf, int in_len, int out_len) { struct rdma_ucm_migrate_id cmd; struct rdma_ucm_migrate_resp resp; + struct ucma_event *uevent, *tmp; struct ucma_context *ctx; + LIST_HEAD(event_list); struct fd f; struct ucma_file *cur_file; int ret = 0; @@ -1668,40 +1611,53 @@ static ssize_t ucma_migrate_id(struct ucma_file *new_file, ret = -EINVAL; goto file_put; } + cur_file = f.file->private_data; /* Validate current fd and prevent destruction of id. */ - ctx = ucma_get_ctx(f.file->private_data, cmd.id); + ctx = ucma_get_ctx(cur_file, cmd.id); if (IS_ERR(ctx)) { ret = PTR_ERR(ctx); goto file_put; } - cur_file = ctx->file; - if (cur_file == new_file) { - resp.events_reported = ctx->events_reported; - goto response; - } - + rdma_lock_handler(ctx->cm_id); /* - * Migrate events between fd's, maintaining order, and avoiding new - * events being added before existing events. + * ctx->file can only be changed under the handler & xa_lock. xa_load() + * must be checked again to ensure the ctx hasn't begun destruction + * since the ucma_get_ctx(). */ - ucma_lock_files(cur_file, new_file); xa_lock(&ctx_table); - - list_move_tail(&ctx->list, &new_file->ctx_list); - ucma_move_events(ctx, new_file); + if (_ucma_find_context(cmd.id, cur_file) != ctx) { + xa_unlock(&ctx_table); + ret = -ENOENT; + goto err_unlock; + } ctx->file = new_file; + xa_unlock(&ctx_table); + + mutex_lock(&cur_file->mut); + list_del(&ctx->list); + /* + * At this point lock_handler() prevents addition of new uevents for + * this ctx. + */ + list_for_each_entry_safe(uevent, tmp, &cur_file->event_list, list) + if (uevent->ctx == ctx) + list_move_tail(&uevent->list, &event_list); resp.events_reported = ctx->events_reported; + mutex_unlock(&cur_file->mut); - xa_unlock(&ctx_table); - ucma_unlock_files(cur_file, new_file); + mutex_lock(&new_file->mut); + list_add_tail(&ctx->list, &new_file->ctx_list); + list_splice_tail(&event_list, &new_file->event_list); + mutex_unlock(&new_file->mut); -response: if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp))) ret = -EFAULT; +err_unlock: + rdma_unlock_handler(ctx->cm_id); ucma_put_ctx(ctx); file_put: fdput(f); @@ -1801,13 +1757,6 @@ static int ucma_open(struct inode *inode, struct file *filp) if (!file) return -ENOMEM; - file->close_wq = alloc_ordered_workqueue("ucma_close_id", - WQ_MEM_RECLAIM); - if (!file->close_wq) { - kfree(file); - return -ENOMEM; - } - INIT_LIST_HEAD(&file->event_list); INIT_LIST_HEAD(&file->ctx_list); init_waitqueue_head(&file->poll_wait); @@ -1822,37 +1771,22 @@ static int ucma_open(struct inode *inode, struct file *filp) static int ucma_close(struct inode *inode, struct file *filp) { struct ucma_file *file = filp->private_data; - struct ucma_context *ctx, *tmp; - mutex_lock(&file->mut); - list_for_each_entry_safe(ctx, tmp, &file->ctx_list, list) { - ctx->destroying = 1; - mutex_unlock(&file->mut); + /* + * All paths that touch ctx_list or ctx_list starting from write() are + * prevented by this being a FD release function. The list_add_tail() in + * ucma_connect_event_handler() can run concurrently, however it only + * adds to the list *after* a listening ID. By only reading the first of + * the list, and relying on __destroy_id() to block + * ucma_connect_event_handler(), no additional locking is needed. + */ + while (!list_empty(&file->ctx_list)) { + struct ucma_context *ctx = list_first_entry( + &file->ctx_list, struct ucma_context, list); xa_erase(&ctx_table, ctx->id); - flush_workqueue(file->close_wq); - /* At that step once ctx was marked as destroying and workqueue - * was flushed we are safe from any inflights handlers that - * might put other closing task. - */ - xa_lock(&ctx_table); - if (!ctx->closing) { - xa_unlock(&ctx_table); - ucma_put_ctx(ctx); - wait_for_completion(&ctx->comp); - /* rdma_destroy_id ensures that no event handlers are - * inflight for that id before releasing it. - */ - rdma_destroy_id(ctx->cm_id); - } else { - xa_unlock(&ctx_table); - } - - ucma_free_ctx(ctx); - mutex_lock(&file->mut); + __destroy_id(ctx); } - mutex_unlock(&file->mut); - destroy_workqueue(file->close_wq); kfree(file); return 0; } diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 831bff8d52e5..e9fecbdf391b 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -39,6 +39,7 @@ #include <linux/export.h> #include <linux/slab.h> #include <linux/pagemap.h> +#include <linux/count_zeros.h> #include <rdma/ib_umem_odp.h> #include "uverbs.h" @@ -60,73 +61,6 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d sg_free_table(&umem->sg_head); } -/* ib_umem_add_sg_table - Add N contiguous pages to scatter table - * - * sg: current scatterlist entry - * page_list: array of npage struct page pointers - * npages: number of pages in page_list - * max_seg_sz: maximum segment size in bytes - * nents: [out] number of entries in the scatterlist - * - * Return new end of scatterlist - */ -static struct scatterlist *ib_umem_add_sg_table(struct scatterlist *sg, - struct page **page_list, - unsigned long npages, - unsigned int max_seg_sz, - int *nents) -{ - unsigned long first_pfn; - unsigned long i = 0; - bool update_cur_sg = false; - bool first = !sg_page(sg); - - /* Check if new page_list is contiguous with end of previous page_list. - * sg->length here is a multiple of PAGE_SIZE and sg->offset is 0. - */ - if (!first && (page_to_pfn(sg_page(sg)) + (sg->length >> PAGE_SHIFT) == - page_to_pfn(page_list[0]))) - update_cur_sg = true; - - while (i != npages) { - unsigned long len; - struct page *first_page = page_list[i]; - - first_pfn = page_to_pfn(first_page); - - /* Compute the number of contiguous pages we have starting - * at i - */ - for (len = 0; i != npages && - first_pfn + len == page_to_pfn(page_list[i]) && - len < (max_seg_sz >> PAGE_SHIFT); - len++) - i++; - - /* Squash N contiguous pages from page_list into current sge */ - if (update_cur_sg) { - if ((max_seg_sz - sg->length) >= (len << PAGE_SHIFT)) { - sg_set_page(sg, sg_page(sg), - sg->length + (len << PAGE_SHIFT), - 0); - update_cur_sg = false; - continue; - } - update_cur_sg = false; - } - - /* Squash N contiguous pages into next sge or first sge */ - if (!first) - sg = sg_next(sg); - - (*nents)++; - sg_set_page(sg, first_page, len << PAGE_SHIFT, 0); - first = false; - } - - return sg; -} - /** * ib_umem_find_best_pgsz - Find best HW page size to use for this MR * @@ -146,18 +80,28 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, unsigned long virt) { struct scatterlist *sg; - unsigned int best_pg_bit; unsigned long va, pgoff; dma_addr_t mask; int i; + /* rdma_for_each_block() has a bug if the page size is smaller than the + * page size used to build the umem. For now prevent smaller page sizes + * from being returned. + */ + pgsz_bitmap &= GENMASK(BITS_PER_LONG - 1, PAGE_SHIFT); + /* At minimum, drivers must support PAGE_SIZE or smaller */ if (WARN_ON(!(pgsz_bitmap & GENMASK(PAGE_SHIFT, 0)))) return 0; - va = virt; - /* max page size not to exceed MR length */ - mask = roundup_pow_of_two(umem->length); + umem->iova = va = virt; + /* The best result is the smallest page size that results in the minimum + * number of required pages. Compute the largest page size that could + * work based on VA address bits that don't change. + */ + mask = pgsz_bitmap & + GENMASK(BITS_PER_LONG - 1, + bits_per((umem->length - 1 + virt) ^ virt)); /* offset into first SGL */ pgoff = umem->address & ~PAGE_MASK; @@ -175,9 +119,14 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, mask |= va; pgoff = 0; } - best_pg_bit = rdma_find_pg_bit(mask, pgsz_bitmap); - return BIT_ULL(best_pg_bit); + /* The mask accumulates 1's in each position where the VA and physical + * address differ, thus the length of trailing 0 is the largest page + * size that can pass the VA through to the physical. + */ + if (mask) + pgsz_bitmap &= GENMASK(count_trailing_zeros(mask), 0); + return rounddown_pow_of_two(pgsz_bitmap); } EXPORT_SYMBOL(ib_umem_find_best_pgsz); @@ -201,7 +150,7 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, struct mm_struct *mm; unsigned long npages; int ret; - struct scatterlist *sg; + struct scatterlist *sg = NULL; unsigned int gup_flags = FOLL_WRITE; /* @@ -224,6 +173,11 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, umem->ibdev = device; umem->length = size; umem->address = addr; + /* + * Drivers should call ib_umem_find_best_pgsz() to set the iova + * correctly. + */ + umem->iova = addr; umem->writable = ib_access_writable(access); umem->owning_mm = mm = current->mm; mmgrab(mm); @@ -251,15 +205,9 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, cur_base = addr & PAGE_MASK; - ret = sg_alloc_table(&umem->sg_head, npages, GFP_KERNEL); - if (ret) - goto vma; - if (!umem->writable) gup_flags |= FOLL_FORCE; - sg = umem->sg_head.sgl; - while (npages) { cond_resched(); ret = pin_user_pages_fast(cur_base, @@ -271,15 +219,19 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, goto umem_release; cur_base += ret * PAGE_SIZE; - npages -= ret; - - sg = ib_umem_add_sg_table(sg, page_list, ret, - dma_get_max_seg_size(device->dma_device), - &umem->sg_nents); + npages -= ret; + sg = __sg_alloc_table_from_pages( + &umem->sg_head, page_list, ret, 0, ret << PAGE_SHIFT, + dma_get_max_seg_size(device->dma_device), sg, npages, + GFP_KERNEL); + umem->sg_nents = umem->sg_head.nents; + if (IS_ERR(sg)) { + unpin_user_pages_dirty_lock(page_list, ret, 0); + ret = PTR_ERR(sg); + goto umem_release; + } } - sg_mark_end(sg); - if (access & IB_ACCESS_RELAXED_ORDERING) dma_attr |= DMA_ATTR_WEAK_ORDERING; @@ -297,7 +249,6 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, umem_release: __ib_umem_release(device, umem, 0); -vma: atomic64_sub(ib_umem_num_pages(umem), &mm->pinned_vm); out: free_page((unsigned long) page_list); @@ -329,18 +280,6 @@ void ib_umem_release(struct ib_umem *umem) } EXPORT_SYMBOL(ib_umem_release); -int ib_umem_page_count(struct ib_umem *umem) -{ - int i, n = 0; - struct scatterlist *sg; - - for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) - n += sg_dma_len(sg) >> PAGE_SHIFT; - - return n; -} -EXPORT_SYMBOL(ib_umem_page_count); - /* * Copy from the given ib_umem's pages to the given buffer. * diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c index cc6b4befde7c..323f6cf00682 100644 --- a/drivers/infiniband/core/umem_odp.c +++ b/drivers/infiniband/core/umem_odp.c @@ -40,6 +40,7 @@ #include <linux/vmalloc.h> #include <linux/hugetlb.h> #include <linux/interval_tree.h> +#include <linux/hmm.h> #include <linux/pagemap.h> #include <rdma/ib_verbs.h> @@ -60,7 +61,7 @@ static inline int ib_init_umem_odp(struct ib_umem_odp *umem_odp, size_t page_size = 1UL << umem_odp->page_shift; unsigned long start; unsigned long end; - size_t pages; + size_t ndmas, npfns; start = ALIGN_DOWN(umem_odp->umem.address, page_size); if (check_add_overflow(umem_odp->umem.address, @@ -71,20 +72,21 @@ static inline int ib_init_umem_odp(struct ib_umem_odp *umem_odp, if (unlikely(end < page_size)) return -EOVERFLOW; - pages = (end - start) >> umem_odp->page_shift; - if (!pages) + ndmas = (end - start) >> umem_odp->page_shift; + if (!ndmas) return -EINVAL; - umem_odp->page_list = kvcalloc( - pages, sizeof(*umem_odp->page_list), GFP_KERNEL); - if (!umem_odp->page_list) + npfns = (end - start) >> PAGE_SHIFT; + umem_odp->pfn_list = kvcalloc( + npfns, sizeof(*umem_odp->pfn_list), GFP_KERNEL); + if (!umem_odp->pfn_list) return -ENOMEM; umem_odp->dma_list = kvcalloc( - pages, sizeof(*umem_odp->dma_list), GFP_KERNEL); + ndmas, sizeof(*umem_odp->dma_list), GFP_KERNEL); if (!umem_odp->dma_list) { ret = -ENOMEM; - goto out_page_list; + goto out_pfn_list; } ret = mmu_interval_notifier_insert(&umem_odp->notifier, @@ -98,8 +100,8 @@ static inline int ib_init_umem_odp(struct ib_umem_odp *umem_odp, out_dma_list: kvfree(umem_odp->dma_list); -out_page_list: - kvfree(umem_odp->page_list); +out_pfn_list: + kvfree(umem_odp->pfn_list); return ret; } @@ -276,7 +278,7 @@ void ib_umem_odp_release(struct ib_umem_odp *umem_odp) mutex_unlock(&umem_odp->umem_mutex); mmu_interval_notifier_remove(&umem_odp->notifier); kvfree(umem_odp->dma_list); - kvfree(umem_odp->page_list); + kvfree(umem_odp->pfn_list); } put_pid(umem_odp->tgid); kfree(umem_odp); @@ -287,87 +289,56 @@ EXPORT_SYMBOL(ib_umem_odp_release); * Map for DMA and insert a single page into the on-demand paging page tables. * * @umem: the umem to insert the page to. - * @page_index: index in the umem to add the page to. + * @dma_index: index in the umem to add the dma to. * @page: the page struct to map and add. * @access_mask: access permissions needed for this page. * @current_seq: sequence number for synchronization with invalidations. * the sequence number is taken from * umem_odp->notifiers_seq. * - * The function returns -EFAULT if the DMA mapping operation fails. It returns - * -EAGAIN if a concurrent invalidation prevents us from updating the page. + * The function returns -EFAULT if the DMA mapping operation fails. * - * The page is released via put_page even if the operation failed. For on-demand - * pinning, the page is released whenever it isn't stored in the umem. */ static int ib_umem_odp_map_dma_single_page( struct ib_umem_odp *umem_odp, - unsigned int page_index, + unsigned int dma_index, struct page *page, - u64 access_mask, - unsigned long current_seq) + u64 access_mask) { struct ib_device *dev = umem_odp->umem.ibdev; - dma_addr_t dma_addr; - int ret = 0; + dma_addr_t *dma_addr = &umem_odp->dma_list[dma_index]; - if (mmu_interval_check_retry(&umem_odp->notifier, current_seq)) { - ret = -EAGAIN; - goto out; - } - if (!(umem_odp->dma_list[page_index])) { - dma_addr = - ib_dma_map_page(dev, page, 0, BIT(umem_odp->page_shift), - DMA_BIDIRECTIONAL); - if (ib_dma_mapping_error(dev, dma_addr)) { - ret = -EFAULT; - goto out; - } - umem_odp->dma_list[page_index] = dma_addr | access_mask; - umem_odp->page_list[page_index] = page; - umem_odp->npages++; - } else if (umem_odp->page_list[page_index] == page) { - umem_odp->dma_list[page_index] |= access_mask; - } else { + if (*dma_addr) { /* - * This is a race here where we could have done: - * - * CPU0 CPU1 - * get_user_pages() - * invalidate() - * page_fault() - * mutex_lock(umem_mutex) - * page from GUP != page in ODP - * - * It should be prevented by the retry test above as reading - * the seq number should be reliable under the - * umem_mutex. Thus something is really not working right if - * things get here. + * If the page is already dma mapped it means it went through + * a non-invalidating trasition, like read-only to writable. + * Resync the flags. */ - WARN(true, - "Got different pages in IB device and from get_user_pages. IB device page: %p, gup page: %p\n", - umem_odp->page_list[page_index], page); - ret = -EAGAIN; + *dma_addr = (*dma_addr & ODP_DMA_ADDR_MASK) | access_mask; + return 0; } -out: - put_page(page); - return ret; + *dma_addr = ib_dma_map_page(dev, page, 0, 1 << umem_odp->page_shift, + DMA_BIDIRECTIONAL); + if (ib_dma_mapping_error(dev, *dma_addr)) { + *dma_addr = 0; + return -EFAULT; + } + umem_odp->npages++; + *dma_addr |= access_mask; + return 0; } /** - * ib_umem_odp_map_dma_pages - Pin and DMA map userspace memory in an ODP MR. + * ib_umem_odp_map_dma_and_lock - DMA map userspace memory in an ODP MR and lock it. * - * Pins the range of pages passed in the argument, and maps them to - * DMA addresses. The DMA addresses of the mapped pages is updated in - * umem_odp->dma_list. + * Maps the range passed in the argument to DMA addresses. + * The DMA addresses of the mapped pages is updated in umem_odp->dma_list. + * Upon success the ODP MR will be locked to let caller complete its device + * page table update. * * Returns the number of pages mapped in success, negative error code * for failure. - * An -EAGAIN error code is returned when a concurrent mmu notifier prevents - * the function from completing its task. - * An -ENOENT error code indicates that userspace process is being terminated - * and mm was already destroyed. * @umem_odp: the umem to map and pin * @user_virt: the address from which we need to map. * @bcnt: the minimal number of bytes to pin and map. The mapping might be @@ -376,21 +347,19 @@ out: * the return value. * @access_mask: bit mask of the requested access permissions for the given * range. - * @current_seq: the MMU notifiers sequance value for synchronization with - * invalidations. the sequance number is read from - * umem_odp->notifiers_seq before calling this function + * @fault: is faulting required for the given range */ -int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt, - u64 bcnt, u64 access_mask, - unsigned long current_seq) +int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, + u64 bcnt, u64 access_mask, bool fault) + __acquires(&umem_odp->umem_mutex) { struct task_struct *owning_process = NULL; struct mm_struct *owning_mm = umem_odp->umem.owning_mm; - struct page **local_page_list = NULL; - u64 page_mask, off; - int j, k, ret = 0, start_idx, npages = 0; - unsigned int flags = 0, page_shift; - phys_addr_t p = 0; + int pfn_index, dma_index, ret = 0, start_idx; + unsigned int page_shift, hmm_order, pfn_start_idx; + unsigned long num_pfns, current_seq; + struct hmm_range range = {}; + unsigned long timeout; if (access_mask == 0) return -EINVAL; @@ -399,15 +368,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt, user_virt + bcnt > ib_umem_end(umem_odp)) return -EFAULT; - local_page_list = (struct page **)__get_free_page(GFP_KERNEL); - if (!local_page_list) - return -ENOMEM; - page_shift = umem_odp->page_shift; - page_mask = ~(BIT(page_shift) - 1); - off = user_virt & (~page_mask); - user_virt = user_virt & page_mask; - bcnt += off; /* Charge for the first page offset as well. */ /* * owning_process is allowed to be NULL, this means somehow the mm is @@ -420,99 +381,104 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt, goto out_put_task; } - if (access_mask & ODP_WRITE_ALLOWED_BIT) - flags |= FOLL_WRITE; + range.notifier = &umem_odp->notifier; + range.start = ALIGN_DOWN(user_virt, 1UL << page_shift); + range.end = ALIGN(user_virt + bcnt, 1UL << page_shift); + pfn_start_idx = (range.start - ib_umem_start(umem_odp)) >> PAGE_SHIFT; + num_pfns = (range.end - range.start) >> PAGE_SHIFT; + if (fault) { + range.default_flags = HMM_PFN_REQ_FAULT; - start_idx = (user_virt - ib_umem_start(umem_odp)) >> page_shift; - k = start_idx; + if (access_mask & ODP_WRITE_ALLOWED_BIT) + range.default_flags |= HMM_PFN_REQ_WRITE; + } - while (bcnt > 0) { - const size_t gup_num_pages = min_t(size_t, - ALIGN(bcnt, PAGE_SIZE) / PAGE_SIZE, - PAGE_SIZE / sizeof(struct page *)); + range.hmm_pfns = &(umem_odp->pfn_list[pfn_start_idx]); + timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); - mmap_read_lock(owning_mm); - /* - * Note: this might result in redundent page getting. We can - * avoid this by checking dma_list to be 0 before calling - * get_user_pages. However, this make the code much more - * complex (and doesn't gain us much performance in most use - * cases). - */ - npages = get_user_pages_remote(owning_mm, - user_virt, gup_num_pages, - flags, local_page_list, NULL, NULL); - mmap_read_unlock(owning_mm); - - if (npages < 0) { - if (npages != -EAGAIN) - pr_warn("fail to get %zu user pages with error %d\n", gup_num_pages, npages); - else - pr_debug("fail to get %zu user pages with error %d\n", gup_num_pages, npages); - break; - } +retry: + current_seq = range.notifier_seq = + mmu_interval_read_begin(&umem_odp->notifier); - bcnt -= min_t(size_t, npages << PAGE_SHIFT, bcnt); - mutex_lock(&umem_odp->umem_mutex); - for (j = 0; j < npages; j++, user_virt += PAGE_SIZE) { - if (user_virt & ~page_mask) { - p += PAGE_SIZE; - if (page_to_phys(local_page_list[j]) != p) { - ret = -EFAULT; - break; - } - put_page(local_page_list[j]); - continue; - } + mmap_read_lock(owning_mm); + ret = hmm_range_fault(&range); + mmap_read_unlock(owning_mm); + if (unlikely(ret)) { + if (ret == -EBUSY && !time_after(jiffies, timeout)) + goto retry; + goto out_put_mm; + } - ret = ib_umem_odp_map_dma_single_page( - umem_odp, k, local_page_list[j], - access_mask, current_seq); - if (ret < 0) { - if (ret != -EAGAIN) - pr_warn("ib_umem_odp_map_dma_single_page failed with error %d\n", ret); - else - pr_debug("ib_umem_odp_map_dma_single_page failed with error %d\n", ret); - break; - } + start_idx = (range.start - ib_umem_start(umem_odp)) >> page_shift; + dma_index = start_idx; - p = page_to_phys(local_page_list[j]); - k++; - } + mutex_lock(&umem_odp->umem_mutex); + if (mmu_interval_read_retry(&umem_odp->notifier, current_seq)) { mutex_unlock(&umem_odp->umem_mutex); + goto retry; + } - if (ret < 0) { + for (pfn_index = 0; pfn_index < num_pfns; + pfn_index += 1 << (page_shift - PAGE_SHIFT), dma_index++) { + + if (fault) { /* - * Release pages, remembering that the first page - * to hit an error was already released by - * ib_umem_odp_map_dma_single_page(). + * Since we asked for hmm_range_fault() to populate + * pages it shouldn't return an error entry on success. */ - if (npages - (j + 1) > 0) - release_pages(&local_page_list[j+1], - npages - (j + 1)); + WARN_ON(range.hmm_pfns[pfn_index] & HMM_PFN_ERROR); + WARN_ON(!(range.hmm_pfns[pfn_index] & HMM_PFN_VALID)); + } else { + if (!(range.hmm_pfns[pfn_index] & HMM_PFN_VALID)) { + WARN_ON(umem_odp->dma_list[dma_index]); + continue; + } + access_mask = ODP_READ_ALLOWED_BIT; + if (range.hmm_pfns[pfn_index] & HMM_PFN_WRITE) + access_mask |= ODP_WRITE_ALLOWED_BIT; + } + + hmm_order = hmm_pfn_to_map_order(range.hmm_pfns[pfn_index]); + /* If a hugepage was detected and ODP wasn't set for, the umem + * page_shift will be used, the opposite case is an error. + */ + if (hmm_order + PAGE_SHIFT < page_shift) { + ret = -EINVAL; + ibdev_dbg(umem_odp->umem.ibdev, + "%s: un-expected hmm_order %d, page_shift %d\n", + __func__, hmm_order, page_shift); break; } - } - if (ret >= 0) { - if (npages < 0 && k == start_idx) - ret = npages; - else - ret = k - start_idx; + ret = ib_umem_odp_map_dma_single_page( + umem_odp, dma_index, hmm_pfn_to_page(range.hmm_pfns[pfn_index]), + access_mask); + if (ret < 0) { + ibdev_dbg(umem_odp->umem.ibdev, + "ib_umem_odp_map_dma_single_page failed with error %d\n", ret); + break; + } } + /* upon sucesss lock should stay on hold for the callee */ + if (!ret) + ret = dma_index - start_idx; + else + mutex_unlock(&umem_odp->umem_mutex); +out_put_mm: mmput(owning_mm); out_put_task: if (owning_process) put_task_struct(owning_process); - free_page((unsigned long)local_page_list); return ret; } -EXPORT_SYMBOL(ib_umem_odp_map_dma_pages); +EXPORT_SYMBOL(ib_umem_odp_map_dma_and_lock); void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt, u64 bound) { + dma_addr_t dma_addr; + dma_addr_t dma; int idx; u64 addr; struct ib_device *dev = umem_odp->umem.ibdev; @@ -521,20 +487,16 @@ void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt, virt = max_t(u64, virt, ib_umem_start(umem_odp)); bound = min_t(u64, bound, ib_umem_end(umem_odp)); - /* Note that during the run of this function, the - * notifiers_count of the MR is > 0, preventing any racing - * faults from completion. We might be racing with other - * invalidations, so we must make sure we free each page only - * once. */ for (addr = virt; addr < bound; addr += BIT(umem_odp->page_shift)) { idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; - if (umem_odp->page_list[idx]) { - struct page *page = umem_odp->page_list[idx]; - dma_addr_t dma = umem_odp->dma_list[idx]; - dma_addr_t dma_addr = dma & ODP_DMA_ADDR_MASK; + dma = umem_odp->dma_list[idx]; - WARN_ON(!dma_addr); + /* The access flags guaranteed a valid DMA address in case was NULL */ + if (dma) { + unsigned long pfn_idx = (addr - ib_umem_start(umem_odp)) >> PAGE_SHIFT; + struct page *page = hmm_pfn_to_page(umem_odp->pfn_list[pfn_idx]); + dma_addr = dma & ODP_DMA_ADDR_MASK; ib_dma_unmap_page(dev, dma_addr, BIT(umem_odp->page_shift), DMA_BIDIRECTIONAL); @@ -551,7 +513,6 @@ void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt, */ set_page_dirty(head_page); } - umem_odp->page_list[idx] = NULL; umem_odp->dma_list[idx] = 0; umem_odp->npages--; } diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 2fbc583d5bdd..418d133a8fb0 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -218,10 +218,12 @@ int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs) if (!ucontext) return -ENOMEM; - ucontext->res.type = RDMA_RESTRACK_CTX; ucontext->device = ib_dev; ucontext->ufile = ufile; xa_init_flags(&ucontext->mmap_xa, XA_FLAGS_ALLOC); + + rdma_restrack_new(&ucontext->res, RDMA_RESTRACK_CTX); + rdma_restrack_set_name(&ucontext->res, NULL); attrs->context = ucontext; return 0; } @@ -250,7 +252,7 @@ int ib_init_ucontext(struct uverbs_attr_bundle *attrs) if (ret) goto err_uncharge; - rdma_restrack_uadd(&ucontext->res); + rdma_restrack_add(&ucontext->res); /* * Make sure that ib_uverbs_get_ucontext() sees the pointer update @@ -313,6 +315,7 @@ static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs) err_uobj: rdma_alloc_abort_uobject(uobj, attrs, false); err_ucontext: + rdma_restrack_put(&attrs->context->res); kfree(attrs->context); attrs->context = NULL; return ret; @@ -439,12 +442,14 @@ static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs) pd->device = ib_dev; pd->uobject = uobj; atomic_set(&pd->usecnt, 0); - pd->res.type = RDMA_RESTRACK_PD; + + rdma_restrack_new(&pd->res, RDMA_RESTRACK_PD); + rdma_restrack_set_name(&pd->res, NULL); ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata); if (ret) goto err_alloc; - rdma_restrack_uadd(&pd->res); + rdma_restrack_add(&pd->res); uobj->object = pd; uobj_finalize_uobj_create(uobj, attrs); @@ -453,6 +458,7 @@ static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs) return uverbs_response(attrs, &resp, sizeof(resp)); err_alloc: + rdma_restrack_put(&pd->res); kfree(pd); err: uobj_alloc_abort(uobj, attrs); @@ -742,9 +748,11 @@ static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs) mr->sig_attrs = NULL; mr->uobject = uobj; atomic_inc(&pd->usecnt); - mr->res.type = RDMA_RESTRACK_MR; mr->iova = cmd.hca_va; - rdma_restrack_uadd(&mr->res); + + rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR); + rdma_restrack_set_name(&mr->res, NULL); + rdma_restrack_add(&mr->res); uobj->object = mr; uobj_put_obj_read(pd); @@ -858,7 +866,7 @@ static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs) static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs) { struct ib_uverbs_alloc_mw cmd; - struct ib_uverbs_alloc_mw_resp resp; + struct ib_uverbs_alloc_mw_resp resp = {}; struct ib_uobject *uobj; struct ib_pd *pd; struct ib_mw *mw; @@ -884,15 +892,21 @@ static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs) goto err_put; } - mw = pd->device->ops.alloc_mw(pd, cmd.mw_type, &attrs->driver_udata); - if (IS_ERR(mw)) { - ret = PTR_ERR(mw); + mw = rdma_zalloc_drv_obj(ib_dev, ib_mw); + if (!mw) { + ret = -ENOMEM; goto err_put; } - mw->device = pd->device; - mw->pd = pd; + mw->device = ib_dev; + mw->pd = pd; mw->uobject = uobj; + mw->type = cmd.mw_type; + + ret = pd->device->ops.alloc_mw(mw, &attrs->driver_udata); + if (ret) + goto err_alloc; + atomic_inc(&pd->usecnt); uobj->object = mw; @@ -903,6 +917,8 @@ static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs) resp.mw_handle = uobj->id; return uverbs_response(attrs, &resp, sizeof(resp)); +err_alloc: + kfree(mw); err_put: uobj_put_obj_read(pd); err_free: @@ -994,12 +1010,14 @@ static int create_cq(struct uverbs_attr_bundle *attrs, cq->event_handler = ib_uverbs_cq_event_handler; cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; atomic_set(&cq->usecnt, 0); - cq->res.type = RDMA_RESTRACK_CQ; + + rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ); + rdma_restrack_set_name(&cq->res, NULL); ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata); if (ret) goto err_free; - rdma_restrack_uadd(&cq->res); + rdma_restrack_add(&cq->res); obj->uevent.uobject.object = cq; obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file); @@ -1013,6 +1031,7 @@ static int create_cq(struct uverbs_attr_bundle *attrs, return uverbs_response(attrs, &resp, sizeof(resp)); err_free: + rdma_restrack_put(&cq->res); kfree(cq); err_file: if (ev_file) @@ -1237,8 +1256,21 @@ static int create_qp(struct uverbs_attr_bundle *attrs, bool has_sq = true; struct ib_device *ib_dev; - if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW)) - return -EPERM; + switch (cmd->qp_type) { + case IB_QPT_RAW_PACKET: + if (!capable(CAP_NET_RAW)) + return -EPERM; + break; + case IB_QPT_RC: + case IB_QPT_UC: + case IB_QPT_UD: + case IB_QPT_XRC_INI: + case IB_QPT_XRC_TGT: + case IB_QPT_DRIVER: + break; + default: + return -EINVAL; + } obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs, &ib_dev); @@ -2985,11 +3017,11 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) { struct ib_uverbs_ex_create_rwq_ind_table cmd; struct ib_uverbs_ex_create_rwq_ind_table_resp resp = {}; - struct ib_uobject *uobj; + struct ib_uobject *uobj; int err; struct ib_rwq_ind_table_init_attr init_attr = {}; struct ib_rwq_ind_table *rwq_ind_tbl; - struct ib_wq **wqs = NULL; + struct ib_wq **wqs = NULL; u32 *wqs_handles = NULL; struct ib_wq *wq = NULL; int i, num_read_wqs; @@ -3047,17 +3079,15 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) goto put_wqs; } - init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size; - init_attr.ind_tbl = wqs; - - rwq_ind_tbl = ib_dev->ops.create_rwq_ind_table(ib_dev, &init_attr, - &attrs->driver_udata); - - if (IS_ERR(rwq_ind_tbl)) { - err = PTR_ERR(rwq_ind_tbl); + rwq_ind_tbl = rdma_zalloc_drv_obj(ib_dev, ib_rwq_ind_table); + if (!rwq_ind_tbl) { + err = -ENOMEM; goto err_uobj; } + init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size; + init_attr.ind_tbl = wqs; + rwq_ind_tbl->ind_tbl = wqs; rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size; rwq_ind_tbl->uobject = uobj; @@ -3065,6 +3095,11 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) rwq_ind_tbl->device = ib_dev; atomic_set(&rwq_ind_tbl->usecnt, 0); + err = ib_dev->ops.create_rwq_ind_table(rwq_ind_tbl, &init_attr, + &attrs->driver_udata); + if (err) + goto err_create; + for (i = 0; i < num_wq_handles; i++) rdma_lookup_put_uobject(&wqs[i]->uobject->uevent.uobject, UVERBS_LOOKUP_READ); @@ -3076,6 +3111,8 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) resp.response_length = uverbs_response_length(attrs, sizeof(resp)); return uverbs_response(attrs, &resp, sizeof(resp)); +err_create: + kfree(rwq_ind_tbl); err_uobj: uobj_alloc_abort(uobj, attrs); put_wqs: @@ -3232,8 +3269,8 @@ static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs) goto err_free; } - flow_id = qp->device->ops.create_flow( - qp, flow_attr, IB_FLOW_DOMAIN_USER, &attrs->driver_udata); + flow_id = qp->device->ops.create_flow(qp, flow_attr, + &attrs->driver_udata); if (IS_ERR(flow_id)) { err = PTR_ERR(flow_id); diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 37794d88b1f3..4bb7c642f80c 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -108,8 +108,11 @@ int uverbs_dealloc_mw(struct ib_mw *mw) int ret; ret = mw->device->ops.dealloc_mw(mw); - if (!ret) - atomic_dec(&pd->usecnt); + if (ret) + return ret; + + atomic_dec(&pd->usecnt); + kfree(mw); return ret; } @@ -845,8 +848,6 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile) * will only be one mm, so no big deal. */ mmap_read_lock(mm); - if (!mmget_still_valid(mm)) - goto skip_mm; mutex_lock(&ufile->umap_lock); list_for_each_entry_safe (priv, next_priv, &ufile->umaps, list) { @@ -865,7 +866,6 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile) } } mutex_unlock(&ufile->umap_lock); - skip_mm: mmap_read_unlock(mm); mmput(mm); } diff --git a/drivers/infiniband/core/uverbs_std_types.c b/drivers/infiniband/core/uverbs_std_types.c index 08c39cfb1bd9..0658101fca00 100644 --- a/drivers/infiniband/core/uverbs_std_types.c +++ b/drivers/infiniband/core/uverbs_std_types.c @@ -81,12 +81,20 @@ static int uverbs_free_rwq_ind_tbl(struct ib_uobject *uobject, { struct ib_rwq_ind_table *rwq_ind_tbl = uobject->object; struct ib_wq **ind_tbl = rwq_ind_tbl->ind_tbl; - int ret; + u32 table_size = (1 << rwq_ind_tbl->log_ind_tbl_size); + int ret, i; + + if (atomic_read(&rwq_ind_tbl->usecnt)) + return -EBUSY; - ret = ib_destroy_rwq_ind_table(rwq_ind_tbl); + ret = rwq_ind_tbl->device->ops.destroy_rwq_ind_table(rwq_ind_tbl); if (ib_is_destroy_retryable(ret, why, uobject)) return ret; + for (i = 0; i < table_size; i++) + atomic_dec(&ind_tbl[i]->usecnt); + + kfree(rwq_ind_tbl); kfree(ind_tbl); return ret; } @@ -122,8 +130,7 @@ static int uverbs_free_pd(struct ib_uobject *uobject, if (ret) return ret; - ib_dealloc_pd_user(pd, &attrs->driver_udata); - return 0; + return ib_dealloc_pd_user(pd, &attrs->driver_udata); } void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue) diff --git a/drivers/infiniband/core/uverbs_std_types_counters.c b/drivers/infiniband/core/uverbs_std_types_counters.c index c7e7438752bc..b3c6c066b601 100644 --- a/drivers/infiniband/core/uverbs_std_types_counters.c +++ b/drivers/infiniband/core/uverbs_std_types_counters.c @@ -46,7 +46,9 @@ static int uverbs_free_counters(struct ib_uobject *uobject, if (ret) return ret; - counters->device->ops.destroy_counters(counters); + ret = counters->device->ops.destroy_counters(counters); + if (ret) + return ret; kfree(counters); return 0; } diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c index b1c7dacc02de..8dabd05988b2 100644 --- a/drivers/infiniband/core/uverbs_std_types_cq.c +++ b/drivers/infiniband/core/uverbs_std_types_cq.c @@ -33,6 +33,7 @@ #include <rdma/uverbs_std_types.h> #include "rdma_core.h" #include "uverbs.h" +#include "restrack.h" static int uverbs_free_cq(struct ib_uobject *uobject, enum rdma_remove_reason why, @@ -123,7 +124,9 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( cq->event_handler = ib_uverbs_cq_event_handler; cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; atomic_set(&cq->usecnt, 0); - cq->res.type = RDMA_RESTRACK_CQ; + + rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ); + rdma_restrack_set_name(&cq->res, NULL); ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata); if (ret) @@ -131,7 +134,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( obj->uevent.uobject.object = cq; obj->uevent.uobject.user_handle = user_handle; - rdma_restrack_uadd(&cq->res); + rdma_restrack_add(&cq->res); uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_CREATE_CQ_HANDLE); ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_CQ_RESP_CQE, &cq->cqe, @@ -139,6 +142,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( return ret; err_free: + rdma_restrack_put(&cq->res); kfree(cq); err_event_file: if (obj->uevent.event_file) diff --git a/drivers/infiniband/core/uverbs_std_types_device.c b/drivers/infiniband/core/uverbs_std_types_device.c index 75df2094a010..f367d523a46b 100644 --- a/drivers/infiniband/core/uverbs_std_types_device.c +++ b/drivers/infiniband/core/uverbs_std_types_device.c @@ -3,11 +3,13 @@ * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. */ +#include <linux/overflow.h> #include <rdma/uverbs_std_types.h> #include "rdma_core.h" #include "uverbs.h" #include <rdma/uverbs_ioctl.h> #include <rdma/opa_addr.h> +#include <rdma/ib_cache.h> /* * This ioctl method allows calling any defined write or write_ex @@ -165,7 +167,8 @@ void copy_port_attr_to_resp(struct ib_port_attr *attr, resp->subnet_timeout = attr->subnet_timeout; resp->init_type_reply = attr->init_type_reply; resp->active_width = attr->active_width; - resp->active_speed = attr->active_speed; + /* This ABI needs to be extended to provide any speed more than IB_SPEED_NDR */ + resp->active_speed = min_t(u16, attr->active_speed, IB_SPEED_NDR); resp->phys_state = attr->phys_state; resp->link_layer = rdma_port_get_link_layer(ib_dev, port_num); } @@ -265,6 +268,172 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_CONTEXT)( return ucontext->device->ops.query_ucontext(ucontext, attrs); } +static int copy_gid_entries_to_user(struct uverbs_attr_bundle *attrs, + struct ib_uverbs_gid_entry *entries, + size_t num_entries, size_t user_entry_size) +{ + const struct uverbs_attr *attr; + void __user *user_entries; + size_t copy_len; + int ret; + int i; + + if (user_entry_size == sizeof(*entries)) { + ret = uverbs_copy_to(attrs, + UVERBS_ATTR_QUERY_GID_TABLE_RESP_ENTRIES, + entries, sizeof(*entries) * num_entries); + return ret; + } + + copy_len = min_t(size_t, user_entry_size, sizeof(*entries)); + attr = uverbs_attr_get(attrs, UVERBS_ATTR_QUERY_GID_TABLE_RESP_ENTRIES); + if (IS_ERR(attr)) + return PTR_ERR(attr); + + user_entries = u64_to_user_ptr(attr->ptr_attr.data); + for (i = 0; i < num_entries; i++) { + if (copy_to_user(user_entries, entries, copy_len)) + return -EFAULT; + + if (user_entry_size > sizeof(*entries)) { + if (clear_user(user_entries + sizeof(*entries), + user_entry_size - sizeof(*entries))) + return -EFAULT; + } + + entries++; + user_entries += user_entry_size; + } + + return uverbs_output_written(attrs, + UVERBS_ATTR_QUERY_GID_TABLE_RESP_ENTRIES); +} + +static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_GID_TABLE)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_uverbs_gid_entry *entries; + struct ib_ucontext *ucontext; + struct ib_device *ib_dev; + size_t user_entry_size; + ssize_t num_entries; + size_t max_entries; + size_t num_bytes; + u32 flags; + int ret; + + ret = uverbs_get_flags32(&flags, attrs, + UVERBS_ATTR_QUERY_GID_TABLE_FLAGS, 0); + if (ret) + return ret; + + ret = uverbs_get_const(&user_entry_size, attrs, + UVERBS_ATTR_QUERY_GID_TABLE_ENTRY_SIZE); + if (ret) + return ret; + + max_entries = uverbs_attr_ptr_get_array_size( + attrs, UVERBS_ATTR_QUERY_GID_TABLE_RESP_ENTRIES, + user_entry_size); + if (max_entries <= 0) + return -EINVAL; + + ucontext = ib_uverbs_get_ucontext(attrs); + if (IS_ERR(ucontext)) + return PTR_ERR(ucontext); + ib_dev = ucontext->device; + + if (check_mul_overflow(max_entries, sizeof(*entries), &num_bytes)) + return -EINVAL; + + entries = uverbs_zalloc(attrs, num_bytes); + if (!entries) + return -ENOMEM; + + num_entries = rdma_query_gid_table(ib_dev, entries, max_entries); + if (num_entries < 0) + return -EINVAL; + + ret = copy_gid_entries_to_user(attrs, entries, num_entries, + user_entry_size); + if (ret) + return ret; + + ret = uverbs_copy_to(attrs, + UVERBS_ATTR_QUERY_GID_TABLE_RESP_NUM_ENTRIES, + &num_entries, sizeof(num_entries)); + return ret; +} + +static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_GID_ENTRY)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_uverbs_gid_entry entry = {}; + const struct ib_gid_attr *gid_attr; + struct ib_ucontext *ucontext; + struct ib_device *ib_dev; + struct net_device *ndev; + u32 gid_index; + u32 port_num; + u32 flags; + int ret; + + ret = uverbs_get_flags32(&flags, attrs, + UVERBS_ATTR_QUERY_GID_ENTRY_FLAGS, 0); + if (ret) + return ret; + + ret = uverbs_get_const(&port_num, attrs, + UVERBS_ATTR_QUERY_GID_ENTRY_PORT); + if (ret) + return ret; + + ret = uverbs_get_const(&gid_index, attrs, + UVERBS_ATTR_QUERY_GID_ENTRY_GID_INDEX); + if (ret) + return ret; + + ucontext = ib_uverbs_get_ucontext(attrs); + if (IS_ERR(ucontext)) + return PTR_ERR(ucontext); + ib_dev = ucontext->device; + + if (!rdma_is_port_valid(ib_dev, port_num)) + return -EINVAL; + + if (!rdma_ib_or_roce(ib_dev, port_num)) + return -EOPNOTSUPP; + + gid_attr = rdma_get_gid_attr(ib_dev, port_num, gid_index); + if (IS_ERR(gid_attr)) + return PTR_ERR(gid_attr); + + memcpy(&entry.gid, &gid_attr->gid, sizeof(gid_attr->gid)); + entry.gid_index = gid_attr->index; + entry.port_num = gid_attr->port_num; + entry.gid_type = gid_attr->gid_type; + + rcu_read_lock(); + ndev = rdma_read_gid_attr_ndev_rcu(gid_attr); + if (IS_ERR(ndev)) { + if (PTR_ERR(ndev) != -ENODEV) { + ret = PTR_ERR(ndev); + rcu_read_unlock(); + goto out; + } + } else { + entry.netdev_ifindex = ndev->ifindex; + } + rcu_read_unlock(); + + ret = uverbs_copy_to_struct_or_zero( + attrs, UVERBS_ATTR_QUERY_GID_ENTRY_RESP_ENTRY, &entry, + sizeof(entry)); +out: + rdma_put_gid_attr(gid_attr); + return ret; +} + DECLARE_UVERBS_NAMED_METHOD( UVERBS_METHOD_GET_CONTEXT, UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS, @@ -299,12 +468,38 @@ DECLARE_UVERBS_NAMED_METHOD( reserved), UA_MANDATORY)); +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_QUERY_GID_TABLE, + UVERBS_ATTR_CONST_IN(UVERBS_ATTR_QUERY_GID_TABLE_ENTRY_SIZE, u64, + UA_MANDATORY), + UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_QUERY_GID_TABLE_FLAGS, u32, + UA_OPTIONAL), + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_GID_TABLE_RESP_ENTRIES, + UVERBS_ATTR_MIN_SIZE(0), UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_GID_TABLE_RESP_NUM_ENTRIES, + UVERBS_ATTR_TYPE(u64), UA_MANDATORY)); + +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_QUERY_GID_ENTRY, + UVERBS_ATTR_CONST_IN(UVERBS_ATTR_QUERY_GID_ENTRY_PORT, u32, + UA_MANDATORY), + UVERBS_ATTR_CONST_IN(UVERBS_ATTR_QUERY_GID_ENTRY_GID_INDEX, u32, + UA_MANDATORY), + UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_QUERY_GID_ENTRY_FLAGS, u32, + UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_GID_ENTRY_RESP_ENTRY, + UVERBS_ATTR_STRUCT(struct ib_uverbs_gid_entry, + netdev_ifindex), + UA_MANDATORY)); + DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DEVICE, &UVERBS_METHOD(UVERBS_METHOD_GET_CONTEXT), &UVERBS_METHOD(UVERBS_METHOD_INVOKE_WRITE), &UVERBS_METHOD(UVERBS_METHOD_INFO_HANDLES), &UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT), - &UVERBS_METHOD(UVERBS_METHOD_QUERY_CONTEXT)); + &UVERBS_METHOD(UVERBS_METHOD_QUERY_CONTEXT), + &UVERBS_METHOD(UVERBS_METHOD_QUERY_GID_TABLE), + &UVERBS_METHOD(UVERBS_METHOD_QUERY_GID_ENTRY)); const struct uapi_definition uverbs_def_obj_device[] = { UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DEVICE), diff --git a/drivers/infiniband/core/uverbs_std_types_wq.c b/drivers/infiniband/core/uverbs_std_types_wq.c index cad842ede077..f2e6a625724a 100644 --- a/drivers/infiniband/core/uverbs_std_types_wq.c +++ b/drivers/infiniband/core/uverbs_std_types_wq.c @@ -16,7 +16,7 @@ static int uverbs_free_wq(struct ib_uobject *uobject, container_of(uobject, struct ib_uwq_object, uevent.uobject); int ret; - ret = ib_destroy_wq(wq, &attrs->driver_udata); + ret = ib_destroy_wq_user(wq, &attrs->driver_udata); if (ib_is_destroy_retryable(ret, why, uobject)) return ret; diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 307886737646..740f8454b6b4 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -272,15 +272,16 @@ struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags, atomic_set(&pd->usecnt, 0); pd->flags = flags; - pd->res.type = RDMA_RESTRACK_PD; - rdma_restrack_set_task(&pd->res, caller); + rdma_restrack_new(&pd->res, RDMA_RESTRACK_PD); + rdma_restrack_set_name(&pd->res, caller); ret = device->ops.alloc_pd(pd, NULL); if (ret) { + rdma_restrack_put(&pd->res); kfree(pd); return ERR_PTR(ret); } - rdma_restrack_kadd(&pd->res); + rdma_restrack_add(&pd->res); if (device->attrs.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) pd->local_dma_lkey = device->local_dma_lkey; @@ -329,7 +330,7 @@ EXPORT_SYMBOL(__ib_alloc_pd); * exist. The caller is responsible to synchronously destroy them and * guarantee no new allocations will happen. */ -void ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata) +int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata) { int ret; @@ -343,9 +344,13 @@ void ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata) requires the caller to guarantee we can't race here. */ WARN_ON(atomic_read(&pd->usecnt)); + ret = pd->device->ops.dealloc_pd(pd, udata); + if (ret) + return ret; + rdma_restrack_del(&pd->res); - pd->device->ops.dealloc_pd(pd, udata); kfree(pd); + return ret; } EXPORT_SYMBOL(ib_dealloc_pd_user); @@ -728,7 +733,7 @@ int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr, (struct in6_addr *)dgid); return 0; } else if (net_type == RDMA_NETWORK_IPV6 || - net_type == RDMA_NETWORK_IB) { + net_type == RDMA_NETWORK_IB || RDMA_NETWORK_ROCE_V1) { *dgid = hdr->ibgrh.dgid; *sgid = hdr->ibgrh.sgid; return 0; @@ -964,18 +969,22 @@ int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata) { const struct ib_gid_attr *sgid_attr = ah->sgid_attr; struct ib_pd *pd; + int ret; might_sleep_if(flags & RDMA_DESTROY_AH_SLEEPABLE); pd = ah->pd; - ah->device->ops.destroy_ah(ah, flags); + ret = ah->device->ops.destroy_ah(ah, flags); + if (ret) + return ret; + atomic_dec(&pd->usecnt); if (sgid_attr) rdma_put_gid_attr(sgid_attr); kfree(ah); - return 0; + return ret; } EXPORT_SYMBOL(rdma_destroy_ah_user); @@ -1060,10 +1069,14 @@ EXPORT_SYMBOL(ib_query_srq); int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata) { + int ret; + if (atomic_read(&srq->usecnt)) return -EBUSY; - srq->device->ops.destroy_srq(srq, udata); + ret = srq->device->ops.destroy_srq(srq, udata); + if (ret) + return ret; atomic_dec(&srq->pd->usecnt); if (srq->srq_type == IB_SRQT_XRC) @@ -1072,7 +1085,7 @@ int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata) atomic_dec(&srq->ext.cq->usecnt); kfree(srq); - return 0; + return ret; } EXPORT_SYMBOL(ib_destroy_srq_user); @@ -1781,7 +1794,7 @@ int ib_modify_qp_with_udata(struct ib_qp *ib_qp, struct ib_qp_attr *attr, } EXPORT_SYMBOL(ib_modify_qp_with_udata); -int ib_get_eth_speed(struct ib_device *dev, u8 port_num, u8 *speed, u8 *width) +int ib_get_eth_speed(struct ib_device *dev, u8 port_num, u16 *speed, u8 *width) { int rc; u32 netdev_speed; @@ -1984,16 +1997,18 @@ struct ib_cq *__ib_create_cq(struct ib_device *device, cq->event_handler = event_handler; cq->cq_context = cq_context; atomic_set(&cq->usecnt, 0); - cq->res.type = RDMA_RESTRACK_CQ; - rdma_restrack_set_task(&cq->res, caller); + + rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ); + rdma_restrack_set_name(&cq->res, caller); ret = device->ops.create_cq(cq, cq_attr, NULL); if (ret) { + rdma_restrack_put(&cq->res); kfree(cq); return ERR_PTR(ret); } - rdma_restrack_kadd(&cq->res); + rdma_restrack_add(&cq->res); return cq; } EXPORT_SYMBOL(__ib_create_cq); @@ -2011,16 +2026,21 @@ EXPORT_SYMBOL(rdma_set_cq_moderation); int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata) { + int ret; + if (WARN_ON_ONCE(cq->shared)) return -EOPNOTSUPP; if (atomic_read(&cq->usecnt)) return -EBUSY; + ret = cq->device->ops.destroy_cq(cq, udata); + if (ret) + return ret; + rdma_restrack_del(&cq->res); - cq->device->ops.destroy_cq(cq, udata); kfree(cq); - return 0; + return ret; } EXPORT_SYMBOL(ib_destroy_cq_user); @@ -2059,8 +2079,10 @@ struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, mr->pd = pd; mr->dm = NULL; atomic_inc(&pd->usecnt); - mr->res.type = RDMA_RESTRACK_MR; - rdma_restrack_kadd(&mr->res); + + rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR); + rdma_restrack_parent_name(&mr->res, &pd->res); + rdma_restrack_add(&mr->res); return mr; } @@ -2139,11 +2161,12 @@ struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, mr->uobject = NULL; atomic_inc(&pd->usecnt); mr->need_inval = false; - mr->res.type = RDMA_RESTRACK_MR; - rdma_restrack_kadd(&mr->res); mr->type = mr_type; mr->sig_attrs = NULL; + rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR); + rdma_restrack_parent_name(&mr->res, &pd->res); + rdma_restrack_add(&mr->res); out: trace_mr_alloc(pd, mr_type, max_num_sg, mr); return mr; @@ -2199,11 +2222,12 @@ struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd, mr->uobject = NULL; atomic_inc(&pd->usecnt); mr->need_inval = false; - mr->res.type = RDMA_RESTRACK_MR; - rdma_restrack_kadd(&mr->res); mr->type = IB_MR_TYPE_INTEGRITY; mr->sig_attrs = sig_attrs; + rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR); + rdma_restrack_parent_name(&mr->res, &pd->res); + rdma_restrack_add(&mr->res); out: trace_mr_integ_alloc(pd, max_num_data_sg, max_num_meta_sg, mr); return mr; @@ -2328,13 +2352,17 @@ EXPORT_SYMBOL(ib_alloc_xrcd_user); */ int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata) { + int ret; + if (atomic_read(&xrcd->usecnt)) return -EBUSY; WARN_ON(!xa_empty(&xrcd->tgt_qps)); - xrcd->device->ops.dealloc_xrcd(xrcd, udata); + ret = xrcd->device->ops.dealloc_xrcd(xrcd, udata); + if (ret) + return ret; kfree(xrcd); - return 0; + return ret; } EXPORT_SYMBOL(ib_dealloc_xrcd_user); @@ -2378,25 +2406,28 @@ struct ib_wq *ib_create_wq(struct ib_pd *pd, EXPORT_SYMBOL(ib_create_wq); /** - * ib_destroy_wq - Destroys the specified user WQ. + * ib_destroy_wq_user - Destroys the specified user WQ. * @wq: The WQ to destroy. * @udata: Valid user data */ -int ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata) +int ib_destroy_wq_user(struct ib_wq *wq, struct ib_udata *udata) { struct ib_cq *cq = wq->cq; struct ib_pd *pd = wq->pd; + int ret; if (atomic_read(&wq->usecnt)) return -EBUSY; - wq->device->ops.destroy_wq(wq, udata); + ret = wq->device->ops.destroy_wq(wq, udata); + if (ret) + return ret; + atomic_dec(&pd->usecnt); atomic_dec(&cq->usecnt); - - return 0; + return ret; } -EXPORT_SYMBOL(ib_destroy_wq); +EXPORT_SYMBOL(ib_destroy_wq_user); /** * ib_modify_wq - Modifies the specified WQ. @@ -2419,29 +2450,6 @@ int ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr, } EXPORT_SYMBOL(ib_modify_wq); -/* - * ib_destroy_rwq_ind_table - Destroys the specified Indirection Table. - * @wq_ind_table: The Indirection Table to destroy. -*/ -int ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table) -{ - int err, i; - u32 table_size = (1 << rwq_ind_table->log_ind_tbl_size); - struct ib_wq **ind_tbl = rwq_ind_table->ind_tbl; - - if (atomic_read(&rwq_ind_table->usecnt)) - return -EBUSY; - - err = rwq_ind_table->device->ops.destroy_rwq_ind_table(rwq_ind_table); - if (!err) { - for (i = 0; i < table_size; i++) - atomic_dec(&ind_tbl[i]->usecnt); - } - - return err; -} -EXPORT_SYMBOL(ib_destroy_rwq_ind_table); - int ib_check_mr_status(struct ib_mr *mr, u32 check_mask, struct ib_mr_status *mr_status) { diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h index a300588634c5..b930ea3dab7a 100644 --- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h +++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h @@ -150,7 +150,7 @@ struct bnxt_re_dev { struct delayed_work worker; u8 cur_prio_map; - u8 active_speed; + u16 active_speed; u8 active_width; /* FP Notification Queue (CQ & SRQ) */ diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 1d7a9ca5240c..cf3db9628397 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -532,7 +532,7 @@ fail: } /* Protection Domains */ -void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) +int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) { struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); struct bnxt_re_dev *rdev = pd->rdev; @@ -542,6 +542,7 @@ void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) if (pd->qplib_pd.id) bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, &pd->qplib_pd); + return 0; } int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) @@ -601,13 +602,14 @@ fail: } /* Address Handles */ -void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags) +int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags) { struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); struct bnxt_re_dev *rdev = ah->rdev; bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, !(flags & RDMA_DESTROY_AH_SLEEPABLE)); + return 0; } static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype) @@ -938,9 +940,7 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, return PTR_ERR(umem); qp->sumem = umem; - qplib_qp->sq.sg_info.sghead = umem->sg_head.sgl; - qplib_qp->sq.sg_info.npages = ib_umem_num_pages(umem); - qplib_qp->sq.sg_info.nmap = umem->nmap; + qplib_qp->sq.sg_info.umem = umem; qplib_qp->sq.sg_info.pgsize = PAGE_SIZE; qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT; qplib_qp->qp_handle = ureq.qp_handle; @@ -953,9 +953,7 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, if (IS_ERR(umem)) goto rqfail; qp->rumem = umem; - qplib_qp->rq.sg_info.sghead = umem->sg_head.sgl; - qplib_qp->rq.sg_info.npages = ib_umem_num_pages(umem); - qplib_qp->rq.sg_info.nmap = umem->nmap; + qplib_qp->rq.sg_info.umem = umem; qplib_qp->rq.sg_info.pgsize = PAGE_SIZE; qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT; } @@ -1568,7 +1566,7 @@ static enum ib_mtu __to_ib_mtu(u32 mtu) } /* Shared Receive Queues */ -void bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) +int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) { struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq); @@ -1583,6 +1581,7 @@ void bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) atomic_dec(&rdev->srq_count); if (nq) nq->budget--; + return 0; } static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, @@ -1608,9 +1607,7 @@ static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, return PTR_ERR(umem); srq->umem = umem; - qplib_srq->sg_info.sghead = umem->sg_head.sgl; - qplib_srq->sg_info.npages = ib_umem_num_pages(umem); - qplib_srq->sg_info.nmap = umem->nmap; + qplib_srq->sg_info.umem = umem; qplib_srq->sg_info.pgsize = PAGE_SIZE; qplib_srq->sg_info.pgshft = PAGE_SHIFT; qplib_srq->srq_handle = ureq.srq_handle; @@ -2800,7 +2797,7 @@ int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, } /* Completion Queues */ -void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { struct bnxt_re_cq *cq; struct bnxt_qplib_nq *nq; @@ -2816,6 +2813,7 @@ void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) atomic_dec(&rdev->cq_count); nq->budget--; kfree(cq->cql); + return 0; } int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, @@ -2860,9 +2858,7 @@ int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, rc = PTR_ERR(cq->umem); goto fail; } - cq->qplib_cq.sg_info.sghead = cq->umem->sg_head.sgl; - cq->qplib_cq.sg_info.npages = ib_umem_num_pages(cq->umem); - cq->qplib_cq.sg_info.nmap = cq->umem->nmap; + cq->qplib_cq.sg_info.umem = cq->umem; cq->qplib_cq.dpi = &uctx->dpi; } else { cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL); @@ -3774,23 +3770,6 @@ int bnxt_re_dealloc_mw(struct ib_mw *ib_mw) return rc; } -static int bnxt_re_page_size_ok(int page_shift) -{ - switch (page_shift) { - case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K: - case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K: - case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K: - case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M: - case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K: - case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M: - case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M: - case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G: - return 1; - default: - return 0; - } -} - static int fill_umem_pbl_tbl(struct ib_umem *umem, u64 *pbl_tbl_orig, int page_shift) { @@ -3798,7 +3777,7 @@ static int fill_umem_pbl_tbl(struct ib_umem *umem, u64 *pbl_tbl_orig, u64 page_size = BIT_ULL(page_shift); struct ib_block_iter biter; - rdma_for_each_block(umem->sg_head.sgl, &biter, umem->nmap, page_size) + rdma_umem_for_each_dma_block(umem, &biter, page_size) *pbl_tbl++ = rdma_block_iter_dma_address(&biter); return pbl_tbl - pbl_tbl_orig; @@ -3814,7 +3793,8 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, struct bnxt_re_mr *mr; struct ib_umem *umem; u64 *pbl_tbl = NULL; - int umem_pgs, page_shift, rc; + unsigned long page_size; + int umem_pgs, rc; if (length > BNXT_RE_MAX_MR_SIZE) { ibdev_err(&rdev->ibdev, "MR Size: %lld > Max supported:%lld\n", @@ -3848,42 +3828,34 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, mr->ib_umem = umem; mr->qplib_mr.va = virt_addr; - umem_pgs = ib_umem_page_count(umem); - if (!umem_pgs) { - ibdev_err(&rdev->ibdev, "umem is invalid!"); - rc = -EINVAL; - goto free_umem; - } - mr->qplib_mr.total_size = length; - - pbl_tbl = kcalloc(umem_pgs, sizeof(u64 *), GFP_KERNEL); - if (!pbl_tbl) { - rc = -ENOMEM; - goto free_umem; - } - - page_shift = __ffs(ib_umem_find_best_pgsz(umem, - BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M, - virt_addr)); - - if (!bnxt_re_page_size_ok(page_shift)) { + page_size = ib_umem_find_best_pgsz( + umem, BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M, virt_addr); + if (!page_size) { ibdev_err(&rdev->ibdev, "umem page size unsupported!"); rc = -EFAULT; - goto fail; + goto free_umem; } + mr->qplib_mr.total_size = length; - if (page_shift == BNXT_RE_PAGE_SHIFT_4K && + if (page_size == BNXT_RE_PAGE_SIZE_4K && length > BNXT_RE_MAX_MR_SIZE_LOW) { ibdev_err(&rdev->ibdev, "Requested MR Sz:%llu Max sup:%llu", length, (u64)BNXT_RE_MAX_MR_SIZE_LOW); rc = -EINVAL; - goto fail; + goto free_umem; + } + + umem_pgs = ib_umem_num_dma_blocks(umem, page_size); + pbl_tbl = kcalloc(umem_pgs, sizeof(*pbl_tbl), GFP_KERNEL); + if (!pbl_tbl) { + rc = -ENOMEM; + goto free_umem; } /* Map umem buf ptrs to the PBL */ - umem_pgs = fill_umem_pbl_tbl(umem, pbl_tbl, page_shift); + umem_pgs = fill_umem_pbl_tbl(umem, pbl_tbl, order_base_2(page_size)); rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, pbl_tbl, - umem_pgs, false, 1 << page_shift); + umem_pgs, false, page_size); if (rc) { ibdev_err(&rdev->ibdev, "Failed to register user MR"); goto fail; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index 1daeb30e06fd..9a8130b79256 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -163,12 +163,12 @@ int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, u8 port_num); int bnxt_re_alloc_pd(struct ib_pd *pd, struct ib_udata *udata); -void bnxt_re_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); +int bnxt_re_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); int bnxt_re_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata); int bnxt_re_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr); int bnxt_re_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr); -void bnxt_re_destroy_ah(struct ib_ah *ah, u32 flags); +int bnxt_re_destroy_ah(struct ib_ah *ah, u32 flags); int bnxt_re_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *srq_init_attr, struct ib_udata *udata); @@ -176,7 +176,7 @@ int bnxt_re_modify_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr, enum ib_srq_attr_mask srq_attr_mask, struct ib_udata *udata); int bnxt_re_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr); -void bnxt_re_destroy_srq(struct ib_srq *srq, struct ib_udata *udata); +int bnxt_re_destroy_srq(struct ib_srq *srq, struct ib_udata *udata); int bnxt_re_post_srq_recv(struct ib_srq *srq, const struct ib_recv_wr *recv_wr, const struct ib_recv_wr **bad_recv_wr); struct ib_qp *bnxt_re_create_qp(struct ib_pd *pd, @@ -193,7 +193,7 @@ int bnxt_re_post_recv(struct ib_qp *qp, const struct ib_recv_wr *recv_wr, const struct ib_recv_wr **bad_recv_wr); int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata); -void bnxt_re_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +int bnxt_re_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc); int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags); diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index 53aee5a42ab8..04621ba8fa76 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -736,7 +736,8 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) if (ret) return ret; - return ib_register_device(ibdev, "bnxt_re%d"); + dma_set_max_seg_size(&rdev->en_dev->pdev->dev, UINT_MAX); + return ib_register_device(ibdev, "bnxt_re%d", &rdev->en_dev->pdev->dev); } static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev) diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c index f78da54a0bc5..995d4633b0a1 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -295,9 +295,9 @@ static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq, u16 cnq_events) } } -static void bnxt_qplib_service_nq(unsigned long data) +static void bnxt_qplib_service_nq(struct tasklet_struct *t) { - struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data; + struct bnxt_qplib_nq *nq = from_tasklet(nq, t, nq_tasklet); struct bnxt_qplib_hwq *hwq = &nq->hwq; int num_srqne_processed = 0; int num_cqne_processed = 0; @@ -448,8 +448,7 @@ int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx, nq->msix_vec = msix_vector; if (need_init) - tasklet_init(&nq->nq_tasklet, bnxt_qplib_service_nq, - (unsigned long)nq); + tasklet_setup(&nq->nq_tasklet, bnxt_qplib_service_nq); else tasklet_enable(&nq->nq_tasklet); diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c index f7736e34ac64..441eb421e5e5 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c @@ -50,7 +50,7 @@ #include "qplib_sp.h" #include "qplib_fp.h" -static void bnxt_qplib_service_creq(unsigned long data); +static void bnxt_qplib_service_creq(struct tasklet_struct *t); /* Hardware communication channel */ static int __wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) @@ -79,7 +79,7 @@ static int __block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) goto done; do { mdelay(1); /* 1m sec */ - bnxt_qplib_service_creq((unsigned long)rcfw); + bnxt_qplib_service_creq(&rcfw->creq.creq_tasklet); } while (test_bit(cbit, cmdq->cmdq_bitmap) && --count); done: return count ? 0 : -ETIMEDOUT; @@ -370,9 +370,9 @@ static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw, } /* SP - CREQ Completion handlers */ -static void bnxt_qplib_service_creq(unsigned long data) +static void bnxt_qplib_service_creq(struct tasklet_struct *t) { - struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data; + struct bnxt_qplib_rcfw *rcfw = from_tasklet(rcfw, t, creq.creq_tasklet); struct bnxt_qplib_creq_ctx *creq = &rcfw->creq; u32 type, budget = CREQ_ENTRY_POLL_BUDGET; struct bnxt_qplib_hwq *hwq = &creq->hwq; @@ -687,8 +687,7 @@ int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector, creq->msix_vec = msix_vector; if (need_init) - tasklet_init(&creq->creq_tasklet, - bnxt_qplib_service_creq, (unsigned long)rcfw); + tasklet_setup(&creq->creq_tasklet, bnxt_qplib_service_creq); else tasklet_enable(&creq->creq_tasklet); rc = request_irq(creq->msix_vec, bnxt_qplib_creq_irq, 0, diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index 7efa6e5dce62..fa7878336100 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -45,6 +45,9 @@ #include <linux/dma-mapping.h> #include <linux/if_vlan.h> #include <linux/vmalloc.h> +#include <rdma/ib_verbs.h> +#include <rdma/ib_umem.h> + #include "roce_hsi.h" #include "qplib_res.h" #include "qplib_sp.h" @@ -87,12 +90,11 @@ static void __free_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_pbl *pbl, static void bnxt_qplib_fill_user_dma_pages(struct bnxt_qplib_pbl *pbl, struct bnxt_qplib_sg_info *sginfo) { - struct scatterlist *sghead = sginfo->sghead; - struct sg_dma_page_iter sg_iter; + struct ib_block_iter biter; int i = 0; - for_each_sg_dma_page(sghead, &sg_iter, sginfo->nmap, 0) { - pbl->pg_map_arr[i] = sg_page_iter_dma_address(&sg_iter); + rdma_umem_for_each_dma_block(sginfo->umem, &biter, sginfo->pgsize) { + pbl->pg_map_arr[i] = rdma_block_iter_dma_address(&biter); pbl->pg_arr[i] = NULL; pbl->pg_count++; i++; @@ -104,15 +106,16 @@ static int __alloc_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_sg_info *sginfo) { struct pci_dev *pdev = res->pdev; - struct scatterlist *sghead; bool is_umem = false; u32 pages; int i; if (sginfo->nopte) return 0; - pages = sginfo->npages; - sghead = sginfo->sghead; + if (sginfo->umem) + pages = ib_umem_num_dma_blocks(sginfo->umem, sginfo->pgsize); + else + pages = sginfo->npages; /* page ptr arrays */ pbl->pg_arr = vmalloc(pages * sizeof(void *)); if (!pbl->pg_arr) @@ -127,7 +130,7 @@ static int __alloc_pbl(struct bnxt_qplib_res *res, pbl->pg_count = 0; pbl->pg_size = sginfo->pgsize; - if (!sghead) { + if (!sginfo->umem) { for (i = 0; i < pages; i++) { pbl->pg_arr[i] = dma_alloc_coherent(&pdev->dev, pbl->pg_size, @@ -183,14 +186,12 @@ int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq, struct bnxt_qplib_sg_info sginfo = {}; u32 depth, stride, npbl, npde; dma_addr_t *src_phys_ptr, **dst_virt_ptr; - struct scatterlist *sghead = NULL; struct bnxt_qplib_res *res; struct pci_dev *pdev; int i, rc, lvl; res = hwq_attr->res; pdev = res->pdev; - sghead = hwq_attr->sginfo->sghead; pg_size = hwq_attr->sginfo->pgsize; hwq->level = PBL_LVL_MAX; @@ -204,7 +205,7 @@ int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq, aux_pages++; } - if (!sghead) { + if (!hwq_attr->sginfo->umem) { hwq->is_user = false; npages = (depth * stride) / pg_size + aux_pages; if ((depth * stride) % pg_size) @@ -213,11 +214,14 @@ int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq, return -EINVAL; hwq_attr->sginfo->npages = npages; } else { + unsigned long sginfo_num_pages = ib_umem_num_dma_blocks( + hwq_attr->sginfo->umem, hwq_attr->sginfo->pgsize); + hwq->is_user = true; - npages = hwq_attr->sginfo->npages; + npages = sginfo_num_pages; npages = (npages * PAGE_SIZE) / BIT_ULL(hwq_attr->sginfo->pgshft); - if ((hwq_attr->sginfo->npages * PAGE_SIZE) % + if ((sginfo_num_pages * PAGE_SIZE) % BIT_ULL(hwq_attr->sginfo->pgshft)) if (!npages) npages++; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h index 9da470d1e4a3..7a1ab38b95da 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h @@ -126,8 +126,7 @@ struct bnxt_qplib_pbl { }; struct bnxt_qplib_sg_info { - struct scatterlist *sghead; - u32 nmap; + struct ib_umem *umem; u32 npages; u32 pgshft; u32 pgsize; diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c index 1f288c73ccfc..8769e7aa097f 100644 --- a/drivers/infiniband/hw/cxgb4/cm.c +++ b/drivers/infiniband/hw/cxgb4/cm.c @@ -77,9 +77,9 @@ static int enable_ecn; module_param(enable_ecn, int, 0644); MODULE_PARM_DESC(enable_ecn, "Enable ECN (default=0/disabled)"); -static int dack_mode = 1; +static int dack_mode; module_param(dack_mode, int, 0644); -MODULE_PARM_DESC(dack_mode, "Delayed ack mode (default=1)"); +MODULE_PARM_DESC(dack_mode, "Delayed ack mode (default=0)"); uint c4iw_max_read_depth = 32; module_param(c4iw_max_read_depth, int, 0644); diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c index 352b8af1998a..28349ed50885 100644 --- a/drivers/infiniband/hw/cxgb4/cq.c +++ b/drivers/infiniband/hw/cxgb4/cq.c @@ -967,7 +967,7 @@ int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) return !err || err == -ENODATA ? npolled : err; } -void c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { struct c4iw_cq *chp; struct c4iw_ucontext *ucontext; @@ -985,6 +985,7 @@ void c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) ucontext ? &ucontext->uctx : &chp->cq.rdev->uctx, chp->destroy_skb, chp->wr_waitp); c4iw_put_wr_wait(chp->wr_waitp); + return 0; } int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h index 2b2b009b371a..a27899402f59 100644 --- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h +++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h @@ -985,21 +985,20 @@ int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset); int c4iw_dealloc_mw(struct ib_mw *mw); void c4iw_dealloc(struct uld_ctx *ctx); -struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, - struct ib_udata *udata); +int c4iw_alloc_mw(struct ib_mw *mw, struct ib_udata *udata); struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt, int acc, struct ib_udata *udata); struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc); int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata); -void c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); +int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata); int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); int c4iw_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *attr, enum ib_srq_attr_mask srq_attr_mask, struct ib_udata *udata); -void c4iw_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata); +int c4iw_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata); int c4iw_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *attrs, struct ib_udata *udata); int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c index 73936c3341b7..42234df896fb 100644 --- a/drivers/infiniband/hw/cxgb4/mem.c +++ b/drivers/infiniband/hw/cxgb4/mem.c @@ -510,7 +510,7 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, __be64 *pages; int shift, n, i; int err = -ENOMEM; - struct sg_dma_page_iter sg_iter; + struct ib_block_iter biter; struct c4iw_dev *rhp; struct c4iw_pd *php; struct c4iw_mr *mhp; @@ -548,7 +548,7 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, shift = PAGE_SHIFT; - n = ib_umem_num_pages(mhp->umem); + n = ib_umem_num_dma_blocks(mhp->umem, 1 << shift); err = alloc_pbl(mhp, n); if (err) goto err_umem_release; @@ -561,8 +561,8 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, i = n = 0; - for_each_sg_dma_page(mhp->umem->sg_head.sgl, &sg_iter, mhp->umem->nmap, 0) { - pages[i++] = cpu_to_be64(sg_page_iter_dma_address(&sg_iter)); + rdma_umem_for_each_dma_block(mhp->umem, &biter, 1 << shift) { + pages[i++] = cpu_to_be64(rdma_block_iter_dma_address(&biter)); if (i == PAGE_SIZE / sizeof(*pages)) { err = write_pbl(&mhp->rhp->rdev, pages, mhp->attr.pbl_addr + (n << 3), i, @@ -611,30 +611,23 @@ err_free_mhp: return ERR_PTR(err); } -struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, - struct ib_udata *udata) +int c4iw_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) { + struct c4iw_mw *mhp = to_c4iw_mw(ibmw); struct c4iw_dev *rhp; struct c4iw_pd *php; - struct c4iw_mw *mhp; u32 mmid; u32 stag = 0; int ret; - if (type != IB_MW_TYPE_1) - return ERR_PTR(-EINVAL); + if (ibmw->type != IB_MW_TYPE_1) + return -EINVAL; - php = to_c4iw_pd(pd); + php = to_c4iw_pd(ibmw->pd); rhp = php->rhp; - mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); - if (!mhp) - return ERR_PTR(-ENOMEM); - mhp->wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL); - if (!mhp->wr_waitp) { - ret = -ENOMEM; - goto free_mhp; - } + if (!mhp->wr_waitp) + return -ENOMEM; mhp->dereg_skb = alloc_skb(SGE_MAX_WR_LEN, GFP_KERNEL); if (!mhp->dereg_skb) { @@ -645,18 +638,19 @@ struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, ret = allocate_window(&rhp->rdev, &stag, php->pdid, mhp->wr_waitp); if (ret) goto free_skb; + mhp->rhp = rhp; mhp->attr.pdid = php->pdid; mhp->attr.type = FW_RI_STAG_MW; mhp->attr.stag = stag; mmid = (stag) >> 8; - mhp->ibmw.rkey = stag; + ibmw->rkey = stag; if (xa_insert_irq(&rhp->mrs, mmid, mhp, GFP_KERNEL)) { ret = -ENOMEM; goto dealloc_win; } pr_debug("mmid 0x%x mhp %p stag 0x%x\n", mmid, mhp, stag); - return &(mhp->ibmw); + return 0; dealloc_win: deallocate_window(&rhp->rdev, mhp->attr.stag, mhp->dereg_skb, @@ -665,9 +659,7 @@ free_skb: kfree_skb(mhp->dereg_skb); free_wr_wait: c4iw_put_wr_wait(mhp->wr_waitp); -free_mhp: - kfree(mhp); - return ERR_PTR(ret); + return ret; } int c4iw_dealloc_mw(struct ib_mw *mw) @@ -684,8 +676,6 @@ int c4iw_dealloc_mw(struct ib_mw *mw) mhp->wr_waitp); kfree_skb(mhp->dereg_skb); c4iw_put_wr_wait(mhp->wr_waitp); - pr_debug("ib_mw %p mmid 0x%x ptr %p\n", mw, mmid, mhp); - kfree(mhp); return 0; } diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index 6c579d2d3997..8138c57a1e43 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -190,7 +190,7 @@ static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) return ret; } -static void c4iw_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata) +static int c4iw_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata) { struct c4iw_dev *rhp; struct c4iw_pd *php; @@ -202,6 +202,7 @@ static void c4iw_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata) mutex_lock(&rhp->rdev.stats.lock); rhp->rdev.stats.pd.cur--; mutex_unlock(&rhp->rdev.stats.lock); + return 0; } static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata) @@ -497,8 +498,10 @@ static const struct ib_device_ops c4iw_dev_ops = { .query_qp = c4iw_ib_query_qp, .reg_user_mr = c4iw_reg_user_mr, .req_notify_cq = c4iw_arm_cq, - INIT_RDMA_OBJ_SIZE(ib_pd, c4iw_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_cq, c4iw_cq, ibcq), + INIT_RDMA_OBJ_SIZE(ib_mw, c4iw_mw, ibmw), + INIT_RDMA_OBJ_SIZE(ib_pd, c4iw_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_srq, c4iw_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_ucontext, c4iw_ucontext, ibucontext), }; @@ -567,7 +570,9 @@ void c4iw_register_device(struct work_struct *work) ret = set_netdevs(&dev->ibdev, &dev->rdev); if (ret) goto err_dealloc_ctx; - ret = ib_register_device(&dev->ibdev, "cxgb4_%d"); + dma_set_max_seg_size(&dev->rdev.lldi.pdev->dev, UINT_MAX); + ret = ib_register_device(&dev->ibdev, "cxgb4_%d", + &dev->rdev.lldi.pdev->dev); if (ret) goto err_dealloc_ctx; return; diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c index cbddb20c6121..f20379e4e2ec 100644 --- a/drivers/infiniband/hw/cxgb4/qp.c +++ b/drivers/infiniband/hw/cxgb4/qp.c @@ -2797,7 +2797,7 @@ err_free_wr_wait: return ret; } -void c4iw_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) +int c4iw_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) { struct c4iw_dev *rhp; struct c4iw_srq *srq; @@ -2813,4 +2813,5 @@ void c4iw_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) srq->wr_waitp); c4iw_free_srq_idx(&rhp->rdev, srq->idx); c4iw_put_wr_wait(srq->wr_waitp); + return 0; } diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h index 1889dd172a25..e5d9712e98c4 100644 --- a/drivers/infiniband/hw/efa/efa.h +++ b/drivers/infiniband/hw/efa/efa.h @@ -33,7 +33,8 @@ struct efa_irq { char name[EFA_IRQNAME_SIZE]; }; -struct efa_sw_stats { +/* Don't use anything other than atomic64 */ +struct efa_stats { atomic64_t alloc_pd_err; atomic64_t create_qp_err; atomic64_t create_cq_err; @@ -41,11 +42,6 @@ struct efa_sw_stats { atomic64_t alloc_ucontext_err; atomic64_t create_ah_err; atomic64_t mmap_err; -}; - -/* Don't use anything other than atomic64 */ -struct efa_stats { - struct efa_sw_stats sw_stats; atomic64_t keep_alive_rcvd; }; @@ -134,12 +130,12 @@ int efa_query_gid(struct ib_device *ibdev, u8 port, int index, int efa_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey); int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); -void efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); +int efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); struct ib_qp *efa_create_qp(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata); -void efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); +int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata); struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, @@ -156,7 +152,7 @@ void efa_mmap_free(struct rdma_user_mmap_entry *rdma_entry); int efa_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata); -void efa_destroy_ah(struct ib_ah *ibah, u32 flags); +int efa_destroy_ah(struct ib_ah *ibah, u32 flags); int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_udata *udata); enum rdma_link_layer efa_port_link_layer(struct ib_device *ibdev, diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h index 5484b08bbc5d..b199e4ac6cf9 100644 --- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h @@ -61,6 +61,8 @@ enum efa_admin_qp_state { enum efa_admin_get_stats_type { EFA_ADMIN_GET_STATS_TYPE_BASIC = 0, + EFA_ADMIN_GET_STATS_TYPE_MESSAGES = 1, + EFA_ADMIN_GET_STATS_TYPE_RDMA_READ = 2, }; enum efa_admin_get_stats_scope { @@ -68,14 +70,6 @@ enum efa_admin_get_stats_scope { EFA_ADMIN_GET_STATS_SCOPE_QUEUE = 1, }; -enum efa_admin_modify_qp_mask_bits { - EFA_ADMIN_QP_STATE_BIT = 0, - EFA_ADMIN_CUR_QP_STATE_BIT = 1, - EFA_ADMIN_QKEY_BIT = 2, - EFA_ADMIN_SQ_PSN_BIT = 3, - EFA_ADMIN_SQ_DRAINED_ASYNC_NOTIFY_BIT = 4, -}; - /* * QP allocation sizes, converted by fabric QueuePair (QP) create command * from QP capabilities. @@ -199,8 +193,14 @@ struct efa_admin_modify_qp_cmd { struct efa_admin_aq_common_desc aq_common_desc; /* - * Mask indicating which fields should be updated see enum - * efa_admin_modify_qp_mask_bits + * Mask indicating which fields should be updated + * 0 : qp_state + * 1 : cur_qp_state + * 2 : qkey + * 3 : sq_psn + * 4 : sq_drained_async_notify + * 5 : rnr_retry + * 31:6 : reserved */ u32 modify_mask; @@ -222,8 +222,8 @@ struct efa_admin_modify_qp_cmd { /* Enable async notification when SQ is drained */ u8 sq_drained_async_notify; - /* MBZ */ - u8 reserved1; + /* Number of RNR retries (valid only for SRD QPs) */ + u8 rnr_retry; /* MBZ */ u16 reserved2; @@ -258,8 +258,8 @@ struct efa_admin_query_qp_resp { /* Indicates that draining is in progress */ u8 sq_draining; - /* MBZ */ - u8 reserved1; + /* Number of RNR retries (valid only for SRD QPs) */ + u8 rnr_retry; /* MBZ */ u16 reserved2; @@ -530,10 +530,36 @@ struct efa_admin_basic_stats { u64 rx_drops; }; +struct efa_admin_messages_stats { + u64 send_bytes; + + u64 send_wrs; + + u64 recv_bytes; + + u64 recv_wrs; +}; + +struct efa_admin_rdma_read_stats { + u64 read_wrs; + + u64 read_bytes; + + u64 read_wr_err; + + u64 read_resp_bytes; +}; + struct efa_admin_acq_get_stats_resp { struct efa_admin_acq_common_desc acq_common_desc; - struct efa_admin_basic_stats basic_stats; + union { + struct efa_admin_basic_stats basic_stats; + + struct efa_admin_messages_stats messages_stats; + + struct efa_admin_rdma_read_stats rdma_read_stats; + } u; }; struct efa_admin_get_set_feature_common_desc { @@ -576,7 +602,9 @@ struct efa_admin_feature_device_attr_desc { /* * 0 : rdma_read - If set, RDMA Read is supported on * TX queues - * 31:1 : reserved - MBZ + * 1 : rnr_retry - If set, RNR retry is supported on + * modify QP command + * 31:2 : reserved - MBZ */ u32 device_caps; @@ -862,6 +890,14 @@ struct efa_admin_host_info { #define EFA_ADMIN_CREATE_QP_CMD_SQ_VIRT_MASK BIT(0) #define EFA_ADMIN_CREATE_QP_CMD_RQ_VIRT_MASK BIT(1) +/* modify_qp_cmd */ +#define EFA_ADMIN_MODIFY_QP_CMD_QP_STATE_MASK BIT(0) +#define EFA_ADMIN_MODIFY_QP_CMD_CUR_QP_STATE_MASK BIT(1) +#define EFA_ADMIN_MODIFY_QP_CMD_QKEY_MASK BIT(2) +#define EFA_ADMIN_MODIFY_QP_CMD_SQ_PSN_MASK BIT(3) +#define EFA_ADMIN_MODIFY_QP_CMD_SQ_DRAINED_ASYNC_NOTIFY_MASK BIT(4) +#define EFA_ADMIN_MODIFY_QP_CMD_RNR_RETRY_MASK BIT(5) + /* reg_mr_cmd */ #define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK GENMASK(4, 0) #define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_MASK BIT(7) @@ -878,6 +914,7 @@ struct efa_admin_host_info { /* feature_device_attr_desc */ #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_READ_MASK BIT(0) +#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RNR_RETRY_MASK BIT(1) /* host_info */ #define EFA_ADMIN_HOST_INFO_DRIVER_MODULE_TYPE_MASK GENMASK(7, 0) diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c index 6ac23627f65a..f752ef64159c 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.c +++ b/drivers/infiniband/hw/efa/efa_com_cmd.c @@ -76,6 +76,7 @@ int efa_com_modify_qp(struct efa_com_dev *edev, cmd.qkey = params->qkey; cmd.sq_psn = params->sq_psn; cmd.sq_drained_async_notify = params->sq_drained_async_notify; + cmd.rnr_retry = params->rnr_retry; err = efa_com_cmd_exec(aq, (struct efa_admin_aq_entry *)&cmd, @@ -121,6 +122,7 @@ int efa_com_query_qp(struct efa_com_dev *edev, result->qkey = resp.qkey; result->sq_draining = resp.sq_draining; result->sq_psn = resp.sq_psn; + result->rnr_retry = resp.rnr_retry; return 0; } @@ -750,11 +752,27 @@ int efa_com_get_stats(struct efa_com_dev *edev, return err; } - result->basic_stats.tx_bytes = resp.basic_stats.tx_bytes; - result->basic_stats.tx_pkts = resp.basic_stats.tx_pkts; - result->basic_stats.rx_bytes = resp.basic_stats.rx_bytes; - result->basic_stats.rx_pkts = resp.basic_stats.rx_pkts; - result->basic_stats.rx_drops = resp.basic_stats.rx_drops; + switch (cmd.type) { + case EFA_ADMIN_GET_STATS_TYPE_BASIC: + result->basic_stats.tx_bytes = resp.u.basic_stats.tx_bytes; + result->basic_stats.tx_pkts = resp.u.basic_stats.tx_pkts; + result->basic_stats.rx_bytes = resp.u.basic_stats.rx_bytes; + result->basic_stats.rx_pkts = resp.u.basic_stats.rx_pkts; + result->basic_stats.rx_drops = resp.u.basic_stats.rx_drops; + break; + case EFA_ADMIN_GET_STATS_TYPE_MESSAGES: + result->messages_stats.send_bytes = resp.u.messages_stats.send_bytes; + result->messages_stats.send_wrs = resp.u.messages_stats.send_wrs; + result->messages_stats.recv_bytes = resp.u.messages_stats.recv_bytes; + result->messages_stats.recv_wrs = resp.u.messages_stats.recv_wrs; + break; + case EFA_ADMIN_GET_STATS_TYPE_RDMA_READ: + result->rdma_read_stats.read_wrs = resp.u.rdma_read_stats.read_wrs; + result->rdma_read_stats.read_bytes = resp.u.rdma_read_stats.read_bytes; + result->rdma_read_stats.read_wr_err = resp.u.rdma_read_stats.read_wr_err; + result->rdma_read_stats.read_resp_bytes = resp.u.rdma_read_stats.read_resp_bytes; + break; + } return 0; } diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h index 190bac23f585..eea4ebfbe6ec 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.h +++ b/drivers/infiniband/hw/efa/efa_com_cmd.h @@ -47,6 +47,7 @@ struct efa_com_modify_qp_params { u32 qkey; u32 sq_psn; u8 sq_drained_async_notify; + u8 rnr_retry; }; struct efa_com_query_qp_params { @@ -58,6 +59,7 @@ struct efa_com_query_qp_result { u32 qkey; u32 sq_draining; u32 sq_psn; + u8 rnr_retry; }; struct efa_com_destroy_qp_params { @@ -238,8 +240,24 @@ struct efa_com_basic_stats { u64 rx_drops; }; +struct efa_com_messages_stats { + u64 send_bytes; + u64 send_wrs; + u64 recv_bytes; + u64 recv_wrs; +}; + +struct efa_com_rdma_read_stats { + u64 read_wrs; + u64 read_bytes; + u64 read_wr_err; + u64 read_resp_bytes; +}; + union efa_com_get_stats_result { struct efa_com_basic_stats basic_stats; + struct efa_com_messages_stats messages_stats; + struct efa_com_rdma_read_stats rdma_read_stats; }; void efa_com_set_dma_addr(dma_addr_t addr, u32 *addr_high, u32 *addr_low); diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index 92d701146320..6faed3a81e08 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -331,7 +331,7 @@ static int efa_ib_device_add(struct efa_dev *dev) ib_set_device_ops(&dev->ibdev, &efa_dev_ops); - err = ib_register_device(&dev->ibdev, "efa_%d"); + err = ib_register_device(&dev->ibdev, "efa_%d", &pdev->dev); if (err) goto err_release_doorbell_bar; @@ -418,7 +418,7 @@ static int efa_device_init(struct efa_com_dev *edev, struct pci_dev *pdev) err); return err; } - + dma_set_max_seg_size(&pdev->dev, UINT_MAX); return 0; } diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 9e201f169289..191e0843f090 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -4,6 +4,7 @@ */ #include <linux/vmalloc.h> +#include <linux/log2.h> #include <rdma/ib_addr.h> #include <rdma/ib_umem.h> @@ -35,6 +36,14 @@ struct efa_user_mmap_entry { op(EFA_RX_BYTES, "rx_bytes") \ op(EFA_RX_PKTS, "rx_pkts") \ op(EFA_RX_DROPS, "rx_drops") \ + op(EFA_SEND_BYTES, "send_bytes") \ + op(EFA_SEND_WRS, "send_wrs") \ + op(EFA_RECV_BYTES, "recv_bytes") \ + op(EFA_RECV_WRS, "recv_wrs") \ + op(EFA_RDMA_READ_WRS, "rdma_read_wrs") \ + op(EFA_RDMA_READ_BYTES, "rdma_read_bytes") \ + op(EFA_RDMA_READ_WR_ERR, "rdma_read_wr_err") \ + op(EFA_RDMA_READ_RESP_BYTES, "rdma_read_resp_bytes") \ op(EFA_SUBMITTED_CMDS, "submitted_cmds") \ op(EFA_COMPLETED_CMDS, "completed_cmds") \ op(EFA_CMDS_ERR, "cmds_err") \ @@ -142,10 +151,9 @@ to_emmap(struct rdma_user_mmap_entry *rdma_entry) return container_of(rdma_entry, struct efa_user_mmap_entry, rdma_entry); } -static inline bool is_rdma_read_cap(struct efa_dev *dev) -{ - return dev->dev_attr.device_caps & EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_READ_MASK; -} +#define EFA_DEV_CAP(dev, cap) \ + ((dev)->dev_attr.device_caps & \ + EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_##cap##_MASK) #define is_reserved_cleared(reserved) \ !memchr_inv(reserved, 0, sizeof(reserved)) @@ -221,9 +229,12 @@ int efa_query_device(struct ib_device *ibdev, resp.max_rq_wr = dev_attr->max_rq_depth; resp.max_rdma_size = dev_attr->max_rdma_size; - if (is_rdma_read_cap(dev)) + if (EFA_DEV_CAP(dev, RDMA_READ)) resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RDMA_READ; + if (EFA_DEV_CAP(dev, RNR_RETRY)) + resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RNR_RETRY; + err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); if (err) { @@ -269,7 +280,7 @@ int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, #define EFA_QUERY_QP_SUPP_MASK \ (IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | \ - IB_QP_QKEY | IB_QP_SQ_PSN | IB_QP_CAP) + IB_QP_QKEY | IB_QP_SQ_PSN | IB_QP_CAP | IB_QP_RNR_RETRY) if (qp_attr_mask & ~EFA_QUERY_QP_SUPP_MASK) { ibdev_dbg(&dev->ibdev, @@ -291,6 +302,7 @@ int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, qp_attr->sq_psn = result.sq_psn; qp_attr->sq_draining = result.sq_draining; qp_attr->port_num = 1; + qp_attr->rnr_retry = result.rnr_retry; qp_attr->cap.max_send_wr = qp->max_send_wr; qp_attr->cap.max_recv_wr = qp->max_recv_wr; @@ -376,17 +388,18 @@ int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) err_dealloc_pd: efa_pd_dealloc(dev, result.pdn); err_out: - atomic64_inc(&dev->stats.sw_stats.alloc_pd_err); + atomic64_inc(&dev->stats.alloc_pd_err); return err; } -void efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +int efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct efa_dev *dev = to_edev(ibpd->device); struct efa_pd *pd = to_epd(ibpd); ibdev_dbg(&dev->ibdev, "Dealloc pd[%d]\n", pd->pdn); efa_pd_dealloc(dev, pd->pdn); + return 0; } static int efa_destroy_qp_handle(struct efa_dev *dev, u32 qp_handle) @@ -737,18 +750,130 @@ err_free_mapped: err_free_qp: kfree(qp); err_out: - atomic64_inc(&dev->stats.sw_stats.create_qp_err); + atomic64_inc(&dev->stats.create_qp_err); return ERR_PTR(err); } +static const struct { + int valid; + enum ib_qp_attr_mask req_param; + enum ib_qp_attr_mask opt_param; +} srd_qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = { + [IB_QPS_RESET] = { + [IB_QPS_RESET] = { .valid = 1 }, + [IB_QPS_INIT] = { + .valid = 1, + .req_param = IB_QP_PKEY_INDEX | + IB_QP_PORT | + IB_QP_QKEY, + }, + }, + [IB_QPS_INIT] = { + [IB_QPS_RESET] = { .valid = 1 }, + [IB_QPS_ERR] = { .valid = 1 }, + [IB_QPS_INIT] = { + .valid = 1, + .opt_param = IB_QP_PKEY_INDEX | + IB_QP_PORT | + IB_QP_QKEY, + }, + [IB_QPS_RTR] = { + .valid = 1, + .opt_param = IB_QP_PKEY_INDEX | + IB_QP_QKEY, + }, + }, + [IB_QPS_RTR] = { + [IB_QPS_RESET] = { .valid = 1 }, + [IB_QPS_ERR] = { .valid = 1 }, + [IB_QPS_RTS] = { + .valid = 1, + .req_param = IB_QP_SQ_PSN, + .opt_param = IB_QP_CUR_STATE | + IB_QP_QKEY | + IB_QP_RNR_RETRY, + + } + }, + [IB_QPS_RTS] = { + [IB_QPS_RESET] = { .valid = 1 }, + [IB_QPS_ERR] = { .valid = 1 }, + [IB_QPS_RTS] = { + .valid = 1, + .opt_param = IB_QP_CUR_STATE | + IB_QP_QKEY, + }, + [IB_QPS_SQD] = { + .valid = 1, + .opt_param = IB_QP_EN_SQD_ASYNC_NOTIFY, + }, + }, + [IB_QPS_SQD] = { + [IB_QPS_RESET] = { .valid = 1 }, + [IB_QPS_ERR] = { .valid = 1 }, + [IB_QPS_RTS] = { + .valid = 1, + .opt_param = IB_QP_CUR_STATE | + IB_QP_QKEY, + }, + [IB_QPS_SQD] = { + .valid = 1, + .opt_param = IB_QP_PKEY_INDEX | + IB_QP_QKEY, + } + }, + [IB_QPS_SQE] = { + [IB_QPS_RESET] = { .valid = 1 }, + [IB_QPS_ERR] = { .valid = 1 }, + [IB_QPS_RTS] = { + .valid = 1, + .opt_param = IB_QP_CUR_STATE | + IB_QP_QKEY, + } + }, + [IB_QPS_ERR] = { + [IB_QPS_RESET] = { .valid = 1 }, + [IB_QPS_ERR] = { .valid = 1 }, + } +}; + +static bool efa_modify_srd_qp_is_ok(enum ib_qp_state cur_state, + enum ib_qp_state next_state, + enum ib_qp_attr_mask mask) +{ + enum ib_qp_attr_mask req_param, opt_param; + + if (mask & IB_QP_CUR_STATE && + cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS && + cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE) + return false; + + if (!srd_qp_state_table[cur_state][next_state].valid) + return false; + + req_param = srd_qp_state_table[cur_state][next_state].req_param; + opt_param = srd_qp_state_table[cur_state][next_state].opt_param; + + if ((mask & req_param) != req_param) + return false; + + if (mask & ~(req_param | opt_param | IB_QP_STATE)) + return false; + + return true; +} + static int efa_modify_qp_validate(struct efa_dev *dev, struct efa_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, enum ib_qp_state cur_state, enum ib_qp_state new_state) { + int err; + #define EFA_MODIFY_QP_SUPP_MASK \ (IB_QP_STATE | IB_QP_CUR_STATE | IB_QP_EN_SQD_ASYNC_NOTIFY | \ - IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_QKEY | IB_QP_SQ_PSN) + IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_QKEY | IB_QP_SQ_PSN | \ + IB_QP_RNR_RETRY) if (qp_attr_mask & ~EFA_MODIFY_QP_SUPP_MASK) { ibdev_dbg(&dev->ibdev, @@ -757,8 +882,14 @@ static int efa_modify_qp_validate(struct efa_dev *dev, struct efa_qp *qp, return -EOPNOTSUPP; } - if (!ib_modify_qp_is_ok(cur_state, new_state, IB_QPT_UD, - qp_attr_mask)) { + if (qp->ibqp.qp_type == IB_QPT_DRIVER) + err = !efa_modify_srd_qp_is_ok(cur_state, new_state, + qp_attr_mask); + else + err = !ib_modify_qp_is_ok(cur_state, new_state, IB_QPT_UD, + qp_attr_mask); + + if (err) { ibdev_dbg(&dev->ibdev, "Invalid modify QP parameters\n"); return -EINVAL; } @@ -805,28 +936,36 @@ int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, params.qp_handle = qp->qp_handle; if (qp_attr_mask & IB_QP_STATE) { - params.modify_mask |= BIT(EFA_ADMIN_QP_STATE_BIT) | - BIT(EFA_ADMIN_CUR_QP_STATE_BIT); + EFA_SET(¶ms.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_QP_STATE, + 1); + EFA_SET(¶ms.modify_mask, + EFA_ADMIN_MODIFY_QP_CMD_CUR_QP_STATE, 1); params.cur_qp_state = qp_attr->cur_qp_state; params.qp_state = qp_attr->qp_state; } if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { - params.modify_mask |= - BIT(EFA_ADMIN_SQ_DRAINED_ASYNC_NOTIFY_BIT); + EFA_SET(¶ms.modify_mask, + EFA_ADMIN_MODIFY_QP_CMD_SQ_DRAINED_ASYNC_NOTIFY, 1); params.sq_drained_async_notify = qp_attr->en_sqd_async_notify; } if (qp_attr_mask & IB_QP_QKEY) { - params.modify_mask |= BIT(EFA_ADMIN_QKEY_BIT); + EFA_SET(¶ms.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_QKEY, 1); params.qkey = qp_attr->qkey; } if (qp_attr_mask & IB_QP_SQ_PSN) { - params.modify_mask |= BIT(EFA_ADMIN_SQ_PSN_BIT); + EFA_SET(¶ms.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_SQ_PSN, 1); params.sq_psn = qp_attr->sq_psn; } + if (qp_attr_mask & IB_QP_RNR_RETRY) { + EFA_SET(¶ms.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_RNR_RETRY, + 1); + params.rnr_retry = qp_attr->rnr_retry; + } + err = efa_com_modify_qp(&dev->edev, ¶ms); if (err) return err; @@ -843,7 +982,7 @@ static int efa_destroy_cq_idx(struct efa_dev *dev, int cq_idx) return efa_com_destroy_cq(&dev->edev, ¶ms); } -void efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct efa_dev *dev = to_edev(ibcq->device); struct efa_cq *cq = to_ecq(ibcq); @@ -856,6 +995,7 @@ void efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) efa_destroy_cq_idx(dev, cq->cq_idx); efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size, DMA_FROM_DEVICE); + return 0; } static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq, @@ -996,7 +1136,7 @@ err_free_mapped: DMA_FROM_DEVICE); err_out: - atomic64_inc(&dev->stats.sw_stats.create_cq_err); + atomic64_inc(&dev->stats.create_cq_err); return err; } @@ -1013,8 +1153,7 @@ static int umem_to_page_list(struct efa_dev *dev, ibdev_dbg(&dev->ibdev, "hp_cnt[%u], pages_in_hp[%u]\n", hp_cnt, pages_in_hp); - rdma_for_each_block(umem->sg_head.sgl, &biter, umem->nmap, - BIT(hp_shift)) + rdma_umem_for_each_dma_block(umem, &biter, BIT(hp_shift)) page_list[hp_idx++] = rdma_block_iter_dma_address(&biter); return 0; @@ -1026,7 +1165,7 @@ static struct scatterlist *efa_vmalloc_buf_to_sg(u64 *buf, int page_cnt) struct page *pg; int i; - sglist = kcalloc(page_cnt, sizeof(*sglist), GFP_KERNEL); + sglist = kmalloc_array(page_cnt, sizeof(*sglist), GFP_KERNEL); if (!sglist) return NULL; sg_init_table(sglist, page_cnt); @@ -1370,7 +1509,7 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, supp_access_flags = IB_ACCESS_LOCAL_WRITE | - (is_rdma_read_cap(dev) ? IB_ACCESS_REMOTE_READ : 0); + (EFA_DEV_CAP(dev, RDMA_READ) ? IB_ACCESS_REMOTE_READ : 0); access_flags &= ~IB_ACCESS_OPTIONAL; if (access_flags & ~supp_access_flags) { @@ -1410,9 +1549,8 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, goto err_unmap; } - params.page_shift = __ffs(pg_sz); - params.page_num = DIV_ROUND_UP(length + (start & (pg_sz - 1)), - pg_sz); + params.page_shift = order_base_2(pg_sz); + params.page_num = ib_umem_num_dma_blocks(mr->umem, pg_sz); ibdev_dbg(&dev->ibdev, "start %#llx length %#llx params.page_shift %u params.page_num %u\n", @@ -1451,7 +1589,7 @@ err_unmap: err_free: kfree(mr); err_out: - atomic64_inc(&dev->stats.sw_stats.reg_mr_err); + atomic64_inc(&dev->stats.reg_mr_err); return ERR_PTR(err); } @@ -1569,19 +1707,17 @@ int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata) resp.max_tx_batch = dev->dev_attr.max_tx_batch; resp.min_sq_wr = dev->dev_attr.min_sq_depth; - if (udata && udata->outlen) { - err = ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen)); - if (err) - goto err_dealloc_uar; - } + err = ib_copy_to_udata(udata, &resp, + min(sizeof(resp), udata->outlen)); + if (err) + goto err_dealloc_uar; return 0; err_dealloc_uar: efa_dealloc_uar(dev, result.uarn); err_out: - atomic64_inc(&dev->stats.sw_stats.alloc_ucontext_err); + atomic64_inc(&dev->stats.alloc_ucontext_err); return err; } @@ -1614,7 +1750,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, ibdev_dbg(&dev->ibdev, "pgoff[%#lx] does not have valid entry\n", vma->vm_pgoff); - atomic64_inc(&dev->stats.sw_stats.mmap_err); + atomic64_inc(&dev->stats.mmap_err); return -EINVAL; } entry = to_emmap(rdma_entry); @@ -1656,7 +1792,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n", entry->address, rdma_entry->npages * PAGE_SIZE, entry->mmap_flag, err); - atomic64_inc(&dev->stats.sw_stats.mmap_err); + atomic64_inc(&dev->stats.mmap_err); } rdma_user_mmap_entry_put(rdma_entry); @@ -1741,11 +1877,11 @@ int efa_create_ah(struct ib_ah *ibah, err_destroy_ah: efa_ah_destroy(dev, ah); err_out: - atomic64_inc(&dev->stats.sw_stats.create_ah_err); + atomic64_inc(&dev->stats.create_ah_err); return err; } -void efa_destroy_ah(struct ib_ah *ibah, u32 flags) +int efa_destroy_ah(struct ib_ah *ibah, u32 flags) { struct efa_dev *dev = to_edev(ibah->pd->device); struct efa_ah *ah = to_eah(ibah); @@ -1755,10 +1891,11 @@ void efa_destroy_ah(struct ib_ah *ibah, u32 flags) if (!(flags & RDMA_DESTROY_AH_SLEEPABLE)) { ibdev_dbg(&dev->ibdev, "Destroy address handle is not supported in atomic context\n"); - return; + return -EOPNOTSUPP; } efa_ah_destroy(dev, ah); + return 0; } struct rdma_hw_stats *efa_alloc_hw_stats(struct ib_device *ibdev, u8 port_num) @@ -1774,13 +1911,15 @@ int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, struct efa_com_get_stats_params params = {}; union efa_com_get_stats_result result; struct efa_dev *dev = to_edev(ibdev); + struct efa_com_rdma_read_stats *rrs; + struct efa_com_messages_stats *ms; struct efa_com_basic_stats *bs; struct efa_com_stats_admin *as; struct efa_stats *s; int err; - params.type = EFA_ADMIN_GET_STATS_TYPE_BASIC; params.scope = EFA_ADMIN_GET_STATS_SCOPE_ALL; + params.type = EFA_ADMIN_GET_STATS_TYPE_BASIC; err = efa_com_get_stats(&dev->edev, ¶ms, &result); if (err) @@ -1793,6 +1932,28 @@ int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, stats->value[EFA_RX_PKTS] = bs->rx_pkts; stats->value[EFA_RX_DROPS] = bs->rx_drops; + params.type = EFA_ADMIN_GET_STATS_TYPE_MESSAGES; + err = efa_com_get_stats(&dev->edev, ¶ms, &result); + if (err) + return err; + + ms = &result.messages_stats; + stats->value[EFA_SEND_BYTES] = ms->send_bytes; + stats->value[EFA_SEND_WRS] = ms->send_wrs; + stats->value[EFA_RECV_BYTES] = ms->recv_bytes; + stats->value[EFA_RECV_WRS] = ms->recv_wrs; + + params.type = EFA_ADMIN_GET_STATS_TYPE_RDMA_READ; + err = efa_com_get_stats(&dev->edev, ¶ms, &result); + if (err) + return err; + + rrs = &result.rdma_read_stats; + stats->value[EFA_RDMA_READ_WRS] = rrs->read_wrs; + stats->value[EFA_RDMA_READ_BYTES] = rrs->read_bytes; + stats->value[EFA_RDMA_READ_WR_ERR] = rrs->read_wr_err; + stats->value[EFA_RDMA_READ_RESP_BYTES] = rrs->read_resp_bytes; + as = &dev->edev.aq.stats; stats->value[EFA_SUBMITTED_CMDS] = atomic64_read(&as->submitted_cmd); stats->value[EFA_COMPLETED_CMDS] = atomic64_read(&as->completed_cmd); @@ -1801,13 +1962,14 @@ int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, s = &dev->stats; stats->value[EFA_KEEP_ALIVE_RCVD] = atomic64_read(&s->keep_alive_rcvd); - stats->value[EFA_ALLOC_PD_ERR] = atomic64_read(&s->sw_stats.alloc_pd_err); - stats->value[EFA_CREATE_QP_ERR] = atomic64_read(&s->sw_stats.create_qp_err); - stats->value[EFA_CREATE_CQ_ERR] = atomic64_read(&s->sw_stats.create_cq_err); - stats->value[EFA_REG_MR_ERR] = atomic64_read(&s->sw_stats.reg_mr_err); - stats->value[EFA_ALLOC_UCONTEXT_ERR] = atomic64_read(&s->sw_stats.alloc_ucontext_err); - stats->value[EFA_CREATE_AH_ERR] = atomic64_read(&s->sw_stats.create_ah_err); - stats->value[EFA_MMAP_ERR] = atomic64_read(&s->sw_stats.mmap_err); + stats->value[EFA_ALLOC_PD_ERR] = atomic64_read(&s->alloc_pd_err); + stats->value[EFA_CREATE_QP_ERR] = atomic64_read(&s->create_qp_err); + stats->value[EFA_CREATE_CQ_ERR] = atomic64_read(&s->create_cq_err); + stats->value[EFA_REG_MR_ERR] = atomic64_read(&s->reg_mr_err); + stats->value[EFA_ALLOC_UCONTEXT_ERR] = + atomic64_read(&s->alloc_ucontext_err); + stats->value[EFA_CREATE_AH_ERR] = atomic64_read(&s->create_ah_err); + stats->value[EFA_MMAP_ERR] = atomic64_read(&s->mmap_err); return ARRAY_SIZE(efa_stats_names); } diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c index 04575c9afd61..a307d4c8b15a 100644 --- a/drivers/infiniband/hw/hfi1/sdma.c +++ b/drivers/infiniband/hw/hfi1/sdma.c @@ -232,11 +232,11 @@ static const struct sdma_set_state_action sdma_action_table[] = { static void sdma_complete(struct kref *); static void sdma_finalput(struct sdma_state *); static void sdma_get(struct sdma_state *); -static void sdma_hw_clean_up_task(unsigned long); +static void sdma_hw_clean_up_task(struct tasklet_struct *); static void sdma_put(struct sdma_state *); static void sdma_set_state(struct sdma_engine *, enum sdma_states); static void sdma_start_hw_clean_up(struct sdma_engine *); -static void sdma_sw_clean_up_task(unsigned long); +static void sdma_sw_clean_up_task(struct tasklet_struct *); static void sdma_sendctrl(struct sdma_engine *, unsigned); static void init_sdma_regs(struct sdma_engine *, u32, uint); static void sdma_process_event( @@ -545,9 +545,10 @@ static void sdma_err_progress_check(struct timer_list *t) schedule_work(&sde->err_halt_worker); } -static void sdma_hw_clean_up_task(unsigned long opaque) +static void sdma_hw_clean_up_task(struct tasklet_struct *t) { - struct sdma_engine *sde = (struct sdma_engine *)opaque; + struct sdma_engine *sde = from_tasklet(sde, t, + sdma_hw_clean_up_task); u64 statuscsr; while (1) { @@ -604,9 +605,9 @@ static void sdma_flush_descq(struct sdma_engine *sde) sdma_desc_avail(sde, sdma_descq_freecnt(sde)); } -static void sdma_sw_clean_up_task(unsigned long opaque) +static void sdma_sw_clean_up_task(struct tasklet_struct *t) { - struct sdma_engine *sde = (struct sdma_engine *)opaque; + struct sdma_engine *sde = from_tasklet(sde, t, sdma_sw_clean_up_task); unsigned long flags; spin_lock_irqsave(&sde->tail_lock, flags); @@ -1454,11 +1455,10 @@ int sdma_init(struct hfi1_devdata *dd, u8 port) sde->tail_csr = get_kctxt_csr_addr(dd, this_idx, SD(TAIL)); - tasklet_init(&sde->sdma_hw_clean_up_task, sdma_hw_clean_up_task, - (unsigned long)sde); - - tasklet_init(&sde->sdma_sw_clean_up_task, sdma_sw_clean_up_task, - (unsigned long)sde); + tasklet_setup(&sde->sdma_hw_clean_up_task, + sdma_hw_clean_up_task); + tasklet_setup(&sde->sdma_sw_clean_up_task, + sdma_sw_clean_up_task); INIT_WORK(&sde->err_halt_worker, sdma_err_halt_wait); INIT_WORK(&sde->flush_worker, sdma_field_flush); diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c index 30865635b449..3591923abebb 100644 --- a/drivers/infiniband/hw/hfi1/verbs.c +++ b/drivers/infiniband/hw/hfi1/verbs.c @@ -1424,7 +1424,7 @@ static int query_port(struct rvt_dev_info *rdi, u8 port_num, props->gid_tbl_len = HFI1_GUIDS_PER_PORT; props->active_width = (u8)opa_width_to_ib(ppd->link_width_active); /* see rate_show() in ib core/sysfs.c */ - props->active_speed = (u8)opa_speed_to_ib(ppd->link_speed_active); + props->active_speed = opa_speed_to_ib(ppd->link_speed_active); props->max_vl_num = ppd->vls_supported; /* Once we are a "first class" citizen and have added the OPA MTUs to diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c index 5b2f9314edd3..75b06db60f7c 100644 --- a/drivers/infiniband/hw/hns/hns_roce_ah.c +++ b/drivers/infiniband/hw/hns/hns_roce_ah.c @@ -39,6 +39,22 @@ #define HNS_ROCE_VLAN_SL_BIT_MASK 7 #define HNS_ROCE_VLAN_SL_SHIFT 13 +static inline u16 get_ah_udp_sport(const struct rdma_ah_attr *ah_attr) +{ + u32 fl = ah_attr->grh.flow_label; + u16 sport; + + if (!fl) + sport = get_random_u32() % + (IB_ROCE_UDP_ENCAP_VALID_PORT_MAX + 1 - + IB_ROCE_UDP_ENCAP_VALID_PORT_MIN) + + IB_ROCE_UDP_ENCAP_VALID_PORT_MIN; + else + sport = rdma_flow_label_to_udp_sport(fl); + + return sport; +} + int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata) { @@ -79,6 +95,8 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, memcpy(ah->av.dgid, grh->dgid.raw, HNS_ROCE_GID_SIZE); ah->av.sl = rdma_ah_get_sl(ah_attr); + ah->av.flowlabel = grh->flow_label; + ah->av.udp_sport = get_ah_udp_sport(ah_attr); return 0; } @@ -98,8 +116,3 @@ int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) return 0; } - -void hns_roce_destroy_ah(struct ib_ah *ah, u32 flags) -{ - return; -} diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c index a522cb2d29ea..a6b23dec1adc 100644 --- a/drivers/infiniband/hw/hns/hns_roce_alloc.c +++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c @@ -268,8 +268,7 @@ int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, } /* convert system page cnt to hw page cnt */ - rdma_for_each_block(umem->sg_head.sgl, &biter, umem->nmap, - 1 << page_shift) { + rdma_umem_for_each_dma_block(umem, &biter, 1 << page_shift) { addr = rdma_block_iter_dma_address(&biter); if (idx >= start) { bufs[total++] = addr; diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index e87d616f7988..809b22aa5056 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -150,7 +150,7 @@ static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, int err; buf_attr.page_shift = hr_dev->caps.cqe_buf_pg_sz + HNS_HW_PAGE_SHIFT; - buf_attr.region[0].size = hr_cq->cq_depth * hr_dev->caps.cq_entry_sz; + buf_attr.region[0].size = hr_cq->cq_depth * hr_cq->cqe_size; buf_attr.region[0].hopnum = hr_dev->caps.cqe_hop_num; buf_attr.region_count = 1; buf_attr.fixed_page = true; @@ -224,6 +224,21 @@ static void free_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, } } +static void set_cqe_size(struct hns_roce_cq *hr_cq, struct ib_udata *udata, + struct hns_roce_ib_create_cq *ucmd) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); + + if (udata) { + if (udata->inlen >= offsetofend(typeof(*ucmd), cqe_size)) + hr_cq->cqe_size = ucmd->cqe_size; + else + hr_cq->cqe_size = HNS_ROCE_V2_CQE_SIZE; + } else { + hr_cq->cqe_size = hr_dev->caps.cqe_sz; + } +} + int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, struct ib_udata *udata) { @@ -258,7 +273,8 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, INIT_LIST_HEAD(&hr_cq->rq_list); if (udata) { - ret = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)); + ret = ib_copy_from_udata(&ucmd, udata, + min(sizeof(ucmd), udata->inlen)); if (ret) { ibdev_err(ibdev, "Failed to copy CQ udata, err %d\n", ret); @@ -266,6 +282,8 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, } } + set_cqe_size(hr_cq, udata, &ucmd); + ret = alloc_cq_buf(hr_dev, hr_cq, udata, ucmd.buf_addr); if (ret) { ibdev_err(ibdev, "Failed to alloc CQ buf, err %d\n", ret); @@ -287,7 +305,7 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, /* * For the QP created by kernel space, tptr value should be initialized * to zero; For the QP created by user space, it will cause synchronous - * problems if tptr is set to zero here, so we initialze it in user + * problems if tptr is set to zero here, so we initialize it in user * space. */ if (!udata && hr_cq->tptr_addr) @@ -311,7 +329,7 @@ err_cq_buf: return ret; } -void hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +int hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); @@ -322,6 +340,7 @@ void hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) free_cq_buf(hr_dev, hr_cq); free_cq_db(hr_dev, hr_cq, udata); free_cqc(hr_dev, hr_cq); + return 0; } void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn) diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 6edcbdcd8f43..6d2acff69f98 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -37,8 +37,8 @@ #define DRV_NAME "hns_roce" -/* hip08 is a pci device */ #define PCI_REVISION_ID_HIP08 0x21 +#define PCI_REVISION_ID_HIP09 0x30 #define HNS_ROCE_HW_VER1 ('h' << 24 | 'i' << 16 | '0' << 8 | '6') @@ -57,7 +57,6 @@ /* Hardware specification only for v1 engine */ #define HNS_ROCE_MAX_INNER_MTPT_NUM 0x7 #define HNS_ROCE_MAX_MTPT_PBL_NUM 0x100000 -#define HNS_ROCE_MAX_SGE_NUM 2 #define HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS 20 #define HNS_ROCE_MAX_FREE_CQ_WAIT_CNT \ @@ -76,15 +75,18 @@ #define HNS_ROCE_CEQ 0 #define HNS_ROCE_AEQ 1 -#define HNS_ROCE_CEQ_ENTRY_SIZE 0x4 -#define HNS_ROCE_AEQ_ENTRY_SIZE 0x10 +#define HNS_ROCE_CEQE_SIZE 0x4 +#define HNS_ROCE_AEQE_SIZE 0x10 -#define HNS_ROCE_SL_SHIFT 28 -#define HNS_ROCE_TCLASS_SHIFT 20 -#define HNS_ROCE_FLOW_LABEL_MASK 0xfffff +#define HNS_ROCE_V3_EQE_SIZE 0x40 + +#define HNS_ROCE_V2_CQE_SIZE 32 +#define HNS_ROCE_V3_CQE_SIZE 64 + +#define HNS_ROCE_V2_QPC_SZ 256 +#define HNS_ROCE_V3_QPC_SZ 512 #define HNS_ROCE_MAX_PORTS 6 -#define HNS_ROCE_MAX_GID_NUM 16 #define HNS_ROCE_GID_SIZE 16 #define HNS_ROCE_SGE_SIZE 16 @@ -112,8 +114,6 @@ #define PAGES_SHIFT_24 24 #define PAGES_SHIFT_32 32 -#define HNS_ROCE_PCI_BAR_NUM 2 - #define HNS_ROCE_IDX_QUE_ENTRY_SZ 4 #define SRQ_DB_REG 0x230 @@ -467,6 +467,7 @@ struct hns_roce_cq { void __iomem *cq_db_l; u16 *tptr_addr; int arm_sn; + int cqe_size; unsigned long cqn; u32 vector; atomic_t refcount; @@ -535,17 +536,18 @@ struct hns_roce_raq_table { }; struct hns_roce_av { - u8 port; - u8 gid_index; - u8 stat_rate; - u8 hop_limit; - u32 flowlabel; - u8 sl; - u8 tclass; - u8 dgid[HNS_ROCE_GID_SIZE]; - u8 mac[ETH_ALEN]; - u16 vlan_id; - bool vlan_en; + u8 port; + u8 gid_index; + u8 stat_rate; + u8 hop_limit; + u32 flowlabel; + u16 udp_sport; + u8 sl; + u8 tclass; + u8 dgid[HNS_ROCE_GID_SIZE]; + u8 mac[ETH_ALEN]; + u16 vlan_id; + bool vlan_en; }; struct hns_roce_ah { @@ -655,6 +657,8 @@ struct hns_roce_qp { struct hns_roce_sge sge; u32 next_sge; + enum ib_mtu path_mtu; + u32 max_inline_data; /* 0: flush needed, 1: unneeded */ unsigned long flush_flag; @@ -678,7 +682,8 @@ enum { }; struct hns_roce_ceqe { - __le32 comp; + __le32 comp; + __le32 rsv[15]; }; struct hns_roce_aeqe { @@ -715,6 +720,7 @@ struct hns_roce_aeqe { u8 rsv0; } __packed cmd; } event; + __le32 rsv[12]; }; struct hns_roce_eq { @@ -791,15 +797,15 @@ struct hns_roce_caps { int num_pds; int reserved_pds; u32 mtt_entry_sz; - u32 cq_entry_sz; + u32 cqe_sz; u32 page_size_cap; u32 reserved_lkey; int mtpt_entry_sz; - int qpc_entry_sz; + int qpc_sz; int irrl_entry_sz; int trrl_entry_sz; int cqc_entry_sz; - int sccc_entry_sz; + int sccc_sz; int qpc_timer_entry_sz; int cqc_timer_entry_sz; int srqc_entry_sz; @@ -809,6 +815,8 @@ struct hns_roce_caps { u32 pbl_hop_num; int aeqe_depth; int ceqe_depth; + u32 aeqe_size; + u32 ceqe_size; enum ib_mtu max_mtu; u32 qpc_bt_num; u32 qpc_timer_bt_num; @@ -930,7 +938,7 @@ struct hns_roce_hw { int (*poll_cq)(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, struct ib_udata *udata); - void (*destroy_cq)(struct ib_cq *ibcq, struct ib_udata *udata); + int (*destroy_cq)(struct ib_cq *ibcq, struct ib_udata *udata); int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period); int (*init_eq)(struct hns_roce_dev *hr_dev); void (*cleanup_eq)(struct hns_roce_dev *hr_dev); @@ -1178,10 +1186,13 @@ void hns_roce_bitmap_free_range(struct hns_roce_bitmap *bitmap, int hns_roce_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata); int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); -void hns_roce_destroy_ah(struct ib_ah *ah, u32 flags); +static inline int hns_roce_destroy_ah(struct ib_ah *ah, u32 flags) +{ + return 0; +} int hns_roce_alloc_pd(struct ib_pd *pd, struct ib_udata *udata); -void hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); +int hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc); struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, @@ -1200,8 +1211,7 @@ int hns_roce_hw_destroy_mpt(struct hns_roce_dev *hr_dev, unsigned long mpt_index); unsigned long key_to_hw_index(u32 key); -struct ib_mw *hns_roce_alloc_mw(struct ib_pd *pd, enum ib_mw_type, - struct ib_udata *udata); +int hns_roce_alloc_mw(struct ib_mw *mw, struct ib_udata *udata); int hns_roce_dealloc_mw(struct ib_mw *ibmw); void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf); @@ -1220,7 +1230,7 @@ int hns_roce_create_srq(struct ib_srq *srq, int hns_roce_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr, enum ib_srq_attr_mask srq_attr_mask, struct ib_udata *udata); -void hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); +int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); struct ib_qp *hns_roce_create_qp(struct ib_pd *ib_pd, struct ib_qp_init_attr *init_attr, @@ -1247,7 +1257,7 @@ int to_hr_qp_type(int qp_type); int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, struct ib_udata *udata); -void hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); +int hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); int hns_roce_db_map_user(struct hns_roce_ucontext *context, struct ib_udata *udata, unsigned long virt, struct hns_roce_db *db); diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index c8db6f8ae018..7487cf3d2c37 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -338,8 +338,8 @@ static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, void __iomem *bt_cmd; __le32 bt_cmd_val[2]; __le32 bt_cmd_h = 0; - __le32 bt_cmd_l = 0; - u64 bt_ba = 0; + __le32 bt_cmd_l; + u64 bt_ba; int ret = 0; /* Find the HEM(Hardware Entry Memory) entry */ @@ -1027,7 +1027,7 @@ void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) if (hr_dev->caps.cqc_timer_entry_sz) hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cqc_timer_table); - if (hr_dev->caps.sccc_entry_sz) + if (hr_dev->caps.sccc_sz) hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.sccc_table); if (hr_dev->caps.trrl_entry_sz) @@ -1404,7 +1404,7 @@ int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, { const struct hns_roce_buf_region *r; int ofs, end; - int ret = 0; + int ret; int unit; int i; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index aeb3a6fa7d47..5f4d8a32ed6d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -70,15 +70,15 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, struct hns_roce_qp *qp = to_hr_qp(ibqp); struct device *dev = &hr_dev->pdev->dev; struct hns_roce_sq_db sq_db = {}; - int ps_opcode = 0, i = 0; + int ps_opcode, i; unsigned long flags = 0; void *wqe = NULL; __le32 doorbell[2]; - u32 wqe_idx = 0; - int nreq = 0; int ret = 0; - u8 *smac; int loopback; + u32 wqe_idx; + int nreq; + u8 *smac; if (unlikely(ibqp->qp_type != IB_QPT_GSI && ibqp->qp_type != IB_QPT_RC)) { @@ -271,7 +271,6 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, ps_opcode = HNS_ROCE_WQE_OPCODE_SEND; break; case IB_WR_LOCAL_INV: - break; case IB_WR_ATOMIC_CMP_AND_SWP: case IB_WR_ATOMIC_FETCH_AND_ADD: case IB_WR_LSO: @@ -888,7 +887,7 @@ static int hns_roce_db_init(struct hns_roce_dev *hr_dev) u32 odb_ext_mod; u32 sdb_evt_mod; u32 odb_evt_mod; - int ret = 0; + int ret; memset(db, 0, sizeof(*db)); @@ -1148,8 +1147,8 @@ static int hns_roce_raq_init(struct hns_roce_dev *hr_dev) struct hns_roce_v1_priv *priv = hr_dev->priv; struct hns_roce_raq_table *raq = &priv->raq_table; struct device *dev = &hr_dev->pdev->dev; - int raq_shift = 0; dma_addr_t addr; + int raq_shift; __le32 tmp; u32 val; int ret; @@ -1360,7 +1359,7 @@ static int hns_roce_free_mr_init(struct hns_roce_dev *hr_dev) struct hns_roce_v1_priv *priv = hr_dev->priv; struct hns_roce_free_mr *free_mr = &priv->free_mr; struct device *dev = &hr_dev->pdev->dev; - int ret = 0; + int ret; free_mr->free_mr_wq = create_singlethread_workqueue("hns_roce_free_mr"); if (!free_mr->free_mr_wq) { @@ -1440,8 +1439,8 @@ static int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset) static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev) { - int i = 0; struct hns_roce_caps *caps = &hr_dev->caps; + int i; hr_dev->vendor_id = roce_read(hr_dev, ROCEE_VENDOR_ID_REG); hr_dev->vendor_part_id = roce_read(hr_dev, ROCEE_VENDOR_PART_ID_REG); @@ -1471,12 +1470,12 @@ static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev) caps->max_qp_dest_rdma = HNS_ROCE_V1_MAX_QP_DEST_RDMA; caps->max_sq_desc_sz = HNS_ROCE_V1_MAX_SQ_DESC_SZ; caps->max_rq_desc_sz = HNS_ROCE_V1_MAX_RQ_DESC_SZ; - caps->qpc_entry_sz = HNS_ROCE_V1_QPC_ENTRY_SIZE; + caps->qpc_sz = HNS_ROCE_V1_QPC_SIZE; caps->irrl_entry_sz = HNS_ROCE_V1_IRRL_ENTRY_SIZE; caps->cqc_entry_sz = HNS_ROCE_V1_CQC_ENTRY_SIZE; caps->mtpt_entry_sz = HNS_ROCE_V1_MTPT_ENTRY_SIZE; caps->mtt_entry_sz = HNS_ROCE_V1_MTT_ENTRY_SIZE; - caps->cq_entry_sz = HNS_ROCE_V1_CQE_ENTRY_SIZE; + caps->cqe_sz = HNS_ROCE_V1_CQE_SIZE; caps->page_size_cap = HNS_ROCE_V1_PAGE_SIZE_SUPPORT; caps->reserved_lkey = 0; caps->reserved_pds = 0; @@ -1643,7 +1642,7 @@ static int hns_roce_v1_chk_mbox(struct hns_roce_dev *hr_dev, unsigned long timeout) { u8 __iomem *hcr = hr_dev->reg_base + ROCEE_MB1_REG; - unsigned long end = 0; + unsigned long end; u32 status = 0; end = msecs_to_jiffies(timeout) + jiffies; @@ -1671,7 +1670,7 @@ static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, { unsigned long flags; u32 *p = NULL; - u8 gid_idx = 0; + u8 gid_idx; gid_idx = hns_get_gid_index(hr_dev, port, gid_index); @@ -1897,8 +1896,7 @@ static int hns_roce_v1_write_mtpt(struct hns_roce_dev *hr_dev, void *mb_buf, static void *get_cqe(struct hns_roce_cq *hr_cq, int n) { - return hns_roce_buf_offset(hr_cq->mtr.kmem, - n * HNS_ROCE_V1_CQE_ENTRY_SIZE); + return hns_roce_buf_offset(hr_cq->mtr.kmem, n * HNS_ROCE_V1_CQE_SIZE); } static void *get_sw_cqe(struct hns_roce_cq *hr_cq, int n) @@ -2445,7 +2443,7 @@ static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev, struct hns_roce_cmd_mailbox *mailbox; struct device *dev = &hr_dev->pdev->dev; - int ret = 0; + int ret; if (cur_state >= HNS_ROCE_QP_NUM_STATE || new_state >= HNS_ROCE_QP_NUM_STATE || @@ -3394,7 +3392,7 @@ static int hns_roce_v1_q_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); struct device *dev = &hr_dev->pdev->dev; struct hns_roce_qp_context *context; - int tmp_qp_state = 0; + int tmp_qp_state; int ret = 0; int state; @@ -3572,7 +3570,7 @@ int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) return 0; } -static void hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); @@ -3603,6 +3601,7 @@ static void hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) } wait_time++; } + return 0; } static void set_eq_cons_index_v1(struct hns_roce_eq *eq, int req_not) @@ -3775,8 +3774,7 @@ static void hns_roce_v1_db_overflow_handle(struct hns_roce_dev *hr_dev, static struct hns_roce_aeqe *get_aeqe_v1(struct hns_roce_eq *eq, u32 entry) { - unsigned long off = (entry & (eq->entries - 1)) * - HNS_ROCE_AEQ_ENTRY_SIZE; + unsigned long off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQE_SIZE; return (struct hns_roce_aeqe *)((u8 *) (eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) + @@ -3881,8 +3879,7 @@ static int hns_roce_v1_aeq_int(struct hns_roce_dev *hr_dev, static struct hns_roce_ceqe *get_ceqe_v1(struct hns_roce_eq *eq, u32 entry) { - unsigned long off = (entry & (eq->entries - 1)) * - HNS_ROCE_CEQ_ENTRY_SIZE; + unsigned long off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQE_SIZE; return (struct hns_roce_ceqe *)((u8 *) (eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) + @@ -3934,7 +3931,7 @@ static irqreturn_t hns_roce_v1_msix_interrupt_eq(int irq, void *eq_ptr) { struct hns_roce_eq *eq = eq_ptr; struct hns_roce_dev *hr_dev = eq->hr_dev; - int int_work = 0; + int int_work; if (eq->type_flag == HNS_ROCE_CEQ) /* CEQ irq routine, CEQ is pulse irq, not clear */ @@ -4132,9 +4129,9 @@ static int hns_roce_v1_create_eq(struct hns_roce_dev *hr_dev, void __iomem *eqc = hr_dev->eq_table.eqc_base[eq->eqn]; struct device *dev = &hr_dev->pdev->dev; dma_addr_t tmp_dma_addr; - u32 eqconsindx_val = 0; u32 eqcuridx_val = 0; - u32 eqshift_val = 0; + u32 eqconsindx_val; + u32 eqshift_val; __le32 tmp2 = 0; __le32 tmp1 = 0; __le32 tmp = 0; @@ -4253,7 +4250,7 @@ static int hns_roce_v1_init_eq_table(struct hns_roce_dev *hr_dev) CEQ_REG_OFFSET * i; eq->entries = hr_dev->caps.ceqe_depth; eq->log_entries = ilog2(eq->entries); - eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE; + eq->eqe_size = HNS_ROCE_CEQE_SIZE; } else { /* AEQ */ eq_table->eqc_base[i] = hr_dev->reg_base + @@ -4263,7 +4260,7 @@ static int hns_roce_v1_init_eq_table(struct hns_roce_dev *hr_dev) ROCEE_CAEP_AEQE_CONS_IDX_REG; eq->entries = hr_dev->caps.aeqe_depth; eq->log_entries = ilog2(eq->entries); - eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE; + eq->eqe_size = HNS_ROCE_AEQE_SIZE; } } diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h index 52307b2c7100..ffd0156080f5 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h @@ -68,13 +68,13 @@ #define HNS_ROCE_V1_COMP_EQE_NUM 0x8000 #define HNS_ROCE_V1_ASYNC_EQE_NUM 0x400 -#define HNS_ROCE_V1_QPC_ENTRY_SIZE 256 +#define HNS_ROCE_V1_QPC_SIZE 256 #define HNS_ROCE_V1_IRRL_ENTRY_SIZE 8 #define HNS_ROCE_V1_CQC_ENTRY_SIZE 64 #define HNS_ROCE_V1_MTPT_ENTRY_SIZE 64 #define HNS_ROCE_V1_MTT_ENTRY_SIZE 64 -#define HNS_ROCE_V1_CQE_ENTRY_SIZE 32 +#define HNS_ROCE_V1_CQE_SIZE 32 #define HNS_ROCE_V1_PAGE_SIZE_SUPPORT 0xFFFFF000 #define HNS_ROCE_V1_TABLE_CHUNK_SIZE (1 << 17) diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 4cda95ed1fbe..6d30850696c5 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -153,6 +153,67 @@ static void set_atomic_seg(const struct ib_send_wr *wr, V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge); } +static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, + const struct ib_send_wr *wr, + unsigned int *sge_idx, u32 msg_len) +{ + struct ib_device *ibdev = &(to_hr_dev(qp->ibqp.device))->ib_dev; + unsigned int dseg_len = sizeof(struct hns_roce_v2_wqe_data_seg); + unsigned int ext_sge_sz = qp->sq.max_gs * dseg_len; + unsigned int left_len_in_pg; + unsigned int idx = *sge_idx; + unsigned int i = 0; + unsigned int len; + void *addr; + void *dseg; + + if (msg_len > ext_sge_sz) { + ibdev_err(ibdev, + "no enough extended sge space for inline data.\n"); + return -EINVAL; + } + + dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1)); + left_len_in_pg = hr_hw_page_align((uintptr_t)dseg) - (uintptr_t)dseg; + len = wr->sg_list[0].length; + addr = (void *)(unsigned long)(wr->sg_list[0].addr); + + /* When copying data to extended sge space, the left length in page may + * not long enough for current user's sge. So the data should be + * splited into several parts, one in the first page, and the others in + * the subsequent pages. + */ + while (1) { + if (len <= left_len_in_pg) { + memcpy(dseg, addr, len); + + idx += len / dseg_len; + + i++; + if (i >= wr->num_sge) + break; + + left_len_in_pg -= len; + len = wr->sg_list[i].length; + addr = (void *)(unsigned long)(wr->sg_list[i].addr); + dseg += len; + } else { + memcpy(dseg, addr, left_len_in_pg); + + len -= left_len_in_pg; + addr += left_len_in_pg; + idx += left_len_in_pg / dseg_len; + dseg = hns_roce_get_extend_sge(qp, + idx & (qp->sge.sge_cnt - 1)); + left_len_in_pg = 1 << HNS_HW_PAGE_SHIFT; + } + } + + *sge_idx = idx; + + return 0; +} + static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr, unsigned int *sge_ind, unsigned int valid_num_sge) { @@ -177,73 +238,115 @@ static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr, *sge_ind = idx; } +static bool check_inl_data_len(struct hns_roce_qp *qp, unsigned int len) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device); + int mtu = ib_mtu_enum_to_int(qp->path_mtu); + + if (len > qp->max_inline_data || len > mtu) { + ibdev_err(&hr_dev->ib_dev, + "invalid length of data, data len = %u, max inline len = %u, path mtu = %d.\n", + len, qp->max_inline_data, mtu); + return false; + } + + return true; +} + +static int set_rc_inl(struct hns_roce_qp *qp, const struct ib_send_wr *wr, + struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, + unsigned int *sge_idx) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device); + u32 msg_len = le32_to_cpu(rc_sq_wqe->msg_len); + struct ib_device *ibdev = &hr_dev->ib_dev; + unsigned int curr_idx = *sge_idx; + void *dseg = rc_sq_wqe; + unsigned int i; + int ret; + + if (unlikely(wr->opcode == IB_WR_RDMA_READ)) { + ibdev_err(ibdev, "invalid inline parameters!\n"); + return -EINVAL; + } + + if (!check_inl_data_len(qp, msg_len)) + return -EINVAL; + + dseg += sizeof(struct hns_roce_v2_rc_send_wqe); + + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S, 1); + + if (msg_len <= HNS_ROCE_V2_MAX_RC_INL_INN_SZ) { + roce_set_bit(rc_sq_wqe->byte_20, + V2_RC_SEND_WQE_BYTE_20_INL_TYPE_S, 0); + + for (i = 0; i < wr->num_sge; i++) { + memcpy(dseg, ((void *)wr->sg_list[i].addr), + wr->sg_list[i].length); + dseg += wr->sg_list[i].length; + } + } else { + roce_set_bit(rc_sq_wqe->byte_20, + V2_RC_SEND_WQE_BYTE_20_INL_TYPE_S, 1); + + ret = fill_ext_sge_inl_data(qp, wr, &curr_idx, msg_len); + if (ret) + return ret; + + roce_set_field(rc_sq_wqe->byte_16, + V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M, + V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, + curr_idx - *sge_idx); + } + + *sge_idx = curr_idx; + + return 0; +} + static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr, struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, unsigned int *sge_ind, unsigned int valid_num_sge) { - struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); struct hns_roce_v2_wqe_data_seg *dseg = (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe); - struct ib_device *ibdev = &hr_dev->ib_dev; struct hns_roce_qp *qp = to_hr_qp(ibqp); - void *wqe = dseg; int j = 0; int i; - if (wr->send_flags & IB_SEND_INLINE && valid_num_sge) { - if (unlikely(le32_to_cpu(rc_sq_wqe->msg_len) > - hr_dev->caps.max_sq_inline)) { - ibdev_err(ibdev, "inline len(1-%d)=%d, illegal", - rc_sq_wqe->msg_len, - hr_dev->caps.max_sq_inline); - return -EINVAL; - } + roce_set_field(rc_sq_wqe->byte_20, + V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M, + V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S, + (*sge_ind) & (qp->sge.sge_cnt - 1)); - if (unlikely(wr->opcode == IB_WR_RDMA_READ)) { - ibdev_err(ibdev, "Not support inline data!\n"); - return -EINVAL; - } + if (wr->send_flags & IB_SEND_INLINE) + return set_rc_inl(qp, wr, rc_sq_wqe, sge_ind); + if (valid_num_sge <= HNS_ROCE_SGE_IN_WQE) { for (i = 0; i < wr->num_sge; i++) { - memcpy(wqe, ((void *)wr->sg_list[i].addr), - wr->sg_list[i].length); - wqe += wr->sg_list[i].length; + if (likely(wr->sg_list[i].length)) { + set_data_seg_v2(dseg, wr->sg_list + i); + dseg++; + } } - - roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S, - 1); } else { - if (valid_num_sge <= HNS_ROCE_SGE_IN_WQE) { - for (i = 0; i < wr->num_sge; i++) { - if (likely(wr->sg_list[i].length)) { - set_data_seg_v2(dseg, wr->sg_list + i); - dseg++; - } + for (i = 0; i < wr->num_sge && j < HNS_ROCE_SGE_IN_WQE; i++) { + if (likely(wr->sg_list[i].length)) { + set_data_seg_v2(dseg, wr->sg_list + i); + dseg++; + j++; } - } else { - roce_set_field(rc_sq_wqe->byte_20, - V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M, - V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S, - (*sge_ind) & (qp->sge.sge_cnt - 1)); - - for (i = 0; i < wr->num_sge && j < HNS_ROCE_SGE_IN_WQE; - i++) { - if (likely(wr->sg_list[i].length)) { - set_data_seg_v2(dseg, wr->sg_list + i); - dseg++; - j++; - } - } - - set_extend_sge(qp, wr, sge_ind, valid_num_sge); } - roce_set_field(rc_sq_wqe->byte_16, - V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M, - V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge); + set_extend_sge(qp, wr, sge_ind, valid_num_sge); } + roce_set_field(rc_sq_wqe->byte_16, + V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M, + V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge); + return 0; } @@ -292,6 +395,33 @@ static unsigned int calc_wr_sge_num(const struct ib_send_wr *wr, return valid_num; } +static __le32 get_immtdata(const struct ib_send_wr *wr) +{ + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_RDMA_WRITE_WITH_IMM: + return cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); + default: + return 0; + } +} + +static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, + const struct ib_send_wr *wr) +{ + u32 ib_op = wr->opcode; + + if (ib_op != IB_WR_SEND && ib_op != IB_WR_SEND_WITH_IMM) + return -EINVAL; + + ud_sq_wqe->immtdata = get_immtdata(wr); + + roce_set_field(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_OPCODE_M, + V2_UD_SEND_WQE_BYTE_4_OPCODE_S, to_hr_opcode(ib_op)); + + return 0; +} + static inline int set_ud_wqe(struct hns_roce_qp *qp, const struct ib_send_wr *wr, void *wqe, unsigned int *sge_idx, @@ -305,10 +435,15 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp, u32 msg_len = 0; bool loopback; u8 *smac; + int ret; valid_num_sge = calc_wr_sge_num(wr, &msg_len); memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe)); + ret = set_ud_opcode(ud_sq_wqe, wr); + if (WARN_ON(ret)) + return ret; + roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M, V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]); roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M, @@ -329,23 +464,8 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp, roce_set_bit(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback); - roce_set_field(ud_sq_wqe->byte_4, - V2_UD_SEND_WQE_BYTE_4_OPCODE_M, - V2_UD_SEND_WQE_BYTE_4_OPCODE_S, - HNS_ROCE_V2_WQE_OP_SEND); - ud_sq_wqe->msg_len = cpu_to_le32(msg_len); - switch (wr->opcode) { - case IB_WR_SEND_WITH_IMM: - case IB_WR_RDMA_WRITE_WITH_IMM: - ud_sq_wqe->immtdata = cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); - break; - default: - ud_sq_wqe->immtdata = 0; - break; - } - /* Set sig attr */ roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_CQE_S, (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0); @@ -369,7 +489,7 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp, curr_idx & (qp->sge.sge_cnt - 1)); roce_set_field(ud_sq_wqe->byte_24, V2_UD_SEND_WQE_BYTE_24_UDPSPN_M, - V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0); + V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, ah->av.udp_sport); ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ? qp->qkey : ud_wr(wr)->remote_qkey); roce_set_field(ud_sq_wqe->byte_32, V2_UD_SEND_WQE_BYTE_32_DQPN_M, @@ -402,6 +522,46 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp, return 0; } +static int set_rc_opcode(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, + const struct ib_send_wr *wr) +{ + u32 ib_op = wr->opcode; + + rc_sq_wqe->immtdata = get_immtdata(wr); + + switch (ib_op) { + case IB_WR_RDMA_READ: + case IB_WR_RDMA_WRITE: + case IB_WR_RDMA_WRITE_WITH_IMM: + rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey); + rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr); + break; + case IB_WR_SEND: + case IB_WR_SEND_WITH_IMM: + break; + case IB_WR_ATOMIC_CMP_AND_SWP: + case IB_WR_ATOMIC_FETCH_AND_ADD: + rc_sq_wqe->rkey = cpu_to_le32(atomic_wr(wr)->rkey); + rc_sq_wqe->va = cpu_to_le64(atomic_wr(wr)->remote_addr); + break; + case IB_WR_REG_MR: + set_frmr_seg(rc_sq_wqe, reg_wr(wr)); + break; + case IB_WR_LOCAL_INV: + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SO_S, 1); + fallthrough; + case IB_WR_SEND_WITH_INV: + rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey); + break; + default: + return -EINVAL; + } + + roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OPCODE_M, + V2_RC_SEND_WQE_BYTE_4_OPCODE_S, to_hr_opcode(ib_op)); + + return 0; +} static inline int set_rc_wqe(struct hns_roce_qp *qp, const struct ib_send_wr *wr, void *wqe, unsigned int *sge_idx, @@ -411,25 +571,16 @@ static inline int set_rc_wqe(struct hns_roce_qp *qp, unsigned int curr_idx = *sge_idx; unsigned int valid_num_sge; u32 msg_len = 0; - int ret = 0; + int ret; valid_num_sge = calc_wr_sge_num(wr, &msg_len); memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe)); rc_sq_wqe->msg_len = cpu_to_le32(msg_len); - switch (wr->opcode) { - case IB_WR_SEND_WITH_IMM: - case IB_WR_RDMA_WRITE_WITH_IMM: - rc_sq_wqe->immtdata = cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); - break; - case IB_WR_SEND_WITH_INV: - rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey); - break; - default: - rc_sq_wqe->immtdata = 0; - break; - } + ret = set_rc_opcode(rc_sq_wqe, wr); + if (WARN_ON(ret)) + return ret; roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_FENCE_S, (wr->send_flags & IB_SEND_FENCE) ? 1 : 0); @@ -443,33 +594,6 @@ static inline int set_rc_wqe(struct hns_roce_qp *qp, roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S, owner_bit); - switch (wr->opcode) { - case IB_WR_RDMA_READ: - case IB_WR_RDMA_WRITE: - case IB_WR_RDMA_WRITE_WITH_IMM: - rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey); - rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr); - break; - case IB_WR_LOCAL_INV: - roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SO_S, 1); - rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey); - break; - case IB_WR_REG_MR: - set_frmr_seg(rc_sq_wqe, reg_wr(wr)); - break; - case IB_WR_ATOMIC_CMP_AND_SWP: - case IB_WR_ATOMIC_FETCH_AND_ADD: - rc_sq_wqe->rkey = cpu_to_le32(atomic_wr(wr)->rkey); - rc_sq_wqe->va = cpu_to_le64(atomic_wr(wr)->remote_addr); - break; - default: - break; - } - - roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OPCODE_M, - V2_RC_SEND_WQE_BYTE_4_OPCODE_S, - to_hr_opcode(wr->opcode)); - if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) set_atomic_seg(wr, rc_sq_wqe, valid_num_sge); @@ -1682,7 +1806,7 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ; caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ; caps->max_srq_desc_sz = HNS_ROCE_V2_MAX_SRQ_DESC_SZ; - caps->qpc_entry_sz = HNS_ROCE_V2_QPC_ENTRY_SZ; + caps->qpc_sz = HNS_ROCE_V2_QPC_SZ; caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ; caps->trrl_entry_sz = HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ; caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ; @@ -1690,7 +1814,7 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ; caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ; caps->idx_entry_sz = HNS_ROCE_V2_IDX_ENTRY_SZ; - caps->cq_entry_sz = HNS_ROCE_V2_CQE_ENTRY_SIZE; + caps->cqe_sz = HNS_ROCE_V2_CQE_SIZE; caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED; caps->reserved_lkey = 0; caps->reserved_pds = 0; @@ -1739,6 +1863,8 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM; caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM; caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM; + caps->aeqe_size = HNS_ROCE_AEQE_SIZE; + caps->ceqe_size = HNS_ROCE_CEQE_SIZE; caps->local_ca_ack_delay = 0; caps->max_mtu = IB_MTU_4096; @@ -1760,19 +1886,26 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->cqc_timer_buf_pg_sz = 0; caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; - caps->sccc_entry_sz = HNS_ROCE_V2_SCCC_ENTRY_SZ; + caps->sccc_sz = HNS_ROCE_V2_SCCC_SZ; caps->sccc_ba_pg_sz = 0; caps->sccc_buf_pg_sz = 0; caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM; + + if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { + caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE; + caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE; + caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE; + caps->qpc_sz = HNS_ROCE_V3_QPC_SZ; + } } static void calc_pg_sz(int obj_num, int obj_size, int hop_num, int ctx_bt_num, int *buf_page_size, int *bt_page_size, u32 hem_type) { u64 obj_per_chunk; - int bt_chunk_size = 1 << PAGE_SHIFT; - int buf_chunk_size = 1 << PAGE_SHIFT; - int obj_per_chunk_default = buf_chunk_size / obj_size; + u64 bt_chunk_size = PAGE_SIZE; + u64 buf_chunk_size = PAGE_SIZE; + u64 obj_per_chunk_default = buf_chunk_size / obj_size; *buf_page_size = 0; *bt_page_size = 0; @@ -1855,7 +1988,7 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) caps->max_sq_desc_sz = resp_a->max_sq_desc_sz; caps->max_rq_desc_sz = resp_a->max_rq_desc_sz; caps->max_srq_desc_sz = resp_a->max_srq_desc_sz; - caps->cq_entry_sz = resp_a->cq_entry_sz; + caps->cqe_sz = HNS_ROCE_V2_CQE_SIZE; caps->mtpt_entry_sz = resp_b->mtpt_entry_sz; caps->irrl_entry_sz = resp_b->irrl_entry_sz; @@ -1863,9 +1996,9 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) caps->cqc_entry_sz = resp_b->cqc_entry_sz; caps->srqc_entry_sz = resp_b->srqc_entry_sz; caps->idx_entry_sz = resp_b->idx_entry_sz; - caps->sccc_entry_sz = resp_b->scc_ctx_entry_sz; + caps->sccc_sz = resp_b->sccc_sz; caps->max_mtu = resp_b->max_mtu; - caps->qpc_entry_sz = le16_to_cpu(resp_b->qpc_entry_sz); + caps->qpc_sz = HNS_ROCE_V2_QPC_SZ; caps->min_cqes = resp_b->min_cqes; caps->min_wqes = resp_b->min_wqes; caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap); @@ -1958,6 +2091,8 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ; caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ; caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS; + caps->ceqe_size = HNS_ROCE_CEQE_SIZE; + caps->aeqe_size = HNS_ROCE_AEQE_SIZE; caps->mtt_ba_pg_sz = 0; caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS; caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS; @@ -1981,7 +2116,15 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M, V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S); - calc_pg_sz(caps->num_qps, caps->qpc_entry_sz, caps->qpc_hop_num, + if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { + caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE; + caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE; + caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE; + caps->qpc_sz = HNS_ROCE_V3_QPC_SZ; + caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ; + } + + calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num, caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz, HEM_TYPE_QPC); calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num, @@ -1998,7 +2141,7 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; - calc_pg_sz(caps->num_qps, caps->sccc_entry_sz, + calc_pg_sz(caps->num_qps, caps->sccc_sz, caps->sccc_hop_num, caps->sccc_bt_num, &caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz, HEM_TYPE_SCCC); @@ -2018,6 +2161,56 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) return 0; } +static int hns_roce_config_qpc_size(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cmq_desc desc; + struct hns_roce_cfg_entry_size *cfg_size = + (struct hns_roce_cfg_entry_size *)desc.data; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE, + false); + + cfg_size->type = cpu_to_le32(HNS_ROCE_CFG_QPC_SIZE); + cfg_size->size = cpu_to_le32(hr_dev->caps.qpc_sz); + + return hns_roce_cmq_send(hr_dev, &desc, 1); +} + +static int hns_roce_config_sccc_size(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cmq_desc desc; + struct hns_roce_cfg_entry_size *cfg_size = + (struct hns_roce_cfg_entry_size *)desc.data; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE, + false); + + cfg_size->type = cpu_to_le32(HNS_ROCE_CFG_SCCC_SIZE); + cfg_size->size = cpu_to_le32(hr_dev->caps.sccc_sz); + + return hns_roce_cmq_send(hr_dev, &desc, 1); +} + +static int hns_roce_config_entry_size(struct hns_roce_dev *hr_dev) +{ + int ret; + + if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09) + return 0; + + ret = hns_roce_config_qpc_size(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "failed to cfg qpc sz, ret = %d.\n", ret); + return ret; + } + + ret = hns_roce_config_sccc_size(hr_dev); + if (ret) + dev_err(hr_dev->dev, "failed to cfg sccc sz, ret = %d.\n", ret); + + return ret; +} + static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) { struct hns_roce_caps *caps = &hr_dev->caps; @@ -2090,9 +2283,14 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) } ret = hns_roce_v2_set_bt(hr_dev); - if (ret) - dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n", - ret); + if (ret) { + dev_err(hr_dev->dev, + "Configure bt attribute fail, ret = %d.\n", ret); + return ret; + } + + /* Configure the size of QPC, SCCC, etc. */ + ret = hns_roce_config_entry_size(hr_dev); return ret; } @@ -2757,8 +2955,7 @@ static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw) static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n) { - return hns_roce_buf_offset(hr_cq->mtr.kmem, - n * HNS_ROCE_V2_CQE_ENTRY_SIZE); + return hns_roce_buf_offset(hr_cq->mtr.kmem, n * hr_cq->cqe_size); } static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n) @@ -2858,6 +3055,10 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M, V2_CQC_BYTE_8_CQN_S, hr_cq->cqn); + roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQE_SIZE_M, + V2_CQC_BYTE_8_CQE_SIZE_S, hr_cq->cqe_size == + HNS_ROCE_V3_CQE_SIZE ? 1 : 0); + cq_context->cqe_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0])); roce_set_field(cq_context->byte_16_hop_addr, @@ -3025,7 +3226,8 @@ out: } static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, - struct hns_roce_v2_cqe *cqe, struct ib_wc *wc) + struct hns_roce_cq *cq, struct hns_roce_v2_cqe *cqe, + struct ib_wc *wc) { static const struct { u32 cqe_status; @@ -3066,7 +3268,7 @@ static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, ibdev_err(&hr_dev->ib_dev, "error cqe status 0x%x:\n", cqe_status); print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 4, cqe, - sizeof(*cqe), false); + cq->cqe_size, false); /* * For hns ROCEE, GENERAL_ERR is an error type that is not defined in @@ -3163,7 +3365,7 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, ++wq->tail; } - get_cqe_status(hr_dev, *cur_qp, cqe, wc); + get_cqe_status(hr_dev, *cur_qp, hr_cq, cqe, wc); if (unlikely(wc->status != IB_WC_SUCCESS)) return 0; @@ -3514,16 +3716,21 @@ static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev, static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev, struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask, struct hns_roce_qp *hr_qp) { struct hns_roce_cmd_mailbox *mailbox; + int qpc_size; int ret; mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); - memcpy(mailbox->buf, context, sizeof(*context) * 2); + /* The qpc size of HIP08 is only 256B, which is half of HIP09 */ + qpc_size = hr_dev->caps.qpc_sz; + memcpy(mailbox->buf, context, qpc_size); + memcpy(mailbox->buf + qpc_size, qpc_mask, qpc_size); ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0, HNS_ROCE_CMD_MODIFY_QPC, @@ -3641,9 +3848,6 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp, V2_QPC_BYTE_76_SRQ_EN_S, 1); } - roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M, - V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 4); - roce_set_bit(context->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1); hr_qp->access_flags = attr->qp_access_flags; @@ -3954,6 +4158,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, dma_addr_t trrl_ba; dma_addr_t irrl_ba; enum ib_mtu mtu; + u8 lp_pktn_ini; u8 port_num; u64 *mtts; u8 *dmac; @@ -4052,6 +4257,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, V2_QPC_BYTE_52_DMAC_S, 0); mtu = get_mtu(ibqp, attr); + hr_qp->path_mtu = mtu; if (attr_mask & IB_QP_PATH_MTU) { roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M, @@ -4061,13 +4267,21 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, } #define MAX_LP_MSG_LEN 65536 - /* MTU*(2^LP_PKTN_INI) shouldn't be bigger than 64kb */ + /* MTU * (2 ^ LP_PKTN_INI) shouldn't be bigger than 64KB */ + lp_pktn_ini = ilog2(MAX_LP_MSG_LEN / ib_mtu_enum_to_int(mtu)); + roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M, - V2_QPC_BYTE_56_LP_PKTN_INI_S, - ilog2(MAX_LP_MSG_LEN / ib_mtu_enum_to_int(mtu))); + V2_QPC_BYTE_56_LP_PKTN_INI_S, lp_pktn_ini); roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M, V2_QPC_BYTE_56_LP_PKTN_INI_S, 0); + /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */ + roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M, + V2_QPC_BYTE_172_ACK_REQ_FREQ_S, lp_pktn_ini); + roce_set_field(qpc_mask->byte_172_sq_psn, + V2_QPC_BYTE_172_ACK_REQ_FREQ_M, + V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0); + roce_set_bit(qpc_mask->byte_108_rx_reqepsn, V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0); roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M, @@ -4164,6 +4378,14 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, return 0; } +static inline u16 get_udp_sport(u32 fl, u32 lqpn, u32 rqpn) +{ + if (!fl) + fl = rdma_calc_flow_label(lqpn, rqpn); + + return rdma_flow_label_to_udp_sport(fl); +} + static int hns_roce_v2_set_path(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int attr_mask, @@ -4227,7 +4449,8 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M, V2_QPC_BYTE_52_UDPSPN_S, - is_udp ? 0x12b7 : 0); + is_udp ? get_udp_sport(grh->flow_label, ibqp->qp_num, + attr->dest_qp_num) : 0); roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M, V2_QPC_BYTE_52_UDPSPN_S, 0); @@ -4259,11 +4482,19 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, V2_QPC_BYTE_28_FL_S, 0); memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw)); + + hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); + if (unlikely(hr_qp->sl > MAX_SERVICE_LEVEL)) { + ibdev_err(ibdev, + "failed to fill QPC, sl (%d) shouldn't be larger than %d.\n", + hr_qp->sl, MAX_SERVICE_LEVEL); + return -EINVAL; + } + roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M, - V2_QPC_BYTE_28_SL_S, rdma_ah_get_sl(&attr->ah_attr)); + V2_QPC_BYTE_28_SL_S, hr_qp->sl); roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M, V2_QPC_BYTE_28_SL_S, 0); - hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); return 0; } @@ -4309,7 +4540,7 @@ static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp, } if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { - memset(qpc_mask, 0, sizeof(*qpc_mask)); + memset(qpc_mask, 0, hr_dev->caps.qpc_sz); modify_qp_reset_to_init(ibqp, attr, attr_mask, context, qpc_mask); } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) { @@ -4532,8 +4763,9 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, * we should set all bits of the relevant fields in context mask to * 0 at the same time, else set them to 0x1. */ - memset(context, 0, sizeof(*context)); - memset(qpc_mask, 0xff, sizeof(*qpc_mask)); + memset(context, 0, hr_dev->caps.qpc_sz); + memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz); + ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state, new_state, context, qpc_mask); if (ret) @@ -4583,7 +4815,7 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, V2_QPC_BYTE_60_QP_ST_S, 0); /* SW pass context to HW */ - ret = hns_roce_v2_qp_modify(hr_dev, ctx, hr_qp); + ret = hns_roce_v2_qp_modify(hr_dev, context, qpc_mask, hr_qp); if (ret) { ibdev_err(ibdev, "failed to modify QP, ret = %d\n", ret); goto out; @@ -4646,7 +4878,7 @@ static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, if (ret) goto out; - memcpy(hr_context, mailbox->buf, sizeof(*hr_context)); + memcpy(hr_context, mailbox->buf, hr_dev->caps.qpc_sz); out: hns_roce_free_cmd_mailbox(hr_dev, mailbox); @@ -4759,7 +4991,9 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, qp_attr->retry_cnt = roce_get_field(context.byte_212_lsn, V2_QPC_BYTE_212_RETRY_CNT_M, V2_QPC_BYTE_212_RETRY_CNT_S); - qp_attr->rnr_retry = le32_to_cpu(context.rq_rnr_timer); + qp_attr->rnr_retry = roce_get_field(context.byte_244_rnr_rxack, + V2_QPC_BYTE_244_RNR_CNT_M, + V2_QPC_BYTE_244_RNR_CNT_S); done: qp_attr->cur_qp_state = qp_attr->qp_state; @@ -4775,6 +5009,7 @@ done: } qp_init_attr->cap = qp_attr->cap; + qp_init_attr->sq_sig_type = hr_qp->sq_signal_bits; out: mutex_unlock(&hr_qp->mutex); @@ -5004,6 +5239,10 @@ static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq, struct hns_roce_cmd_mailbox *mailbox; int ret; + /* Resizing SRQs is not supported yet */ + if (srq_attr_mask & IB_SRQ_MAX_WR) + return -EINVAL; + if (srq_attr_mask & IB_SRQ_LIMIT) { if (srq_attr->srq_limit >= srq->wqe_cnt) return -EINVAL; @@ -5233,7 +5472,7 @@ static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq) aeqe = hns_roce_buf_offset(eq->mtr.kmem, (eq->cons_index & (eq->entries - 1)) * - HNS_ROCE_AEQ_ENTRY_SIZE); + eq->eqe_size); return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^ !!(eq->cons_index & eq->entries)) ? aeqe : NULL; @@ -5333,7 +5572,8 @@ static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq) ceqe = hns_roce_buf_offset(eq->mtr.kmem, (eq->cons_index & (eq->entries - 1)) * - HNS_ROCE_CEQ_ENTRY_SIZE); + eq->eqe_size); + return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^ (!!(eq->cons_index & eq->entries)) ? ceqe : NULL; } @@ -5374,7 +5614,7 @@ static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr) { struct hns_roce_eq *eq = eq_ptr; struct hns_roce_dev *hr_dev = eq->hr_dev; - int int_work = 0; + int int_work; if (eq->type_flag == HNS_ROCE_CEQ) /* Completion event interrupt */ @@ -5609,14 +5849,16 @@ static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq, roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CONS_INDX_M, HNS_ROCE_EQC_CONS_INDX_S, HNS_ROCE_EQ_INIT_CONS_IDX); - /* set nex_eqe_ba[43:12] */ - roce_set_field(eqc->nxt_eqe_ba0, HNS_ROCE_EQC_NXT_EQE_BA_L_M, + roce_set_field(eqc->byte_40, HNS_ROCE_EQC_NXT_EQE_BA_L_M, HNS_ROCE_EQC_NXT_EQE_BA_L_S, eqe_ba[1] >> 12); - /* set nex_eqe_ba[63:44] */ - roce_set_field(eqc->nxt_eqe_ba1, HNS_ROCE_EQC_NXT_EQE_BA_H_M, + roce_set_field(eqc->byte_44, HNS_ROCE_EQC_NXT_EQE_BA_H_M, HNS_ROCE_EQC_NXT_EQE_BA_H_S, eqe_ba[1] >> 44); + roce_set_field(eqc->byte_44, HNS_ROCE_EQC_EQE_SIZE_M, + HNS_ROCE_EQC_EQE_SIZE_S, + eq->eqe_size == HNS_ROCE_V3_EQE_SIZE ? 1 : 0); + return 0; } @@ -5807,7 +6049,7 @@ static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev) eq_cmd = HNS_ROCE_CMD_CREATE_CEQC; eq->type_flag = HNS_ROCE_CEQ; eq->entries = hr_dev->caps.ceqe_depth; - eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE; + eq->eqe_size = hr_dev->caps.ceqe_size; eq->irq = hr_dev->irq[i + other_num + aeq_num]; eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM; eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL; @@ -5816,7 +6058,7 @@ static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev) eq_cmd = HNS_ROCE_CMD_CREATE_AEQC; eq->type_flag = HNS_ROCE_AEQ; eq->entries = hr_dev->caps.aeqe_depth; - eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE; + eq->eqe_size = hr_dev->caps.aeqe_size; eq->irq = hr_dev->irq[i - comp_num + other_num]; eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM; eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index ac29be43b6bd..29c9dd4bcbc6 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -60,6 +60,7 @@ #define HNS_ROCE_V2_MAX_SQ_SGE_NUM 64 #define HNS_ROCE_V2_MAX_EXTEND_SGE_NUM 0x200000 #define HNS_ROCE_V2_MAX_SQ_INLINE 0x20 +#define HNS_ROCE_V2_MAX_RC_INL_INN_SZ 32 #define HNS_ROCE_V2_UAR_NUM 256 #define HNS_ROCE_V2_PHY_UAR_NUM 1 #define HNS_ROCE_V2_MAX_IRQ_NUM 65 @@ -77,7 +78,6 @@ #define HNS_ROCE_V2_MAX_SQ_DESC_SZ 64 #define HNS_ROCE_V2_MAX_RQ_DESC_SZ 16 #define HNS_ROCE_V2_MAX_SRQ_DESC_SZ 64 -#define HNS_ROCE_V2_QPC_ENTRY_SZ 256 #define HNS_ROCE_V2_IRRL_ENTRY_SZ 64 #define HNS_ROCE_V2_TRRL_ENTRY_SZ 48 #define HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ 100 @@ -86,8 +86,10 @@ #define HNS_ROCE_V2_MTPT_ENTRY_SZ 64 #define HNS_ROCE_V2_MTT_ENTRY_SZ 64 #define HNS_ROCE_V2_IDX_ENTRY_SZ 4 -#define HNS_ROCE_V2_CQE_ENTRY_SIZE 32 -#define HNS_ROCE_V2_SCCC_ENTRY_SZ 32 + +#define HNS_ROCE_V2_SCCC_SZ 32 +#define HNS_ROCE_V3_SCCC_SZ 64 + #define HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ PAGE_SIZE #define HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ PAGE_SIZE #define HNS_ROCE_V2_PAGE_SIZE_SUPPORTED 0xFFFFF000 @@ -229,6 +231,7 @@ enum hns_roce_opcode_type { HNS_ROCE_OPC_CFG_TMOUT_LLM = 0x8404, HNS_ROCE_OPC_QUERY_PF_TIMER_RES = 0x8406, HNS_ROCE_OPC_QUERY_PF_CAPS_NUM = 0x8408, + HNS_ROCE_OPC_CFG_ENTRY_SIZE = 0x8409, HNS_ROCE_OPC_CFG_SGID_TB = 0x8500, HNS_ROCE_OPC_CFG_SMAC_TB = 0x8501, HNS_ROCE_OPC_POST_MB = 0x8504, @@ -309,6 +312,9 @@ struct hns_roce_v2_cq_context { #define V2_CQC_BYTE_8_CQN_S 0 #define V2_CQC_BYTE_8_CQN_M GENMASK(23, 0) +#define V2_CQC_BYTE_8_CQE_SIZE_S 27 +#define V2_CQC_BYTE_8_CQE_SIZE_M GENMASK(28, 27) + #define V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S 0 #define V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M GENMASK(19, 0) @@ -512,6 +518,7 @@ struct hns_roce_v2_qp_context { __le32 byte_248_ack_psn; __le32 byte_252_err_txcqn; __le32 byte_256_sqflush_rqcqe; + __le32 ext[64]; }; #define V2_QPC_BYTE_4_TST_S 0 @@ -896,6 +903,7 @@ struct hns_roce_v2_cqe { u8 smac[4]; __le32 byte_28; __le32 byte_32; + __le32 rsv[8]; }; #define V2_CQE_BYTE_4_OPCODE_S 0 @@ -1187,6 +1195,8 @@ struct hns_roce_v2_rc_send_wqe { #define V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S 0 #define V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M GENMASK(23, 0) +#define V2_RC_SEND_WQE_BYTE_20_INL_TYPE_S 31 + struct hns_roce_wqe_frmr_seg { __le32 pbl_size; __le32 mode_buf_pg_sz; @@ -1537,6 +1547,18 @@ struct hns_roce_cfg_sgid_tb { __le32 vf_sgid_h; __le32 vf_sgid_type_rsv; }; + +enum { + HNS_ROCE_CFG_QPC_SIZE = BIT(0), + HNS_ROCE_CFG_SCCC_SIZE = BIT(1), +}; + +struct hns_roce_cfg_entry_size { + __le32 type; + __le32 rsv[4]; + __le32 size; +}; + #define CFG_SGID_TB_TABLE_IDX_S 0 #define CFG_SGID_TB_TABLE_IDX_M GENMASK(7, 0) @@ -1571,7 +1593,7 @@ struct hns_roce_query_pf_caps_a { u8 max_sq_desc_sz; u8 max_rq_desc_sz; u8 max_srq_desc_sz; - u8 cq_entry_sz; + u8 cqe_sz; }; struct hns_roce_query_pf_caps_b { @@ -1581,9 +1603,9 @@ struct hns_roce_query_pf_caps_b { u8 cqc_entry_sz; u8 srqc_entry_sz; u8 idx_entry_sz; - u8 scc_ctx_entry_sz; + u8 sccc_sz; u8 max_mtu; - __le16 qpc_entry_sz; + __le16 qpc_sz; __le16 qpc_timer_entry_sz; __le16 cqc_timer_entry_sz; u8 min_cqes; @@ -1777,8 +1799,8 @@ struct hns_roce_eq_context { __le32 byte_28; __le32 byte_32; __le32 byte_36; - __le32 nxt_eqe_ba0; - __le32 nxt_eqe_ba1; + __le32 byte_40; + __le32 byte_44; __le32 rsv[5]; }; @@ -1920,6 +1942,9 @@ struct hns_roce_eq_context { #define HNS_ROCE_EQC_NXT_EQE_BA_H_S 0 #define HNS_ROCE_EQC_NXT_EQE_BA_H_M GENMASK(19, 0) +#define HNS_ROCE_EQC_EQE_SIZE_S 20 +#define HNS_ROCE_EQC_EQE_SIZE_M GENMASK(21, 20) + #define HNS_ROCE_V2_CEQE_COMP_CQN_S 0 #define HNS_ROCE_V2_CEQE_COMP_CQN_M GENMASK(23, 0) @@ -1941,6 +1966,8 @@ struct hns_roce_eq_context { #define HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S 0 #define HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M GENMASK(23, 0) +#define MAX_SERVICE_LEVEL 0x7 + struct hns_roce_wqe_atomic_seg { __le64 fetchadd_swap_data; __le64 cmp_data; diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 5907cfd878a6..afeffafc59f9 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -141,8 +141,8 @@ static int hns_roce_netdev_event(struct notifier_block *self, struct net_device *dev = netdev_notifier_info_to_dev(ptr); struct hns_roce_ib_iboe *iboe = NULL; struct hns_roce_dev *hr_dev = NULL; - u8 port = 0; - int ret = 0; + int ret; + u8 port; hr_dev = container_of(self, struct hns_roce_dev, iboe.nb); iboe = &hr_dev->iboe; @@ -323,6 +323,8 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, mutex_init(&context->page_mutex); } + resp.cqe_size = hr_dev->caps.cqe_sz; + ret = ib_copy_to_udata(udata, &resp, sizeof(resp)); if (ret) goto error_fail_copy_to_udata; @@ -454,6 +456,8 @@ static const struct ib_device_ops hns_roce_dev_mr_ops = { static const struct ib_device_ops hns_roce_dev_mw_ops = { .alloc_mw = hns_roce_alloc_mw, .dealloc_mw = hns_roce_dealloc_mw, + + INIT_RDMA_OBJ_SIZE(ib_mw, hns_roce_mw, ibmw), }; static const struct ib_device_ops hns_roce_dev_frmr_ops = { @@ -545,7 +549,8 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) if (ret) return ret; } - ret = ib_register_device(ib_dev, "hns_%d"); + dma_set_max_seg_size(dev, UINT_MAX); + ret = ib_register_device(ib_dev, "hns_%d", dev); if (ret) { dev_err(dev, "ib_register_device failed!\n"); return ret; @@ -587,7 +592,7 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) } ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.qp_table, - HEM_TYPE_QPC, hr_dev->caps.qpc_entry_sz, + HEM_TYPE_QPC, hr_dev->caps.qpc_sz, hr_dev->caps.num_qps, 1); if (ret) { dev_err(dev, "Failed to init QP context memory, aborting.\n"); @@ -638,11 +643,11 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) } } - if (hr_dev->caps.sccc_entry_sz) { + if (hr_dev->caps.sccc_sz) { ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.sccc_table, HEM_TYPE_SCCC, - hr_dev->caps.sccc_entry_sz, + hr_dev->caps.sccc_sz, hr_dev->caps.num_qps, 1); if (ret) { dev_err(dev, @@ -682,7 +687,7 @@ err_unmap_qpc_timer: hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qpc_timer_table); err_unmap_ctx: - if (hr_dev->caps.sccc_entry_sz) + if (hr_dev->caps.sccc_sz) hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.sccc_table); err_unmap_srq: diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index e5df3884b41d..7f81a695e9af 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -589,28 +589,22 @@ err_table: return ret; } -struct ib_mw *hns_roce_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, - struct ib_udata *udata) +int hns_roce_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) { - struct hns_roce_dev *hr_dev = to_hr_dev(ib_pd->device); - struct hns_roce_mw *mw; + struct hns_roce_dev *hr_dev = to_hr_dev(ibmw->device); + struct hns_roce_mw *mw = to_hr_mw(ibmw); unsigned long index = 0; int ret; - mw = kmalloc(sizeof(*mw), GFP_KERNEL); - if (!mw) - return ERR_PTR(-ENOMEM); - /* Allocate a key for mw from bitmap */ ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index); if (ret) - goto err_bitmap; + return ret; mw->rkey = hw_index_to_key(index); - mw->ibmw.rkey = mw->rkey; - mw->ibmw.type = type; - mw->pdn = to_hr_pd(ib_pd)->pdn; + ibmw->rkey = mw->rkey; + mw->pdn = to_hr_pd(ibmw->pd)->pdn; mw->pbl_hop_num = hr_dev->caps.pbl_hop_num; mw->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz; mw->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz; @@ -619,15 +613,11 @@ struct ib_mw *hns_roce_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, if (ret) goto err_mw; - return &mw->ibmw; + return 0; err_mw: hns_roce_mw_free(hr_dev, mw); - -err_bitmap: - kfree(mw); - - return ERR_PTR(ret); + return ret; } int hns_roce_dealloc_mw(struct ib_mw *ibmw) @@ -636,8 +626,6 @@ int hns_roce_dealloc_mw(struct ib_mw *ibmw) struct hns_roce_mw *mw = to_hr_mw(ibmw); hns_roce_mw_free(hr_dev, mw); - kfree(mw); - return 0; } @@ -707,19 +695,6 @@ static inline size_t mtr_bufs_size(struct hns_roce_buf_attr *attr) return size; } -static inline int mtr_umem_page_count(struct ib_umem *umem, - unsigned int page_shift) -{ - int count = ib_umem_page_count(umem); - - if (page_shift >= PAGE_SHIFT) - count >>= page_shift - PAGE_SHIFT; - else - count <<= PAGE_SHIFT - page_shift; - - return count; -} - static inline size_t mtr_kmem_direct_size(bool is_direct, size_t alloc_size, unsigned int page_shift) { @@ -767,13 +742,11 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, struct ib_udata *udata, unsigned long user_addr) { struct ib_device *ibdev = &hr_dev->ib_dev; - unsigned int max_pg_shift = buf_attr->page_shift; - unsigned int best_pg_shift = 0; + unsigned int best_pg_shift; int all_pg_count = 0; size_t direct_size; size_t total_size; - unsigned long tmp; - int ret = 0; + int ret; total_size = mtr_bufs_size(buf_attr); if (total_size < 1) { @@ -782,6 +755,9 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, } if (udata) { + unsigned long pgsz_bitmap; + unsigned long page_size; + mtr->kmem = NULL; mtr->umem = ib_umem_get(ibdev, user_addr, total_size, buf_attr->user_access); @@ -790,15 +766,17 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, PTR_ERR(mtr->umem)); return -ENOMEM; } - if (buf_attr->fixed_page) { - best_pg_shift = max_pg_shift; - } else { - tmp = GENMASK(max_pg_shift, 0); - ret = ib_umem_find_best_pgsz(mtr->umem, tmp, user_addr); - best_pg_shift = (ret <= PAGE_SIZE) ? - PAGE_SHIFT : ilog2(ret); - } - all_pg_count = mtr_umem_page_count(mtr->umem, best_pg_shift); + if (buf_attr->fixed_page) + pgsz_bitmap = 1 << buf_attr->page_shift; + else + pgsz_bitmap = GENMASK(buf_attr->page_shift, PAGE_SHIFT); + + page_size = ib_umem_find_best_pgsz(mtr->umem, pgsz_bitmap, + user_addr); + if (!page_size) + return -EINVAL; + best_pg_shift = order_base_2(page_size); + all_pg_count = ib_umem_num_dma_blocks(mtr->umem, page_size); ret = 0; } else { mtr->umem = NULL; @@ -808,16 +786,15 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, return -ENOMEM; } direct_size = mtr_kmem_direct_size(is_direct, total_size, - max_pg_shift); + buf_attr->page_shift); ret = hns_roce_buf_alloc(hr_dev, total_size, direct_size, - mtr->kmem, max_pg_shift); + mtr->kmem, buf_attr->page_shift); if (ret) { ibdev_err(ibdev, "Failed to alloc kmem, ret %d\n", ret); goto err_alloc_mem; - } else { - best_pg_shift = max_pg_shift; - all_pg_count = mtr->kmem->npages; } + best_pg_shift = buf_attr->page_shift; + all_pg_count = mtr->kmem->npages; } /* must bigger than minimum hardware page shift */ @@ -967,7 +944,7 @@ static int mtr_init_buf_cfg(struct hns_roce_dev *hr_dev, unsigned int *buf_page_shift) { struct hns_roce_buf_region *r; - unsigned int page_shift = 0; + unsigned int page_shift; int page_cnt = 0; size_t buf_size; int region_cnt; diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c index b10c50b8736e..98f69496adb4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_pd.c +++ b/drivers/infiniband/hw/hns/hns_roce_pd.c @@ -82,9 +82,10 @@ int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) return 0; } -void hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) +int hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) { hns_roce_pd_free(to_hr_dev(pd->device), to_hr_pd(pd)->pdn); + return 0; } int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar) diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index c063c450c715..6c081dd985fc 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -41,8 +41,6 @@ #include "hns_roce_hem.h" #include <rdma/hns-abi.h> -#define SQP_NUM (2 * HNS_ROCE_MAX_PORTS) - static void flush_work_handle(struct work_struct *work) { struct hns_roce_work *flush_work = container_of(work, @@ -288,7 +286,7 @@ static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) } } - if (hr_dev->caps.sccc_entry_sz) { + if (hr_dev->caps.sccc_sz) { /* Alloc memory for SCC CTX */ ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table, hr_qp->qpn); @@ -551,10 +549,9 @@ static int set_kernel_sq_size(struct hns_roce_dev *hr_dev, int ret; if (!cap->max_send_wr || cap->max_send_wr > hr_dev->caps.max_wqes || - cap->max_send_sge > hr_dev->caps.max_sq_sg || - cap->max_inline_data > hr_dev->caps.max_sq_inline) { + cap->max_send_sge > hr_dev->caps.max_sq_sg) { ibdev_err(ibdev, - "failed to check SQ WR, SGE or inline num, ret = %d.\n", + "failed to check SQ WR or SGE num, ret = %d.\n", -EINVAL); return -EINVAL; } @@ -577,9 +574,6 @@ static int set_kernel_sq_size(struct hns_roce_dev *hr_dev, cap->max_send_wr = cnt; cap->max_send_sge = hr_qp->sq.max_gs; - /* We don't support inline sends for kernel QPs (yet) */ - cap->max_inline_data = 0; - return 0; } @@ -847,6 +841,11 @@ static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, hr_qp->ibqp.qp_type = init_attr->qp_type; + if (init_attr->cap.max_inline_data > hr_dev->caps.max_sq_inline) + init_attr->cap.max_inline_data = hr_dev->caps.max_sq_inline; + + hr_qp->max_inline_data = init_attr->cap.max_inline_data; + if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR; else @@ -1014,53 +1013,32 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, int ret; switch (init_attr->qp_type) { - case IB_QPT_RC: { - hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL); - if (!hr_qp) - return ERR_PTR(-ENOMEM); - - ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, - hr_qp); - if (ret) { - ibdev_err(ibdev, "Create QP 0x%06lx failed(%d)\n", - hr_qp->qpn, ret); - kfree(hr_qp); - return ERR_PTR(ret); - } - + case IB_QPT_RC: + case IB_QPT_GSI: break; + default: + ibdev_err(ibdev, "not support QP type %d\n", + init_attr->qp_type); + return ERR_PTR(-EOPNOTSUPP); } - case IB_QPT_GSI: { - /* Userspace is not allowed to create special QPs: */ - if (udata) { - ibdev_err(ibdev, "not support usr space GSI\n"); - return ERR_PTR(-EINVAL); - } - hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL); - if (!hr_qp) - return ERR_PTR(-ENOMEM); + hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL); + if (!hr_qp) + return ERR_PTR(-ENOMEM); + if (init_attr->qp_type == IB_QPT_GSI) { hr_qp->port = init_attr->port_num - 1; hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; - - ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, - hr_qp); - if (ret) { - ibdev_err(ibdev, "Create GSI QP failed!\n"); - kfree(hr_qp); - return ERR_PTR(ret); - } - - break; - } - default:{ - ibdev_err(ibdev, "not support QP type %d\n", - init_attr->qp_type); - return ERR_PTR(-EOPNOTSUPP); - } } + ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp); + if (ret) { + ibdev_err(ibdev, "Create QP type 0x%x failed(%d)\n", + init_attr->qp_type, ret); + ibdev_err(ibdev, "Create GSI QP failed!\n"); + kfree(hr_qp); + return ERR_PTR(ret); + } return &hr_qp->ibqp; } @@ -1161,8 +1139,10 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, mutex_lock(&hr_qp->mutex); - cur_state = attr_mask & IB_QP_CUR_STATE ? - attr->cur_qp_state : (enum ib_qp_state)hr_qp->state; + if (attr_mask & IB_QP_CUR_STATE && attr->cur_qp_state != hr_qp->state) + goto out; + + cur_state = hr_qp->state; new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; if (ibqp->uobject && diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index b9e2dbd372b6..8caf74e44efd 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -285,7 +285,7 @@ int hns_roce_create_srq(struct ib_srq *ib_srq, struct hns_roce_srq *srq = to_hr_srq(ib_srq); struct ib_device *ibdev = &hr_dev->ib_dev; struct hns_roce_ib_create_srq ucmd = {}; - int ret = 0; + int ret; u32 cqn; /* Check the actual SRQ wqe and SRQ sge num */ @@ -363,7 +363,7 @@ err_buf_alloc: return ret; } -void hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) +int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); struct hns_roce_srq *srq = to_hr_srq(ibsrq); @@ -372,6 +372,7 @@ void hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) free_srq_idx(hr_dev, srq); free_srq_wrid(srq); free_srq_buf(hr_dev, srq); + return 0; } int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev) diff --git a/drivers/infiniband/hw/i40iw/i40iw.h b/drivers/infiniband/hw/i40iw/i40iw.h index 25747b85a79c..832b80de004f 100644 --- a/drivers/infiniband/hw/i40iw/i40iw.h +++ b/drivers/infiniband/hw/i40iw/i40iw.h @@ -409,8 +409,8 @@ static inline struct i40iw_qp *to_iwqp(struct ib_qp *ibqp) } /* i40iw.c */ -void i40iw_add_ref(struct ib_qp *); -void i40iw_rem_ref(struct ib_qp *); +void i40iw_qp_add_ref(struct ib_qp *ibqp); +void i40iw_qp_rem_ref(struct ib_qp *ibqp); struct ib_qp *i40iw_get_qp(struct ib_device *, int); void i40iw_flush_wqes(struct i40iw_device *iwdev, @@ -554,9 +554,8 @@ enum i40iw_status_code i40iw_manage_qhash(struct i40iw_device *iwdev, bool wait); void i40iw_receive_ilq(struct i40iw_sc_vsi *vsi, struct i40iw_puda_buf *rbuf); void i40iw_free_sqbuf(struct i40iw_sc_vsi *vsi, void *bufp); -void i40iw_free_qp_resources(struct i40iw_device *iwdev, - struct i40iw_qp *iwqp, - u32 qp_num); +void i40iw_free_qp_resources(struct i40iw_qp *iwqp); + enum i40iw_status_code i40iw_obj_aligned_mem(struct i40iw_device *iwdev, struct i40iw_dma_mem *memptr, u32 size, u32 mask); diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c index a3b95805c154..3053c345a5a3 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_cm.c +++ b/drivers/infiniband/hw/i40iw/i40iw_cm.c @@ -2322,7 +2322,7 @@ static void i40iw_rem_ref_cm_node(struct i40iw_cm_node *cm_node) iwqp = cm_node->iwqp; if (iwqp) { iwqp->cm_node = NULL; - i40iw_rem_ref(&iwqp->ibqp); + i40iw_qp_rem_ref(&iwqp->ibqp); cm_node->iwqp = NULL; } else if (cm_node->qhash_set) { i40iw_get_addr_info(cm_node, &nfo); @@ -3452,7 +3452,7 @@ void i40iw_cm_disconn(struct i40iw_qp *iwqp) kfree(work); return; } - i40iw_add_ref(&iwqp->ibqp); + i40iw_qp_add_ref(&iwqp->ibqp); spin_unlock_irqrestore(&iwdev->qptable_lock, flags); work->iwqp = iwqp; @@ -3623,7 +3623,7 @@ static void i40iw_disconnect_worker(struct work_struct *work) kfree(dwork); i40iw_cm_disconn_true(iwqp); - i40iw_rem_ref(&iwqp->ibqp); + i40iw_qp_rem_ref(&iwqp->ibqp); } /** @@ -3745,7 +3745,7 @@ int i40iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) cm_node->lsmm_size = accept.size + conn_param->private_data_len; i40iw_cm_init_tsa_conn(iwqp, cm_node); cm_id->add_ref(cm_id); - i40iw_add_ref(&iwqp->ibqp); + i40iw_qp_add_ref(&iwqp->ibqp); attr.qp_state = IB_QPS_RTS; cm_node->qhash_set = false; @@ -3908,7 +3908,7 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) iwqp->cm_node = cm_node; cm_node->iwqp = iwqp; iwqp->cm_id = cm_id; - i40iw_add_ref(&iwqp->ibqp); + i40iw_qp_add_ref(&iwqp->ibqp); if (cm_node->state != I40IW_CM_STATE_OFFLOADED) { cm_node->state = I40IW_CM_STATE_SYN_SENT; diff --git a/drivers/infiniband/hw/i40iw/i40iw_hw.c b/drivers/infiniband/hw/i40iw/i40iw_hw.c index e1085634b8d9..56fdc161f6f8 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_hw.c +++ b/drivers/infiniband/hw/i40iw/i40iw_hw.c @@ -313,7 +313,7 @@ void i40iw_process_aeq(struct i40iw_device *iwdev) __func__, info->qp_cq_id); continue; } - i40iw_add_ref(&iwqp->ibqp); + i40iw_qp_add_ref(&iwqp->ibqp); spin_unlock_irqrestore(&iwdev->qptable_lock, flags); qp = &iwqp->sc_qp; spin_lock_irqsave(&iwqp->lock, flags); @@ -426,7 +426,7 @@ void i40iw_process_aeq(struct i40iw_device *iwdev) break; } if (info->qp) - i40iw_rem_ref(&iwqp->ibqp); + i40iw_qp_rem_ref(&iwqp->ibqp); } while (1); if (aeqcnt) diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c index 58a433135a03..2408b279e4c2 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_main.c +++ b/drivers/infiniband/hw/i40iw/i40iw_main.c @@ -192,9 +192,9 @@ static void i40iw_enable_intr(struct i40iw_sc_dev *dev, u32 msix_id) * i40iw_dpc - tasklet for aeq and ceq 0 * @data: iwarp device */ -static void i40iw_dpc(unsigned long data) +static void i40iw_dpc(struct tasklet_struct *t) { - struct i40iw_device *iwdev = (struct i40iw_device *)data; + struct i40iw_device *iwdev = from_tasklet(iwdev, t, dpc_tasklet); if (iwdev->msix_shared) i40iw_process_ceq(iwdev, iwdev->ceqlist); @@ -206,9 +206,9 @@ static void i40iw_dpc(unsigned long data) * i40iw_ceq_dpc - dpc handler for CEQ * @data: data points to CEQ */ -static void i40iw_ceq_dpc(unsigned long data) +static void i40iw_ceq_dpc(struct tasklet_struct *t) { - struct i40iw_ceq *iwceq = (struct i40iw_ceq *)data; + struct i40iw_ceq *iwceq = from_tasklet(iwceq, t, dpc_tasklet); struct i40iw_device *iwdev = iwceq->iwdev; i40iw_process_ceq(iwdev, iwceq); @@ -689,10 +689,10 @@ static enum i40iw_status_code i40iw_configure_ceq_vector(struct i40iw_device *iw enum i40iw_status_code status; if (iwdev->msix_shared && !ceq_id) { - tasklet_init(&iwdev->dpc_tasklet, i40iw_dpc, (unsigned long)iwdev); + tasklet_setup(&iwdev->dpc_tasklet, i40iw_dpc); status = request_irq(msix_vec->irq, i40iw_irq_handler, 0, "AEQCEQ", iwdev); } else { - tasklet_init(&iwceq->dpc_tasklet, i40iw_ceq_dpc, (unsigned long)iwceq); + tasklet_setup(&iwceq->dpc_tasklet, i40iw_ceq_dpc); status = request_irq(msix_vec->irq, i40iw_ceq_handler, 0, "CEQ", iwceq); } @@ -841,7 +841,7 @@ static enum i40iw_status_code i40iw_configure_aeq_vector(struct i40iw_device *iw u32 ret = 0; if (!iwdev->msix_shared) { - tasklet_init(&iwdev->dpc_tasklet, i40iw_dpc, (unsigned long)iwdev); + tasklet_setup(&iwdev->dpc_tasklet, i40iw_dpc); ret = request_irq(msix_vec->irq, i40iw_irq_handler, 0, "i40iw", iwdev); } if (ret) { @@ -1573,7 +1573,7 @@ static enum i40iw_status_code i40iw_setup_init_state(struct i40iw_handler *hdl, status = i40iw_save_msix_info(iwdev, ldev); if (status) return status; - iwdev->hw.dev_context = (void *)ldev->pcidev; + iwdev->hw.pcidev = ldev->pcidev; iwdev->hw.hw_addr = ldev->hw_addr; status = i40iw_allocate_dma_mem(&iwdev->hw, &iwdev->obj_mem, 8192, 4096); diff --git a/drivers/infiniband/hw/i40iw/i40iw_pble.c b/drivers/infiniband/hw/i40iw/i40iw_pble.c index 540aab5e502d..5f97643e22e5 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_pble.c +++ b/drivers/infiniband/hw/i40iw/i40iw_pble.c @@ -167,7 +167,7 @@ static enum i40iw_status_code add_sd_direct(struct i40iw_sc_dev *dev, */ static void i40iw_free_vmalloc_mem(struct i40iw_hw *hw, struct i40iw_chunk *chunk) { - struct pci_dev *pcidev = (struct pci_dev *)hw->dev_context; + struct pci_dev *pcidev = hw->pcidev; int i; if (!chunk->pg_cnt) @@ -193,7 +193,7 @@ static enum i40iw_status_code i40iw_get_vmalloc_mem(struct i40iw_hw *hw, struct i40iw_chunk *chunk, int pg_cnt) { - struct pci_dev *pcidev = (struct pci_dev *)hw->dev_context; + struct pci_dev *pcidev = hw->pcidev; struct page *page; u8 *addr; u32 size; diff --git a/drivers/infiniband/hw/i40iw/i40iw_type.h b/drivers/infiniband/hw/i40iw/i40iw_type.h index 54c323c40d96..c3babf3cbb8e 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_type.h +++ b/drivers/infiniband/hw/i40iw/i40iw_type.h @@ -73,6 +73,7 @@ struct i40iw_pd_ops; struct i40iw_priv_qp_ops; struct i40iw_priv_cq_ops; struct i40iw_hmc_ops; +struct pci_dev; enum i40iw_page_size { I40IW_PAGE_SIZE_4K, @@ -261,7 +262,7 @@ struct i40iw_vsi_pestat { struct i40iw_hw { u8 __iomem *hw_addr; - void *dev_context; + struct pci_dev *pcidev; struct i40iw_hmc_info hmc; }; diff --git a/drivers/infiniband/hw/i40iw/i40iw_utils.c b/drivers/infiniband/hw/i40iw/i40iw_utils.c index e07fb37af086..644f8c641aa0 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_utils.c +++ b/drivers/infiniband/hw/i40iw/i40iw_utils.c @@ -478,25 +478,6 @@ void i40iw_cleanup_pending_cqp_op(struct i40iw_device *iwdev) } /** - * i40iw_free_qp - callback after destroy cqp completes - * @cqp_request: cqp request for destroy qp - * @num: not used - */ -static void i40iw_free_qp(struct i40iw_cqp_request *cqp_request, u32 num) -{ - struct i40iw_sc_qp *qp = (struct i40iw_sc_qp *)cqp_request->param; - struct i40iw_qp *iwqp = (struct i40iw_qp *)qp->back_qp; - struct i40iw_device *iwdev; - u32 qp_num = iwqp->ibqp.qp_num; - - iwdev = iwqp->iwdev; - - i40iw_rem_pdusecount(iwqp->iwpd, iwdev); - i40iw_free_qp_resources(iwdev, iwqp, qp_num); - i40iw_rem_devusecount(iwdev); -} - -/** * i40iw_wait_event - wait for completion * @iwdev: iwarp device * @cqp_request: cqp request to wait @@ -616,26 +597,23 @@ void i40iw_rem_pdusecount(struct i40iw_pd *iwpd, struct i40iw_device *iwdev) } /** - * i40iw_add_ref - add refcount for qp + * i40iw_qp_add_ref - add refcount for qp * @ibqp: iqarp qp */ -void i40iw_add_ref(struct ib_qp *ibqp) +void i40iw_qp_add_ref(struct ib_qp *ibqp) { struct i40iw_qp *iwqp = (struct i40iw_qp *)ibqp; - atomic_inc(&iwqp->refcount); + refcount_inc(&iwqp->refcount); } /** - * i40iw_rem_ref - rem refcount for qp and free if 0 + * i40iw_qp_rem_ref - rem refcount for qp and free if 0 * @ibqp: iqarp qp */ -void i40iw_rem_ref(struct ib_qp *ibqp) +void i40iw_qp_rem_ref(struct ib_qp *ibqp) { struct i40iw_qp *iwqp; - enum i40iw_status_code status; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; struct i40iw_device *iwdev; u32 qp_num; unsigned long flags; @@ -643,7 +621,7 @@ void i40iw_rem_ref(struct ib_qp *ibqp) iwqp = to_iwqp(ibqp); iwdev = iwqp->iwdev; spin_lock_irqsave(&iwdev->qptable_lock, flags); - if (!atomic_dec_and_test(&iwqp->refcount)) { + if (!refcount_dec_and_test(&iwqp->refcount)) { spin_unlock_irqrestore(&iwdev->qptable_lock, flags); return; } @@ -651,25 +629,8 @@ void i40iw_rem_ref(struct ib_qp *ibqp) qp_num = iwqp->ibqp.qp_num; iwdev->qp_table[qp_num] = NULL; spin_unlock_irqrestore(&iwdev->qptable_lock, flags); - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, false); - if (!cqp_request) - return; - - cqp_request->callback_fcn = i40iw_free_qp; - cqp_request->param = (void *)&iwqp->sc_qp; - cqp_info = &cqp_request->info; - cqp_info->cqp_cmd = OP_QP_DESTROY; - cqp_info->post_sq = 1; - cqp_info->in.u.qp_destroy.qp = &iwqp->sc_qp; - cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request; - cqp_info->in.u.qp_destroy.remove_hash_idx = true; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (!status) - return; + complete(&iwqp->free_qp); - i40iw_rem_pdusecount(iwqp->iwpd, iwdev); - i40iw_free_qp_resources(iwdev, iwqp, qp_num); - i40iw_rem_devusecount(iwdev); } /** @@ -751,7 +712,7 @@ enum i40iw_status_code i40iw_allocate_dma_mem(struct i40iw_hw *hw, u64 size, u32 alignment) { - struct pci_dev *pcidev = (struct pci_dev *)hw->dev_context; + struct pci_dev *pcidev = hw->pcidev; if (!mem) return I40IW_ERR_PARAM; @@ -770,7 +731,7 @@ enum i40iw_status_code i40iw_allocate_dma_mem(struct i40iw_hw *hw, */ void i40iw_free_dma_mem(struct i40iw_hw *hw, struct i40iw_dma_mem *mem) { - struct pci_dev *pcidev = (struct pci_dev *)hw->dev_context; + struct pci_dev *pcidev = hw->pcidev; if (!mem || !mem->va) return; @@ -936,7 +897,7 @@ static void i40iw_terminate_timeout(struct timer_list *t) struct i40iw_sc_qp *qp = (struct i40iw_sc_qp *)&iwqp->sc_qp; i40iw_terminate_done(qp, 1); - i40iw_rem_ref(&iwqp->ibqp); + i40iw_qp_rem_ref(&iwqp->ibqp); } /** @@ -948,7 +909,7 @@ void i40iw_terminate_start_timer(struct i40iw_sc_qp *qp) struct i40iw_qp *iwqp; iwqp = (struct i40iw_qp *)qp->back_qp; - i40iw_add_ref(&iwqp->ibqp); + i40iw_qp_add_ref(&iwqp->ibqp); timer_setup(&iwqp->terminate_timer, i40iw_terminate_timeout, 0); iwqp->terminate_timer.expires = jiffies + HZ; add_timer(&iwqp->terminate_timer); @@ -964,7 +925,7 @@ void i40iw_terminate_del_timer(struct i40iw_sc_qp *qp) iwqp = (struct i40iw_qp *)qp->back_qp; if (del_timer(&iwqp->terminate_timer)) - i40iw_rem_ref(&iwqp->ibqp); + i40iw_qp_rem_ref(&iwqp->ibqp); } /** diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index b51339328a51..581ecbadf586 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -328,12 +328,13 @@ error: * @ibpd: ptr of pd to be deallocated * @udata: user data or null for kernel object */ -static void i40iw_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +static int i40iw_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct i40iw_pd *iwpd = to_iwpd(ibpd); struct i40iw_device *iwdev = to_iwdev(ibpd->device); i40iw_rem_pdusecount(iwpd, iwdev); + return 0; } /** @@ -363,11 +364,11 @@ static struct i40iw_pbl *i40iw_get_pbl(unsigned long va, * @iwqp: qp ptr (user or kernel) * @qp_num: qp number assigned */ -void i40iw_free_qp_resources(struct i40iw_device *iwdev, - struct i40iw_qp *iwqp, - u32 qp_num) +void i40iw_free_qp_resources(struct i40iw_qp *iwqp) { struct i40iw_pbl *iwpbl = &iwqp->iwpbl; + struct i40iw_device *iwdev = iwqp->iwdev; + u32 qp_num = iwqp->ibqp.qp_num; i40iw_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp); i40iw_dealloc_push_page(iwdev, &iwqp->sc_qp); @@ -379,7 +380,7 @@ void i40iw_free_qp_resources(struct i40iw_device *iwdev, i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwqp->kqp.dma_mem); kfree(iwqp->kqp.wrid_mem); iwqp->kqp.wrid_mem = NULL; - kfree(iwqp->allocated_buffer); + kfree(iwqp); } /** @@ -401,6 +402,10 @@ static void i40iw_clean_cqes(struct i40iw_qp *iwqp, struct i40iw_cq *iwcq) static int i40iw_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) { struct i40iw_qp *iwqp = to_iwqp(ibqp); + struct ib_qp_attr attr; + struct i40iw_device *iwdev = iwqp->iwdev; + + memset(&attr, 0, sizeof(attr)); iwqp->destroyed = 1; @@ -415,7 +420,15 @@ static int i40iw_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) } } - i40iw_rem_ref(&iwqp->ibqp); + attr.qp_state = IB_QPS_ERR; + i40iw_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL); + i40iw_qp_rem_ref(&iwqp->ibqp); + wait_for_completion(&iwqp->free_qp); + i40iw_cqp_qp_destroy_cmd(&iwdev->sc_dev, &iwqp->sc_qp); + i40iw_rem_pdusecount(iwqp->iwpd, iwdev); + i40iw_free_qp_resources(iwqp); + i40iw_rem_devusecount(iwdev); + return 0; } @@ -524,7 +537,6 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, struct i40iw_create_qp_req req; struct i40iw_create_qp_resp uresp; u32 qp_num = 0; - void *mem; enum i40iw_status_code ret; int err_code; int sq_size; @@ -566,16 +578,15 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; - mem = kzalloc(sizeof(*iwqp), GFP_KERNEL); - if (!mem) + iwqp = kzalloc(sizeof(*iwqp), GFP_KERNEL); + if (!iwqp) return ERR_PTR(-ENOMEM); - iwqp = (struct i40iw_qp *)mem; - iwqp->allocated_buffer = mem; qp = &iwqp->sc_qp; qp->back_qp = (void *)iwqp; qp->push_idx = I40IW_INVALID_PUSH_PAGE_INDEX; + iwqp->iwdev = iwdev; iwqp->ctx_info.iwarp_info = &iwqp->iwarp_info; if (i40iw_allocate_dma_mem(dev->hw, @@ -600,7 +611,6 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, goto error; } - iwqp->iwdev = iwdev; iwqp->iwpd = iwpd; iwqp->ibqp.qp_num = qp_num; qp = &iwqp->sc_qp; @@ -714,7 +724,7 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, goto error; } - i40iw_add_ref(&iwqp->ibqp); + refcount_set(&iwqp->refcount, 1); spin_lock_init(&iwqp->lock); iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0; iwdev->qp_table[qp_num] = iwqp; @@ -736,10 +746,11 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, } init_completion(&iwqp->sq_drained); init_completion(&iwqp->rq_drained); + init_completion(&iwqp->free_qp); return &iwqp->ibqp; error: - i40iw_free_qp_resources(iwdev, iwqp, qp_num); + i40iw_free_qp_resources(iwqp); return ERR_PTR(err_code); } @@ -1052,7 +1063,7 @@ void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq) * @ib_cq: cq pointer * @udata: user data or NULL for kernel object */ -static void i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +static int i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { struct i40iw_cq *iwcq; struct i40iw_device *iwdev; @@ -1064,6 +1075,7 @@ static void i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) i40iw_cq_wq_destroy(iwdev, cq); cq_free_resources(iwdev, iwcq); i40iw_rem_devusecount(iwdev); + return 0; } /** @@ -1320,8 +1332,7 @@ static void i40iw_copy_user_pgaddrs(struct i40iw_mr *iwmr, if (iwmr->type == IW_MEMREG_TYPE_QP) iwpbl->qp_mr.sq_page = sg_page(region->sg_head.sgl); - rdma_for_each_block(region->sg_head.sgl, &biter, region->nmap, - iwmr->page_size) { + rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) { *pbl = rdma_block_iter_dma_address(&biter); pbl = i40iw_next_pbl_addr(pbl, &pinfo, &idx); } @@ -1744,15 +1755,12 @@ static struct ib_mr *i40iw_reg_user_mr(struct ib_pd *pd, struct i40iw_mr *iwmr; struct ib_umem *region; struct i40iw_mem_reg_req req; - u64 pbl_depth = 0; u32 stag = 0; u16 access; - u64 region_length; bool use_pbles = false; unsigned long flags; int err = -ENOSYS; int ret; - int pg_shift; if (!udata) return ERR_PTR(-EOPNOTSUPP); @@ -1787,18 +1795,13 @@ static struct ib_mr *i40iw_reg_user_mr(struct ib_pd *pd, if (req.reg_type == IW_MEMREG_TYPE_MEM) iwmr->page_size = ib_umem_find_best_pgsz(region, SZ_4K | SZ_2M, virt); - - region_length = region->length + (start & (iwmr->page_size - 1)); - pg_shift = ffs(iwmr->page_size) - 1; - pbl_depth = region_length >> pg_shift; - pbl_depth += (region_length & (iwmr->page_size - 1)) ? 1 : 0; iwmr->length = region->length; iwpbl->user_base = virt; palloc = &iwpbl->pble_alloc; iwmr->type = req.reg_type; - iwmr->page_cnt = (u32)pbl_depth; + iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); switch (req.reg_type) { case IW_MEMREG_TYPE_QP: @@ -2636,13 +2639,13 @@ static const struct ib_device_ops i40iw_dev_ops = { .get_hw_stats = i40iw_get_hw_stats, .get_port_immutable = i40iw_port_immutable, .iw_accept = i40iw_accept, - .iw_add_ref = i40iw_add_ref, + .iw_add_ref = i40iw_qp_add_ref, .iw_connect = i40iw_connect, .iw_create_listen = i40iw_create_listen, .iw_destroy_listen = i40iw_destroy_listen, .iw_get_qp = i40iw_get_qp, .iw_reject = i40iw_reject, - .iw_rem_ref = i40iw_rem_ref, + .iw_rem_ref = i40iw_qp_rem_ref, .map_mr_sg = i40iw_map_mr_sg, .mmap = i40iw_mmap, .modify_qp = i40iw_modify_qp, @@ -2668,7 +2671,7 @@ static struct i40iw_ib_device *i40iw_init_rdma_device(struct i40iw_device *iwdev { struct i40iw_ib_device *iwibdev; struct net_device *netdev = iwdev->netdev; - struct pci_dev *pcidev = (struct pci_dev *)iwdev->hw.dev_context; + struct pci_dev *pcidev = iwdev->hw.pcidev; iwibdev = ib_alloc_device(i40iw_ib_device, ibdev); if (!iwibdev) { @@ -2758,7 +2761,8 @@ int i40iw_register_rdma_device(struct i40iw_device *iwdev) if (ret) goto error; - ret = ib_register_device(&iwibdev->ibdev, "i40iw%d"); + dma_set_max_seg_size(&iwdev->hw.pcidev->dev, UINT_MAX); + ret = ib_register_device(&iwibdev->ibdev, "i40iw%d", &iwdev->hw.pcidev->dev); if (ret) goto error; diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.h b/drivers/infiniband/hw/i40iw/i40iw_verbs.h index 331bc21cbcc7..bab71f3e5637 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.h +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.h @@ -139,7 +139,7 @@ struct i40iw_qp { struct i40iw_qp_host_ctx_info ctx_info; struct i40iwarp_offload_info iwarp_info; void *allocated_buffer; - atomic_t refcount; + refcount_t refcount; struct iw_cm_id *cm_id; void *cm_node; struct ib_mr *lsmm_mr; @@ -174,5 +174,6 @@ struct i40iw_qp { struct i40iw_dma_mem ietf_mem; struct completion sq_drained; struct completion rq_drained; + struct completion free_qp; }; #endif diff --git a/drivers/infiniband/hw/mlx4/ah.c b/drivers/infiniband/hw/mlx4/ah.c index 5f8f8d5c0ce0..7321d6ab5fe1 100644 --- a/drivers/infiniband/hw/mlx4/ah.c +++ b/drivers/infiniband/hw/mlx4/ah.c @@ -232,8 +232,3 @@ int mlx4_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) return 0; } - -void mlx4_ib_destroy_ah(struct ib_ah *ah, u32 flags) -{ - return; -} diff --git a/drivers/infiniband/hw/mlx4/cm.c b/drivers/infiniband/hw/mlx4/cm.c index b591861934b3..4aff1c8298b1 100644 --- a/drivers/infiniband/hw/mlx4/cm.c +++ b/drivers/infiniband/hw/mlx4/cm.c @@ -54,11 +54,20 @@ struct id_map_entry { struct delayed_work timeout; }; +struct rej_tmout_entry { + int slave; + u32 rem_pv_cm_id; + struct delayed_work timeout; + struct xarray *xa_rej_tmout; +}; + struct cm_generic_msg { struct ib_mad_hdr hdr; __be32 local_comm_id; __be32 remote_comm_id; + unsigned char unused[2]; + __be16 rej_reason; }; struct cm_sidr_generic_msg { @@ -280,11 +289,15 @@ static void schedule_delayed(struct ib_device *ibdev, struct id_map_entry *id) if (!sriov->is_going_down && !id->scheduled_delete) { id->scheduled_delete = 1; schedule_delayed_work(&id->timeout, CM_CLEANUP_CACHE_TIMEOUT); + } else if (id->scheduled_delete) { + /* Adjust timeout if already scheduled */ + mod_delayed_work(system_wq, &id->timeout, CM_CLEANUP_CACHE_TIMEOUT); } spin_unlock_irqrestore(&sriov->going_down_lock, flags); spin_unlock(&sriov->id_map_lock); } +#define REJ_REASON(m) be16_to_cpu(((struct cm_generic_msg *)(m))->rej_reason) int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id, struct ib_mad *mad) { @@ -293,8 +306,10 @@ int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id int pv_cm_id = -1; if (mad->mad_hdr.attr_id == CM_REQ_ATTR_ID || - mad->mad_hdr.attr_id == CM_REP_ATTR_ID || - mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID) { + mad->mad_hdr.attr_id == CM_REP_ATTR_ID || + mad->mad_hdr.attr_id == CM_MRA_ATTR_ID || + mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID || + (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID && REJ_REASON(mad) == IB_CM_REJ_TIMEOUT)) { sl_cm_id = get_local_comm_id(mad); id = id_map_get(ibdev, &pv_cm_id, slave_id, sl_cm_id); if (id) @@ -314,8 +329,8 @@ int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id } if (!id) { - pr_debug("id{slave: %d, sl_cm_id: 0x%x} is NULL!\n", - slave_id, sl_cm_id); + pr_debug("id{slave: %d, sl_cm_id: 0x%x} is NULL! attr_id: 0x%x\n", + slave_id, sl_cm_id, be16_to_cpu(mad->mad_hdr.attr_id)); return -EINVAL; } @@ -327,11 +342,94 @@ cont: return 0; } +static void rej_tmout_timeout(struct work_struct *work) +{ + struct delayed_work *delay = to_delayed_work(work); + struct rej_tmout_entry *item = container_of(delay, struct rej_tmout_entry, timeout); + struct rej_tmout_entry *deleted; + + deleted = xa_cmpxchg(item->xa_rej_tmout, item->rem_pv_cm_id, item, NULL, 0); + + if (deleted != item) + pr_debug("deleted(%p) != item(%p)\n", deleted, item); + + kfree(item); +} + +static int alloc_rej_tmout(struct mlx4_ib_sriov *sriov, u32 rem_pv_cm_id, int slave) +{ + struct rej_tmout_entry *item; + struct rej_tmout_entry *old; + int ret = 0; + + xa_lock(&sriov->xa_rej_tmout); + item = xa_load(&sriov->xa_rej_tmout, (unsigned long)rem_pv_cm_id); + + if (item) { + if (xa_err(item)) + ret = xa_err(item); + else + /* If a retry, adjust delayed work */ + mod_delayed_work(system_wq, &item->timeout, CM_CLEANUP_CACHE_TIMEOUT); + goto err_or_exists; + } + xa_unlock(&sriov->xa_rej_tmout); + + item = kmalloc(sizeof(*item), GFP_KERNEL); + if (!item) + return -ENOMEM; + + INIT_DELAYED_WORK(&item->timeout, rej_tmout_timeout); + item->slave = slave; + item->rem_pv_cm_id = rem_pv_cm_id; + item->xa_rej_tmout = &sriov->xa_rej_tmout; + + old = xa_cmpxchg(&sriov->xa_rej_tmout, (unsigned long)rem_pv_cm_id, NULL, item, GFP_KERNEL); + if (old) { + pr_debug( + "Non-null old entry (%p) or error (%d) when inserting\n", + old, xa_err(old)); + kfree(item); + return xa_err(old); + } + + schedule_delayed_work(&item->timeout, CM_CLEANUP_CACHE_TIMEOUT); + + return 0; + +err_or_exists: + xa_unlock(&sriov->xa_rej_tmout); + return ret; +} + +static int lookup_rej_tmout_slave(struct mlx4_ib_sriov *sriov, u32 rem_pv_cm_id) +{ + struct rej_tmout_entry *item; + int slave; + + xa_lock(&sriov->xa_rej_tmout); + item = xa_load(&sriov->xa_rej_tmout, (unsigned long)rem_pv_cm_id); + + if (!item || xa_err(item)) { + pr_debug("Could not find slave. rem_pv_cm_id 0x%x error: %d\n", + rem_pv_cm_id, xa_err(item)); + slave = !item ? -ENOENT : xa_err(item); + } else { + slave = item->slave; + } + xa_unlock(&sriov->xa_rej_tmout); + + return slave; +} + int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave, struct ib_mad *mad) { + struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; + u32 rem_pv_cm_id = get_local_comm_id(mad); u32 pv_cm_id; struct id_map_entry *id; + int sts; if (mad->mad_hdr.attr_id == CM_REQ_ATTR_ID || mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID) { @@ -347,6 +445,13 @@ int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave, be64_to_cpu(gid.global.interface_id)); return -ENOENT; } + + sts = alloc_rej_tmout(sriov, rem_pv_cm_id, *slave); + if (sts) + /* Even if this fails, we pass on the REQ to the slave */ + pr_debug("Could not allocate rej_tmout entry. rem_pv_cm_id 0x%x slave %d status %d\n", + rem_pv_cm_id, *slave, sts); + return 0; } @@ -354,7 +459,14 @@ int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave, id = id_map_get(ibdev, (int *)&pv_cm_id, -1, -1); if (!id) { - pr_debug("Couldn't find an entry for pv_cm_id 0x%x\n", pv_cm_id); + if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID && + REJ_REASON(mad) == IB_CM_REJ_TIMEOUT && slave) { + *slave = lookup_rej_tmout_slave(sriov, rem_pv_cm_id); + + return (*slave < 0) ? *slave : 0; + } + pr_debug("Couldn't find an entry for pv_cm_id 0x%x, attr_id 0x%x\n", + pv_cm_id, be16_to_cpu(mad->mad_hdr.attr_id)); return -ENOENT; } @@ -375,6 +487,34 @@ void mlx4_ib_cm_paravirt_init(struct mlx4_ib_dev *dev) INIT_LIST_HEAD(&dev->sriov.cm_list); dev->sriov.sl_id_map = RB_ROOT; xa_init_flags(&dev->sriov.pv_id_table, XA_FLAGS_ALLOC); + xa_init(&dev->sriov.xa_rej_tmout); +} + +static void rej_tmout_xa_cleanup(struct mlx4_ib_sriov *sriov, int slave) +{ + struct rej_tmout_entry *item; + bool flush_needed = false; + unsigned long id; + int cnt = 0; + + xa_lock(&sriov->xa_rej_tmout); + xa_for_each(&sriov->xa_rej_tmout, id, item) { + if (slave < 0 || slave == item->slave) { + mod_delayed_work(system_wq, &item->timeout, 0); + flush_needed = true; + ++cnt; + } + } + xa_unlock(&sriov->xa_rej_tmout); + + if (flush_needed) { + flush_scheduled_work(); + pr_debug("Deleted %d entries in xarray for slave %d during cleanup\n", + cnt, slave); + } + + if (slave < 0) + WARN_ON(!xa_empty(&sriov->xa_rej_tmout)); } /* slave = -1 ==> all slaves */ @@ -444,4 +584,6 @@ void mlx4_ib_cm_paravirt_clean(struct mlx4_ib_dev *dev, int slave) list_del(&map->list); kfree(map); } + + rej_tmout_xa_cleanup(sriov, slave); } diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c index 8a3436994f80..e9b5a4d57fb1 100644 --- a/drivers/infiniband/hw/mlx4/cq.c +++ b/drivers/infiniband/hw/mlx4/cq.c @@ -149,7 +149,6 @@ static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev, struct ib_udata *udata, if (IS_ERR(*umem)) return PTR_ERR(*umem); - n = ib_umem_page_count(*umem); shift = mlx4_ib_umem_calc_optimal_mtt_size(*umem, 0, &n); err = mlx4_mtt_init(dev->dev, n, shift, &buf->mtt); @@ -475,7 +474,7 @@ out: return err; } -void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(cq->device); struct mlx4_ib_cq *mcq = to_mcq(cq); @@ -495,6 +494,7 @@ void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) mlx4_db_free(dev->dev, &mcq->db); } ib_umem_release(mcq->umem); + return 0; } static void dump_cqe(void *cqe) diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c index abe68708d6d6..8bd16474708f 100644 --- a/drivers/infiniband/hw/mlx4/mad.c +++ b/drivers/infiniband/hw/mlx4/mad.c @@ -500,6 +500,13 @@ static int get_gids_from_l3_hdr(struct ib_grh *grh, union ib_gid *sgid, sgid, dgid); } +static int is_proxy_qp0(struct mlx4_ib_dev *dev, int qpn, int slave) +{ + int proxy_start = dev->dev->phys_caps.base_proxy_sqpn + 8 * slave; + + return (qpn >= proxy_start && qpn <= proxy_start + 1); +} + int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port, enum ib_qp_type dest_qpt, struct ib_wc *wc, struct ib_grh *grh, struct ib_mad *mad) @@ -520,8 +527,10 @@ int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port, u16 cached_pkey; u8 is_eth = dev->dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH; - if (dest_qpt > IB_QPT_GSI) + if (dest_qpt > IB_QPT_GSI) { + pr_debug("dest_qpt (%d) > IB_QPT_GSI\n", dest_qpt); return -EINVAL; + } tun_ctx = dev->sriov.demux[port-1].tun[slave]; @@ -538,12 +547,20 @@ int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port, if (dest_qpt) { u16 pkey_ix; ret = ib_get_cached_pkey(&dev->ib_dev, port, wc->pkey_index, &cached_pkey); - if (ret) + if (ret) { + pr_debug("unable to get %s cached pkey for index %d, ret %d\n", + is_proxy_qp0(dev, wc->src_qp, slave) ? "SMI" : "GSI", + wc->pkey_index, ret); return -EINVAL; + } ret = find_slave_port_pkey_ix(dev, slave, port, cached_pkey, &pkey_ix); - if (ret) + if (ret) { + pr_debug("unable to get %s pkey ix for pkey 0x%x, ret %d\n", + is_proxy_qp0(dev, wc->src_qp, slave) ? "SMI" : "GSI", + cached_pkey, ret); return -EINVAL; + } tun_pkey_ix = pkey_ix; } else tun_pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][0]; @@ -715,7 +732,8 @@ static int mlx4_ib_demux_mad(struct ib_device *ibdev, u8 port, err = mlx4_ib_send_to_slave(dev, slave, port, wc->qp->qp_type, wc, grh, mad); if (err) - pr_debug("failed sending to slave %d via tunnel qp (%d)\n", + pr_debug("failed sending %s to slave %d via tunnel qp (%d)\n", + is_proxy_qp0(dev, wc->src_qp, slave) ? "SMI" : "GSI", slave, err); return 0; } @@ -794,7 +812,8 @@ static int mlx4_ib_demux_mad(struct ib_device *ibdev, u8 port, err = mlx4_ib_send_to_slave(dev, slave, port, wc->qp->qp_type, wc, grh, mad); if (err) - pr_debug("failed sending to slave %d via tunnel qp (%d)\n", + pr_debug("failed sending %s to slave %d via tunnel qp (%d)\n", + is_proxy_qp0(dev, wc->src_qp, slave) ? "SMI" : "GSI", slave, err); return 0; } @@ -807,27 +826,6 @@ static int ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, int err; struct ib_port_attr pattr; - if (in_wc && in_wc->qp) { - pr_debug("received MAD: port:%d slid:%d sqpn:%d " - "dlid_bits:%d dqpn:%d wc_flags:0x%x tid:%016llx cls:%x mtd:%x atr:%x\n", - port_num, - in_wc->slid, in_wc->src_qp, - in_wc->dlid_path_bits, - in_wc->qp->qp_num, - in_wc->wc_flags, - be64_to_cpu(in_mad->mad_hdr.tid), - in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.method, - be16_to_cpu(in_mad->mad_hdr.attr_id)); - if (in_wc->wc_flags & IB_WC_GRH) { - pr_debug("sgid_hi:0x%016llx sgid_lo:0x%016llx\n", - be64_to_cpu(in_grh->sgid.global.subnet_prefix), - be64_to_cpu(in_grh->sgid.global.interface_id)); - pr_debug("dgid_hi:0x%016llx dgid_lo:0x%016llx\n", - be64_to_cpu(in_grh->dgid.global.subnet_prefix), - be64_to_cpu(in_grh->dgid.global.interface_id)); - } - } - slid = in_wc ? ib_lid_cpu16(in_wc->slid) : be16_to_cpu(IB_LID_PERMISSIVE); if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP && slid == 0) { @@ -1299,6 +1297,18 @@ static void mlx4_ib_tunnel_comp_handler(struct ib_cq *cq, void *arg) spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); } +static void mlx4_ib_wire_comp_handler(struct ib_cq *cq, void *arg) +{ + unsigned long flags; + struct mlx4_ib_demux_pv_ctx *ctx = cq->cq_context; + struct mlx4_ib_dev *dev = to_mdev(ctx->ib_dev); + + spin_lock_irqsave(&dev->sriov.going_down_lock, flags); + if (!dev->sriov.is_going_down && ctx->state == DEMUX_PV_STATE_ACTIVE) + queue_work(ctx->wi_wq, &ctx->work); + spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); +} + static int mlx4_ib_post_pv_qp_buf(struct mlx4_ib_demux_pv_ctx *ctx, struct mlx4_ib_demux_pv_qp *tun_qp, int index) @@ -1341,14 +1351,6 @@ static int mlx4_ib_multiplex_sa_handler(struct ib_device *ibdev, int port, return ret; } -static int is_proxy_qp0(struct mlx4_ib_dev *dev, int qpn, int slave) -{ - int proxy_start = dev->dev->phys_caps.base_proxy_sqpn + 8 * slave; - - return (qpn >= proxy_start && qpn <= proxy_start + 1); -} - - int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port, enum ib_qp_type dest_qpt, u16 pkey_index, u32 remote_qpn, u32 qkey, struct rdma_ah_attr *attr, @@ -1401,10 +1403,10 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port, spin_lock(&sqp->tx_lock); if (sqp->tx_ix_head - sqp->tx_ix_tail >= - (MLX4_NUM_TUNNEL_BUFS - 1)) + (MLX4_NUM_WIRE_BUFS - 1)) ret = -EAGAIN; else - wire_tx_ix = (++sqp->tx_ix_head) & (MLX4_NUM_TUNNEL_BUFS - 1); + wire_tx_ix = (++sqp->tx_ix_head) & (MLX4_NUM_WIRE_BUFS - 1); spin_unlock(&sqp->tx_lock); if (ret) goto out; @@ -1484,6 +1486,7 @@ static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct ib_wc u16 vlan_id; u8 qos; u8 *dmac; + int sts; /* Get slave that sent this packet */ if (wc->src_qp < dev->dev->phys_caps.base_proxy_sqpn || @@ -1580,13 +1583,17 @@ static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct ib_wc &vlan_id, &qos)) rdma_ah_set_sl(&ah_attr, qos); - mlx4_ib_send_to_wire(dev, slave, ctx->port, - is_proxy_qp0(dev, wc->src_qp, slave) ? - IB_QPT_SMI : IB_QPT_GSI, - be16_to_cpu(tunnel->hdr.pkey_index), - be32_to_cpu(tunnel->hdr.remote_qpn), - be32_to_cpu(tunnel->hdr.qkey), - &ah_attr, wc->smac, vlan_id, &tunnel->mad); + sts = mlx4_ib_send_to_wire(dev, slave, ctx->port, + is_proxy_qp0(dev, wc->src_qp, slave) ? + IB_QPT_SMI : IB_QPT_GSI, + be16_to_cpu(tunnel->hdr.pkey_index), + be32_to_cpu(tunnel->hdr.remote_qpn), + be32_to_cpu(tunnel->hdr.qkey), + &ah_attr, wc->smac, vlan_id, &tunnel->mad); + if (sts) + pr_debug("failed sending %s to wire on behalf of slave %d (%d)\n", + is_proxy_qp0(dev, wc->src_qp, slave) ? "SMI" : "GSI", + slave, sts); } static int mlx4_ib_alloc_pv_bufs(struct mlx4_ib_demux_pv_ctx *ctx, @@ -1595,19 +1602,20 @@ static int mlx4_ib_alloc_pv_bufs(struct mlx4_ib_demux_pv_ctx *ctx, int i; struct mlx4_ib_demux_pv_qp *tun_qp; int rx_buf_size, tx_buf_size; + const int nmbr_bufs = is_tun ? MLX4_NUM_TUNNEL_BUFS : MLX4_NUM_WIRE_BUFS; if (qp_type > IB_QPT_GSI) return -EINVAL; tun_qp = &ctx->qp[qp_type]; - tun_qp->ring = kcalloc(MLX4_NUM_TUNNEL_BUFS, + tun_qp->ring = kcalloc(nmbr_bufs, sizeof(struct mlx4_ib_buf), GFP_KERNEL); if (!tun_qp->ring) return -ENOMEM; - tun_qp->tx_ring = kcalloc(MLX4_NUM_TUNNEL_BUFS, + tun_qp->tx_ring = kcalloc(nmbr_bufs, sizeof (struct mlx4_ib_tun_tx_buf), GFP_KERNEL); if (!tun_qp->tx_ring) { @@ -1624,7 +1632,7 @@ static int mlx4_ib_alloc_pv_bufs(struct mlx4_ib_demux_pv_ctx *ctx, tx_buf_size = sizeof (struct mlx4_mad_snd_buf); } - for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { + for (i = 0; i < nmbr_bufs; i++) { tun_qp->ring[i].addr = kmalloc(rx_buf_size, GFP_KERNEL); if (!tun_qp->ring[i].addr) goto err; @@ -1638,7 +1646,7 @@ static int mlx4_ib_alloc_pv_bufs(struct mlx4_ib_demux_pv_ctx *ctx, } } - for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { + for (i = 0; i < nmbr_bufs; i++) { tun_qp->tx_ring[i].buf.addr = kmalloc(tx_buf_size, GFP_KERNEL); if (!tun_qp->tx_ring[i].buf.addr) @@ -1669,7 +1677,7 @@ tx_err: tx_buf_size, DMA_TO_DEVICE); kfree(tun_qp->tx_ring[i].buf.addr); } - i = MLX4_NUM_TUNNEL_BUFS; + i = nmbr_bufs; err: while (i > 0) { --i; @@ -1690,6 +1698,7 @@ static void mlx4_ib_free_pv_qp_bufs(struct mlx4_ib_demux_pv_ctx *ctx, int i; struct mlx4_ib_demux_pv_qp *tun_qp; int rx_buf_size, tx_buf_size; + const int nmbr_bufs = is_tun ? MLX4_NUM_TUNNEL_BUFS : MLX4_NUM_WIRE_BUFS; if (qp_type > IB_QPT_GSI) return; @@ -1704,13 +1713,13 @@ static void mlx4_ib_free_pv_qp_bufs(struct mlx4_ib_demux_pv_ctx *ctx, } - for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { + for (i = 0; i < nmbr_bufs; i++) { ib_dma_unmap_single(ctx->ib_dev, tun_qp->ring[i].map, rx_buf_size, DMA_FROM_DEVICE); kfree(tun_qp->ring[i].addr); } - for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { + for (i = 0; i < nmbr_bufs; i++) { ib_dma_unmap_single(ctx->ib_dev, tun_qp->tx_ring[i].buf.map, tx_buf_size, DMA_TO_DEVICE); kfree(tun_qp->tx_ring[i].buf.addr); @@ -1744,9 +1753,6 @@ static void mlx4_ib_tunnel_comp_worker(struct work_struct *work) "buf:%lld\n", wc.wr_id); break; case IB_WC_SEND: - pr_debug("received tunnel send completion:" - "wrid=0x%llx, status=0x%x\n", - wc.wr_id, wc.status); rdma_destroy_ah(tun_qp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah, 0); tun_qp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah @@ -1793,6 +1799,7 @@ static int create_pv_sqp(struct mlx4_ib_demux_pv_ctx *ctx, struct mlx4_ib_qp_tunnel_init_attr qp_init_attr; struct ib_qp_attr attr; int qp_attr_mask_INIT; + const int nmbr_bufs = create_tun ? MLX4_NUM_TUNNEL_BUFS : MLX4_NUM_WIRE_BUFS; if (qp_type > IB_QPT_GSI) return -EINVAL; @@ -1803,8 +1810,8 @@ static int create_pv_sqp(struct mlx4_ib_demux_pv_ctx *ctx, qp_init_attr.init_attr.send_cq = ctx->cq; qp_init_attr.init_attr.recv_cq = ctx->cq; qp_init_attr.init_attr.sq_sig_type = IB_SIGNAL_ALL_WR; - qp_init_attr.init_attr.cap.max_send_wr = MLX4_NUM_TUNNEL_BUFS; - qp_init_attr.init_attr.cap.max_recv_wr = MLX4_NUM_TUNNEL_BUFS; + qp_init_attr.init_attr.cap.max_send_wr = nmbr_bufs; + qp_init_attr.init_attr.cap.max_recv_wr = nmbr_bufs; qp_init_attr.init_attr.cap.max_send_sge = 1; qp_init_attr.init_attr.cap.max_recv_sge = 1; if (create_tun) { @@ -1866,7 +1873,7 @@ static int create_pv_sqp(struct mlx4_ib_demux_pv_ctx *ctx, goto err_qp; } - for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { + for (i = 0; i < nmbr_bufs; i++) { ret = mlx4_ib_post_pv_qp_buf(ctx, tun_qp, i); if (ret) { pr_err(" mlx4_ib_post_pv_buf error" @@ -1902,8 +1909,8 @@ static void mlx4_ib_sqp_comp_worker(struct work_struct *work) switch (wc.opcode) { case IB_WC_SEND: kfree(sqp->tx_ring[wc.wr_id & - (MLX4_NUM_TUNNEL_BUFS - 1)].ah); - sqp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah + (MLX4_NUM_WIRE_BUFS - 1)].ah); + sqp->tx_ring[wc.wr_id & (MLX4_NUM_WIRE_BUFS - 1)].ah = NULL; spin_lock(&sqp->tx_lock); sqp->tx_ix_tail++; @@ -1912,13 +1919,13 @@ static void mlx4_ib_sqp_comp_worker(struct work_struct *work) case IB_WC_RECV: mad = (struct ib_mad *) &(((struct mlx4_mad_rcv_buf *) (sqp->ring[wc.wr_id & - (MLX4_NUM_TUNNEL_BUFS - 1)].addr))->payload); + (MLX4_NUM_WIRE_BUFS - 1)].addr))->payload); grh = &(((struct mlx4_mad_rcv_buf *) (sqp->ring[wc.wr_id & - (MLX4_NUM_TUNNEL_BUFS - 1)].addr))->grh); + (MLX4_NUM_WIRE_BUFS - 1)].addr))->grh); mlx4_ib_demux_mad(ctx->ib_dev, ctx->port, &wc, grh, mad); if (mlx4_ib_post_pv_qp_buf(ctx, sqp, wc.wr_id & - (MLX4_NUM_TUNNEL_BUFS - 1))) + (MLX4_NUM_WIRE_BUFS - 1))) pr_err("Failed reposting SQP " "buf:%lld\n", wc.wr_id); break; @@ -1931,8 +1938,8 @@ static void mlx4_ib_sqp_comp_worker(struct work_struct *work) ctx->slave, wc.status, wc.wr_id); if (!MLX4_TUN_IS_RECV(wc.wr_id)) { kfree(sqp->tx_ring[wc.wr_id & - (MLX4_NUM_TUNNEL_BUFS - 1)].ah); - sqp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah + (MLX4_NUM_WIRE_BUFS - 1)].ah); + sqp->tx_ring[wc.wr_id & (MLX4_NUM_WIRE_BUFS - 1)].ah = NULL; spin_lock(&sqp->tx_lock); sqp->tx_ix_tail++; @@ -1972,6 +1979,7 @@ static int create_pv_resources(struct ib_device *ibdev, int slave, int port, { int ret, cq_size; struct ib_cq_init_attr cq_attr = {}; + const int nmbr_bufs = create_tun ? MLX4_NUM_TUNNEL_BUFS : MLX4_NUM_WIRE_BUFS; if (ctx->state != DEMUX_PV_STATE_DOWN) return -EEXIST; @@ -1996,12 +2004,13 @@ static int create_pv_resources(struct ib_device *ibdev, int slave, int port, goto err_out_qp0; } - cq_size = 2 * MLX4_NUM_TUNNEL_BUFS; + cq_size = 2 * nmbr_bufs; if (ctx->has_smi) cq_size *= 2; cq_attr.cqe = cq_size; - ctx->cq = ib_create_cq(ctx->ib_dev, mlx4_ib_tunnel_comp_handler, + ctx->cq = ib_create_cq(ctx->ib_dev, + create_tun ? mlx4_ib_tunnel_comp_handler : mlx4_ib_wire_comp_handler, NULL, ctx, &cq_attr); if (IS_ERR(ctx->cq)) { ret = PTR_ERR(ctx->cq); @@ -2038,6 +2047,7 @@ static int create_pv_resources(struct ib_device *ibdev, int slave, int port, INIT_WORK(&ctx->work, mlx4_ib_sqp_comp_worker); ctx->wq = to_mdev(ibdev)->sriov.demux[port - 1].wq; + ctx->wi_wq = to_mdev(ibdev)->sriov.demux[port - 1].wi_wq; ret = ib_req_notify_cq(ctx->cq, IB_CQ_NEXT_COMP); if (ret) { @@ -2181,7 +2191,7 @@ static int mlx4_ib_alloc_demux_ctx(struct mlx4_ib_dev *dev, goto err_mcg; } - snprintf(name, sizeof name, "mlx4_ibt%d", port); + snprintf(name, sizeof(name), "mlx4_ibt%d", port); ctx->wq = alloc_ordered_workqueue(name, WQ_MEM_RECLAIM); if (!ctx->wq) { pr_err("Failed to create tunnelling WQ for port %d\n", port); @@ -2189,7 +2199,15 @@ static int mlx4_ib_alloc_demux_ctx(struct mlx4_ib_dev *dev, goto err_wq; } - snprintf(name, sizeof name, "mlx4_ibud%d", port); + snprintf(name, sizeof(name), "mlx4_ibwi%d", port); + ctx->wi_wq = alloc_ordered_workqueue(name, WQ_MEM_RECLAIM); + if (!ctx->wi_wq) { + pr_err("Failed to create wire WQ for port %d\n", port); + ret = -ENOMEM; + goto err_wiwq; + } + + snprintf(name, sizeof(name), "mlx4_ibud%d", port); ctx->ud_wq = alloc_ordered_workqueue(name, WQ_MEM_RECLAIM); if (!ctx->ud_wq) { pr_err("Failed to create up/down WQ for port %d\n", port); @@ -2200,6 +2218,10 @@ static int mlx4_ib_alloc_demux_ctx(struct mlx4_ib_dev *dev, return 0; err_udwq: + destroy_workqueue(ctx->wi_wq); + ctx->wi_wq = NULL; + +err_wiwq: destroy_workqueue(ctx->wq); ctx->wq = NULL; @@ -2247,12 +2269,14 @@ static void mlx4_ib_free_demux_ctx(struct mlx4_ib_demux_ctx *ctx) ctx->tun[i]->state = DEMUX_PV_STATE_DOWNING; } flush_workqueue(ctx->wq); + flush_workqueue(ctx->wi_wq); for (i = 0; i < dev->dev->caps.sqp_demux; i++) { destroy_pv_resources(dev, i, ctx->port, ctx->tun[i], 0); free_pv_object(dev, i, ctx->port); } kfree(ctx->tun); destroy_workqueue(ctx->ud_wq); + destroy_workqueue(ctx->wi_wq); destroy_workqueue(ctx->wq); } } diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index bd4f975e7f9a..cd0fba6b0964 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -1215,9 +1215,10 @@ static int mlx4_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) return 0; } -static void mlx4_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) +static int mlx4_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) { mlx4_pd_free(to_mdev(pd->device)->dev, to_mpd(pd)->pdn); + return 0; } static int mlx4_ib_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) @@ -1256,11 +1257,12 @@ err2: return err; } -static void mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata) +static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata) { ib_destroy_cq(to_mxrcd(xrcd)->cq); ib_dealloc_pd(to_mxrcd(xrcd)->pd); mlx4_xrcd_free(to_mdev(xrcd->device)->dev, to_mxrcd(xrcd)->xrcdn); + return 0; } static int add_gid_entry(struct ib_qp *ibqp, union ib_gid *gid) @@ -1533,23 +1535,11 @@ static int __mlx4_ib_create_flow(struct ib_qp *qp, struct ib_flow_attr *flow_att struct mlx4_net_trans_rule_hw_ctrl *ctrl; int default_flow; - static const u16 __mlx4_domain[] = { - [IB_FLOW_DOMAIN_USER] = MLX4_DOMAIN_UVERBS, - [IB_FLOW_DOMAIN_ETHTOOL] = MLX4_DOMAIN_ETHTOOL, - [IB_FLOW_DOMAIN_RFS] = MLX4_DOMAIN_RFS, - [IB_FLOW_DOMAIN_NIC] = MLX4_DOMAIN_NIC, - }; - if (flow_attr->priority > MLX4_IB_FLOW_MAX_PRIO) { pr_err("Invalid priority value %d\n", flow_attr->priority); return -EINVAL; } - if (domain >= IB_FLOW_DOMAIN_NUM) { - pr_err("Invalid domain value %d\n", domain); - return -EINVAL; - } - if (mlx4_map_sw_to_hw_steering_mode(mdev->dev, flow_type) < 0) return -EINVAL; @@ -1558,8 +1548,7 @@ static int __mlx4_ib_create_flow(struct ib_qp *qp, struct ib_flow_attr *flow_att return PTR_ERR(mailbox); ctrl = mailbox->buf; - ctrl->prio = cpu_to_be16(__mlx4_domain[domain] | - flow_attr->priority); + ctrl->prio = cpu_to_be16(domain | flow_attr->priority); ctrl->type = mlx4_map_sw_to_hw_steering_mode(mdev->dev, flow_type); ctrl->port = flow_attr->port; ctrl->qpn = cpu_to_be32(qp->qp_num); @@ -1701,8 +1690,8 @@ static int mlx4_ib_add_dont_trap_rule(struct mlx4_dev *dev, } static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp, - struct ib_flow_attr *flow_attr, - int domain, struct ib_udata *udata) + struct ib_flow_attr *flow_attr, + struct ib_udata *udata) { int err = 0, i = 0, j = 0; struct mlx4_ib_flow *mflow; @@ -1768,8 +1757,8 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp, } while (i < ARRAY_SIZE(type) && type[i]) { - err = __mlx4_ib_create_flow(qp, flow_attr, domain, type[i], - &mflow->reg_id[i].id); + err = __mlx4_ib_create_flow(qp, flow_attr, MLX4_DOMAIN_UVERBS, + type[i], &mflow->reg_id[i].id); if (err) goto err_create_flow; if (is_bonded) { @@ -1778,7 +1767,7 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp, */ flow_attr->port = 2; err = __mlx4_ib_create_flow(qp, flow_attr, - domain, type[j], + MLX4_DOMAIN_UVERBS, type[j], &mflow->reg_id[j].mirror); flow_attr->port = 1; if (err) @@ -2589,11 +2578,16 @@ static const struct ib_device_ops mlx4_ib_dev_wq_ops = { .destroy_rwq_ind_table = mlx4_ib_destroy_rwq_ind_table, .destroy_wq = mlx4_ib_destroy_wq, .modify_wq = mlx4_ib_modify_wq, + + INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mlx4_ib_rwq_ind_table, + ib_rwq_ind_tbl), }; static const struct ib_device_ops mlx4_ib_dev_mw_ops = { .alloc_mw = mlx4_ib_alloc_mw, .dealloc_mw = mlx4_ib_dealloc_mw, + + INIT_RDMA_OBJ_SIZE(ib_mw, mlx4_ib_mw, ibmw), }; static const struct ib_device_ops mlx4_ib_dev_xrc_ops = { @@ -2847,7 +2841,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) goto err_steer_free_bitmap; rdma_set_device_sysfs_group(&ibdev->ib_dev, &mlx4_attr_group); - if (ib_register_device(&ibdev->ib_dev, "mlx4_%d")) + if (ib_register_device(&ibdev->ib_dev, "mlx4_%d", + &dev->persist->pdev->dev)) goto err_diag_counters; if (mlx4_ib_mad_init(ibdev)) @@ -2989,10 +2984,8 @@ int mlx4_ib_steer_qp_reg(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp, /* Add an empty rule for IB L2 */ memset(&ib_spec->mask, 0, sizeof(ib_spec->mask)); - err = __mlx4_ib_create_flow(&mqp->ibqp, flow, - IB_FLOW_DOMAIN_NIC, - MLX4_FS_REGULAR, - &mqp->reg_id); + err = __mlx4_ib_create_flow(&mqp->ibqp, flow, MLX4_DOMAIN_NIC, + MLX4_FS_REGULAR, &mqp->reg_id); } else { err = __mlx4_ib_destroy_flow(mdev->dev, mqp->reg_id); } diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index 38e87a700a2a..58df06492d69 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -233,7 +233,8 @@ enum mlx4_ib_mad_ifc_flags { }; enum { - MLX4_NUM_TUNNEL_BUFS = 256, + MLX4_NUM_TUNNEL_BUFS = 512, + MLX4_NUM_WIRE_BUFS = 2048, }; struct mlx4_ib_tunnel_header { @@ -298,6 +299,26 @@ struct mlx4_ib_rss { u8 rss_key[MLX4_EN_RSS_KEY_SIZE]; }; +enum { + /* + * Largest possible UD header: send with GRH and immediate + * data plus 18 bytes for an Ethernet header with VLAN/802.1Q + * tag. (LRH would only use 8 bytes, so Ethernet is the + * biggest case) + */ + MLX4_IB_UD_HEADER_SIZE = 82, + MLX4_IB_LSO_HEADER_SPARE = 128, +}; + +struct mlx4_ib_sqp { + int pkey_index; + u32 qkey; + u32 send_psn; + struct ib_ud_header ud_header; + u8 header_buf[MLX4_IB_UD_HEADER_SIZE]; + struct ib_qp *roce_v2_gsi; +}; + struct mlx4_ib_qp { union { struct ib_qp ibqp; @@ -343,7 +364,10 @@ struct mlx4_ib_qp { struct mlx4_wqn_range *wqn_range; /* Number of RSS QP parents that uses this WQ */ u32 rss_usecnt; - struct mlx4_ib_rss *rss_ctx; + union { + struct mlx4_ib_rss *rss_ctx; + struct mlx4_ib_sqp *sqp; + }; }; struct mlx4_ib_srq { @@ -366,6 +390,10 @@ struct mlx4_ib_ah { union mlx4_ext_av av; }; +struct mlx4_ib_rwq_ind_table { + struct ib_rwq_ind_table ib_rwq_ind_tbl; +}; + /****************************************/ /* alias guid support */ /****************************************/ @@ -454,6 +482,7 @@ struct mlx4_ib_demux_pv_ctx { struct ib_pd *pd; struct work_struct work; struct workqueue_struct *wq; + struct workqueue_struct *wi_wq; struct mlx4_ib_demux_pv_qp qp[2]; }; @@ -461,6 +490,7 @@ struct mlx4_ib_demux_ctx { struct ib_device *ib_dev; int port; struct workqueue_struct *wq; + struct workqueue_struct *wi_wq; struct workqueue_struct *ud_wq; spinlock_t ud_lock; atomic64_t subnet_prefix; @@ -492,6 +522,7 @@ struct mlx4_ib_sriov { spinlock_t id_map_lock; struct rb_root sl_id_map; struct list_head cm_list; + struct xarray xa_rej_tmout; }; struct gid_cache_context { @@ -725,8 +756,7 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt_addr, int access_flags, struct ib_udata *udata); int mlx4_ib_dereg_mr(struct ib_mr *mr, struct ib_udata *udata); -struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, - struct ib_udata *udata); +int mlx4_ib_alloc_mw(struct ib_mw *mw, struct ib_udata *udata); int mlx4_ib_dealloc_mw(struct ib_mw *mw); struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, u32 max_num_sg); @@ -736,7 +766,7 @@ int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period); int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata); int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata); -void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int mlx4_ib_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq); @@ -747,14 +777,17 @@ int mlx4_ib_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr, int mlx4_ib_create_ah_slave(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, int slave_sgid_index, u8 *s_mac, u16 vlan_tag); int mlx4_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); -void mlx4_ib_destroy_ah(struct ib_ah *ah, u32 flags); +static inline int mlx4_ib_destroy_ah(struct ib_ah *ah, u32 flags) +{ + return 0; +} int mlx4_ib_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *init_attr, struct ib_udata *udata); int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); int mlx4_ib_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr); -void mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata); +int mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata); void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index); int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr); @@ -890,15 +923,18 @@ void mlx4_ib_sl2vl_update(struct mlx4_ib_dev *mdev, int port); struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd, struct ib_wq_init_attr *init_attr, struct ib_udata *udata); -void mlx4_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata); +int mlx4_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata); int mlx4_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr, u32 wq_attr_mask, struct ib_udata *udata); -struct ib_rwq_ind_table -*mlx4_ib_create_rwq_ind_table(struct ib_device *device, - struct ib_rwq_ind_table_init_attr *init_attr, - struct ib_udata *udata); -int mlx4_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table); +int mlx4_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl, + struct ib_rwq_ind_table_init_attr *init_attr, + struct ib_udata *udata); +static inline int +mlx4_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table) +{ + return 0; +} int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, u64 start_va, int *num_of_mtts); diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index 1d5ef0de12c9..426fed005d53 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -271,6 +271,8 @@ int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, u64 start_va, u64 total_len = 0; int i; + *num_of_mtts = ib_umem_num_dma_blocks(umem, PAGE_SIZE); + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { /* * Initialization - save the first chunk start as the @@ -421,7 +423,6 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, goto err_free; } - n = ib_umem_page_count(mr->umem); shift = mlx4_ib_umem_calc_optimal_mtt_size(mr->umem, start, &n); err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, virt_addr, length, @@ -511,7 +512,7 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, mmr->umem = NULL; goto release_mpt_entry; } - n = ib_umem_page_count(mmr->umem); + n = ib_umem_num_dma_blocks(mmr->umem, PAGE_SIZE); shift = PAGE_SHIFT; err = mlx4_mr_rereg_mem_write(dev->dev, &mmr->mmr, @@ -610,37 +611,27 @@ int mlx4_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) return 0; } -struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, - struct ib_udata *udata) +int mlx4_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) { - struct mlx4_ib_dev *dev = to_mdev(pd->device); - struct mlx4_ib_mw *mw; + struct mlx4_ib_dev *dev = to_mdev(ibmw->device); + struct mlx4_ib_mw *mw = to_mmw(ibmw); int err; - mw = kmalloc(sizeof(*mw), GFP_KERNEL); - if (!mw) - return ERR_PTR(-ENOMEM); - - err = mlx4_mw_alloc(dev->dev, to_mpd(pd)->pdn, - to_mlx4_type(type), &mw->mmw); + err = mlx4_mw_alloc(dev->dev, to_mpd(ibmw->pd)->pdn, + to_mlx4_type(ibmw->type), &mw->mmw); if (err) - goto err_free; + return err; err = mlx4_mw_enable(dev->dev, &mw->mmw); if (err) goto err_mw; - mw->ibmw.rkey = mw->mmw.key; - - return &mw->ibmw; + ibmw->rkey = mw->mmw.key; + return 0; err_mw: mlx4_mw_free(dev->dev, &mw->mmw); - -err_free: - kfree(mw); - - return ERR_PTR(err); + return err; } int mlx4_ib_dealloc_mw(struct ib_mw *ibmw) @@ -648,8 +639,6 @@ int mlx4_ib_dealloc_mw(struct ib_mw *ibmw) struct mlx4_ib_mw *mw = to_mmw(ibmw); mlx4_mw_free(to_mdev(ibmw->device)->dev, &mw->mmw); - kfree(mw); - return 0; } diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index 2975f350b9fd..5cb8e602294c 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -68,27 +68,6 @@ enum { }; enum { - /* - * Largest possible UD header: send with GRH and immediate - * data plus 18 bytes for an Ethernet header with VLAN/802.1Q - * tag. (LRH would only use 8 bytes, so Ethernet is the - * biggest case) - */ - MLX4_IB_UD_HEADER_SIZE = 82, - MLX4_IB_LSO_HEADER_SPARE = 128, -}; - -struct mlx4_ib_sqp { - struct mlx4_ib_qp qp; - int pkey_index; - u32 qkey; - u32 send_psn; - struct ib_ud_header ud_header; - u8 header_buf[MLX4_IB_UD_HEADER_SIZE]; - struct ib_qp *roce_v2_gsi; -}; - -enum { MLX4_IB_MIN_SQ_STRIDE = 6, MLX4_IB_CACHE_LINE_SIZE = 64, }; @@ -123,11 +102,6 @@ enum mlx4_ib_source_type { MLX4_IB_RWQ_SRC = 1, }; -static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp) -{ - return container_of(mqp, struct mlx4_ib_sqp, qp); -} - static int is_tunnel_qp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) { if (!mlx4_is_master(dev->dev)) @@ -656,8 +630,6 @@ static int create_qp_rss(struct mlx4_ib_dev *dev, if (err) goto err_qpn; - mutex_init(&qp->mutex); - INIT_LIST_HEAD(&qp->gid_list); INIT_LIST_HEAD(&qp->steering_rules); @@ -696,80 +668,72 @@ err_qpn: return err; } -static struct ib_qp *_mlx4_ib_create_qp_rss(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata) +static int _mlx4_ib_create_qp_rss(struct ib_pd *pd, struct mlx4_ib_qp *qp, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) { - struct mlx4_ib_qp *qp; struct mlx4_ib_create_qp_rss ucmd = {}; size_t required_cmd_sz; int err; if (!udata) { pr_debug("RSS QP with NULL udata\n"); - return ERR_PTR(-EINVAL); + return -EINVAL; } if (udata->outlen) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; required_cmd_sz = offsetof(typeof(ucmd), reserved1) + sizeof(ucmd.reserved1); if (udata->inlen < required_cmd_sz) { pr_debug("invalid inlen\n"); - return ERR_PTR(-EINVAL); + return -EINVAL; } if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) { pr_debug("copy failed\n"); - return ERR_PTR(-EFAULT); + return -EFAULT; } if (memchr_inv(ucmd.reserved, 0, sizeof(ucmd.reserved))) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; if (ucmd.comp_mask || ucmd.reserved1) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; if (udata->inlen > sizeof(ucmd) && !ib_is_udata_cleared(udata, sizeof(ucmd), udata->inlen - sizeof(ucmd))) { pr_debug("inlen is not supported\n"); - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; } if (init_attr->qp_type != IB_QPT_RAW_PACKET) { pr_debug("RSS QP with unsupported QP type %d\n", init_attr->qp_type); - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; } if (init_attr->create_flags) { pr_debug("RSS QP doesn't support create flags\n"); - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; } if (init_attr->send_cq || init_attr->cap.max_send_wr) { pr_debug("RSS QP with unsupported send attributes\n"); - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; } - qp = kzalloc(sizeof(*qp), GFP_KERNEL); - if (!qp) - return ERR_PTR(-ENOMEM); - qp->pri.vid = 0xFFFF; qp->alt.vid = 0xFFFF; err = create_qp_rss(to_mdev(pd->device), init_attr, &ucmd, qp); - if (err) { - kfree(qp); - return ERR_PTR(err); - } + if (err) + return err; qp->ibqp.qp_num = qp->mqp.qpn; - - return &qp->ibqp; + return 0; } /* @@ -873,7 +837,6 @@ static int create_rq(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, qp->mlx4_ib_qp_type = MLX4_IB_QPT_RAW_PACKET; - mutex_init(&qp->mutex); spin_lock_init(&qp->sq.lock); spin_lock_init(&qp->rq.lock); INIT_LIST_HEAD(&qp->gid_list); @@ -922,7 +885,6 @@ static int create_rq(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, goto err; } - n = ib_umem_page_count(qp->umem); shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n); err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt); @@ -989,13 +951,11 @@ err: static int create_qp_common(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata, int sqpn, - struct mlx4_ib_qp **caller_qp) + struct mlx4_ib_qp *qp) { struct mlx4_ib_dev *dev = to_mdev(pd->device); int qpn; int err; - struct mlx4_ib_sqp *sqp = NULL; - struct mlx4_ib_qp *qp; struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context( udata, struct mlx4_ib_ucontext, ibucontext); enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type; @@ -1043,27 +1003,18 @@ static int create_qp_common(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, sqpn = qpn; } - if (!*caller_qp) { - if (qp_type == MLX4_IB_QPT_SMI || qp_type == MLX4_IB_QPT_GSI || - (qp_type & (MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_SMI_OWNER | - MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) { - sqp = kzalloc(sizeof(struct mlx4_ib_sqp), GFP_KERNEL); - if (!sqp) - return -ENOMEM; - qp = &sqp->qp; - } else { - qp = kzalloc(sizeof(struct mlx4_ib_qp), GFP_KERNEL); - if (!qp) - return -ENOMEM; - } - qp->pri.vid = 0xFFFF; - qp->alt.vid = 0xFFFF; - } else - qp = *caller_qp; + if (init_attr->qp_type == IB_QPT_SMI || + init_attr->qp_type == IB_QPT_GSI || qp_type == MLX4_IB_QPT_SMI || + qp_type == MLX4_IB_QPT_GSI || + (qp_type & (MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_SMI_OWNER | + MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) { + qp->sqp = kzalloc(sizeof(struct mlx4_ib_sqp), GFP_KERNEL); + if (!qp->sqp) + return -ENOMEM; + } qp->mlx4_ib_qp_type = qp_type; - mutex_init(&qp->mutex); spin_lock_init(&qp->sq.lock); spin_lock_init(&qp->rq.lock); INIT_LIST_HEAD(&qp->gid_list); @@ -1117,7 +1068,6 @@ static int create_qp_common(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, goto err; } - n = ib_umem_page_count(qp->umem); shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n); err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt); @@ -1239,9 +1189,6 @@ static int create_qp_common(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, qp->mqp.event = mlx4_ib_qp_event; - if (!*caller_qp) - *caller_qp = qp; - spin_lock_irqsave(&dev->reset_flow_resource_lock, flags); mlx4_ib_lock_cqs(to_mcq(init_attr->send_cq), to_mcq(init_attr->recv_cq)); @@ -1293,10 +1240,7 @@ err_db: mlx4_db_free(dev->dev, &qp->db); err: - if (!sqp && !*caller_qp) - kfree(qp); - kfree(sqp); - + kfree(qp->sqp); return err; } @@ -1410,7 +1354,6 @@ static void destroy_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) mlx4_qp_free(dev->dev, &qp->mqp); mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1); del_gid_entries(qp); - kfree(qp->rss_ctx); } static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, @@ -1529,17 +1472,16 @@ static u32 get_sqp_num(struct mlx4_ib_dev *dev, struct ib_qp_init_attr *attr) return dev->dev->caps.spec_qps[attr->port_num - 1].qp1_proxy; } -static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata) +static int _mlx4_ib_create_qp(struct ib_pd *pd, struct mlx4_ib_qp *qp, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) { - struct mlx4_ib_qp *qp = NULL; int err; int sup_u_create_flags = MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK; u16 xrcdn = 0; if (init_attr->rwq_ind_tbl) - return _mlx4_ib_create_qp_rss(pd, init_attr, udata); + return _mlx4_ib_create_qp_rss(pd, qp, init_attr, udata); /* * We only support LSO, vendor flag1, and multicast loopback blocking, @@ -1551,16 +1493,16 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd, MLX4_IB_SRIOV_SQP | MLX4_IB_QP_NETIF | MLX4_IB_QP_CREATE_ROCE_V2_GSI)) - return ERR_PTR(-EINVAL); + return -EINVAL; if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) { if (init_attr->qp_type != IB_QPT_UD) - return ERR_PTR(-EINVAL); + return -EINVAL; } if (init_attr->create_flags) { if (udata && init_attr->create_flags & ~(sup_u_create_flags)) - return ERR_PTR(-EINVAL); + return -EINVAL; if ((init_attr->create_flags & ~(MLX4_IB_SRIOV_SQP | MLX4_IB_QP_CREATE_ROCE_V2_GSI | @@ -1570,7 +1512,7 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd, init_attr->qp_type > IB_QPT_GSI) || (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI && init_attr->qp_type != IB_QPT_GSI)) - return ERR_PTR(-EINVAL); + return -EINVAL; } switch (init_attr->qp_type) { @@ -1581,53 +1523,43 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd, fallthrough; case IB_QPT_XRC_INI: if (!(to_mdev(pd->device)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)) - return ERR_PTR(-ENOSYS); + return -ENOSYS; init_attr->recv_cq = init_attr->send_cq; fallthrough; case IB_QPT_RC: case IB_QPT_UC: case IB_QPT_RAW_PACKET: - qp = kzalloc(sizeof(*qp), GFP_KERNEL); - if (!qp) - return ERR_PTR(-ENOMEM); + case IB_QPT_UD: qp->pri.vid = 0xFFFF; qp->alt.vid = 0xFFFF; - fallthrough; - case IB_QPT_UD: - { - err = create_qp_common(pd, init_attr, udata, 0, &qp); - if (err) { - kfree(qp); - return ERR_PTR(err); - } + err = create_qp_common(pd, init_attr, udata, 0, qp); + if (err) + return err; qp->ibqp.qp_num = qp->mqp.qpn; qp->xrcdn = xrcdn; - break; - } case IB_QPT_SMI: case IB_QPT_GSI: { int sqpn; - /* Userspace is not allowed to create special QPs: */ - if (udata) - return ERR_PTR(-EINVAL); if (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI) { int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev, 1, 1, &sqpn, 0, MLX4_RES_USAGE_DRIVER); if (res) - return ERR_PTR(res); + return res; } else { sqpn = get_sqp_num(to_mdev(pd->device), init_attr); } - err = create_qp_common(pd, init_attr, udata, sqpn, &qp); + qp->pri.vid = 0xFFFF; + qp->alt.vid = 0xFFFF; + err = create_qp_common(pd, init_attr, udata, sqpn, qp); if (err) - return ERR_PTR(err); + return err; qp->port = init_attr->port_num; qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : @@ -1636,25 +1568,33 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd, } default: /* Don't support raw QPs */ - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; } - - return &qp->ibqp; + return 0; } struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) { struct ib_device *device = pd ? pd->device : init_attr->xrcd->device; - struct ib_qp *ibqp; struct mlx4_ib_dev *dev = to_mdev(device); + struct mlx4_ib_qp *qp; + int ret; - ibqp = _mlx4_ib_create_qp(pd, init_attr, udata); + qp = kzalloc(sizeof(*qp), GFP_KERNEL); + if (!qp) + return ERR_PTR(-ENOMEM); - if (!IS_ERR(ibqp) && - (init_attr->qp_type == IB_QPT_GSI) && + mutex_init(&qp->mutex); + ret = _mlx4_ib_create_qp(pd, qp, init_attr, udata); + if (ret) { + kfree(qp); + return ERR_PTR(ret); + } + + if (init_attr->qp_type == IB_QPT_GSI && !(init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI)) { - struct mlx4_ib_sqp *sqp = to_msqp((to_mqp(ibqp))); + struct mlx4_ib_sqp *sqp = qp->sqp; int is_eth = rdma_cap_eth_ah(&dev->ib_dev, init_attr->port_num); if (is_eth && @@ -1666,14 +1606,14 @@ struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd, pr_err("Failed to create GSI QP for RoCEv2 (%ld)\n", PTR_ERR(sqp->roce_v2_gsi)); sqp->roce_v2_gsi = NULL; } else { - sqp = to_msqp(to_mqp(sqp->roce_v2_gsi)); - sqp->qp.flags |= MLX4_IB_ROCE_V2_GSI_QP; + to_mqp(sqp->roce_v2_gsi)->flags |= + MLX4_IB_ROCE_V2_GSI_QP; } init_attr->create_flags &= ~MLX4_IB_QP_CREATE_ROCE_V2_GSI; } } - return ibqp; + return &qp->ibqp; } static int _mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) @@ -1700,10 +1640,8 @@ static int _mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) destroy_qp_common(dev, mqp, MLX4_IB_QP_SRC, udata); } - if (is_sqp(dev, mqp)) - kfree(to_msqp(mqp)); - else - kfree(mqp); + kfree(mqp->sqp); + kfree(mqp); return 0; } @@ -1713,7 +1651,7 @@ int mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) struct mlx4_ib_qp *mqp = to_mqp(qp); if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) { - struct mlx4_ib_sqp *sqp = to_msqp(mqp); + struct mlx4_ib_sqp *sqp = mqp->sqp; if (sqp->roce_v2_gsi) ib_destroy_qp(sqp->roce_v2_gsi); @@ -2575,7 +2513,7 @@ static int __mlx4_ib_modify_qp(void *src, enum mlx4_ib_source_type src_type, qp->alt_port = attr->alt_port_num; if (is_sqp(dev, qp)) - store_sqp_attrs(to_msqp(qp), attr, attr_mask); + store_sqp_attrs(qp->sqp, attr, attr_mask); /* * If we moved QP0 to RTR, bring the IB link up; if we moved @@ -2852,7 +2790,7 @@ int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, ret = _mlx4_ib_modify_qp(ibqp, attr, attr_mask, udata); if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) { - struct mlx4_ib_sqp *sqp = to_msqp(mqp); + struct mlx4_ib_sqp *sqp = mqp->sqp; int err = 0; if (sqp->roce_v2_gsi) @@ -2877,12 +2815,13 @@ static int vf_get_qp0_qkey(struct mlx4_dev *dev, int qpn, u32 *qkey) return -EINVAL; } -static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp, +static int build_sriov_qp0_header(struct mlx4_ib_qp *qp, const struct ib_ud_wr *wr, void *wqe, unsigned *mlx_seg_len) { - struct mlx4_ib_dev *mdev = to_mdev(sqp->qp.ibqp.device); - struct ib_device *ib_dev = &mdev->ib_dev; + struct mlx4_ib_dev *mdev = to_mdev(qp->ibqp.device); + struct mlx4_ib_sqp *sqp = qp->sqp; + struct ib_device *ib_dev = qp->ibqp.device; struct mlx4_wqe_mlx_seg *mlx = wqe; struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx; struct mlx4_ib_ah *ah = to_mah(wr->ah); @@ -2904,12 +2843,12 @@ static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp, /* for proxy-qp0 sends, need to add in size of tunnel header */ /* for tunnel-qp0 sends, tunnel header is already in s/g list */ - if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) + if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) send_size += sizeof (struct mlx4_ib_tunnel_header); ib_ud_header_init(send_size, 1, 0, 0, 0, 0, 0, 0, &sqp->ud_header); - if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) { + if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) { sqp->ud_header.lrh.service_level = be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28; sqp->ud_header.lrh.destination_lid = @@ -2926,26 +2865,26 @@ static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp, sqp->ud_header.lrh.virtual_lane = 0; sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED); - err = ib_get_cached_pkey(ib_dev, sqp->qp.port, 0, &pkey); + err = ib_get_cached_pkey(ib_dev, qp->port, 0, &pkey); if (err) return err; sqp->ud_header.bth.pkey = cpu_to_be16(pkey); - if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER) + if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER) sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn); else sqp->ud_header.bth.destination_qpn = - cpu_to_be32(mdev->dev->caps.spec_qps[sqp->qp.port - 1].qp0_tunnel); + cpu_to_be32(mdev->dev->caps.spec_qps[qp->port - 1].qp0_tunnel); sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1)); if (mlx4_is_master(mdev->dev)) { - if (mlx4_get_parav_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey)) + if (mlx4_get_parav_qkey(mdev->dev, qp->mqp.qpn, &qkey)) return -EINVAL; } else { - if (vf_get_qp0_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey)) + if (vf_get_qp0_qkey(mdev->dev, qp->mqp.qpn, &qkey)) return -EINVAL; } sqp->ud_header.deth.qkey = cpu_to_be32(qkey); - sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.mqp.qpn); + sqp->ud_header.deth.source_qpn = cpu_to_be32(qp->mqp.qpn); sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY; sqp->ud_header.immediate_present = 0; @@ -3029,10 +2968,11 @@ static int fill_gid_by_hw_index(struct mlx4_ib_dev *ibdev, u8 port_num, } #define MLX4_ROCEV2_QP1_SPORT 0xC000 -static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr, +static int build_mlx_header(struct mlx4_ib_qp *qp, const struct ib_ud_wr *wr, void *wqe, unsigned *mlx_seg_len) { - struct ib_device *ib_dev = sqp->qp.ibqp.device; + struct mlx4_ib_sqp *sqp = qp->sqp; + struct ib_device *ib_dev = qp->ibqp.device; struct mlx4_ib_dev *ibdev = to_mdev(ib_dev); struct mlx4_wqe_mlx_seg *mlx = wqe; struct mlx4_wqe_ctrl_seg *ctrl = wqe; @@ -3056,7 +2996,7 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr, for (i = 0; i < wr->wr.num_sge; ++i) send_size += wr->wr.sg_list[i].length; - is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET; + is_eth = rdma_port_get_link_layer(qp->ibqp.device, qp->port) == IB_LINK_LAYER_ETHERNET; is_grh = mlx4_ib_ah_grh_present(ah); if (is_eth) { enum ib_gid_type gid_type; @@ -3070,9 +3010,9 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr, if (err) return err; } else { - err = fill_gid_by_hw_index(ibdev, sqp->qp.port, - ah->av.ib.gid_index, - &sgid, &gid_type); + err = fill_gid_by_hw_index(ibdev, qp->port, + ah->av.ib.gid_index, &sgid, + &gid_type); if (!err) { is_udp = gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP; if (is_udp) { @@ -3117,13 +3057,18 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr, * indexes don't necessarily match the hw ones, so * we must use our own cache */ - sqp->ud_header.grh.source_gid.global.subnet_prefix = - cpu_to_be64(atomic64_read(&(to_mdev(ib_dev)->sriov. - demux[sqp->qp.port - 1]. - subnet_prefix))); - sqp->ud_header.grh.source_gid.global.interface_id = - to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1]. - guid_cache[ah->av.ib.gid_index]; + sqp->ud_header.grh.source_gid.global + .subnet_prefix = + cpu_to_be64(atomic64_read( + &(to_mdev(ib_dev) + ->sriov + .demux[qp->port - 1] + .subnet_prefix))); + sqp->ud_header.grh.source_gid.global + .interface_id = + to_mdev(ib_dev) + ->sriov.demux[qp->port - 1] + .guid_cache[ah->av.ib.gid_index]; } else { sqp->ud_header.grh.source_gid = ah->ibah.sgid_attr->gid; @@ -3155,10 +3100,13 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr, mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); if (!is_eth) { - mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) | - (sqp->ud_header.lrh.destination_lid == - IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) | - (sqp->ud_header.lrh.service_level << 8)); + mlx->flags |= + cpu_to_be32((!qp->ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) | + (sqp->ud_header.lrh.destination_lid == + IB_LID_PERMISSIVE ? + MLX4_WQE_MLX_SLR : + 0) | + (sqp->ud_header.lrh.service_level << 8)); if (ah->av.ib.port_pd & cpu_to_be32(0x80000000)) mlx->flags |= cpu_to_be32(0x1); /* force loopback */ mlx->rlid = sqp->ud_header.lrh.destination_lid; @@ -3204,21 +3152,23 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr, sqp->ud_header.vlan.tag = cpu_to_be16(vlan | pcp); } } else { - sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : - sl_to_vl(to_mdev(ib_dev), - sqp->ud_header.lrh.service_level, - sqp->qp.port); - if (sqp->qp.ibqp.qp_num && sqp->ud_header.lrh.virtual_lane == 15) + sqp->ud_header.lrh.virtual_lane = + !qp->ibqp.qp_num ? + 15 : + sl_to_vl(to_mdev(ib_dev), + sqp->ud_header.lrh.service_level, + qp->port); + if (qp->ibqp.qp_num && sqp->ud_header.lrh.virtual_lane == 15) return -EINVAL; if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE) sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE; } sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED); - if (!sqp->qp.ibqp.qp_num) - err = ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, + if (!qp->ibqp.qp_num) + err = ib_get_cached_pkey(ib_dev, qp->port, sqp->pkey_index, &pkey); else - err = ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->pkey_index, + err = ib_get_cached_pkey(ib_dev, qp->port, wr->pkey_index, &pkey); if (err) return err; @@ -3228,7 +3178,7 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr, sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1)); sqp->ud_header.deth.qkey = cpu_to_be32(wr->remote_qkey & 0x80000000 ? sqp->qkey : wr->remote_qkey); - sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num); + sqp->ud_header.deth.source_qpn = cpu_to_be32(qp->ibqp.qp_num); header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf); @@ -3551,14 +3501,14 @@ static int _mlx4_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, struct mlx4_ib_dev *mdev = to_mdev(ibqp->device); if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) { - struct mlx4_ib_sqp *sqp = to_msqp(qp); + struct mlx4_ib_sqp *sqp = qp->sqp; if (sqp->roce_v2_gsi) { struct mlx4_ib_ah *ah = to_mah(ud_wr(wr)->ah); enum ib_gid_type gid_type; union ib_gid gid; - if (!fill_gid_by_hw_index(mdev, sqp->qp.port, + if (!fill_gid_by_hw_index(mdev, qp->port, ah->av.ib.gid_index, &gid, &gid_type)) qp = (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? @@ -3678,8 +3628,8 @@ static int _mlx4_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, break; case MLX4_IB_QPT_TUN_SMI_OWNER: - err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr), - ctrl, &seglen); + err = build_sriov_qp0_header(qp, ud_wr(wr), ctrl, + &seglen); if (unlikely(err)) { *bad_wr = wr; goto out; @@ -3715,8 +3665,8 @@ static int _mlx4_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, break; case MLX4_IB_QPT_PROXY_SMI_OWNER: - err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr), - ctrl, &seglen); + err = build_sriov_qp0_header(qp, ud_wr(wr), ctrl, + &seglen); if (unlikely(err)) { *bad_wr = wr; goto out; @@ -3749,8 +3699,7 @@ static int _mlx4_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, case MLX4_IB_QPT_SMI: case MLX4_IB_QPT_GSI: - err = build_mlx_header(to_msqp(qp), ud_wr(wr), ctrl, - &seglen); + err = build_mlx_header(qp, ud_wr(wr), ctrl, &seglen); if (unlikely(err)) { *bad_wr = wr; goto out; @@ -4172,6 +4121,7 @@ struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd, if (!qp) return ERR_PTR(-ENOMEM); + mutex_init(&qp->mutex); qp->pri.vid = 0xFFFF; qp->alt.vid = 0xFFFF; @@ -4327,7 +4277,7 @@ int mlx4_ib_modify_wq(struct ib_wq *ibwq, struct ib_wq_attr *wq_attr, return err; } -void mlx4_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata) +int mlx4_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(ibwq->device); struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq); @@ -4338,36 +4288,35 @@ void mlx4_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata) destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, udata); kfree(qp); + return 0; } -struct ib_rwq_ind_table -*mlx4_ib_create_rwq_ind_table(struct ib_device *device, - struct ib_rwq_ind_table_init_attr *init_attr, - struct ib_udata *udata) +int mlx4_ib_create_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table, + struct ib_rwq_ind_table_init_attr *init_attr, + struct ib_udata *udata) { - struct ib_rwq_ind_table *rwq_ind_table; struct mlx4_ib_create_rwq_ind_tbl_resp resp = {}; unsigned int ind_tbl_size = 1 << init_attr->log_ind_tbl_size; + struct ib_device *device = rwq_ind_table->device; unsigned int base_wqn; size_t min_resp_len; - int i; - int err; + int i, err = 0; if (udata->inlen > 0 && !ib_is_udata_cleared(udata, 0, udata->inlen)) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved); if (udata->outlen && udata->outlen < min_resp_len) - return ERR_PTR(-EINVAL); + return -EINVAL; if (ind_tbl_size > device->attrs.rss_caps.max_rwq_indirection_table_size) { pr_debug("log_ind_tbl_size = %d is bigger than supported = %d\n", ind_tbl_size, device->attrs.rss_caps.max_rwq_indirection_table_size); - return ERR_PTR(-EINVAL); + return -EINVAL; } base_wqn = init_attr->ind_tbl[0]->wq_num; @@ -4375,39 +4324,23 @@ struct ib_rwq_ind_table if (base_wqn % ind_tbl_size) { pr_debug("WQN=0x%x isn't aligned with indirection table size\n", base_wqn); - return ERR_PTR(-EINVAL); + return -EINVAL; } for (i = 1; i < ind_tbl_size; i++) { if (++base_wqn != init_attr->ind_tbl[i]->wq_num) { pr_debug("indirection table's WQNs aren't consecutive\n"); - return ERR_PTR(-EINVAL); + return -EINVAL; } } - rwq_ind_table = kzalloc(sizeof(*rwq_ind_table), GFP_KERNEL); - if (!rwq_ind_table) - return ERR_PTR(-ENOMEM); - if (udata->outlen) { resp.response_length = offsetof(typeof(resp), response_length) + sizeof(resp.response_length); err = ib_copy_to_udata(udata, &resp, resp.response_length); - if (err) - goto err; } - return rwq_ind_table; - -err: - kfree(rwq_ind_table); - return ERR_PTR(err); -} - -int mlx4_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl) -{ - kfree(ib_rwq_ind_tbl); - return 0; + return err; } struct mlx4_ib_drain_cqe { diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c index 8f9d5035142d..bf618529e734 100644 --- a/drivers/infiniband/hw/mlx4/srq.c +++ b/drivers/infiniband/hw/mlx4/srq.c @@ -115,8 +115,9 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq, if (IS_ERR(srq->umem)) return PTR_ERR(srq->umem); - err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem), - PAGE_SHIFT, &srq->mtt); + err = mlx4_mtt_init( + dev->dev, ib_umem_num_dma_blocks(srq->umem, PAGE_SIZE), + PAGE_SHIFT, &srq->mtt); if (err) goto err_buf; @@ -260,7 +261,7 @@ int mlx4_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) return 0; } -void mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) +int mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(srq->device); struct mlx4_ib_srq *msrq = to_msrq(srq); @@ -282,6 +283,7 @@ void mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) mlx4_db_free(dev->dev, &msrq->db); } ib_umem_release(msrq->umem); + return 0; } void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index) diff --git a/drivers/infiniband/hw/mlx5/ah.c b/drivers/infiniband/hw/mlx5/ah.c index 59e5ec39b447..505bc47fd575 100644 --- a/drivers/infiniband/hw/mlx5/ah.c +++ b/drivers/infiniband/hw/mlx5/ah.c @@ -106,8 +106,8 @@ int mlx5_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, if (ah_type == RDMA_AH_ATTR_TYPE_ROCE && udata) { int err; struct mlx5_ib_create_ah_resp resp = {}; - u32 min_resp_len = offsetof(typeof(resp), dmac) + - sizeof(resp.dmac); + u32 min_resp_len = + offsetofend(struct mlx5_ib_create_ah_resp, dmac); if (udata->outlen < min_resp_len) return -EINVAL; @@ -147,8 +147,3 @@ int mlx5_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) return 0; } - -void mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags) -{ - return; -} diff --git a/drivers/infiniband/hw/mlx5/cmd.c b/drivers/infiniband/hw/mlx5/cmd.c index ebb2f108b64f..234f29912ba9 100644 --- a/drivers/infiniband/hw/mlx5/cmd.c +++ b/drivers/infiniband/hw/mlx5/cmd.c @@ -168,14 +168,14 @@ void mlx5_cmd_destroy_tis(struct mlx5_core_dev *dev, u32 tisn, u16 uid) mlx5_cmd_exec_in(dev, destroy_tis, in); } -void mlx5_cmd_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn, u16 uid) +int mlx5_cmd_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn, u16 uid) { u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {}; MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT); MLX5_SET(destroy_rqt_in, in, rqtn, rqtn); MLX5_SET(destroy_rqt_in, in, uid, uid); - mlx5_cmd_exec_in(dev, destroy_rqt, in); + return mlx5_cmd_exec_in(dev, destroy_rqt, in); } int mlx5_cmd_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn, @@ -209,14 +209,14 @@ void mlx5_cmd_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn, mlx5_cmd_exec_in(dev, dealloc_transport_domain, in); } -void mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid) +int mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid) { u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)] = {}; MLX5_SET(dealloc_pd_in, in, opcode, MLX5_CMD_OP_DEALLOC_PD); MLX5_SET(dealloc_pd_in, in, pd, pdn); MLX5_SET(dealloc_pd_in, in, uid, uid); - mlx5_cmd_exec_in(dev, dealloc_pd, in); + return mlx5_cmd_exec_in(dev, dealloc_pd, in); } int mlx5_cmd_attach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid, diff --git a/drivers/infiniband/hw/mlx5/cmd.h b/drivers/infiniband/hw/mlx5/cmd.h index 1d192a8ca87d..88ea6ef8f2cb 100644 --- a/drivers/infiniband/hw/mlx5/cmd.h +++ b/drivers/infiniband/hw/mlx5/cmd.h @@ -44,10 +44,10 @@ int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point, int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr, u64 length, u32 alignment); void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length); -void mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid); +int mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid); void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid); void mlx5_cmd_destroy_tis(struct mlx5_core_dev *dev, u32 tisn, u16 uid); -void mlx5_cmd_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn, u16 uid); +int mlx5_cmd_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn, u16 uid); int mlx5_cmd_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn, u16 uid); void mlx5_cmd_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn, diff --git a/drivers/infiniband/hw/mlx5/counters.c b/drivers/infiniband/hw/mlx5/counters.c index 145f3cb40ccb..70c8fd67ee2f 100644 --- a/drivers/infiniband/hw/mlx5/counters.c +++ b/drivers/infiniband/hw/mlx5/counters.c @@ -117,7 +117,7 @@ err_bound: return ret; } -static void mlx5_ib_destroy_counters(struct ib_counters *counters) +static int mlx5_ib_destroy_counters(struct ib_counters *counters) { struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); @@ -125,6 +125,7 @@ static void mlx5_ib_destroy_counters(struct ib_counters *counters) if (mcounters->hw_cntrs_hndl) mlx5_fc_destroy(to_mdev(counters->device)->mdev, mcounters->hw_cntrs_hndl); + return 0; } static int mlx5_ib_create_counters(struct ib_counters *counters, @@ -456,12 +457,12 @@ static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev, cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts); num_counters += ARRAY_SIZE(ext_ppcnt_cnts); } - cnts->names = kcalloc(num_counters, sizeof(cnts->names), GFP_KERNEL); + cnts->names = kcalloc(num_counters, sizeof(*cnts->names), GFP_KERNEL); if (!cnts->names) return -ENOMEM; cnts->offsets = kcalloc(num_counters, - sizeof(cnts->offsets), GFP_KERNEL); + sizeof(*cnts->offsets), GFP_KERNEL); if (!cnts->offsets) goto err_names; diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index dceb0eb2bed1..fb62f1d04afa 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -168,7 +168,7 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe, { enum rdma_link_layer ll = rdma_port_get_link_layer(qp->ibqp.device, 1); struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device); - struct mlx5_ib_srq *srq; + struct mlx5_ib_srq *srq = NULL; struct mlx5_ib_wq *wq; u16 wqe_ctr; u8 roce_packet_type; @@ -180,7 +180,8 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe, if (qp->ibqp.xrcd) { msrq = mlx5_cmd_get_srq(dev, be32_to_cpu(cqe->srqn)); - srq = to_mibsrq(msrq); + if (msrq) + srq = to_mibsrq(msrq); } else { srq = to_msrq(qp->ibqp.srq); } @@ -254,7 +255,7 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe, switch (roce_packet_type) { case MLX5_CQE_ROCE_L3_HEADER_TYPE_GRH: - wc->network_hdr_type = RDMA_NETWORK_IB; + wc->network_hdr_type = RDMA_NETWORK_ROCE_V1; break; case MLX5_CQE_ROCE_L3_HEADER_TYPE_IPV6: wc->network_hdr_type = RDMA_NETWORK_IPV6; @@ -1023,16 +1024,21 @@ err_cqb: return err; } -void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { struct mlx5_ib_dev *dev = to_mdev(cq->device); struct mlx5_ib_cq *mcq = to_mcq(cq); + int ret; + + ret = mlx5_core_destroy_cq(dev->mdev, &mcq->mcq); + if (ret) + return ret; - mlx5_core_destroy_cq(dev->mdev, &mcq->mcq); if (udata) destroy_cq_user(mcq, udata); else destroy_cq_kernel(dev, mcq); + return 0; } static int is_equal_rsn(struct mlx5_cqe64 *cqe64, u32 rsn) diff --git a/drivers/infiniband/hw/mlx5/fs.c b/drivers/infiniband/hw/mlx5/fs.c index e9cfb9a2ef41..492cfe063bca 100644 --- a/drivers/infiniband/hw/mlx5/fs.c +++ b/drivers/infiniband/hw/mlx5/fs.c @@ -136,12 +136,9 @@ static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask) #define LAST_COUNTERS_FIELD counters /* Field is the last supported field */ -#define FIELDS_NOT_SUPPORTED(filter, field)\ - memchr_inv((void *)&filter.field +\ - sizeof(filter.field), 0,\ - sizeof(filter) -\ - offsetof(typeof(filter), field) -\ - sizeof(filter.field)) +#define FIELDS_NOT_SUPPORTED(filter, field) \ + memchr_inv((void *)&filter.field + sizeof(filter.field), 0, \ + sizeof(filter) - offsetofend(typeof(filter), field)) int parse_flow_flow_action(struct mlx5_ib_flow_action *maction, bool is_egress, @@ -767,6 +764,7 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, { bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP; struct mlx5_flow_namespace *ns = NULL; + enum mlx5_flow_namespace_type fn_type; struct mlx5_ib_flow_prio *prio; struct mlx5_flow_table *ft; int max_table_size; @@ -780,11 +778,9 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, log_max_ft_size)); esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != DEVLINK_ESWITCH_ENCAP_MODE_NONE; - if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { - enum mlx5_flow_namespace_type fn_type; - - if (flow_is_multicast_only(flow_attr) && - !dont_trap) + switch (flow_attr->type) { + case IB_FLOW_ATTR_NORMAL: + if (flow_is_multicast_only(flow_attr) && !dont_trap) priority = MLX5_IB_FLOW_MCAST_PRIO; else priority = ib_prio_to_core_prio(flow_attr->priority, @@ -797,12 +793,11 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; if (!dev->is_rep && !esw_encap && MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, - reformat_l3_tunnel_to_l2)) + reformat_l3_tunnel_to_l2)) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; } else { - max_table_size = - BIT(MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, - log_max_ft_size)); + max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX( + dev->mdev, log_max_ft_size)); fn_type = MLX5_FLOW_NAMESPACE_EGRESS; prio = &dev->flow_db->egress_prios[priority]; if (!dev->is_rep && !esw_encap && @@ -812,27 +807,31 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, ns = mlx5_get_flow_namespace(dev->mdev, fn_type); num_entries = MLX5_FS_MAX_ENTRIES; num_groups = MLX5_FS_MAX_TYPES; - } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || - flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) { + break; + case IB_FLOW_ATTR_ALL_DEFAULT: + case IB_FLOW_ATTR_MC_DEFAULT: ns = mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_LEFTOVERS); - build_leftovers_ft_param(&priority, - &num_entries, - &num_groups); + build_leftovers_ft_param(&priority, &num_entries, &num_groups); prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO]; - } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { + break; + case IB_FLOW_ATTR_SNIFFER: if (!MLX5_CAP_FLOWTABLE(dev->mdev, allow_sniffer_and_nic_rx_shared_tir)) return ERR_PTR(-EOPNOTSUPP); - ns = mlx5_get_flow_namespace(dev->mdev, ft_type == MLX5_IB_FT_RX ? - MLX5_FLOW_NAMESPACE_SNIFFER_RX : - MLX5_FLOW_NAMESPACE_SNIFFER_TX); + ns = mlx5_get_flow_namespace( + dev->mdev, ft_type == MLX5_IB_FT_RX ? + MLX5_FLOW_NAMESPACE_SNIFFER_RX : + MLX5_FLOW_NAMESPACE_SNIFFER_TX); prio = &dev->flow_db->sniffer[ft_type]; priority = 0; num_entries = 1; num_groups = 1; + break; + default: + break; } if (!ns) @@ -954,7 +953,7 @@ static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, if (!flow_is_multicast_only(flow_attr)) set_underlay_qp(dev, spec, underlay_qpn); - if (dev->is_rep) { + if (dev->is_rep && flow_attr->type != IB_FLOW_ATTR_SNIFFER) { struct mlx5_eswitch_rep *rep; rep = dev->port[flow_attr->port - 1].rep; @@ -1116,6 +1115,7 @@ static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev, int err; static const struct ib_flow_attr flow_attr = { .num_of_specs = 0, + .type = IB_FLOW_ATTR_SNIFFER, .size = sizeof(flow_attr) }; @@ -1143,10 +1143,8 @@ err: return ERR_PTR(err); } - static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, struct ib_flow_attr *flow_attr, - int domain, struct ib_udata *udata) { struct mlx5_ib_dev *dev = to_mdev(qp->device); @@ -1162,8 +1160,7 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, int underlay_qpn; if (udata && udata->inlen) { - min_ucmd_sz = offsetof(typeof(ucmd_hdr), reserved) + - sizeof(ucmd_hdr.reserved); + min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved); if (udata->inlen < min_ucmd_sz) return ERR_PTR(-EOPNOTSUPP); @@ -1197,10 +1194,9 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, goto free_ucmd; } - if (domain != IB_FLOW_DOMAIN_USER || - flow_attr->port > dev->num_ports || - (flow_attr->flags & ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | - IB_FLOW_ATTR_FLAGS_EGRESS))) { + if (flow_attr->port > dev->num_ports || + (flow_attr->flags & + ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | IB_FLOW_ATTR_FLAGS_EGRESS))) { err = -EINVAL; goto free_ucmd; } @@ -1245,19 +1241,22 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, dst->tir_num = mqp->raw_packet_qp.rq.tirn; } - if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { + switch (flow_attr->type) { + case IB_FLOW_ATTR_NORMAL: underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ? mqp->underlay_qpn : 0; handler = _create_flow_rule(dev, ft_prio, flow_attr, dst, underlay_qpn, ucmd); - } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || - flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) { - handler = create_leftovers_rule(dev, ft_prio, flow_attr, - dst); - } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { + break; + case IB_FLOW_ATTR_ALL_DEFAULT: + case IB_FLOW_ATTR_MC_DEFAULT: + handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst); + break; + case IB_FLOW_ATTR_SNIFFER: handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst); - } else { + break; + default: err = -EINVAL; goto destroy_ft; } @@ -1305,39 +1304,47 @@ _get_flow_table(struct mlx5_ib_dev *dev, esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != DEVLINK_ESWITCH_ENCAP_MODE_NONE; - if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { - max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, - log_max_ft_size)); + switch (fs_matcher->ns_type) { + case MLX5_FLOW_NAMESPACE_BYPASS: + max_table_size = BIT( + MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size)); if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, reformat_l3_tunnel_to_l2) && !esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; - } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) { + break; + case MLX5_FLOW_NAMESPACE_EGRESS: max_table_size = BIT( MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size)); - if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && !esw_encap) + if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && + !esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; - } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) { + break; + case MLX5_FLOW_NAMESPACE_FDB: max_table_size = BIT( MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; - if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, reformat_l3_tunnel_to_l2) && + if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, + reformat_l3_tunnel_to_l2) && esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; priority = FDB_BYPASS_PATH; - } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) { - max_table_size = - BIT(MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, - log_max_ft_size)); + break; + case MLX5_FLOW_NAMESPACE_RDMA_RX: + max_table_size = BIT( + MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, log_max_ft_size)); priority = fs_matcher->priority; - } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) { - max_table_size = - BIT(MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, - log_max_ft_size)); + break; + case MLX5_FLOW_NAMESPACE_RDMA_TX: + max_table_size = BIT( + MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, log_max_ft_size)); priority = fs_matcher->priority; + break; + default: + break; } max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES); @@ -1346,16 +1353,24 @@ _get_flow_table(struct mlx5_ib_dev *dev, if (!ns) return ERR_PTR(-EOPNOTSUPP); - if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) + switch (fs_matcher->ns_type) { + case MLX5_FLOW_NAMESPACE_BYPASS: prio = &dev->flow_db->prios[priority]; - else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) + break; + case MLX5_FLOW_NAMESPACE_EGRESS: prio = &dev->flow_db->egress_prios[priority]; - else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) + break; + case MLX5_FLOW_NAMESPACE_FDB: prio = &dev->flow_db->fdb; - else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) + break; + case MLX5_FLOW_NAMESPACE_RDMA_RX: prio = &dev->flow_db->rdma_rx[priority]; - else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) + break; + case MLX5_FLOW_NAMESPACE_RDMA_TX: prio = &dev->flow_db->rdma_tx[priority]; + break; + default: return ERR_PTR(-EINVAL); + } if (!prio) return ERR_PTR(-EINVAL); @@ -1488,20 +1503,25 @@ static struct mlx5_ib_flow_handler *raw_fs_rule_add( goto unlock; } - if (dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR) { + switch (dest_type) { + case MLX5_FLOW_DESTINATION_TYPE_TIR: dst[dst_num].type = dest_type; dst[dst_num++].tir_num = dest_id; flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; - } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) { + break; + case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE: dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM; dst[dst_num++].ft_num = dest_id; flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; - } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_PORT) { + break; + case MLX5_FLOW_DESTINATION_TYPE_PORT: dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT; flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; + break; + default: + break; } - if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; dst[dst_num].counter_id = counter_id; diff --git a/drivers/infiniband/hw/mlx5/gsi.c b/drivers/infiniband/hw/mlx5/gsi.c index 40d418153891..7fcad9135276 100644 --- a/drivers/infiniband/hw/mlx5/gsi.c +++ b/drivers/infiniband/hw/mlx5/gsi.c @@ -35,44 +35,19 @@ struct mlx5_ib_gsi_wr { struct ib_cqe cqe; struct ib_wc wc; - int send_flags; bool completed:1; }; -struct mlx5_ib_gsi_qp { - struct ib_qp ibqp; - struct ib_qp *rx_qp; - u8 port_num; - struct ib_qp_cap cap; - enum ib_sig_type sq_sig_type; - /* Serialize qp state modifications */ - struct mutex mutex; - struct ib_cq *cq; - struct mlx5_ib_gsi_wr *outstanding_wrs; - u32 outstanding_pi, outstanding_ci; - int num_qps; - /* Protects access to the tx_qps. Post send operations synchronize - * with tx_qp creation in setup_qp(). Also protects the - * outstanding_wrs array and indices. - */ - spinlock_t lock; - struct ib_qp **tx_qps; -}; - -static struct mlx5_ib_gsi_qp *gsi_qp(struct ib_qp *qp) -{ - return container_of(qp, struct mlx5_ib_gsi_qp, ibqp); -} - static bool mlx5_ib_deth_sqpn_cap(struct mlx5_ib_dev *dev) { return MLX5_CAP_GEN(dev->mdev, set_deth_sqpn); } /* Call with gsi->lock locked */ -static void generate_completions(struct mlx5_ib_gsi_qp *gsi) +static void generate_completions(struct mlx5_ib_qp *mqp) { - struct ib_cq *gsi_cq = gsi->ibqp.send_cq; + struct mlx5_ib_gsi_qp *gsi = &mqp->gsi; + struct ib_cq *gsi_cq = mqp->ibqp.send_cq; struct mlx5_ib_gsi_wr *wr; u32 index; @@ -83,10 +58,7 @@ static void generate_completions(struct mlx5_ib_gsi_qp *gsi) if (!wr->completed) break; - if (gsi->sq_sig_type == IB_SIGNAL_ALL_WR || - wr->send_flags & IB_SEND_SIGNALED) - WARN_ON_ONCE(mlx5_ib_generate_wc(gsi_cq, &wr->wc)); - + WARN_ON_ONCE(mlx5_ib_generate_wc(gsi_cq, &wr->wc)); wr->completed = false; } @@ -98,6 +70,7 @@ static void handle_single_completion(struct ib_cq *cq, struct ib_wc *wc) struct mlx5_ib_gsi_qp *gsi = cq->cq_context; struct mlx5_ib_gsi_wr *wr = container_of(wc->wr_cqe, struct mlx5_ib_gsi_wr, cqe); + struct mlx5_ib_qp *mqp = container_of(gsi, struct mlx5_ib_qp, gsi); u64 wr_id; unsigned long flags; @@ -106,19 +79,19 @@ static void handle_single_completion(struct ib_cq *cq, struct ib_wc *wc) wr_id = wr->wc.wr_id; wr->wc = *wc; wr->wc.wr_id = wr_id; - wr->wc.qp = &gsi->ibqp; + wr->wc.qp = &mqp->ibqp; - generate_completions(gsi); + generate_completions(mqp); spin_unlock_irqrestore(&gsi->lock, flags); } -struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr) +int mlx5_ib_create_gsi(struct ib_pd *pd, struct mlx5_ib_qp *mqp, + struct ib_qp_init_attr *attr) { struct mlx5_ib_dev *dev = to_mdev(pd->device); struct mlx5_ib_gsi_qp *gsi; - struct ib_qp_init_attr hw_init_attr = *init_attr; - const u8 port_num = init_attr->port_num; + struct ib_qp_init_attr hw_init_attr = *attr; + const u8 port_num = attr->port_num; int num_qps = 0; int ret; @@ -130,26 +103,19 @@ struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd, num_qps = MLX5_MAX_PORTS; } - gsi = kzalloc(sizeof(*gsi), GFP_KERNEL); - if (!gsi) - return ERR_PTR(-ENOMEM); - + gsi = &mqp->gsi; gsi->tx_qps = kcalloc(num_qps, sizeof(*gsi->tx_qps), GFP_KERNEL); - if (!gsi->tx_qps) { - ret = -ENOMEM; - goto err_free; - } + if (!gsi->tx_qps) + return -ENOMEM; - gsi->outstanding_wrs = kcalloc(init_attr->cap.max_send_wr, - sizeof(*gsi->outstanding_wrs), - GFP_KERNEL); + gsi->outstanding_wrs = + kcalloc(attr->cap.max_send_wr, sizeof(*gsi->outstanding_wrs), + GFP_KERNEL); if (!gsi->outstanding_wrs) { ret = -ENOMEM; goto err_free_tx; } - mutex_init(&gsi->mutex); - mutex_lock(&dev->devr.mutex); if (dev->devr.ports[port_num - 1].gsi) { @@ -161,12 +127,10 @@ struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd, gsi->num_qps = num_qps; spin_lock_init(&gsi->lock); - gsi->cap = init_attr->cap; - gsi->sq_sig_type = init_attr->sq_sig_type; - gsi->ibqp.qp_num = 1; + gsi->cap = attr->cap; gsi->port_num = port_num; - gsi->cq = ib_alloc_cq(pd->device, gsi, init_attr->cap.max_send_wr, 0, + gsi->cq = ib_alloc_cq(pd->device, gsi, attr->cap.max_send_wr, 0, IB_POLL_SOFTIRQ); if (IS_ERR(gsi->cq)) { mlx5_ib_warn(dev, "unable to create send CQ for GSI QP. error %ld\n", @@ -182,19 +146,31 @@ struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd, hw_init_attr.cap.max_send_sge = 0; hw_init_attr.cap.max_inline_data = 0; } - gsi->rx_qp = ib_create_qp(pd, &hw_init_attr); + + gsi->rx_qp = mlx5_ib_create_qp(pd, &hw_init_attr, NULL); if (IS_ERR(gsi->rx_qp)) { mlx5_ib_warn(dev, "unable to create hardware GSI QP. error %ld\n", PTR_ERR(gsi->rx_qp)); ret = PTR_ERR(gsi->rx_qp); goto err_destroy_cq; } + gsi->rx_qp->device = pd->device; + gsi->rx_qp->pd = pd; + gsi->rx_qp->real_qp = gsi->rx_qp; + + gsi->rx_qp->qp_type = hw_init_attr.qp_type; + gsi->rx_qp->send_cq = hw_init_attr.send_cq; + gsi->rx_qp->recv_cq = hw_init_attr.recv_cq; + gsi->rx_qp->event_handler = hw_init_attr.event_handler; + spin_lock_init(&gsi->rx_qp->mr_lock); + INIT_LIST_HEAD(&gsi->rx_qp->rdma_mrs); + INIT_LIST_HEAD(&gsi->rx_qp->sig_mrs); - dev->devr.ports[init_attr->port_num - 1].gsi = gsi; + dev->devr.ports[attr->port_num - 1].gsi = gsi; mutex_unlock(&dev->devr.mutex); - return &gsi->ibqp; + return 0; err_destroy_cq: ib_free_cq(gsi->cq); @@ -203,23 +179,19 @@ err_free_wrs: kfree(gsi->outstanding_wrs); err_free_tx: kfree(gsi->tx_qps); -err_free: - kfree(gsi); - return ERR_PTR(ret); + return ret; } -int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp) +int mlx5_ib_destroy_gsi(struct mlx5_ib_qp *mqp) { - struct mlx5_ib_dev *dev = to_mdev(qp->device); - struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp); + struct mlx5_ib_dev *dev = to_mdev(mqp->ibqp.device); + struct mlx5_ib_gsi_qp *gsi = &mqp->gsi; const int port_num = gsi->port_num; int qp_index; int ret; - mlx5_ib_dbg(dev, "destroying GSI QP\n"); - mutex_lock(&dev->devr.mutex); - ret = ib_destroy_qp(gsi->rx_qp); + ret = mlx5_ib_destroy_qp(gsi->rx_qp, NULL); if (ret) { mlx5_ib_warn(dev, "unable to destroy hardware GSI QP. error %d\n", ret); @@ -241,7 +213,7 @@ int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp) kfree(gsi->outstanding_wrs); kfree(gsi->tx_qps); - kfree(gsi); + kfree(mqp); return 0; } @@ -259,7 +231,6 @@ static struct ib_qp *create_gsi_ud_qp(struct mlx5_ib_gsi_qp *gsi) .max_send_sge = gsi->cap.max_send_sge, .max_inline_data = gsi->cap.max_inline_data, }, - .sq_sig_type = gsi->sq_sig_type, .qp_type = IB_QPT_UD, .create_flags = MLX5_IB_QP_CREATE_SQPN_QP1, }; @@ -370,56 +341,54 @@ err_destroy_qp: static void setup_qps(struct mlx5_ib_gsi_qp *gsi) { + struct mlx5_ib_dev *dev = to_mdev(gsi->rx_qp->device); u16 qp_index; + mutex_lock(&dev->devr.mutex); for (qp_index = 0; qp_index < gsi->num_qps; ++qp_index) setup_qp(gsi, qp_index); + mutex_unlock(&dev->devr.mutex); } int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr, int attr_mask) { struct mlx5_ib_dev *dev = to_mdev(qp->device); - struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp); + struct mlx5_ib_qp *mqp = to_mqp(qp); + struct mlx5_ib_gsi_qp *gsi = &mqp->gsi; int ret; mlx5_ib_dbg(dev, "modifying GSI QP to state %d\n", attr->qp_state); - mutex_lock(&gsi->mutex); ret = ib_modify_qp(gsi->rx_qp, attr, attr_mask); if (ret) { mlx5_ib_warn(dev, "unable to modify GSI rx QP: %d\n", ret); - goto unlock; + return ret; } if (to_mqp(gsi->rx_qp)->state == IB_QPS_RTS) setup_qps(gsi); - -unlock: - mutex_unlock(&gsi->mutex); - - return ret; + return 0; } int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) { - struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp); + struct mlx5_ib_qp *mqp = to_mqp(qp); + struct mlx5_ib_gsi_qp *gsi = &mqp->gsi; int ret; - mutex_lock(&gsi->mutex); ret = ib_query_qp(gsi->rx_qp, qp_attr, qp_attr_mask, qp_init_attr); qp_init_attr->cap = gsi->cap; - mutex_unlock(&gsi->mutex); - return ret; } /* Call with gsi->lock locked */ -static int mlx5_ib_add_outstanding_wr(struct mlx5_ib_gsi_qp *gsi, +static int mlx5_ib_add_outstanding_wr(struct mlx5_ib_qp *mqp, struct ib_ud_wr *wr, struct ib_wc *wc) { + struct mlx5_ib_gsi_qp *gsi = &mqp->gsi; struct mlx5_ib_dev *dev = to_mdev(gsi->rx_qp->device); struct mlx5_ib_gsi_wr *gsi_wr; @@ -448,22 +417,21 @@ static int mlx5_ib_add_outstanding_wr(struct mlx5_ib_gsi_qp *gsi, } /* Call with gsi->lock locked */ -static int mlx5_ib_gsi_silent_drop(struct mlx5_ib_gsi_qp *gsi, - struct ib_ud_wr *wr) +static int mlx5_ib_gsi_silent_drop(struct mlx5_ib_qp *mqp, struct ib_ud_wr *wr) { struct ib_wc wc = { { .wr_id = wr->wr.wr_id }, .status = IB_WC_SUCCESS, .opcode = IB_WC_SEND, - .qp = &gsi->ibqp, + .qp = &mqp->ibqp, }; int ret; - ret = mlx5_ib_add_outstanding_wr(gsi, wr, &wc); + ret = mlx5_ib_add_outstanding_wr(mqp, wr, &wc); if (ret) return ret; - generate_completions(gsi); + generate_completions(mqp); return 0; } @@ -490,7 +458,8 @@ static struct ib_qp *get_tx_qp(struct mlx5_ib_gsi_qp *gsi, struct ib_ud_wr *wr) int mlx5_ib_gsi_post_send(struct ib_qp *qp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { - struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp); + struct mlx5_ib_qp *mqp = to_mqp(qp); + struct mlx5_ib_gsi_qp *gsi = &mqp->gsi; struct ib_qp *tx_qp; unsigned long flags; int ret; @@ -503,14 +472,14 @@ int mlx5_ib_gsi_post_send(struct ib_qp *qp, const struct ib_send_wr *wr, spin_lock_irqsave(&gsi->lock, flags); tx_qp = get_tx_qp(gsi, &cur_wr); if (!tx_qp) { - ret = mlx5_ib_gsi_silent_drop(gsi, &cur_wr); + ret = mlx5_ib_gsi_silent_drop(mqp, &cur_wr); if (ret) goto err; spin_unlock_irqrestore(&gsi->lock, flags); continue; } - ret = mlx5_ib_add_outstanding_wr(gsi, &cur_wr, NULL); + ret = mlx5_ib_add_outstanding_wr(mqp, &cur_wr, NULL); if (ret) goto err; @@ -534,7 +503,8 @@ err: int mlx5_ib_gsi_post_recv(struct ib_qp *qp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { - struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp); + struct mlx5_ib_qp *mqp = to_mqp(qp); + struct mlx5_ib_gsi_qp *gsi = &mqp->gsi; return ib_post_recv(gsi->rx_qp, wr, bad_wr); } @@ -544,7 +514,5 @@ void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi) if (!gsi) return; - mutex_lock(&gsi->mutex); setup_qps(gsi); - mutex_unlock(&gsi->mutex); } diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index d60d63221b14..89e04ca62ae0 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -326,8 +326,8 @@ out: spin_unlock(&port->mp.mpi_lock); } -static int translate_eth_legacy_proto_oper(u32 eth_proto_oper, u8 *active_speed, - u8 *active_width) +static int translate_eth_legacy_proto_oper(u32 eth_proto_oper, + u16 *active_speed, u8 *active_width) { switch (eth_proto_oper) { case MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII): @@ -384,7 +384,7 @@ static int translate_eth_legacy_proto_oper(u32 eth_proto_oper, u8 *active_speed, return 0; } -static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u8 *active_speed, +static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u16 *active_speed, u8 *active_width) { switch (eth_proto_oper) { @@ -436,7 +436,7 @@ static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u8 *active_speed, return 0; } -static int translate_eth_proto_oper(u32 eth_proto_oper, u8 *active_speed, +static int translate_eth_proto_oper(u32 eth_proto_oper, u16 *active_speed, u8 *active_width, bool ext) { return ext ? @@ -546,7 +546,7 @@ static int set_roce_addr(struct mlx5_ib_dev *dev, u8 port_num, unsigned int index, const union ib_gid *gid, const struct ib_gid_attr *attr) { - enum ib_gid_type gid_type = IB_GID_TYPE_IB; + enum ib_gid_type gid_type = IB_GID_TYPE_ROCE; u16 vlan_id = 0xffff; u8 roce_version = 0; u8 roce_l3_type = 0; @@ -561,7 +561,7 @@ static int set_roce_addr(struct mlx5_ib_dev *dev, u8 port_num, } switch (gid_type) { - case IB_GID_TYPE_IB: + case IB_GID_TYPE_ROCE: roce_version = MLX5_ROCE_VERSION_1; break; case IB_GID_TYPE_ROCE_UDP_ENCAP: @@ -840,7 +840,9 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, /* We support 'Gappy' memory registration too */ props->device_cap_flags |= IB_DEVICE_SG_GAPS_REG; } - props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; + /* IB_WR_REG_MR always requires changing the entity size with UMR */ + if (!MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled)) + props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; if (MLX5_CAP_GEN(mdev, sho)) { props->device_cap_flags |= IB_DEVICE_INTEGRITY_HANDOVER; /* At this stage no support for signature handover */ @@ -1175,32 +1177,24 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, return 0; } -enum mlx5_ib_width { - MLX5_IB_WIDTH_1X = 1 << 0, - MLX5_IB_WIDTH_2X = 1 << 1, - MLX5_IB_WIDTH_4X = 1 << 2, - MLX5_IB_WIDTH_8X = 1 << 3, - MLX5_IB_WIDTH_12X = 1 << 4 -}; - -static void translate_active_width(struct ib_device *ibdev, u8 active_width, - u8 *ib_width) +static void translate_active_width(struct ib_device *ibdev, u16 active_width, + u8 *ib_width) { struct mlx5_ib_dev *dev = to_mdev(ibdev); - if (active_width & MLX5_IB_WIDTH_1X) + if (active_width & MLX5_PTYS_WIDTH_1X) *ib_width = IB_WIDTH_1X; - else if (active_width & MLX5_IB_WIDTH_2X) + else if (active_width & MLX5_PTYS_WIDTH_2X) *ib_width = IB_WIDTH_2X; - else if (active_width & MLX5_IB_WIDTH_4X) + else if (active_width & MLX5_PTYS_WIDTH_4X) *ib_width = IB_WIDTH_4X; - else if (active_width & MLX5_IB_WIDTH_8X) + else if (active_width & MLX5_PTYS_WIDTH_8X) *ib_width = IB_WIDTH_8X; - else if (active_width & MLX5_IB_WIDTH_12X) + else if (active_width & MLX5_PTYS_WIDTH_12X) *ib_width = IB_WIDTH_12X; else { mlx5_ib_dbg(dev, "Invalid active_width %d, setting width to default value: 4x\n", - (int)active_width); + active_width); *ib_width = IB_WIDTH_4X; } @@ -1277,7 +1271,7 @@ static int mlx5_query_hca_port(struct ib_device *ibdev, u8 port, u16 max_mtu; u16 oper_mtu; int err; - u8 ib_link_width_oper; + u16 ib_link_width_oper; u8 vl_hw_cap; rep = kzalloc(sizeof(*rep), GFP_KERNEL); @@ -1310,16 +1304,13 @@ static int mlx5_query_hca_port(struct ib_device *ibdev, u8 port, if (props->port_cap_flags & IB_PORT_CAP_MASK2_SUP) props->port_cap_flags2 = rep->cap_mask2; - err = mlx5_query_port_link_width_oper(mdev, &ib_link_width_oper, port); + err = mlx5_query_ib_port_oper(mdev, &ib_link_width_oper, + &props->active_speed, port); if (err) goto out; translate_active_width(ibdev, ib_link_width_oper, &props->active_width); - err = mlx5_query_port_ib_proto_oper(mdev, &props->active_speed, port); - if (err) - goto out; - mlx5_query_port_max_mtu(mdev, &max_mtu, port); props->max_mtu = mlx5_mtu_to_ib_mtu(max_mtu); @@ -2354,7 +2345,9 @@ static inline int check_dm_type_support(struct mlx5_ib_dev *dev, return -EPERM; if (!(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) || - MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, sw_owner))) + MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, sw_owner) || + MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2) || + MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, sw_owner_v2))) return -EOPNOTSUPP; break; } @@ -2569,12 +2562,12 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) return 0; } -static void mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) +static int mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) { struct mlx5_ib_dev *mdev = to_mdev(pd->device); struct mlx5_ib_pd *mpd = to_mpd(pd); - mlx5_cmd_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid); + return mlx5_cmd_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid); } static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) @@ -2699,9 +2692,7 @@ static void pkey_change_handler(struct work_struct *work) container_of(work, struct mlx5_ib_port_resources, pkey_change_work); - mutex_lock(&ports->devr->mutex); mlx5_ib_gsi_pkey_change(ports->gsi); - mutex_unlock(&ports->devr->mutex); } static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev) @@ -3127,11 +3118,9 @@ static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev) atomic_inc(&devr->p0->usecnt); atomic_set(&devr->s1->usecnt, 0); - for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) { + for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) INIT_WORK(&devr->ports[port].pkey_change_work, pkey_change_handler); - devr->ports[port].devr = devr; - } return 0; @@ -4098,6 +4087,8 @@ static const struct ib_device_ops mlx5_ib_dev_sriov_ops = { static const struct ib_device_ops mlx5_ib_dev_mw_ops = { .alloc_mw = mlx5_ib_alloc_mw, .dealloc_mw = mlx5_ib_dealloc_mw, + + INIT_RDMA_OBJ_SIZE(ib_mw, mlx5_ib_mw, ibmw), }; static const struct ib_device_ops mlx5_ib_dev_xrc_ops = { @@ -4268,6 +4259,9 @@ static const struct ib_device_ops mlx5_ib_dev_common_roce_ops = { .destroy_wq = mlx5_ib_destroy_wq, .get_netdev = mlx5_ib_get_netdev, .modify_wq = mlx5_ib_modify_wq, + + INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mlx5_ib_rwq_ind_table, + ib_rwq_ind_tbl), }; static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev) @@ -4386,7 +4380,7 @@ static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev) name = "mlx5_%d"; else name = "mlx5_bond_%d"; - return ib_register_device(&dev->ib_dev, name); + return ib_register_device(&dev->ib_dev, name, &dev->mdev->pdev->dev); } static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev) diff --git a/drivers/infiniband/hw/mlx5/mem.c b/drivers/infiniband/hw/mlx5/mem.c index c19ec9fd8a63..13de3d2edd34 100644 --- a/drivers/infiniband/hw/mlx5/mem.c +++ b/drivers/infiniband/hw/mlx5/mem.c @@ -169,8 +169,8 @@ void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem, int page_shift, __be64 *pas, int access_flags) { return __mlx5_ib_populate_pas(dev, umem, page_shift, 0, - ib_umem_num_pages(umem), pas, - access_flags); + ib_umem_num_dma_blocks(umem, PAGE_SIZE), + pas, access_flags); } int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset) { diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 5287fc868662..b1f2b34e5955 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -384,6 +384,22 @@ struct mlx5_ib_dct { u32 *in; }; +struct mlx5_ib_gsi_qp { + struct ib_qp *rx_qp; + u8 port_num; + struct ib_qp_cap cap; + struct ib_cq *cq; + struct mlx5_ib_gsi_wr *outstanding_wrs; + u32 outstanding_pi, outstanding_ci; + int num_qps; + /* Protects access to the tx_qps. Post send operations synchronize + * with tx_qp creation in setup_qp(). Also protects the + * outstanding_wrs array and indices. + */ + spinlock_t lock; + struct ib_qp **tx_qps; +}; + struct mlx5_ib_qp { struct ib_qp ibqp; union { @@ -391,6 +407,7 @@ struct mlx5_ib_qp { struct mlx5_ib_raw_packet_qp raw_packet_qp; struct mlx5_ib_rss_qp rss_qp; struct mlx5_ib_dct dct; + struct mlx5_ib_gsi_qp gsi; }; struct mlx5_frag_buf buf; @@ -693,10 +710,7 @@ struct mlx5_mr_cache { unsigned long last_add; }; -struct mlx5_ib_gsi_qp; - struct mlx5_ib_port_resources { - struct mlx5_ib_resources *devr; struct mlx5_ib_gsi_qp *gsi; struct work_struct pkey_change_work; }; @@ -1119,13 +1133,16 @@ void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index); int mlx5_ib_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata); int mlx5_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); -void mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags); +static inline int mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags) +{ + return 0; +} int mlx5_ib_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *init_attr, struct ib_udata *udata); int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr); -void mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata); +int mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata); int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr); int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp); @@ -1148,7 +1165,7 @@ int mlx5_ib_read_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index, void *buffer, size_t buflen, size_t *bc); int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata); -void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period); @@ -1163,8 +1180,7 @@ int mlx5_ib_advise_mr(struct ib_pd *pd, struct ib_sge *sg_list, u32 num_sge, struct uverbs_attr_bundle *attrs); -struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, - struct ib_udata *udata); +int mlx5_ib_alloc_mw(struct ib_mw *mw, struct ib_udata *udata); int mlx5_ib_dealloc_mw(struct ib_mw *mw); int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, int page_shift, int flags); @@ -1193,7 +1209,7 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, const struct ib_mad *in, struct ib_mad *out, size_t *out_mad_size, u16 *out_mad_pkey_index); int mlx5_ib_alloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata); -void mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata); +int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata); int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset); int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port); int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev, @@ -1229,7 +1245,7 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev); int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev); struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, - unsigned int entry); + unsigned int entry, int access_flags); void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr); int mlx5_mr_cache_invalidate(struct mlx5_ib_mr *mr); @@ -1238,12 +1254,12 @@ int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd, struct ib_wq_init_attr *init_attr, struct ib_udata *udata); -void mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata); +int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata); int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr, u32 wq_attr_mask, struct ib_udata *udata); -struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device, - struct ib_rwq_ind_table_init_attr *init_attr, - struct ib_udata *udata); +int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table, + struct ib_rwq_ind_table_init_attr *init_attr, + struct ib_udata *udata); int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table); struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev, struct ib_ucontext *context, @@ -1267,6 +1283,7 @@ void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice, u32 flags, struct ib_sge *sg_list, u32 num_sge); +int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, bool enable); #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ static inline void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev) { @@ -1288,6 +1305,10 @@ mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, { return -EOPNOTSUPP; } +static inline int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, bool enable) +{ + return -EOPNOTSUPP; +} #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ extern const struct mmu_interval_notifier_ops mlx5_mn_ops; @@ -1318,9 +1339,9 @@ void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num); void mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num); /* GSI QP helper functions */ -struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr); -int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp); +int mlx5_ib_create_gsi(struct ib_pd *pd, struct mlx5_ib_qp *mqp, + struct ib_qp_init_attr *attr); +int mlx5_ib_destroy_gsi(struct mlx5_ib_qp *mqp); int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr, int attr_mask); int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, @@ -1358,7 +1379,7 @@ static inline void init_query_mad(struct ib_smp *mad) static inline int is_qp1(enum ib_qp_type qp_type) { - return qp_type == MLX5_IB_QPT_HW_GSI; + return qp_type == MLX5_IB_QPT_HW_GSI || qp_type == IB_QPT_GSI; } #define MLX5_MAX_UMR_SHIFT 16 @@ -1442,25 +1463,54 @@ int bfregn_to_uar_index(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, u32 bfregn, bool dyn_bfreg); -static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev, - bool do_modify_atomic, int access_flags) +static inline bool mlx5_ib_can_load_pas_with_umr(struct mlx5_ib_dev *dev, + size_t length) { + /* + * umr_check_mkey_mask() rejects MLX5_MKEY_MASK_PAGE_SIZE which is + * always set if MLX5_IB_SEND_UMR_UPDATE_TRANSLATION (aka + * MLX5_IB_UPD_XLT_ADDR and MLX5_IB_UPD_XLT_ENABLE) is set. Thus, a mkey + * can never be enabled without this capability. Simplify this weird + * quirky hardware by just saying it can't use PAS lists with UMR at + * all. + */ if (MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled)) return false; - if (do_modify_atomic && + /* + * length is the size of the MR in bytes when mlx5_ib_update_xlt() is + * used. + */ + if (!MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset) && + length >= MLX5_MAX_UMR_PAGES * PAGE_SIZE) + return false; + return true; +} + +/* + * true if an existing MR can be reconfigured to new access_flags using UMR. + * Older HW cannot use UMR to update certain elements of the MKC. See + * umr_check_mkey_mask(), get_umr_update_access_mask() and umr_check_mkey_mask() + */ +static inline bool mlx5_ib_can_reconfig_with_umr(struct mlx5_ib_dev *dev, + unsigned int current_access_flags, + unsigned int target_access_flags) +{ + unsigned int diffs = current_access_flags ^ target_access_flags; + + if ((diffs & IB_ACCESS_REMOTE_ATOMIC) && MLX5_CAP_GEN(dev->mdev, atomic) && MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled)) return false; - if (access_flags & IB_ACCESS_RELAXED_ORDERING && + if ((diffs & IB_ACCESS_RELAXED_ORDERING) && MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write) && !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr)) return false; - if (access_flags & IB_ACCESS_RELAXED_ORDERING && - MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) && - !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)) + if ((diffs & IB_ACCESS_RELAXED_ORDERING) && + MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) && + !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)) return false; return true; diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 3e6f2f9c6655..b261797b258f 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -50,6 +50,29 @@ enum { static void create_mkey_callback(int status, struct mlx5_async_work *context); +static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr, + struct ib_pd *pd) +{ + struct mlx5_ib_dev *dev = to_mdev(pd->device); + + MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC)); + MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE)); + MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ)); + MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE)); + MLX5_SET(mkc, mkc, lr, 1); + + if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) + MLX5_SET(mkc, mkc, relaxed_ordering_write, + !!(acc & IB_ACCESS_RELAXED_ORDERING)); + if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) + MLX5_SET(mkc, mkc, relaxed_ordering_read, + !!(acc & IB_ACCESS_RELAXED_ORDERING)); + + MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); + MLX5_SET(mkc, mkc, qpn, 0xffffff); + MLX5_SET64(mkc, mkc, start_addr, start_addr); +} + static void assign_mkey_variant(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey, u32 *in) @@ -100,7 +123,8 @@ static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) return mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey); } -static bool use_umr_mtt_update(struct mlx5_ib_mr *mr, u64 start, u64 length) +static inline bool mlx5_ib_pas_fits_in_mr(struct mlx5_ib_mr *mr, u64 start, + u64 length) { return ((u64)1 << mr->order) * MLX5_ADAPTER_PAGE_SIZE >= length + (start & (MLX5_ADAPTER_PAGE_SIZE - 1)); @@ -152,12 +176,12 @@ static struct mlx5_ib_mr *alloc_cache_mr(struct mlx5_cache_ent *ent, void *mkc) mr->cache_ent = ent; mr->dev = ent->dev; + set_mkc_access_pd_addr_fields(mkc, 0, 0, ent->dev->umrc.pd); MLX5_SET(mkc, mkc, free, 1); MLX5_SET(mkc, mkc, umr_en, 1); MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3); MLX5_SET(mkc, mkc, access_mode_4_2, (ent->access_mode >> 2) & 0x7); - MLX5_SET(mkc, mkc, qpn, 0xffffff); MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt); MLX5_SET(mkc, mkc, log_page_size, ent->page); return mr; @@ -534,7 +558,7 @@ static void cache_work_func(struct work_struct *work) /* Allocate a special entry from the cache */ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, - unsigned int entry) + unsigned int entry, int access_flags) { struct mlx5_mr_cache *cache = &dev->cache; struct mlx5_cache_ent *ent; @@ -544,6 +568,10 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, entry >= ARRAY_SIZE(cache->ent))) return ERR_PTR(-EINVAL); + /* Matches access in alloc_cache_mr() */ + if (!mlx5_ib_can_reconfig_with_umr(dev, 0, access_flags)) + return ERR_PTR(-EOPNOTSUPP); + ent = &cache->ent[entry]; spin_lock_irq(&ent->lock); if (list_empty(&ent->head)) { @@ -558,6 +586,7 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, queue_adjust_cache_locked(ent); spin_unlock_irq(&ent->lock); } + mr->access_flags = access_flags; return mr; } @@ -730,8 +759,8 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev) MLX5_IB_UMR_OCTOWORD; ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) && - !dev->is_rep && - mlx5_core_is_pf(dev->mdev)) + !dev->is_rep && mlx5_core_is_pf(dev->mdev) && + mlx5_ib_can_load_pas_with_umr(dev, 0)) ent->limit = dev->mdev->profile->mr_cache[i].limit; else ent->limit = 0; @@ -774,29 +803,6 @@ int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev) return 0; } -static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr, - struct ib_pd *pd) -{ - struct mlx5_ib_dev *dev = to_mdev(pd->device); - - MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC)); - MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE)); - MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ)); - MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE)); - MLX5_SET(mkc, mkc, lr, 1); - - if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) - MLX5_SET(mkc, mkc, relaxed_ordering_write, - !!(acc & IB_ACCESS_RELAXED_ORDERING)); - if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) - MLX5_SET(mkc, mkc, relaxed_ordering_read, - !!(acc & IB_ACCESS_RELAXED_ORDERING)); - - MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); - MLX5_SET(mkc, mkc, qpn, 0xffffff); - MLX5_SET64(mkc, mkc, start_addr, start_addr); -} - struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc) { struct mlx5_ib_dev *dev = to_mdev(pd->device); @@ -979,6 +985,11 @@ alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr, if (!ent) return ERR_PTR(-E2BIG); + + /* Matches access in alloc_cache_mr() */ + if (!mlx5_ib_can_reconfig_with_umr(dev, 0, access_flags)) + return ERR_PTR(-EOPNOTSUPP); + mr = get_cache_mr(ent); if (!mr) { mr = create_cache_mr(ent); @@ -1181,38 +1192,31 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, goto err_1; } pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); - if (populate && !(access_flags & IB_ACCESS_ON_DEMAND)) + if (populate) { + if (WARN_ON(access_flags & IB_ACCESS_ON_DEMAND)) { + err = -EINVAL; + goto err_2; + } mlx5_ib_populate_pas(dev, umem, page_shift, pas, pg_cap ? MLX5_IB_MTT_PRESENT : 0); + } /* The pg_access bit allows setting the access flags * in the page list submitted with the command. */ MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap)); mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); + set_mkc_access_pd_addr_fields(mkc, access_flags, virt_addr, + populate ? pd : dev->umrc.pd); MLX5_SET(mkc, mkc, free, !populate); MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); - if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) - MLX5_SET(mkc, mkc, relaxed_ordering_write, - !!(access_flags & IB_ACCESS_RELAXED_ORDERING)); - if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) - MLX5_SET(mkc, mkc, relaxed_ordering_read, - !!(access_flags & IB_ACCESS_RELAXED_ORDERING)); - MLX5_SET(mkc, mkc, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC)); - MLX5_SET(mkc, mkc, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE)); - MLX5_SET(mkc, mkc, rr, !!(access_flags & IB_ACCESS_REMOTE_READ)); - MLX5_SET(mkc, mkc, lw, !!(access_flags & IB_ACCESS_LOCAL_WRITE)); - MLX5_SET(mkc, mkc, lr, 1); MLX5_SET(mkc, mkc, umr_en, 1); - MLX5_SET64(mkc, mkc, start_addr, virt_addr); MLX5_SET64(mkc, mkc, len, length); - MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); MLX5_SET(mkc, mkc, bsf_octword_size, 0); MLX5_SET(mkc, mkc, translations_octword_size, get_octo_len(virt_addr, length, page_shift)); MLX5_SET(mkc, mkc, log_page_size, page_shift); - MLX5_SET(mkc, mkc, qpn, 0xffffff); if (populate) { MLX5_SET(create_mkey_in, in, translations_octword_actual_size, get_octo_len(virt_addr, length, page_shift)); @@ -1308,7 +1312,8 @@ int mlx5_ib_advise_mr(struct ib_pd *pd, struct uverbs_attr_bundle *attrs) { if (advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH && - advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE) + advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE && + advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT) return -EOPNOTSUPP; return mlx5_ib_advise_mr_prefetch(pd, advice, flags, @@ -1353,7 +1358,7 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, { struct mlx5_ib_dev *dev = to_mdev(pd->device); struct mlx5_ib_mr *mr = NULL; - bool use_umr; + bool xlt_with_umr; struct ib_umem *umem; int page_shift; int npages; @@ -1367,6 +1372,11 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n", start, virt_addr, length, access_flags); + xlt_with_umr = mlx5_ib_can_load_pas_with_umr(dev, length); + /* ODP requires xlt update via umr to work. */ + if (!xlt_with_umr && (access_flags & IB_ACCESS_ON_DEMAND)) + return ERR_PTR(-EINVAL); + if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && !start && length == U64_MAX) { if (virt_addr != start) @@ -1387,28 +1397,17 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (err < 0) return ERR_PTR(err); - use_umr = mlx5_ib_can_use_umr(dev, true, access_flags); - - if (order <= mr_cache_max_order(dev) && use_umr) { + if (xlt_with_umr) { mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont, page_shift, order, access_flags); - if (PTR_ERR(mr) == -EAGAIN) { - mlx5_ib_dbg(dev, "cache empty for order %d\n", order); + if (IS_ERR(mr)) mr = NULL; - } - } else if (!MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) { - if (access_flags & IB_ACCESS_ON_DEMAND) { - err = -EINVAL; - pr_err("Got MR registration for ODP MR > 512MB, not supported for Connect-IB\n"); - goto error; - } - use_umr = false; } if (!mr) { mutex_lock(&dev->slow_path_mutex); mr = reg_create(NULL, pd, virt_addr, length, umem, ncont, - page_shift, access_flags, !use_umr); + page_shift, access_flags, !xlt_with_umr); mutex_unlock(&dev->slow_path_mutex); } @@ -1422,15 +1421,16 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, mr->umem = umem; set_mr_fields(dev, mr, npages, length, access_flags); - if (use_umr) { + if (xlt_with_umr && !(access_flags & IB_ACCESS_ON_DEMAND)) { + /* + * If the MR was created with reg_create then it will be + * configured properly but left disabled. It is safe to go ahead + * and configure it again via UMR while enabling it. + */ int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE; - if (access_flags & IB_ACCESS_ON_DEMAND) - update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP; - err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift, update_xlt_flags); - if (err) { dereg_mr(dev, mr); return ERR_PTR(err); @@ -1448,6 +1448,12 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, dereg_mr(dev, mr); return ERR_PTR(err); } + + err = mlx5_ib_init_odp_mr(mr, xlt_with_umr); + if (err) { + dereg_mr(dev, mr); + return ERR_PTR(err); + } } return &mr->ibmr; @@ -1555,8 +1561,11 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, goto err; } - if (!mlx5_ib_can_use_umr(dev, true, access_flags) || - (flags & IB_MR_REREG_TRANS && !use_umr_mtt_update(mr, addr, len))) { + if (!mlx5_ib_can_reconfig_with_umr(dev, mr->access_flags, + access_flags) || + !mlx5_ib_can_load_pas_with_umr(dev, len) || + (flags & IB_MR_REREG_TRANS && + !mlx5_ib_pas_fits_in_mr(mr, addr, len))) { /* * UMR can't be used - MKey needs to be replaced. */ @@ -1727,9 +1736,9 @@ static void mlx5_set_umr_free_mkey(struct ib_pd *pd, u32 *in, int ndescs, mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); + /* This is only used from the kernel, so setting the PD is OK. */ + set_mkc_access_pd_addr_fields(mkc, 0, 0, pd); MLX5_SET(mkc, mkc, free, 1); - MLX5_SET(mkc, mkc, qpn, 0xffffff); - MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); MLX5_SET(mkc, mkc, translations_octword_size, ndescs); MLX5_SET(mkc, mkc, access_mode_1_0, access_mode & 0x3); MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode >> 2) & 0x7); @@ -1973,12 +1982,11 @@ struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd, max_num_meta_sg); } -struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, - struct ib_udata *udata) +int mlx5_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) { - struct mlx5_ib_dev *dev = to_mdev(pd->device); + struct mlx5_ib_dev *dev = to_mdev(ibmw->device); int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); - struct mlx5_ib_mw *mw = NULL; + struct mlx5_ib_mw *mw = to_mmw(ibmw); u32 *in = NULL; void *mkc; int ndescs; @@ -1991,21 +1999,20 @@ struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req))); if (err) - return ERR_PTR(err); + return err; if (req.comp_mask || req.reserved1 || req.reserved2) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; if (udata->inlen > sizeof(req) && !ib_is_udata_cleared(udata, sizeof(req), udata->inlen - sizeof(req))) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4); - mw = kzalloc(sizeof(*mw), GFP_KERNEL); in = kzalloc(inlen, GFP_KERNEL); - if (!mw || !in) { + if (!in) { err = -ENOMEM; goto free; } @@ -2014,11 +2021,11 @@ struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, MLX5_SET(mkc, mkc, free, 1); MLX5_SET(mkc, mkc, translations_octword_size, ndescs); - MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); + MLX5_SET(mkc, mkc, pd, to_mpd(ibmw->pd)->pdn); MLX5_SET(mkc, mkc, umr_en, 1); MLX5_SET(mkc, mkc, lr, 1); MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS); - MLX5_SET(mkc, mkc, en_rinval, !!((type == IB_MW_TYPE_2))); + MLX5_SET(mkc, mkc, en_rinval, !!((ibmw->type == IB_MW_TYPE_2))); MLX5_SET(mkc, mkc, qpn, 0xffffff); err = mlx5_ib_create_mkey(dev, &mw->mmkey, in, inlen); @@ -2026,17 +2033,15 @@ struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, goto free; mw->mmkey.type = MLX5_MKEY_MW; - mw->ibmw.rkey = mw->mmkey.key; + ibmw->rkey = mw->mmkey.key; mw->ndescs = ndescs; - resp.response_length = min(offsetof(typeof(resp), response_length) + - sizeof(resp.response_length), udata->outlen); + resp.response_length = + min(offsetofend(typeof(resp), response_length), udata->outlen); if (resp.response_length) { err = ib_copy_to_udata(udata, &resp, resp.response_length); - if (err) { - mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey); - goto free; - } + if (err) + goto free_mkey; } if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) { @@ -2048,21 +2053,19 @@ struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, } kfree(in); - return &mw->ibmw; + return 0; free_mkey: mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey); free: - kfree(mw); kfree(in); - return ERR_PTR(err); + return err; } int mlx5_ib_dealloc_mw(struct ib_mw *mw) { struct mlx5_ib_dev *dev = to_mdev(mw->device); struct mlx5_ib_mw *mmw = to_mmw(mw); - int err; if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) { xa_erase(&dev->odp_mkeys, mlx5_base_mkey(mmw->mmkey.key)); @@ -2073,11 +2076,7 @@ int mlx5_ib_dealloc_mw(struct ib_mw *mw) synchronize_srcu(&dev->odp_srcu); } - err = mlx5_core_destroy_mkey(dev->mdev, &mmw->mmkey); - if (err) - return err; - kfree(mmw); - return 0; + return mlx5_core_destroy_mkey(dev->mdev, &mmw->mmkey); } int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index cfd7efab114e..5c853ec1b0d8 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -382,7 +382,7 @@ void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev) memset(caps, 0, sizeof(*caps)); if (!MLX5_CAP_GEN(dev->mdev, pg) || - !mlx5_ib_can_use_umr(dev, true, 0)) + !mlx5_ib_can_load_pas_with_umr(dev, 0)) return; caps->general_caps = IB_ODP_SUPPORT; @@ -476,12 +476,12 @@ static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, if (IS_ERR(odp)) return ERR_CAST(odp); - ret = mr = mlx5_mr_cache_alloc(imr->dev, MLX5_IMR_MTT_CACHE_ENTRY); + ret = mr = mlx5_mr_cache_alloc(imr->dev, MLX5_IMR_MTT_CACHE_ENTRY, + imr->access_flags); if (IS_ERR(mr)) goto out_umem; mr->ibmr.pd = imr->ibmr.pd; - mr->access_flags = imr->access_flags; mr->umem = &odp->umem; mr->ibmr.lkey = mr->mmkey.key; mr->ibmr.rkey = mr->mmkey.key; @@ -540,14 +540,13 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, if (IS_ERR(umem_odp)) return ERR_CAST(umem_odp); - imr = mlx5_mr_cache_alloc(dev, MLX5_IMR_KSM_CACHE_ENTRY); + imr = mlx5_mr_cache_alloc(dev, MLX5_IMR_KSM_CACHE_ENTRY, access_flags); if (IS_ERR(imr)) { err = PTR_ERR(imr); goto out_umem; } imr->ibmr.pd = &pd->ibpd; - imr->access_flags = access_flags; imr->mmkey.iova = 0; imr->umem = &umem_odp->umem; imr->ibmr.lkey = imr->mmkey.key; @@ -666,15 +665,21 @@ void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr) } #define MLX5_PF_FLAGS_DOWNGRADE BIT(1) +#define MLX5_PF_FLAGS_SNAPSHOT BIT(2) +#define MLX5_PF_FLAGS_ENABLE BIT(3) static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, u64 user_va, size_t bcnt, u32 *bytes_mapped, u32 flags) { int page_shift, ret, np; bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE; - unsigned long current_seq; u64 access_mask; u64 start_idx; + bool fault = !(flags & MLX5_PF_FLAGS_SNAPSHOT); + u32 xlt_flags = MLX5_IB_UPD_XLT_ATOMIC; + + if (flags & MLX5_PF_FLAGS_ENABLE) + xlt_flags |= MLX5_IB_UPD_XLT_ENABLE; page_shift = odp->page_shift; start_idx = (user_va - ib_umem_start(odp)) >> page_shift; @@ -683,25 +688,15 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, if (odp->umem.writable && !downgrade) access_mask |= ODP_WRITE_ALLOWED_BIT; - current_seq = mmu_interval_read_begin(&odp->notifier); - - np = ib_umem_odp_map_dma_pages(odp, user_va, bcnt, access_mask, - current_seq); + np = ib_umem_odp_map_dma_and_lock(odp, user_va, bcnt, access_mask, fault); if (np < 0) return np; - mutex_lock(&odp->umem_mutex); - if (!mmu_interval_read_retry(&odp->notifier, current_seq)) { - /* - * No need to check whether the MTTs really belong to - * this MR, since ib_umem_odp_map_dma_pages already - * checks this. - */ - ret = mlx5_ib_update_xlt(mr, start_idx, np, - page_shift, MLX5_IB_UPD_XLT_ATOMIC); - } else { - ret = -EAGAIN; - } + /* + * No need to check whether the MTTs really belong to this MR, since + * ib_umem_odp_map_dma_and_lock already checks this. + */ + ret = mlx5_ib_update_xlt(mr, start_idx, np, page_shift, xlt_flags); mutex_unlock(&odp->umem_mutex); if (ret < 0) { @@ -836,6 +831,20 @@ static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, flags); } +int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, bool enable) +{ + u32 flags = MLX5_PF_FLAGS_SNAPSHOT; + int ret; + + if (enable) + flags |= MLX5_PF_FLAGS_ENABLE; + + ret = pagefault_real_mr(mr, to_ib_umem_odp(mr->umem), + mr->umem->address, mr->umem->length, NULL, + flags); + return ret >= 0 ? 0 : ret; +} + struct pf_frame { struct pf_frame *next; u32 key; @@ -1862,6 +1871,9 @@ int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH) pf_flags |= MLX5_PF_FLAGS_DOWNGRADE; + if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT) + pf_flags |= MLX5_PF_FLAGS_SNAPSHOT; + if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH) return mlx5_ib_prefetch_sg_list(pd, advice, pf_flags, sg_list, num_sge); diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 5758dbe64045..600e056798c0 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -1477,7 +1477,8 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_RQN; resp->tirn = rq->tirn; resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN; - if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner)) { + if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) || + MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2)) { resp->tir_icm_addr = MLX5_GET( create_tir_out, out, icm_address_31_0); resp->tir_icm_addr |= @@ -1739,7 +1740,8 @@ create_tir: if (mucontext->devx_uid) { params->resp.comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN; params->resp.tirn = qp->rss_qp.tirn; - if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner)) { + if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) || + MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2)) { params->resp.tir_icm_addr = MLX5_GET(create_tir_out, out, icm_address_31_0); params->resp.tir_icm_addr |= @@ -2409,6 +2411,9 @@ static int create_dct(struct mlx5_ib_dev *dev, struct ib_pd *pd, u32 uidx = params->uidx; void *dctc; + if (mlx5_lag_is_active(dev->mdev) && !MLX5_CAP_GEN(dev->mdev, lag_dct)) + return -EOPNOTSUPP; + qp->dct.in = kzalloc(MLX5_ST_SZ_BYTES(create_dct_in), GFP_KERNEL); if (!qp->dct.in) return -ENOMEM; @@ -2506,18 +2511,6 @@ static int check_valid_flow(struct mlx5_ib_dev *dev, struct ib_pd *pd, return -EINVAL; } - switch (attr->qp_type) { - case IB_QPT_SMI: - case MLX5_IB_QPT_HW_GSI: - case MLX5_IB_QPT_REG_UMR: - case IB_QPT_GSI: - mlx5_ib_dbg(dev, "Kernel doesn't support QP type %d\n", - attr->qp_type); - return -EINVAL; - default: - break; - } - /* * We don't need to see this warning, it means that kernel code * missing ib_pd. Placed here to catch developer's mistakes. @@ -2780,21 +2773,23 @@ static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, goto out; } - if (qp->type == MLX5_IB_QPT_DCT) { + switch (qp->type) { + case MLX5_IB_QPT_DCT: err = create_dct(dev, pd, qp, params); - goto out; - } - - if (qp->type == IB_QPT_XRC_TGT) { + break; + case IB_QPT_XRC_TGT: err = create_xrc_tgt_qp(dev, qp, params); - goto out; + break; + case IB_QPT_GSI: + err = mlx5_ib_create_gsi(pd, qp, params->attr); + break; + default: + if (params->udata) + err = create_user_qp(dev, pd, qp, params); + else + err = create_kernel_qp(dev, pd, qp, params); } - if (params->udata) - err = create_user_qp(dev, pd, qp, params); - else - err = create_kernel_qp(dev, pd, qp, params); - out: if (err) { mlx5_ib_err(dev, "Create QP type %d failed\n", qp->type); @@ -2934,9 +2929,6 @@ struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attr, if (err) return ERR_PTR(err); - if (attr->qp_type == IB_QPT_GSI) - return mlx5_ib_gsi_create_qp(pd, attr); - params.udata = udata; params.uidx = MLX5_IB_DEFAULT_UIDX; params.attr = attr; @@ -3005,9 +2997,14 @@ struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attr, return &qp->ibqp; destroy_qp: - if (qp->type == MLX5_IB_QPT_DCT) { + switch (qp->type) { + case MLX5_IB_QPT_DCT: mlx5_ib_destroy_dct(qp); - } else { + break; + case IB_QPT_GSI: + mlx5_ib_destroy_gsi(qp); + break; + default: /* * These lines below are temp solution till QP allocation * will be moved to be under IB/core responsiblity. @@ -3032,7 +3029,7 @@ int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) struct mlx5_ib_qp *mqp = to_mqp(qp); if (unlikely(qp->qp_type == IB_QPT_GSI)) - return mlx5_ib_gsi_destroy_qp(qp); + return mlx5_ib_destroy_gsi(mqp); if (mqp->type == MLX5_IB_QPT_DCT) return mlx5_ib_destroy_dct(mqp); @@ -3088,20 +3085,44 @@ enum { MLX5_PATH_FLAG_COUNTER = 1 << 2, }; +static int ib_to_mlx5_rate_map(u8 rate) +{ + switch (rate) { + case IB_RATE_PORT_CURRENT: + return 0; + case IB_RATE_56_GBPS: + return 1; + case IB_RATE_25_GBPS: + return 2; + case IB_RATE_100_GBPS: + return 3; + case IB_RATE_200_GBPS: + return 4; + case IB_RATE_50_GBPS: + return 5; + default: + return rate + MLX5_STAT_RATE_OFFSET; + }; + + return 0; +} + static int ib_rate_to_mlx5(struct mlx5_ib_dev *dev, u8 rate) { + u32 stat_rate_support; + if (rate == IB_RATE_PORT_CURRENT) return 0; if (rate < IB_RATE_2_5_GBPS || rate > IB_RATE_600_GBPS) return -EINVAL; + stat_rate_support = MLX5_CAP_GEN(dev->mdev, stat_rate_support); while (rate != IB_RATE_PORT_CURRENT && - !(1 << (rate + MLX5_STAT_RATE_OFFSET) & - MLX5_CAP_GEN(dev->mdev, stat_rate_support))) + !(1 << ib_to_mlx5_rate_map(rate) & stat_rate_support)) --rate; - return rate ? rate + MLX5_STAT_RATE_OFFSET : rate; + return ib_to_mlx5_rate_map(rate); } static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev, @@ -3643,14 +3664,12 @@ static unsigned int get_tx_affinity_rr(struct mlx5_ib_dev *dev, MLX5_MAX_PORTS + 1; } -static bool qp_supports_affinity(struct ib_qp *qp) +static bool qp_supports_affinity(struct mlx5_ib_qp *qp) { - if ((qp->qp_type == IB_QPT_RC) || - (qp->qp_type == IB_QPT_UD) || - (qp->qp_type == IB_QPT_UC) || - (qp->qp_type == IB_QPT_RAW_PACKET) || - (qp->qp_type == IB_QPT_XRC_INI) || - (qp->qp_type == IB_QPT_XRC_TGT)) + if ((qp->type == IB_QPT_RC) || (qp->type == IB_QPT_UD) || + (qp->type == IB_QPT_UC) || (qp->type == IB_QPT_RAW_PACKET) || + (qp->type == IB_QPT_XRC_INI) || (qp->type == IB_QPT_XRC_TGT) || + (qp->type == MLX5_IB_QPT_DCI)) return true; return false; } @@ -3668,7 +3687,7 @@ static unsigned int get_tx_affinity(struct ib_qp *qp, unsigned int tx_affinity; if (!(mlx5_ib_lag_should_assign_affinity(dev) && - qp_supports_affinity(qp))) + qp_supports_affinity(mqp))) return 0; if (mqp->flags & MLX5_IB_QP_CREATE_SQPN_QP1) @@ -4161,7 +4180,11 @@ static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr, MLX5_SET(dctc, dctc, rae, 1); } MLX5_SET(dctc, dctc, pkey_index, attr->pkey_index); - MLX5_SET(dctc, dctc, port, attr->port_num); + if (mlx5_lag_is_active(dev->mdev)) + MLX5_SET(dctc, dctc, port, + get_tx_affinity_rr(dev, udata)); + else + MLX5_SET(dctc, dctc, port, attr->port_num); set_id = mlx5_ib_get_counters_id(dev, attr->port_num - 1); MLX5_SET(dctc, dctc, counter_set_id, set_id); @@ -4716,12 +4739,12 @@ int mlx5_ib_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) return mlx5_cmd_xrcd_alloc(dev->mdev, &xrcd->xrcdn, 0); } -void mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata) +int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata) { struct mlx5_ib_dev *dev = to_mdev(xrcd->device); u32 xrcdn = to_mxrcd(xrcd)->xrcdn; - mlx5_cmd_xrcd_dealloc(dev->mdev, xrcdn, 0); + return mlx5_cmd_xrcd_dealloc(dev->mdev, xrcdn, 0); } static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type) @@ -4921,8 +4944,8 @@ static int prepare_user_rq(struct ib_pd *pd, int err; size_t required_cmd_sz; - required_cmd_sz = offsetof(typeof(ucmd), single_stride_log_num_of_bytes) - + sizeof(ucmd.single_stride_log_num_of_bytes); + required_cmd_sz = offsetofend(struct mlx5_ib_create_wq, + single_stride_log_num_of_bytes); if (udata->inlen < required_cmd_sz) { mlx5_ib_dbg(dev, "invalid inlen\n"); return -EINVAL; @@ -5006,7 +5029,7 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd, if (!udata) return ERR_PTR(-ENOSYS); - min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved); + min_resp_len = offsetofend(struct mlx5_ib_create_wq_resp, reserved); if (udata->outlen && udata->outlen < min_resp_len) return ERR_PTR(-EINVAL); @@ -5036,8 +5059,8 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd, rwq->ibwq.wq_num = rwq->core_qp.qpn; rwq->ibwq.state = IB_WQS_RESET; if (udata->outlen) { - resp.response_length = offsetof(typeof(resp), response_length) + - sizeof(resp.response_length); + resp.response_length = offsetofend( + struct mlx5_ib_create_wq_resp, response_length); err = ib_copy_to_udata(udata, &resp, resp.response_length); if (err) goto err_copy; @@ -5056,22 +5079,27 @@ err: return ERR_PTR(err); } -void mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata) +int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata) { struct mlx5_ib_dev *dev = to_mdev(wq->device); struct mlx5_ib_rwq *rwq = to_mrwq(wq); + int ret; - mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp); + ret = mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp); + if (ret) + return ret; destroy_user_rq(dev, wq->pd, rwq, udata); kfree(rwq); + return 0; } -struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device, - struct ib_rwq_ind_table_init_attr *init_attr, - struct ib_udata *udata) +int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table, + struct ib_rwq_ind_table_init_attr *init_attr, + struct ib_udata *udata) { - struct mlx5_ib_dev *dev = to_mdev(device); - struct mlx5_ib_rwq_ind_table *rwq_ind_tbl; + struct mlx5_ib_rwq_ind_table *rwq_ind_tbl = + to_mrwq_ind_table(ib_rwq_ind_table); + struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_table->device); int sz = 1 << init_attr->log_ind_tbl_size; struct mlx5_ib_create_rwq_ind_tbl_resp resp = {}; size_t min_resp_len; @@ -5084,30 +5112,25 @@ struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device, if (udata->inlen > 0 && !ib_is_udata_cleared(udata, 0, udata->inlen)) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; if (init_attr->log_ind_tbl_size > MLX5_CAP_GEN(dev->mdev, log_max_rqt_size)) { mlx5_ib_dbg(dev, "log_ind_tbl_size = %d is bigger than supported = %d\n", init_attr->log_ind_tbl_size, MLX5_CAP_GEN(dev->mdev, log_max_rqt_size)); - return ERR_PTR(-EINVAL); + return -EINVAL; } - min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved); + min_resp_len = + offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp, reserved); if (udata->outlen && udata->outlen < min_resp_len) - return ERR_PTR(-EINVAL); - - rwq_ind_tbl = kzalloc(sizeof(*rwq_ind_tbl), GFP_KERNEL); - if (!rwq_ind_tbl) - return ERR_PTR(-ENOMEM); + return -EINVAL; inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz; in = kvzalloc(inlen, GFP_KERNEL); - if (!in) { - err = -ENOMEM; - goto err; - } + if (!in) + return -ENOMEM; rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); @@ -5122,26 +5145,24 @@ struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device, err = mlx5_core_create_rqt(dev->mdev, in, inlen, &rwq_ind_tbl->rqtn); kvfree(in); - if (err) - goto err; + return err; rwq_ind_tbl->ib_rwq_ind_tbl.ind_tbl_num = rwq_ind_tbl->rqtn; if (udata->outlen) { - resp.response_length = offsetof(typeof(resp), response_length) + - sizeof(resp.response_length); + resp.response_length = + offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp, + response_length); err = ib_copy_to_udata(udata, &resp, resp.response_length); if (err) goto err_copy; } - return &rwq_ind_tbl->ib_rwq_ind_tbl; + return 0; err_copy: mlx5_cmd_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn, rwq_ind_tbl->uid); -err: - kfree(rwq_ind_tbl); - return ERR_PTR(err); + return err; } int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl) @@ -5149,10 +5170,7 @@ int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl) struct mlx5_ib_rwq_ind_table *rwq_ind_tbl = to_mrwq_ind_table(ib_rwq_ind_tbl); struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_tbl->device); - mlx5_cmd_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn, rwq_ind_tbl->uid); - - kfree(rwq_ind_tbl); - return 0; + return mlx5_cmd_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn, rwq_ind_tbl->uid); } int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr, @@ -5169,7 +5187,7 @@ int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr, void *rqc; void *in; - required_cmd_sz = offsetof(typeof(ucmd), reserved) + sizeof(ucmd.reserved); + required_cmd_sz = offsetofend(struct mlx5_ib_modify_wq, reserved); if (udata->inlen < required_cmd_sz) return -EINVAL; diff --git a/drivers/infiniband/hw/mlx5/qp.h b/drivers/infiniband/hw/mlx5/qp.h index ba899df44c5b..5d4e140db99c 100644 --- a/drivers/infiniband/hw/mlx5/qp.h +++ b/drivers/infiniband/hw/mlx5/qp.h @@ -26,8 +26,8 @@ int mlx5_core_dct_query(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct, int mlx5_core_set_delay_drop(struct mlx5_ib_dev *dev, u32 timeout_usec); -void mlx5_core_destroy_rq_tracked(struct mlx5_ib_dev *dev, - struct mlx5_core_qp *rq); +int mlx5_core_destroy_rq_tracked(struct mlx5_ib_dev *dev, + struct mlx5_core_qp *rq); int mlx5_core_create_sq_tracked(struct mlx5_ib_dev *dev, u32 *in, int inlen, struct mlx5_core_qp *sq); void mlx5_core_destroy_sq_tracked(struct mlx5_ib_dev *dev, diff --git a/drivers/infiniband/hw/mlx5/qpc.c b/drivers/infiniband/hw/mlx5/qpc.c index 7c3968ef9cd1..c683d7000168 100644 --- a/drivers/infiniband/hw/mlx5/qpc.c +++ b/drivers/infiniband/hw/mlx5/qpc.c @@ -576,11 +576,12 @@ err_destroy_rq: return err; } -void mlx5_core_destroy_rq_tracked(struct mlx5_ib_dev *dev, - struct mlx5_core_qp *rq) +int mlx5_core_destroy_rq_tracked(struct mlx5_ib_dev *dev, + struct mlx5_core_qp *rq) { destroy_resource_common(dev, rq); destroy_rq_tracked(dev, rq->qpn, rq->uid); + return 0; } static void destroy_sq_tracked(struct mlx5_ib_dev *dev, u32 sqn, u16 uid) diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c index 7e10cbcb6d5c..e2f720eec1e1 100644 --- a/drivers/infiniband/hw/mlx5/srq.c +++ b/drivers/infiniband/hw/mlx5/srq.c @@ -389,24 +389,21 @@ out_box: return ret; } -void mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) +int mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) { struct mlx5_ib_dev *dev = to_mdev(srq->device); struct mlx5_ib_srq *msrq = to_msrq(srq); + int ret; + + ret = mlx5_cmd_destroy_srq(dev, &msrq->msrq); + if (ret) + return ret; - mlx5_cmd_destroy_srq(dev, &msrq->msrq); - - if (srq->uobject) { - mlx5_ib_db_unmap_user( - rdma_udata_to_drv_context( - udata, - struct mlx5_ib_ucontext, - ibucontext), - &msrq->db); - ib_umem_release(msrq->umem); - } else { + if (udata) + destroy_srq_user(srq->pd, msrq, udata); + else destroy_srq_kernel(dev, msrq); - } + return 0; } void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index) diff --git a/drivers/infiniband/hw/mlx5/srq.h b/drivers/infiniband/hw/mlx5/srq.h index af197c36d757..2c3627b2509d 100644 --- a/drivers/infiniband/hw/mlx5/srq.h +++ b/drivers/infiniband/hw/mlx5/srq.h @@ -56,7 +56,7 @@ struct mlx5_srq_table { int mlx5_cmd_create_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, struct mlx5_srq_attr *in); -void mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq); +int mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq); int mlx5_cmd_query_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, struct mlx5_srq_attr *out); int mlx5_cmd_arm_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, diff --git a/drivers/infiniband/hw/mlx5/srq_cmd.c b/drivers/infiniband/hw/mlx5/srq_cmd.c index 37aaacebd3f2..db889ec3fd48 100644 --- a/drivers/infiniband/hw/mlx5/srq_cmd.c +++ b/drivers/infiniband/hw/mlx5/srq_cmd.c @@ -590,22 +590,32 @@ err_destroy_srq_split: return err; } -void mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq) +int mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq) { struct mlx5_srq_table *table = &dev->srq_table; struct mlx5_core_srq *tmp; int err; - tmp = xa_erase_irq(&table->array, srq->srqn); - if (!tmp || tmp != srq) - return; + /* Delete entry, but leave index occupied */ + tmp = xa_cmpxchg_irq(&table->array, srq->srqn, srq, XA_ZERO_ENTRY, 0); + if (WARN_ON(tmp != srq)) + return xa_err(tmp) ?: -EINVAL; err = destroy_srq_split(dev, srq); - if (err) - return; + if (err) { + /* + * We don't need to check returned result for an error, + * because we are storing in pre-allocated space xarray + * entry and it can't fail at this stage. + */ + xa_cmpxchg_irq(&table->array, srq->srqn, XA_ZERO_ENTRY, srq, 0); + return err; + } + xa_erase_irq(&table->array, srq->srqn); mlx5_core_res_put(&srq->common); wait_for_completion(&srq->common.free); + return 0; } int mlx5_cmd_query_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, diff --git a/drivers/infiniband/hw/mlx5/wr.c b/drivers/infiniband/hw/mlx5/wr.c index 43880973a512..d6038fb6c50c 100644 --- a/drivers/infiniband/hw/mlx5/wr.c +++ b/drivers/infiniband/hw/mlx5/wr.c @@ -398,7 +398,8 @@ static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg) seg->status = MLX5_MKEY_STATUS_FREE; } -static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg, +static void set_reg_mkey_segment(struct mlx5_ib_dev *dev, + struct mlx5_mkey_seg *seg, const struct ib_send_wr *wr) { const struct mlx5_umr_wr *umrwr = umr_wr(wr); @@ -414,10 +415,12 @@ static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg, MLX5_SET(mkc, seg, rr, !!(umrwr->access_flags & IB_ACCESS_REMOTE_READ)); MLX5_SET(mkc, seg, lw, !!(umrwr->access_flags & IB_ACCESS_LOCAL_WRITE)); MLX5_SET(mkc, seg, lr, 1); - MLX5_SET(mkc, seg, relaxed_ordering_write, - !!(umrwr->access_flags & IB_ACCESS_RELAXED_ORDERING)); - MLX5_SET(mkc, seg, relaxed_ordering_read, - !!(umrwr->access_flags & IB_ACCESS_RELAXED_ORDERING)); + if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr)) + MLX5_SET(mkc, seg, relaxed_ordering_write, + !!(umrwr->access_flags & IB_ACCESS_RELAXED_ORDERING)); + if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)) + MLX5_SET(mkc, seg, relaxed_ordering_read, + !!(umrwr->access_flags & IB_ACCESS_RELAXED_ORDERING)); if (umrwr->pd) MLX5_SET(mkc, seg, pd, to_mpd(umrwr->pd)->pdn); @@ -863,13 +866,11 @@ static int set_reg_wr(struct mlx5_ib_qp *qp, bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC; u8 flags = 0; - if (!mlx5_ib_can_use_umr(dev, atomic, wr->access)) { - mlx5_ib_warn(to_mdev(qp->ibqp.device), - "Fast update of %s for MR is disabled\n", - (MLX5_CAP_GEN(dev->mdev, - umr_modify_entity_size_disabled)) ? - "entity size" : - "atomic access"); + /* Matches access in mlx5_set_umr_free_mkey() */ + if (!mlx5_ib_can_reconfig_with_umr(dev, 0, wr->access)) { + mlx5_ib_warn( + to_mdev(qp->ibqp.device), + "Fast update for MR access flags is not possible\n"); return -EINVAL; } @@ -1263,7 +1264,7 @@ static int handle_qpt_reg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; handle_post_send_edge(&qp->sq, seg, *size, cur_edge); - set_reg_mkey_segment(*seg, wr); + set_reg_mkey_segment(dev, *seg, wr); *seg += sizeof(struct mlx5_mkey_seg); *size += sizeof(struct mlx5_mkey_seg) / 16; handle_post_send_edge(&qp->sq, seg, *size, cur_edge); diff --git a/drivers/infiniband/hw/mthca/mthca_dev.h b/drivers/infiniband/hw/mthca/mthca_dev.h index 7550e9d03dec..9dbbf4d16796 100644 --- a/drivers/infiniband/hw/mthca/mthca_dev.h +++ b/drivers/infiniband/hw/mthca/mthca_dev.h @@ -548,7 +548,7 @@ int mthca_alloc_sqp(struct mthca_dev *dev, struct ib_qp_cap *cap, int qpn, int port, - struct mthca_sqp *sqp, + struct mthca_qp *qp, struct ib_udata *udata); void mthca_free_qp(struct mthca_dev *dev, struct mthca_qp *qp); int mthca_create_ah(struct mthca_dev *dev, diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index 9fa2f9164a47..c4d9cdc4ee97 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -373,9 +373,10 @@ static int mthca_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) return 0; } -static void mthca_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) +static int mthca_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) { mthca_pd_free(to_mdev(pd->device), to_mpd(pd)); + return 0; } static int mthca_ah_create(struct ib_ah *ibah, @@ -389,9 +390,10 @@ static int mthca_ah_create(struct ib_ah *ibah, init_attr->ah_attr, ah); } -static void mthca_ah_destroy(struct ib_ah *ah, u32 flags) +static int mthca_ah_destroy(struct ib_ah *ah, u32 flags) { mthca_destroy_ah(to_mdev(ah->device), to_mah(ah)); + return 0; } static int mthca_create_srq(struct ib_srq *ibsrq, @@ -440,7 +442,7 @@ static int mthca_create_srq(struct ib_srq *ibsrq, return 0; } -static void mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) +static int mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) { if (udata) { struct mthca_ucontext *context = @@ -454,6 +456,7 @@ static void mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) } mthca_free_srq(to_mdev(srq->device), to_msrq(srq)); + return 0; } static struct ib_qp *mthca_create_qp(struct ib_pd *pd, @@ -532,13 +535,14 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd, case IB_QPT_SMI: case IB_QPT_GSI: { - /* Don't allow userspace to create special QPs */ - if (udata) - return ERR_PTR(-EINVAL); - - qp = kzalloc(sizeof(struct mthca_sqp), GFP_KERNEL); + qp = kzalloc(sizeof(*qp), GFP_KERNEL); if (!qp) return ERR_PTR(-ENOMEM); + qp->sqp = kzalloc(sizeof(struct mthca_sqp), GFP_KERNEL); + if (!qp->sqp) { + kfree(qp); + return ERR_PTR(-ENOMEM); + } qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1; @@ -547,7 +551,7 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd, to_mcq(init_attr->recv_cq), init_attr->sq_sig_type, &init_attr->cap, qp->ibqp.qp_num, init_attr->port_num, - to_msqp(qp), udata); + qp, udata); break; } default: @@ -556,6 +560,7 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd, } if (err) { + kfree(qp->sqp); kfree(qp); return ERR_PTR(err); } @@ -588,7 +593,8 @@ static int mthca_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) to_mqp(qp)->rq.db_index); } mthca_free_qp(to_mdev(qp->device), to_mqp(qp)); - kfree(qp); + kfree(to_mqp(qp)->sqp); + kfree(to_mqp(qp)); return 0; } @@ -789,7 +795,7 @@ out: return ret; } -static void mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +static int mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { if (udata) { struct mthca_ucontext *context = @@ -808,6 +814,7 @@ static void mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) to_mcq(cq)->set_ci_db_index); } mthca_free_cq(to_mdev(cq->device), to_mcq(cq)); + return 0; } static inline u32 convert_access(int acc) @@ -846,7 +853,7 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt, int acc, struct ib_udata *udata) { struct mthca_dev *dev = to_mdev(pd->device); - struct sg_dma_page_iter sg_iter; + struct ib_block_iter biter; struct mthca_ucontext *context = rdma_udata_to_drv_context( udata, struct mthca_ucontext, ibucontext); struct mthca_mr *mr; @@ -877,7 +884,7 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, goto err; } - n = ib_umem_num_pages(mr->umem); + n = ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE); mr->mtt = mthca_alloc_mtt(dev, n); if (IS_ERR(mr->mtt)) { @@ -895,8 +902,8 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, write_mtt_size = min(mthca_write_mtt_size(dev), (int) (PAGE_SIZE / sizeof *pages)); - for_each_sg_dma_page(mr->umem->sg_head.sgl, &sg_iter, mr->umem->nmap, 0) { - pages[i++] = sg_page_iter_dma_address(&sg_iter); + rdma_umem_for_each_dma_block(mr->umem, &biter, PAGE_SIZE) { + pages[i++] = rdma_block_iter_dma_address(&biter); /* * Be friendly to write_mtt and pass it chunks @@ -1199,7 +1206,7 @@ int mthca_register_device(struct mthca_dev *dev) mutex_init(&dev->cap_mask_mutex); rdma_set_device_sysfs_group(&dev->ib_dev, &mthca_attr_group); - ret = ib_register_device(&dev->ib_dev, "mthca%d"); + ret = ib_register_device(&dev->ib_dev, "mthca%d", &dev->pdev->dev); if (ret) return ret; diff --git a/drivers/infiniband/hw/mthca/mthca_provider.h b/drivers/infiniband/hw/mthca/mthca_provider.h index 84c64bff0d92..8a77483bb33c 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.h +++ b/drivers/infiniband/hw/mthca/mthca_provider.h @@ -240,6 +240,16 @@ struct mthca_wq { __be32 *db; }; +struct mthca_sqp { + int pkey_index; + u32 qkey; + u32 send_psn; + struct ib_ud_header ud_header; + int header_buf_size; + void *header_buf; + dma_addr_t header_dma; +}; + struct mthca_qp { struct ib_qp ibqp; int refcount; @@ -265,17 +275,7 @@ struct mthca_qp { wait_queue_head_t wait; struct mutex mutex; -}; - -struct mthca_sqp { - struct mthca_qp qp; - int pkey_index; - u32 qkey; - u32 send_psn; - struct ib_ud_header ud_header; - int header_buf_size; - void *header_buf; - dma_addr_t header_dma; + struct mthca_sqp *sqp; }; static inline struct mthca_ucontext *to_mucontext(struct ib_ucontext *ibucontext) @@ -313,9 +313,4 @@ static inline struct mthca_qp *to_mqp(struct ib_qp *ibqp) return container_of(ibqp, struct mthca_qp, ibqp); } -static inline struct mthca_sqp *to_msqp(struct mthca_qp *qp) -{ - return container_of(qp, struct mthca_sqp, qp); -} - #endif /* MTHCA_PROVIDER_H */ diff --git a/drivers/infiniband/hw/mthca/mthca_qp.c b/drivers/infiniband/hw/mthca/mthca_qp.c index c6e95d0d760a..08a2a7afafd3 100644 --- a/drivers/infiniband/hw/mthca/mthca_qp.c +++ b/drivers/infiniband/hw/mthca/mthca_qp.c @@ -809,7 +809,7 @@ static int __mthca_modify_qp(struct ib_qp *ibqp, qp->alt_port = attr->alt_port_num; if (is_sqp(dev, qp)) - store_attrs(to_msqp(qp), attr, attr_mask); + store_attrs(qp->sqp, attr, attr_mask); /* * If we moved QP0 to RTR, bring the IB link up; if we moved @@ -1368,39 +1368,40 @@ int mthca_alloc_sqp(struct mthca_dev *dev, struct ib_qp_cap *cap, int qpn, int port, - struct mthca_sqp *sqp, + struct mthca_qp *qp, struct ib_udata *udata) { u32 mqpn = qpn * 2 + dev->qp_table.sqp_start + port - 1; int err; - sqp->qp.transport = MLX; - err = mthca_set_qp_size(dev, cap, pd, &sqp->qp); + qp->transport = MLX; + err = mthca_set_qp_size(dev, cap, pd, qp); if (err) return err; - sqp->header_buf_size = sqp->qp.sq.max * MTHCA_UD_HEADER_SIZE; - sqp->header_buf = dma_alloc_coherent(&dev->pdev->dev, sqp->header_buf_size, - &sqp->header_dma, GFP_KERNEL); - if (!sqp->header_buf) + qp->sqp->header_buf_size = qp->sq.max * MTHCA_UD_HEADER_SIZE; + qp->sqp->header_buf = + dma_alloc_coherent(&dev->pdev->dev, qp->sqp->header_buf_size, + &qp->sqp->header_dma, GFP_KERNEL); + if (!qp->sqp->header_buf) return -ENOMEM; spin_lock_irq(&dev->qp_table.lock); if (mthca_array_get(&dev->qp_table.qp, mqpn)) err = -EBUSY; else - mthca_array_set(&dev->qp_table.qp, mqpn, sqp); + mthca_array_set(&dev->qp_table.qp, mqpn, qp->sqp); spin_unlock_irq(&dev->qp_table.lock); if (err) goto err_out; - sqp->qp.port = port; - sqp->qp.qpn = mqpn; - sqp->qp.transport = MLX; + qp->port = port; + qp->qpn = mqpn; + qp->transport = MLX; err = mthca_alloc_qp_common(dev, pd, send_cq, recv_cq, - send_policy, &sqp->qp, udata); + send_policy, qp, udata); if (err) goto err_out_free; @@ -1421,10 +1422,9 @@ int mthca_alloc_sqp(struct mthca_dev *dev, mthca_unlock_cqs(send_cq, recv_cq); - err_out: - dma_free_coherent(&dev->pdev->dev, sqp->header_buf_size, - sqp->header_buf, sqp->header_dma); - +err_out: + dma_free_coherent(&dev->pdev->dev, qp->sqp->header_buf_size, + qp->sqp->header_buf, qp->sqp->header_dma); return err; } @@ -1487,20 +1487,19 @@ void mthca_free_qp(struct mthca_dev *dev, if (is_sqp(dev, qp)) { atomic_dec(&(to_mpd(qp->ibqp.pd)->sqp_count)); - dma_free_coherent(&dev->pdev->dev, - to_msqp(qp)->header_buf_size, - to_msqp(qp)->header_buf, - to_msqp(qp)->header_dma); + dma_free_coherent(&dev->pdev->dev, qp->sqp->header_buf_size, + qp->sqp->header_buf, qp->sqp->header_dma); } else mthca_free(&dev->qp_table.alloc, qp->qpn); } /* Create UD header for an MLX send and build a data segment for it */ -static int build_mlx_header(struct mthca_dev *dev, struct mthca_sqp *sqp, - int ind, const struct ib_ud_wr *wr, +static int build_mlx_header(struct mthca_dev *dev, struct mthca_qp *qp, int ind, + const struct ib_ud_wr *wr, struct mthca_mlx_seg *mlx, struct mthca_data_seg *data) { + struct mthca_sqp *sqp = qp->sqp; int header_size; int err; u16 pkey; @@ -1513,7 +1512,7 @@ static int build_mlx_header(struct mthca_dev *dev, struct mthca_sqp *sqp, if (err) return err; mlx->flags &= ~cpu_to_be32(MTHCA_NEXT_SOLICIT | 1); - mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MTHCA_MLX_VL15 : 0) | + mlx->flags |= cpu_to_be32((!qp->ibqp.qp_num ? MTHCA_MLX_VL15 : 0) | (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE ? MTHCA_MLX_SLR : 0) | (sqp->ud_header.lrh.service_level << 8)); @@ -1534,29 +1533,29 @@ static int build_mlx_header(struct mthca_dev *dev, struct mthca_sqp *sqp, return -EINVAL; } - sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0; + sqp->ud_header.lrh.virtual_lane = !qp->ibqp.qp_num ? 15 : 0; if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE) sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE; sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED); - if (!sqp->qp.ibqp.qp_num) - ib_get_cached_pkey(&dev->ib_dev, sqp->qp.port, - sqp->pkey_index, &pkey); + if (!qp->ibqp.qp_num) + ib_get_cached_pkey(&dev->ib_dev, qp->port, sqp->pkey_index, + &pkey); else - ib_get_cached_pkey(&dev->ib_dev, sqp->qp.port, - wr->pkey_index, &pkey); + ib_get_cached_pkey(&dev->ib_dev, qp->port, wr->pkey_index, + &pkey); sqp->ud_header.bth.pkey = cpu_to_be16(pkey); sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn); sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1)); sqp->ud_header.deth.qkey = cpu_to_be32(wr->remote_qkey & 0x80000000 ? sqp->qkey : wr->remote_qkey); - sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num); + sqp->ud_header.deth.source_qpn = cpu_to_be32(qp->ibqp.qp_num); header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf + ind * MTHCA_UD_HEADER_SIZE); data->byte_count = cpu_to_be32(header_size); - data->lkey = cpu_to_be32(to_mpd(sqp->qp.ibqp.pd)->ntmr.ibmr.lkey); + data->lkey = cpu_to_be32(to_mpd(qp->ibqp.pd)->ntmr.ibmr.lkey); data->addr = cpu_to_be64(sqp->header_dma + ind * MTHCA_UD_HEADER_SIZE); @@ -1735,9 +1734,9 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, break; case MLX: - err = build_mlx_header(dev, to_msqp(qp), ind, ud_wr(wr), - wqe - sizeof (struct mthca_next_seg), - wqe); + err = build_mlx_header( + dev, qp, ind, ud_wr(wr), + wqe - sizeof(struct mthca_next_seg), wqe); if (err) { *bad_wr = wr; goto out; @@ -2065,9 +2064,9 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, break; case MLX: - err = build_mlx_header(dev, to_msqp(qp), ind, ud_wr(wr), - wqe - sizeof (struct mthca_next_seg), - wqe); + err = build_mlx_header( + dev, qp, ind, ud_wr(wr), + wqe - sizeof(struct mthca_next_seg), wqe); if (err) { *bad_wr = wr; goto out; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma.h b/drivers/infiniband/hw/ocrdma/ocrdma.h index fcfe0e82197a..5eb61c110090 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma.h +++ b/drivers/infiniband/hw/ocrdma/ocrdma.h @@ -185,7 +185,6 @@ struct ocrdma_hw_mr { u32 num_pbes; u32 pbl_size; u32 pbe_size; - u64 fbo; u64 va; }; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c index 6eea02b18968..699a8b719ed6 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c @@ -215,12 +215,13 @@ av_err: return status; } -void ocrdma_destroy_ah(struct ib_ah *ibah, u32 flags) +int ocrdma_destroy_ah(struct ib_ah *ibah, u32 flags) { struct ocrdma_ah *ah = get_ocrdma_ah(ibah); struct ocrdma_dev *dev = get_ocrdma_dev(ibah->device); ocrdma_free_av(dev, ah); + return 0; } int ocrdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr) diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.h b/drivers/infiniband/hw/ocrdma/ocrdma_ah.h index 8b73b3489f3a..35cf2e2ff391 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_ah.h +++ b/drivers/infiniband/hw/ocrdma/ocrdma_ah.h @@ -53,7 +53,7 @@ enum { int ocrdma_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata); -void ocrdma_destroy_ah(struct ib_ah *ah, u32 flags); +int ocrdma_destroy_ah(struct ib_ah *ah, u32 flags); int ocrdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr); int ocrdma_process_mad(struct ib_device *dev, int process_mad_flags, diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c index e07bf0b2209a..c51c3f40700e 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c @@ -1962,6 +1962,7 @@ static int ocrdma_mbx_reg_mr(struct ocrdma_dev *dev, struct ocrdma_hw_mr *hwmr, int i; struct ocrdma_reg_nsmr *cmd; struct ocrdma_reg_nsmr_rsp *rsp; + u64 fbo = hwmr->va & (hwmr->pbe_size - 1); cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_REGISTER_NSMR, sizeof(*cmd)); if (!cmd) @@ -1987,8 +1988,8 @@ static int ocrdma_mbx_reg_mr(struct ocrdma_dev *dev, struct ocrdma_hw_mr *hwmr, OCRDMA_REG_NSMR_HPAGE_SIZE_SHIFT; cmd->totlen_low = hwmr->len; cmd->totlen_high = upper_32_bits(hwmr->len); - cmd->fbo_low = (u32) (hwmr->fbo & 0xffffffff); - cmd->fbo_high = (u32) upper_32_bits(hwmr->fbo); + cmd->fbo_low = lower_32_bits(fbo); + cmd->fbo_high = upper_32_bits(fbo); cmd->va_loaddr = (u32) hwmr->va; cmd->va_hiaddr = (u32) upper_32_bits(hwmr->va); diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c index d8c47d24d6d6..9b96661a7143 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c @@ -255,7 +255,9 @@ static int ocrdma_register_device(struct ocrdma_dev *dev) if (ret) return ret; - return ib_register_device(&dev->ibdev, "ocrdma%d"); + dma_set_max_seg_size(&dev->nic_info.pdev->dev, UINT_MAX); + return ib_register_device(&dev->ibdev, "ocrdma%d", + &dev->nic_info.pdev->dev); } static int ocrdma_alloc_resources(struct ocrdma_dev *dev) diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index c1751c9a0f62..7350fe16f164 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -112,7 +112,7 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, } static inline void get_link_speed_and_width(struct ocrdma_dev *dev, - u8 *ib_speed, u8 *ib_width) + u16 *ib_speed, u8 *ib_width) { int status; u8 speed; @@ -664,7 +664,7 @@ exit: return status; } -void ocrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +int ocrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); @@ -682,10 +682,11 @@ void ocrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) if (is_ucontext_pd(uctx, pd)) { ocrdma_release_ucontext_pd(uctx); - return; + return 0; } } _ocrdma_dealloc_pd(dev, pd); + return 0; } static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr, @@ -810,14 +811,12 @@ static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr) return status; } -static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr, - u32 num_pbes) +static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr) { struct ocrdma_pbe *pbe; - struct sg_dma_page_iter sg_iter; + struct ib_block_iter biter; struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table; - struct ib_umem *umem = mr->umem; - int pbe_cnt, total_num_pbes = 0; + int pbe_cnt; u64 pg_addr; if (!mr->hwmr.num_pbes) @@ -826,19 +825,14 @@ static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr, pbe = (struct ocrdma_pbe *)pbl_tbl->va; pbe_cnt = 0; - for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { + rdma_umem_for_each_dma_block (mr->umem, &biter, PAGE_SIZE) { /* store the page address in pbe */ - pg_addr = sg_page_iter_dma_address(&sg_iter); + pg_addr = rdma_block_iter_dma_address(&biter); pbe->pa_lo = cpu_to_le32(pg_addr); pbe->pa_hi = cpu_to_le32(upper_32_bits(pg_addr)); pbe_cnt += 1; - total_num_pbes += 1; pbe++; - /* if done building pbes, issue the mbx cmd. */ - if (total_num_pbes == num_pbes) - return; - /* if the given pbl is full storing the pbes, * move to next pbl. */ @@ -857,7 +851,6 @@ struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); struct ocrdma_mr *mr; struct ocrdma_pd *pd; - u32 num_pbes; pd = get_ocrdma_pd(ibpd); @@ -872,13 +865,12 @@ struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, status = -EFAULT; goto umem_err; } - num_pbes = ib_umem_page_count(mr->umem); - status = ocrdma_get_pbl_info(dev, mr, num_pbes); + status = ocrdma_get_pbl_info( + dev, mr, ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE)); if (status) goto umem_err; mr->hwmr.pbe_size = PAGE_SIZE; - mr->hwmr.fbo = ib_umem_offset(mr->umem); mr->hwmr.va = usr_addr; mr->hwmr.len = len; mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; @@ -889,7 +881,7 @@ struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, status = ocrdma_build_pbl_tbl(dev, &mr->hwmr); if (status) goto umem_err; - build_user_pbes(dev, mr, num_pbes); + build_user_pbes(dev, mr); status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc); if (status) goto mbx_err; @@ -1056,7 +1048,7 @@ static void ocrdma_flush_cq(struct ocrdma_cq *cq) spin_unlock_irqrestore(&cq->cq_lock, flags); } -void ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); struct ocrdma_eq *eq = NULL; @@ -1081,6 +1073,7 @@ void ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) ocrdma_get_db_addr(dev, pdid), dev->nic_info.db_page_size); } + return 0; } static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) @@ -1857,7 +1850,7 @@ int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) return status; } -void ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) +int ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) { struct ocrdma_srq *srq; struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device); @@ -1872,6 +1865,7 @@ void ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) kfree(srq->idx_bit_fields); kfree(srq->rqe_wr_id_tbl); + return 0; } /* unprivileged verbs and their support functions. */ diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h index df8e3b923a44..425d554e7f3f 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h @@ -67,12 +67,12 @@ void ocrdma_dealloc_ucontext(struct ib_ucontext *uctx); int ocrdma_mmap(struct ib_ucontext *, struct vm_area_struct *vma); int ocrdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata); -void ocrdma_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); +int ocrdma_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata); int ocrdma_resize_cq(struct ib_cq *, int cqe, struct ib_udata *); -void ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); +int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); struct ib_qp *ocrdma_create_qp(struct ib_pd *, struct ib_qp_init_attr *attrs, @@ -92,7 +92,7 @@ int ocrdma_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *attr, int ocrdma_modify_srq(struct ib_srq *, struct ib_srq_attr *, enum ib_srq_attr_mask, struct ib_udata *); int ocrdma_query_srq(struct ib_srq *, struct ib_srq_attr *); -void ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); +int ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); int ocrdma_post_srq_recv(struct ib_srq *, const struct ib_recv_wr *, const struct ib_recv_wr **bad_recv_wr); diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c index d85f992bac29..967641662b24 100644 --- a/drivers/infiniband/hw/qedr/main.c +++ b/drivers/infiniband/hw/qedr/main.c @@ -177,6 +177,8 @@ static int qedr_iw_register_device(struct qedr_dev *dev) } static const struct ib_device_ops qedr_roce_dev_ops = { + .alloc_xrcd = qedr_alloc_xrcd, + .dealloc_xrcd = qedr_dealloc_xrcd, .get_port_immutable = qedr_roce_port_immutable, .query_pkey = qedr_query_pkey, }; @@ -186,6 +188,10 @@ static void qedr_roce_register_device(struct qedr_dev *dev) dev->ibdev.node_type = RDMA_NODE_IB_CA; ib_set_device_ops(&dev->ibdev, &qedr_roce_dev_ops); + + dev->ibdev.uverbs_cmd_mask |= QEDR_UVERBS(OPEN_XRCD) | + QEDR_UVERBS(CLOSE_XRCD) | + QEDR_UVERBS(CREATE_XSRQ); } static const struct ib_device_ops qedr_dev_ops = { @@ -232,6 +238,7 @@ static const struct ib_device_ops qedr_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_cq, qedr_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, qedr_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_srq, qedr_srq, ibsrq), + INIT_RDMA_OBJ_SIZE(ib_xrcd, qedr_xrcd, ibxrcd), INIT_RDMA_OBJ_SIZE(ib_ucontext, qedr_ucontext, ibucontext), }; @@ -286,7 +293,8 @@ static int qedr_register_device(struct qedr_dev *dev) if (rc) return rc; - return ib_register_device(&dev->ibdev, "qedr%d"); + dma_set_max_seg_size(&dev->pdev->dev, UINT_MAX); + return ib_register_device(&dev->ibdev, "qedr%d", &dev->pdev->dev); } /* This function allocates fast-path status block memory */ @@ -602,7 +610,7 @@ static int qedr_set_device_attr(struct qedr_dev *dev) qed_attr = dev->ops->rdma_query_device(dev->rdma_ctx); /* Part 2 - check capabilities */ - page_size = ~dev->attr.page_size_caps + 1; + page_size = ~qed_attr->page_size_caps + 1; if (page_size > PAGE_SIZE) { DP_ERR(dev, "Kernel PAGE_SIZE is %ld which is smaller than minimum page size (%d) required by qedr\n", @@ -705,6 +713,18 @@ static void qedr_affiliated_event(void *context, u8 e_code, void *fw_handle) event.event = IB_EVENT_SRQ_ERR; event_type = EVENT_TYPE_SRQ; break; + case ROCE_ASYNC_EVENT_XRC_DOMAIN_ERR: + event.event = IB_EVENT_QP_ACCESS_ERR; + event_type = EVENT_TYPE_QP; + break; + case ROCE_ASYNC_EVENT_INVALID_XRCETH_ERR: + event.event = IB_EVENT_QP_ACCESS_ERR; + event_type = EVENT_TYPE_QP; + break; + case ROCE_ASYNC_EVENT_XRC_SRQ_CATASTROPHIC_ERR: + event.event = IB_EVENT_CQ_ERR; + event_type = EVENT_TYPE_CQ; + break; default: DP_ERR(dev, "unsupported event %d on handle=%llx\n", e_code, roce_handle64); @@ -1026,6 +1046,13 @@ static void qedr_notify(struct qedr_dev *dev, enum qede_rdma_event event) case QEDE_CHANGE_ADDR: qedr_mac_address_change(dev); break; + case QEDE_CHANGE_MTU: + if (rdma_protocol_iwarp(&dev->ibdev, 1)) + if (dev->ndev->mtu != dev->iwarp_max_mtu) + DP_NOTICE(dev, + "Mtu was changed from %d to %d. This will not take affect for iWARP until qedr is reloaded\n", + dev->iwarp_max_mtu, dev->ndev->mtu); + break; default: pr_err("Event not supported\n"); } diff --git a/drivers/infiniband/hw/qedr/qedr.h b/drivers/infiniband/hw/qedr/qedr.h index 460292179b32..9dde70373a55 100644 --- a/drivers/infiniband/hw/qedr/qedr.h +++ b/drivers/infiniband/hw/qedr/qedr.h @@ -310,6 +310,11 @@ struct qedr_pd { struct qedr_ucontext *uctx; }; +struct qedr_xrcd { + struct ib_xrcd ibxrcd; + u16 xrcd_id; +}; + struct qedr_qp_hwq_info { /* WQE Elements */ struct qed_chain pbl; @@ -361,6 +366,7 @@ struct qedr_srq { struct ib_umem *prod_umem; u16 srq_id; u32 srq_limit; + bool is_xrc; /* lock to protect srq recv post */ spinlock_t lock; }; @@ -573,6 +579,11 @@ static inline struct qedr_pd *get_qedr_pd(struct ib_pd *ibpd) return container_of(ibpd, struct qedr_pd, ibpd); } +static inline struct qedr_xrcd *get_qedr_xrcd(struct ib_xrcd *ibxrcd) +{ + return container_of(ibxrcd, struct qedr_xrcd, ibxrcd); +} + static inline struct qedr_cq *get_qedr_cq(struct ib_cq *ibcq) { return container_of(ibcq, struct qedr_cq, ibcq); @@ -598,6 +609,28 @@ static inline struct qedr_srq *get_qedr_srq(struct ib_srq *ibsrq) return container_of(ibsrq, struct qedr_srq, ibsrq); } +static inline bool qedr_qp_has_srq(struct qedr_qp *qp) +{ + return qp->srq; +} + +static inline bool qedr_qp_has_sq(struct qedr_qp *qp) +{ + if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_XRC_TGT) + return 0; + + return 1; +} + +static inline bool qedr_qp_has_rq(struct qedr_qp *qp) +{ + if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_XRC_INI || + qp->qp_type == IB_QPT_XRC_TGT || qedr_qp_has_srq(qp)) + return 0; + + return 1; +} + static inline struct qedr_user_mmap_entry * get_qedr_mmap_entry(struct rdma_user_mmap_entry *rdma_entry) { diff --git a/drivers/infiniband/hw/qedr/qedr_iw_cm.c b/drivers/infiniband/hw/qedr/qedr_iw_cm.c index 97fc7dd353b0..c7169d2c69e5 100644 --- a/drivers/infiniband/hw/qedr/qedr_iw_cm.c +++ b/drivers/infiniband/hw/qedr/qedr_iw_cm.c @@ -736,7 +736,7 @@ int qedr_iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) struct qedr_dev *dev = ep->dev; struct qedr_qp *qp; struct qed_iwarp_accept_in params; - int rc = 0; + int rc; DP_DEBUG(dev, QEDR_MSG_IWARP, "Accept on qpid=%d\n", conn_param->qpn); @@ -759,8 +759,10 @@ int qedr_iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) params.ord = conn_param->ord; if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT, - &qp->iwarp_cm_flags)) + &qp->iwarp_cm_flags)) { + rc = -EINVAL; goto err; /* QP already destroyed */ + } rc = dev->ops->iwarp_accept(dev->rdma_ctx, ¶ms); if (rc) { diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index b49bef94637e..019642ff24a7 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -136,6 +136,8 @@ int qedr_query_device(struct ib_device *ibdev, IB_DEVICE_RC_RNR_NAK_GEN | IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS; + if (!rdma_protocol_iwarp(&dev->ibdev, 1)) + attr->device_cap_flags |= IB_DEVICE_XRC; attr->max_send_sge = qattr->max_sge; attr->max_recv_sge = qattr->max_sge; attr->max_sge_rd = qattr->max_sge; @@ -157,13 +159,13 @@ int qedr_query_device(struct ib_device *ibdev, attr->local_ca_ack_delay = qattr->dev_ack_delay; attr->max_fast_reg_page_list_len = qattr->max_mr / 8; - attr->max_pkeys = QEDR_ROCE_PKEY_MAX; + attr->max_pkeys = qattr->max_pkey; attr->max_ah = qattr->max_ah; return 0; } -static inline void get_link_speed_and_width(int speed, u8 *ib_speed, +static inline void get_link_speed_and_width(int speed, u16 *ib_speed, u8 *ib_width) { switch (speed) { @@ -231,15 +233,16 @@ int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr) attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; } attr->max_mtu = IB_MTU_4096; - attr->active_mtu = iboe_get_mtu(dev->ndev->mtu); attr->lid = 0; attr->lmc = 0; attr->sm_lid = 0; attr->sm_sl = 0; attr->ip_gids = true; if (rdma_protocol_iwarp(&dev->ibdev, 1)) { + attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu); attr->gid_tbl_len = 1; } else { + attr->active_mtu = iboe_get_mtu(dev->ndev->mtu); attr->gid_tbl_len = QEDR_MAX_SGID; attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN; } @@ -471,15 +474,33 @@ int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) return 0; } -void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct qedr_dev *dev = get_qedr_dev(ibpd->device); struct qedr_pd *pd = get_qedr_pd(ibpd); DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id); dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id); + return 0; +} + + +int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) +{ + struct qedr_dev *dev = get_qedr_dev(ibxrcd->device); + struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd); + + return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id); } +int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) +{ + struct qedr_dev *dev = get_qedr_dev(ibxrcd->device); + u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id; + + dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id); + return 0; +} static void qedr_free_pbl(struct qedr_dev *dev, struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl) { @@ -600,11 +621,9 @@ static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, struct qedr_pbl_info *pbl_info, u32 pg_shift) { int pbe_cnt, total_num_pbes = 0; - u32 fw_pg_cnt, fw_pg_per_umem_pg; struct qedr_pbl *pbl_tbl; - struct sg_dma_page_iter sg_iter; + struct ib_block_iter biter; struct regpair *pbe; - u64 pg_addr; if (!pbl_info->num_pbes) return; @@ -625,32 +644,25 @@ static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, pbe_cnt = 0; - fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift); + rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) { + u64 pg_addr = rdma_block_iter_dma_address(&biter); - for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { - pg_addr = sg_page_iter_dma_address(&sg_iter); - for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) { - pbe->lo = cpu_to_le32(pg_addr); - pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); + pbe->lo = cpu_to_le32(pg_addr); + pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); - pg_addr += BIT(pg_shift); - pbe_cnt++; - total_num_pbes++; - pbe++; + pbe_cnt++; + total_num_pbes++; + pbe++; - if (total_num_pbes == pbl_info->num_pbes) - return; + if (total_num_pbes == pbl_info->num_pbes) + return; - /* If the given pbl is full storing the pbes, - * move to next pbl. - */ - if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { - pbl_tbl++; - pbe = (struct regpair *)pbl_tbl->va; - pbe_cnt = 0; - } - - fw_pg_cnt++; + /* If the given pbl is full storing the pbes, move to next pbl. + */ + if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { + pbl_tbl++; + pbe = (struct regpair *)pbl_tbl->va; + pbe_cnt = 0; } } } @@ -792,9 +804,7 @@ static inline int qedr_init_user_queue(struct ib_udata *udata, return PTR_ERR(q->umem); } - fw_pages = ib_umem_page_count(q->umem) << - (PAGE_SHIFT - FW_PAGE_SHIFT); - + fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT); rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0); if (rc) goto err0; @@ -999,7 +1009,7 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, /* Generate doorbell address. */ cq->db.data.icid = cq->icid; cq->db_addr = dev->db_addr + db_offset; - cq->db.data.params = DB_AGG_CMD_SET << + cq->db.data.params = DB_AGG_CMD_MAX << RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; /* point to the very last element, passing it we will toggle */ @@ -1051,7 +1061,7 @@ int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) #define QEDR_DESTROY_CQ_ITER_DURATION (10) -void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct qedr_dev *dev = get_qedr_dev(ibcq->device); struct qed_rdma_destroy_cq_out_params oparams; @@ -1066,7 +1076,7 @@ void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) /* GSIs CQs are handled by driver, so they don't exist in the FW */ if (cq->cq_type == QEDR_CQ_TYPE_GSI) { qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data); - return; + return 0; } iparams.icid = cq->icid; @@ -1114,6 +1124,7 @@ void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) * Since the destroy CQ ramrod has also been received on the EQ we can * be certain that there's no event handler in process. */ + return 0; } static inline int get_gid_info_from_table(struct ib_qp *ibqp, @@ -1146,7 +1157,7 @@ static inline int get_gid_info_from_table(struct ib_qp *ibqp, SET_FIELD(qp_params->modify_flags, QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); break; - case RDMA_NETWORK_IB: + case RDMA_NETWORK_ROCE_V1: memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], sizeof(qp_params->sgid)); memcpy(&qp_params->dgid.bytes[0], @@ -1166,6 +1177,8 @@ static inline int get_gid_info_from_table(struct ib_qp *ibqp, QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); qp_params->roce_mode = ROCE_V2_IPV4; break; + default: + return -EINVAL; } for (i = 0; i < 4; i++) { @@ -1186,7 +1199,10 @@ static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, struct qedr_device_attr *qattr = &dev->attr; /* QP0... attrs->qp_type == IB_QPT_GSI */ - if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { + if (attrs->qp_type != IB_QPT_RC && + attrs->qp_type != IB_QPT_GSI && + attrs->qp_type != IB_QPT_XRC_INI && + attrs->qp_type != IB_QPT_XRC_TGT) { DP_DEBUG(dev, QEDR_MSG_QP, "create qp: unsupported qp type=0x%x requested\n", attrs->qp_type); @@ -1221,12 +1237,20 @@ static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, return -EINVAL; } - /* Unprivileged user space cannot create special QP */ - if (udata && attrs->qp_type == IB_QPT_GSI) { - DP_ERR(dev, - "create qp: userspace can't create special QPs of type=0x%x\n", - attrs->qp_type); - return -EINVAL; + /* verify consumer QPs are not trying to use GSI QP's CQ. + * TGT QP isn't associated with RQ/SQ + */ + if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) && + (attrs->qp_type != IB_QPT_XRC_TGT)) { + struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq); + struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq); + + if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) || + (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) { + DP_ERR(dev, + "create qp: consumer QP cannot use GSI CQs.\n"); + return -EINVAL; + } } return 0; @@ -1248,8 +1272,8 @@ static int qedr_copy_srq_uresp(struct qedr_dev *dev, } static void qedr_copy_rq_uresp(struct qedr_dev *dev, - struct qedr_create_qp_uresp *uresp, - struct qedr_qp *qp) + struct qedr_create_qp_uresp *uresp, + struct qedr_qp *qp) { /* iWARP requires two doorbells per RQ. */ if (rdma_protocol_iwarp(&dev->ibdev, 1)) { @@ -1291,8 +1315,12 @@ static int qedr_copy_qp_uresp(struct qedr_dev *dev, int rc; memset(uresp, 0, sizeof(*uresp)); - qedr_copy_sq_uresp(dev, uresp, qp); - qedr_copy_rq_uresp(dev, uresp, qp); + + if (qedr_qp_has_sq(qp)) + qedr_copy_sq_uresp(dev, uresp, qp); + + if (qedr_qp_has_rq(qp)) + qedr_copy_rq_uresp(dev, uresp, qp); uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; uresp->qp_id = qp->qp_id; @@ -1316,18 +1344,25 @@ static void qedr_set_common_qp_params(struct qedr_dev *dev, kref_init(&qp->refcnt); init_completion(&qp->iwarp_cm_comp); } + qp->pd = pd; qp->qp_type = attrs->qp_type; qp->max_inline_data = attrs->cap.max_inline_data; - qp->sq.max_sges = attrs->cap.max_send_sge; qp->state = QED_ROCE_QP_STATE_RESET; qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; - qp->sq_cq = get_qedr_cq(attrs->send_cq); qp->dev = dev; + if (qedr_qp_has_sq(qp)) { + qp->sq.max_sges = attrs->cap.max_send_sge; + qp->sq_cq = get_qedr_cq(attrs->send_cq); + DP_DEBUG(dev, QEDR_MSG_QP, + "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", + qp->sq.max_sges, qp->sq_cq->icid); + } - if (attrs->srq) { + if (attrs->srq) qp->srq = get_qedr_srq(attrs->srq); - } else { + + if (qedr_qp_has_rq(qp)) { qp->rq_cq = get_qedr_cq(attrs->recv_cq); qp->rq.max_sges = attrs->cap.max_recv_sge; DP_DEBUG(dev, QEDR_MSG_QP, @@ -1346,30 +1381,26 @@ static void qedr_set_common_qp_params(struct qedr_dev *dev, static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) { - int rc; + int rc = 0; - qp->sq.db = dev->db_addr + - DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); - qp->sq.db_data.data.icid = qp->icid + 1; - rc = qedr_db_recovery_add(dev, qp->sq.db, - &qp->sq.db_data, - DB_REC_WIDTH_32B, - DB_REC_KERNEL); - if (rc) - return rc; + if (qedr_qp_has_sq(qp)) { + qp->sq.db = dev->db_addr + + DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); + qp->sq.db_data.data.icid = qp->icid + 1; + rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data, + DB_REC_WIDTH_32B, DB_REC_KERNEL); + if (rc) + return rc; + } - if (!qp->srq) { + if (qedr_qp_has_rq(qp)) { qp->rq.db = dev->db_addr + DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); qp->rq.db_data.data.icid = qp->icid; - - rc = qedr_db_recovery_add(dev, qp->rq.db, - &qp->rq.db_data, - DB_REC_WIDTH_32B, - DB_REC_KERNEL); - if (rc) - qedr_db_recovery_del(dev, qp->sq.db, - &qp->sq.db_data); + rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data, + DB_REC_WIDTH_32B, DB_REC_KERNEL); + if (rc && qedr_qp_has_sq(qp)) + qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data); } return rc; @@ -1392,6 +1423,10 @@ static int qedr_check_srq_params(struct qedr_dev *dev, DP_ERR(dev, "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", attrs->attr.max_sge, qattr->max_sge); + } + + if (!udata && attrs->srq_type == IB_SRQT_XRC) { + DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n"); return -EINVAL; } @@ -1516,6 +1551,7 @@ int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, return -EINVAL; srq->dev = dev; + srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC); hw_srq = &srq->hw_srq; spin_lock_init(&srq->lock); @@ -1557,6 +1593,14 @@ int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, in_params.prod_pair_addr = phy_prod_pair_addr; in_params.num_pages = page_cnt; in_params.page_size = page_size; + if (srq->is_xrc) { + struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd); + struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq); + + in_params.is_xrc = 1; + in_params.xrcd_id = xrcd->xrcd_id; + in_params.cq_cid = cq->icid; + } rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); if (rc) @@ -1591,7 +1635,7 @@ err0: return -EFAULT; } -void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) +int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) { struct qed_rdma_destroy_srq_in_params in_params = {}; struct qedr_dev *dev = get_qedr_dev(ibsrq->device); @@ -1599,6 +1643,7 @@ void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) xa_erase_irq(&dev->srqs, srq->srq_id); in_params.srq_id = srq->srq_id; + in_params.is_xrc = srq->is_xrc; dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params); if (ibsrq->uobject) @@ -1609,6 +1654,7 @@ void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) DP_DEBUG(dev, QEDR_MSG_SRQ, "destroy srq: destroyed srq with srq_id=0x%0x\n", srq->srq_id); + return 0; } int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, @@ -1649,6 +1695,20 @@ int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, return 0; } +static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type) +{ + switch (ib_qp_type) { + case IB_QPT_RC: + return QED_RDMA_QP_TYPE_RC; + case IB_QPT_XRC_INI: + return QED_RDMA_QP_TYPE_XRC_INI; + case IB_QPT_XRC_TGT: + return QED_RDMA_QP_TYPE_XRC_TGT; + default: + return QED_RDMA_QP_TYPE_INVAL; + } +} + static inline void qedr_init_common_qp_in_params(struct qedr_dev *dev, struct qedr_pd *pd, @@ -1663,20 +1723,27 @@ qedr_init_common_qp_in_params(struct qedr_dev *dev, params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; - params->pd = pd->pd_id; - params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; - params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid; + params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type); params->stats_queue = 0; - params->srq_id = 0; - params->use_srq = false; - if (!qp->srq) { + if (pd) { + params->pd = pd->pd_id; + params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; + } + + if (qedr_qp_has_sq(qp)) + params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid; + + if (qedr_qp_has_rq(qp)) params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; - } else { + if (qedr_qp_has_srq(qp)) { params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; params->srq_id = qp->srq->srq_id; params->use_srq = true; + } else { + params->srq_id = 0; + params->use_srq = false; } } @@ -1690,8 +1757,10 @@ static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) "rq_len=%zd" "\n", qp, - qp->usq.buf_addr, - qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len); + qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0, + qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0, + qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0, + qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0); } static inline void @@ -1717,11 +1786,15 @@ static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_ucontext *ctx, struct qedr_qp *qp) { - ib_umem_release(qp->usq.umem); - qp->usq.umem = NULL; + if (qedr_qp_has_sq(qp)) { + ib_umem_release(qp->usq.umem); + qp->usq.umem = NULL; + } - ib_umem_release(qp->urq.umem); - qp->urq.umem = NULL; + if (qedr_qp_has_rq(qp)) { + ib_umem_release(qp->urq.umem); + qp->urq.umem = NULL; + } if (rdma_protocol_roce(&dev->ibdev, 1)) { qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl); @@ -1756,28 +1829,38 @@ static int qedr_create_user_qp(struct qedr_dev *dev, { struct qed_rdma_create_qp_in_params in_params; struct qed_rdma_create_qp_out_params out_params; - struct qedr_pd *pd = get_qedr_pd(ibpd); - struct qedr_create_qp_uresp uresp; - struct qedr_ucontext *ctx = pd ? pd->uctx : NULL; - struct qedr_create_qp_ureq ureq; + struct qedr_create_qp_uresp uresp = {}; + struct qedr_create_qp_ureq ureq = {}; int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1); - int rc = -EINVAL; + struct qedr_ucontext *ctx = NULL; + struct qedr_pd *pd = NULL; + int rc = 0; qp->create_type = QEDR_QP_CREATE_USER; - memset(&ureq, 0, sizeof(ureq)); - rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen)); - if (rc) { - DP_ERR(dev, "Problem copying data from user space\n"); - return rc; + + if (ibpd) { + pd = get_qedr_pd(ibpd); + ctx = pd->uctx; } - /* SQ - read access only (0) */ - rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr, - ureq.sq_len, true, 0, alloc_and_init); - if (rc) - return rc; + if (udata) { + rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), + udata->inlen)); + if (rc) { + DP_ERR(dev, "Problem copying data from user space\n"); + return rc; + } + } - if (!qp->srq) { + if (qedr_qp_has_sq(qp)) { + /* SQ - read access only (0) */ + rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr, + ureq.sq_len, true, 0, alloc_and_init); + if (rc) + return rc; + } + + if (qedr_qp_has_rq(qp)) { /* RQ - read access only (0) */ rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr, ureq.rq_len, true, 0, alloc_and_init); @@ -1789,9 +1872,21 @@ static int qedr_create_user_qp(struct qedr_dev *dev, qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); in_params.qp_handle_lo = ureq.qp_handle_lo; in_params.qp_handle_hi = ureq.qp_handle_hi; - in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; - in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; - if (!qp->srq) { + + if (qp->qp_type == IB_QPT_XRC_TGT) { + struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd); + + in_params.xrcd_id = xrcd->xrcd_id; + in_params.qp_handle_lo = qp->qp_id; + in_params.use_srq = 1; + } + + if (qedr_qp_has_sq(qp)) { + in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; + in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; + } + + if (qedr_qp_has_rq(qp)) { in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; } @@ -1813,39 +1908,32 @@ static int qedr_create_user_qp(struct qedr_dev *dev, qp->qp_id = out_params.qp_id; qp->icid = out_params.icid; - rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp); - if (rc) - goto err; + if (udata) { + rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp); + if (rc) + goto err; + } /* db offset was calculated in copy_qp_uresp, now set in the user q */ - ctx = pd->uctx; - qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset; - qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset; - - if (rdma_protocol_iwarp(&dev->ibdev, 1)) { - qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset; - - /* calculate the db_rec_db2 data since it is constant so no - * need to reflect from user - */ - qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid); - qp->urq.db_rec_db2_data.data.value = - cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD); + if (qedr_qp_has_sq(qp)) { + qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset; + rc = qedr_db_recovery_add(dev, qp->usq.db_addr, + &qp->usq.db_rec_data->db_data, + DB_REC_WIDTH_32B, + DB_REC_USER); + if (rc) + goto err; } - rc = qedr_db_recovery_add(dev, qp->usq.db_addr, - &qp->usq.db_rec_data->db_data, - DB_REC_WIDTH_32B, - DB_REC_USER); - if (rc) - goto err; - - rc = qedr_db_recovery_add(dev, qp->urq.db_addr, - &qp->urq.db_rec_data->db_data, - DB_REC_WIDTH_32B, - DB_REC_USER); - if (rc) - goto err; + if (qedr_qp_has_rq(qp)) { + qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset; + rc = qedr_db_recovery_add(dev, qp->urq.db_addr, + &qp->urq.db_rec_data->db_data, + DB_REC_WIDTH_32B, + DB_REC_USER); + if (rc) + goto err; + } if (rdma_protocol_iwarp(&dev->ibdev, 1)) { rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr, @@ -1856,7 +1944,6 @@ static int qedr_create_user_qp(struct qedr_dev *dev, goto err; } qedr_qp_user_print(dev, qp); - return rc; err: rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); @@ -2112,16 +2199,47 @@ static int qedr_create_kernel_qp(struct qedr_dev *dev, return rc; } +static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp, + struct ib_udata *udata) +{ + struct qedr_ucontext *ctx = + rdma_udata_to_drv_context(udata, struct qedr_ucontext, + ibucontext); + int rc; + + if (qp->qp_type != IB_QPT_GSI) { + rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); + if (rc) + return rc; + } + + if (qp->create_type == QEDR_QP_CREATE_USER) + qedr_cleanup_user(dev, ctx, qp); + else + qedr_cleanup_kernel(dev, qp); + + return 0; +} + struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, struct ib_qp_init_attr *attrs, struct ib_udata *udata) { - struct qedr_dev *dev = get_qedr_dev(ibpd->device); - struct qedr_pd *pd = get_qedr_pd(ibpd); + struct qedr_xrcd *xrcd = NULL; + struct qedr_pd *pd = NULL; + struct qedr_dev *dev; struct qedr_qp *qp; struct ib_qp *ibqp; int rc = 0; + if (attrs->qp_type == IB_QPT_XRC_TGT) { + xrcd = get_qedr_xrcd(attrs->xrcd); + dev = get_qedr_dev(xrcd->ibxrcd.device); + } else { + pd = get_qedr_pd(ibpd); + dev = get_qedr_dev(ibpd->device); + } + DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n", udata ? "user library" : "kernel", pd); @@ -2152,25 +2270,27 @@ struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, return ibqp; } - if (udata) + if (udata || xrcd) rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); else rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); if (rc) - goto err; + goto out_free_qp; qp->ibqp.qp_num = qp->qp_id; if (rdma_protocol_iwarp(&dev->ibdev, 1)) { rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL); if (rc) - goto err; + goto out_free_qp_resources; } return &qp->ibqp; -err: +out_free_qp_resources: + qedr_free_qp_resources(dev, qp, udata); +out_free_qp: kfree(qp); return ERR_PTR(-EFAULT); @@ -2636,7 +2756,7 @@ int qedr_query_qp(struct ib_qp *ibqp, qp_attr->cap.max_recv_wr = qp->rq.max_wr; qp_attr->cap.max_send_sge = qp->sq.max_sges; qp_attr->cap.max_recv_sge = qp->rq.max_sges; - qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; + qp_attr->cap.max_inline_data = dev->attr.max_inline; qp_init_attr->cap = qp_attr->cap; qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; @@ -2671,28 +2791,6 @@ err: return rc; } -static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp, - struct ib_udata *udata) -{ - struct qedr_ucontext *ctx = - rdma_udata_to_drv_context(udata, struct qedr_ucontext, - ibucontext); - int rc; - - if (qp->qp_type != IB_QPT_GSI) { - rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); - if (rc) - return rc; - } - - if (qp->create_type == QEDR_QP_CREATE_USER) - qedr_cleanup_user(dev, ctx, qp); - else - qedr_cleanup_kernel(dev, qp); - - return 0; -} - int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) { struct qedr_qp *qp = get_qedr_qp(ibqp); @@ -2752,6 +2850,8 @@ int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) if (rdma_protocol_iwarp(&dev->ibdev, 1)) qedr_iw_qp_rem_ref(&qp->ibqp); + else + kfree(qp); return 0; } @@ -2766,11 +2866,12 @@ int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, return 0; } -void qedr_destroy_ah(struct ib_ah *ibah, u32 flags) +int qedr_destroy_ah(struct ib_ah *ibah, u32 flags) { struct qedr_ah *ah = get_qedr_ah(ibah); rdma_destroy_ah_attr(&ah->attr); + return 0; } static void free_mr_info(struct qedr_dev *dev, struct mr_info *info) @@ -2861,7 +2962,8 @@ struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, goto err0; } - rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); + rc = init_mr_info(dev, &mr->info, + ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1); if (rc) goto err1; @@ -2888,10 +2990,8 @@ struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); mr->hw_mr.page_size_log = PAGE_SHIFT; - mr->hw_mr.fbo = ib_umem_offset(mr->umem); mr->hw_mr.length = len; mr->hw_mr.vaddr = usr_addr; - mr->hw_mr.zbva = false; mr->hw_mr.phy_mr = false; mr->hw_mr.dma_mr = false; @@ -2984,10 +3084,8 @@ static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, mr->hw_mr.pbl_ptr = 0; mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); - mr->hw_mr.fbo = 0; mr->hw_mr.length = 0; mr->hw_mr.vaddr = 0; - mr->hw_mr.zbva = false; mr->hw_mr.phy_mr = true; mr->hw_mr.dma_mr = false; @@ -3765,10 +3863,10 @@ int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, * in first 4 bytes and need to update WQE producer in * next 4 bytes. */ - srq->hw_srq.virt_prod_pair_addr->sge_prod = hw_srq->sge_prod; + srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod); /* Make sure sge producer is updated first */ dma_wmb(); - srq->hw_srq.virt_prod_pair_addr->wqe_prod = hw_srq->wqe_prod; + srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod); wr = wr->next; } diff --git a/drivers/infiniband/hw/qedr/verbs.h b/drivers/infiniband/hw/qedr/verbs.h index 39dd6286ba39..2672c32bc2f7 100644 --- a/drivers/infiniband/hw/qedr/verbs.h +++ b/drivers/infiniband/hw/qedr/verbs.h @@ -47,12 +47,13 @@ void qedr_dealloc_ucontext(struct ib_ucontext *uctx); int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma); void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry); int qedr_alloc_pd(struct ib_pd *pd, struct ib_udata *udata); -void qedr_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); - +int qedr_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); +int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata); +int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata); int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata); int qedr_resize_cq(struct ib_cq *, int cqe, struct ib_udata *); -void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); +int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); struct ib_qp *qedr_create_qp(struct ib_pd *, struct ib_qp_init_attr *attrs, struct ib_udata *); @@ -67,12 +68,12 @@ int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr, int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr); -void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); +int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_recv_wr); int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata); -void qedr_destroy_ah(struct ib_ah *ibah, u32 flags); +int qedr_destroy_ah(struct ib_ah *ibah, u32 flags); int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata); struct ib_mr *qedr_get_dma_mr(struct ib_pd *, int acc); diff --git a/drivers/infiniband/hw/qib/qib.h b/drivers/infiniband/hw/qib/qib.h index 432d6d0fd7f4..ee211423058a 100644 --- a/drivers/infiniband/hw/qib/qib.h +++ b/drivers/infiniband/hw/qib/qib.h @@ -619,11 +619,11 @@ struct qib_pportdata { /* LID mask control */ u8 lmc; u8 link_width_supported; - u8 link_speed_supported; + u16 link_speed_supported; u8 link_width_enabled; - u8 link_speed_enabled; + u16 link_speed_enabled; u8 link_width_active; - u8 link_speed_active; + u16 link_speed_active; u8 vls_supported; u8 vls_operational; /* Rx Polarity inversion (compensate for ~tx on partner) */ diff --git a/drivers/infiniband/hw/qib/qib_iba7322.c b/drivers/infiniband/hw/qib/qib_iba7322.c index a10eab89aee4..189a0ce6056a 100644 --- a/drivers/infiniband/hw/qib/qib_iba7322.c +++ b/drivers/infiniband/hw/qib/qib_iba7322.c @@ -1733,9 +1733,9 @@ done: return; } -static void qib_error_tasklet(unsigned long data) +static void qib_error_tasklet(struct tasklet_struct *t) { - struct qib_devdata *dd = (struct qib_devdata *)data; + struct qib_devdata *dd = from_tasklet(dd, t, error_tasklet); handle_7322_errors(dd); qib_write_kreg(dd, kr_errmask, dd->cspec->errormask); @@ -3537,8 +3537,7 @@ try_intx: for (i = 0; i < ARRAY_SIZE(redirect); i++) qib_write_kreg(dd, kr_intredirect + i, redirect[i]); dd->cspec->main_int_mask = mask; - tasklet_init(&dd->error_tasklet, qib_error_tasklet, - (unsigned long)dd); + tasklet_setup(&dd->error_tasklet, qib_error_tasklet); } /** diff --git a/drivers/infiniband/hw/qib/qib_mad.c b/drivers/infiniband/hw/qib/qib_mad.c index e7789e724f56..f83e331977f8 100644 --- a/drivers/infiniband/hw/qib/qib_mad.c +++ b/drivers/infiniband/hw/qib/qib_mad.c @@ -2293,76 +2293,50 @@ static int process_cc(struct ib_device *ibdev, int mad_flags, struct ib_mad *out_mad) { struct ib_cc_mad *ccp = (struct ib_cc_mad *)out_mad; - int ret; - *out_mad = *in_mad; if (ccp->class_version != 2) { ccp->status |= IB_SMP_UNSUP_VERSION; - ret = reply((struct ib_smp *)ccp); - goto bail; + return reply((struct ib_smp *)ccp); } switch (ccp->method) { case IB_MGMT_METHOD_GET: switch (ccp->attr_id) { case IB_CC_ATTR_CLASSPORTINFO: - ret = cc_get_classportinfo(ccp, ibdev); - goto bail; - + return cc_get_classportinfo(ccp, ibdev); case IB_CC_ATTR_CONGESTION_INFO: - ret = cc_get_congestion_info(ccp, ibdev, port); - goto bail; - + return cc_get_congestion_info(ccp, ibdev, port); case IB_CC_ATTR_CA_CONGESTION_SETTING: - ret = cc_get_congestion_setting(ccp, ibdev, port); - goto bail; - + return cc_get_congestion_setting(ccp, ibdev, port); case IB_CC_ATTR_CONGESTION_CONTROL_TABLE: - ret = cc_get_congestion_control_table(ccp, ibdev, port); - goto bail; - - fallthrough; + return cc_get_congestion_control_table(ccp, ibdev, port); default: ccp->status |= IB_SMP_UNSUP_METH_ATTR; - ret = reply((struct ib_smp *) ccp); - goto bail; + return reply((struct ib_smp *) ccp); } - case IB_MGMT_METHOD_SET: switch (ccp->attr_id) { case IB_CC_ATTR_CA_CONGESTION_SETTING: - ret = cc_set_congestion_setting(ccp, ibdev, port); - goto bail; - + return cc_set_congestion_setting(ccp, ibdev, port); case IB_CC_ATTR_CONGESTION_CONTROL_TABLE: - ret = cc_set_congestion_control_table(ccp, ibdev, port); - goto bail; - - fallthrough; + return cc_set_congestion_control_table(ccp, ibdev, port); default: ccp->status |= IB_SMP_UNSUP_METH_ATTR; - ret = reply((struct ib_smp *) ccp); - goto bail; + return reply((struct ib_smp *) ccp); } - case IB_MGMT_METHOD_GET_RESP: /* * The ib_mad module will call us to process responses * before checking for other consumers. * Just tell the caller to process it normally. */ - ret = IB_MAD_RESULT_SUCCESS; - goto bail; - - case IB_MGMT_METHOD_TRAP: - default: - ccp->status |= IB_SMP_UNSUP_METHOD; - ret = reply((struct ib_smp *) ccp); + return IB_MAD_RESULT_SUCCESS; } -bail: - return ret; + /* method is unsupported */ + ccp->status |= IB_SMP_UNSUP_METHOD; + return reply((struct ib_smp *) ccp); } /** diff --git a/drivers/infiniband/hw/qib/qib_sdma.c b/drivers/infiniband/hw/qib/qib_sdma.c index 8f8d61736656..5e86cbf7d70e 100644 --- a/drivers/infiniband/hw/qib/qib_sdma.c +++ b/drivers/infiniband/hw/qib/qib_sdma.c @@ -62,7 +62,7 @@ static void sdma_get(struct qib_sdma_state *); static void sdma_put(struct qib_sdma_state *); static void sdma_set_state(struct qib_pportdata *, enum qib_sdma_states); static void sdma_start_sw_clean_up(struct qib_pportdata *); -static void sdma_sw_clean_up_task(unsigned long); +static void sdma_sw_clean_up_task(struct tasklet_struct *); static void unmap_desc(struct qib_pportdata *, unsigned); static void sdma_get(struct qib_sdma_state *ss) @@ -119,9 +119,10 @@ static void clear_sdma_activelist(struct qib_pportdata *ppd) } } -static void sdma_sw_clean_up_task(unsigned long opaque) +static void sdma_sw_clean_up_task(struct tasklet_struct *t) { - struct qib_pportdata *ppd = (struct qib_pportdata *) opaque; + struct qib_pportdata *ppd = from_tasklet(ppd, t, + sdma_sw_clean_up_task); unsigned long flags; spin_lock_irqsave(&ppd->sdma_lock, flags); @@ -436,8 +437,7 @@ int qib_setup_sdma(struct qib_pportdata *ppd) INIT_LIST_HEAD(&ppd->sdma_activelist); - tasklet_init(&ppd->sdma_sw_clean_up_task, sdma_sw_clean_up_task, - (unsigned long)ppd); + tasklet_setup(&ppd->sdma_sw_clean_up_task, sdma_sw_clean_up_task); ret = dd->f_init_sdma_regs(ppd); if (ret) diff --git a/drivers/infiniband/hw/usnic/usnic_ib_main.c b/drivers/infiniband/hw/usnic/usnic_ib_main.c index 662e7fc7f628..aa2e65fc5cd6 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_main.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_main.c @@ -315,7 +315,6 @@ static int usnic_port_immutable(struct ib_device *ibdev, u8 port_num, if (err) return err; - immutable->pkey_tbl_len = attr.pkey_tbl_len; immutable->gid_tbl_len = attr.gid_tbl_len; return 0; @@ -355,7 +354,6 @@ static const struct ib_device_ops usnic_dev_ops = { .modify_qp = usnic_ib_modify_qp, .query_device = usnic_ib_query_device, .query_gid = usnic_ib_query_gid, - .query_pkey = usnic_ib_query_pkey, .query_port = usnic_ib_query_port, .query_qp = usnic_ib_query_qp, .reg_user_mr = usnic_ib_reg_mr, @@ -427,7 +425,8 @@ static void *usnic_ib_device_add(struct pci_dev *dev) if (ret) goto err_fwd_dealloc; - if (ib_register_device(&us_ibdev->ib_dev, "usnic_%d")) + dma_set_max_seg_size(&dev->dev, SZ_2G); + if (ib_register_device(&us_ibdev->ib_dev, "usnic_%d", &dev->dev)) goto err_fwd_dealloc; usnic_fwd_set_mtu(us_ibdev->ufdev, us_ibdev->netdev->mtu); diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c index b8a77ce11590..9e961f8ffa10 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c @@ -367,7 +367,6 @@ int usnic_ib_query_port(struct ib_device *ibdev, u8 port, props->port_cap_flags = 0; props->gid_tbl_len = 1; - props->pkey_tbl_len = 1; props->bad_pkey_cntr = 0; props->qkey_viol_cntr = 0; props->max_mtu = IB_MTU_4096; @@ -437,16 +436,6 @@ int usnic_ib_query_gid(struct ib_device *ibdev, u8 port, int index, return 0; } -int usnic_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, - u16 *pkey) -{ - if (index > 0) - return -EINVAL; - - *pkey = 0xffff; - return 0; -} - int usnic_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct usnic_ib_pd *pd = to_upd(ibpd); @@ -460,9 +449,10 @@ int usnic_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) return 0; } -void usnic_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) +int usnic_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) { usnic_uiom_dealloc_pd((to_upd(pd))->umem_pd); + return 0; } struct ib_qp *usnic_ib_create_qp(struct ib_pd *pd, @@ -596,9 +586,9 @@ int usnic_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, return 0; } -void usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +int usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { - return; + return 0; } struct ib_mr *usnic_ib_reg_mr(struct ib_pd *pd, u64 start, u64 length, diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h index 2aedf78c13cf..11fe1ba6bbc9 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h @@ -48,10 +48,8 @@ int usnic_ib_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, struct ib_qp_init_attr *qp_init_attr); int usnic_ib_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid); -int usnic_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, - u16 *pkey); int usnic_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); -void usnic_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); +int usnic_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); struct ib_qp *usnic_ib_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata); @@ -60,7 +58,7 @@ int usnic_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata); int usnic_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata); -void usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +int usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); struct ib_mr *usnic_ib_reg_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt_addr, int access_flags, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c index 4f6cc0de7ef9..319546a39a0d 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c @@ -142,7 +142,7 @@ int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, goto err_cq; } - npages = ib_umem_page_count(cq->umem); + npages = ib_umem_num_dma_blocks(cq->umem, PAGE_SIZE); } else { /* One extra page for shared ring state */ npages = 1 + (entries * sizeof(struct pvrdma_cqe) + @@ -235,7 +235,7 @@ static void pvrdma_free_cq(struct pvrdma_dev *dev, struct pvrdma_cq *cq) * @cq: the completion queue to destroy. * @udata: user data or null for kernel object */ -void pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +int pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { struct pvrdma_cq *vcq = to_vcq(cq); union pvrdma_cmd_req req; @@ -261,6 +261,7 @@ void pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) pvrdma_free_cq(dev, vcq); atomic_dec(&dev->num_cqs); + return 0; } static inline struct pvrdma_cqe *get_cqe(struct pvrdma_cq *cq, int i) @@ -375,7 +376,7 @@ retry: * pvrdma_poll_cq - poll for work completion queue entries * @ibcq: completion queue * @num_entries: the maximum number of entries - * @entry: pointer to work completion array + * @wc: pointer to work completion array * * @return: number of polled completion entries */ diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c index 780fd2dfc07e..fa2a3fa0c3e4 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c @@ -270,7 +270,7 @@ static int pvrdma_register_device(struct pvrdma_dev *dev) spin_lock_init(&dev->srq_tbl_lock); rdma_set_device_sysfs_group(&dev->ib_dev, &pvrdma_attr_group); - ret = ib_register_device(&dev->ib_dev, "vmw_pvrdma%d"); + ret = ib_register_device(&dev->ib_dev, "vmw_pvrdma%d", &dev->pdev->dev); if (ret) goto err_srq_free; @@ -854,7 +854,7 @@ static int pvrdma_pci_probe(struct pci_dev *pdev, goto err_free_resource; } } - + dma_set_max_seg_size(&pdev->dev, UINT_MAX); pci_set_master(pdev); /* Map register space */ diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c index 7944c58ded0e..ba43ad07898c 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c @@ -182,17 +182,16 @@ int pvrdma_page_dir_insert_dma(struct pvrdma_page_dir *pdir, u64 idx, int pvrdma_page_dir_insert_umem(struct pvrdma_page_dir *pdir, struct ib_umem *umem, u64 offset) { + struct ib_block_iter biter; u64 i = offset; int ret = 0; - struct sg_dma_page_iter sg_iter; if (offset >= pdir->npages) return -EINVAL; - for_each_sg_dma_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { - dma_addr_t addr = sg_page_iter_dma_address(&sg_iter); - - ret = pvrdma_page_dir_insert_dma(pdir, i, addr); + rdma_umem_for_each_dma_block (umem, &biter, PAGE_SIZE) { + ret = pvrdma_page_dir_insert_dma( + pdir, i, rdma_block_iter_dma_address(&biter)); if (ret) goto exit; diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c index 77a010e68208..e80848bfb3bd 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c @@ -133,7 +133,7 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, return ERR_CAST(umem); } - npages = ib_umem_num_pages(umem); + npages = ib_umem_num_dma_blocks(umem, PAGE_SIZE); if (npages < 0 || npages > PVRDMA_PAGE_DIR_MAX_PAGES) { dev_warn(&dev->pdev->dev, "overflow %d pages in mem region\n", npages); @@ -270,6 +270,7 @@ freemr: /** * pvrdma_dereg_mr - deregister a memory region * @ibmr: memory region + * @udata: pointer to user data * * @return: 0 on success. */ diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c index 9a8f2a9507be..428256c55065 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c @@ -232,8 +232,7 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, switch (init_attr->qp_type) { case IB_QPT_GSI: if (init_attr->port_num == 0 || - init_attr->port_num > pd->device->phys_port_cnt || - udata) { + init_attr->port_num > pd->device->phys_port_cnt) { dev_warn(&dev->pdev->dev, "invalid queuepair attrs\n"); ret = -EINVAL; goto err_qp; @@ -298,9 +297,11 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, goto err_qp; } - qp->npages_send = ib_umem_page_count(qp->sumem); + qp->npages_send = + ib_umem_num_dma_blocks(qp->sumem, PAGE_SIZE); if (!is_srq) - qp->npages_recv = ib_umem_page_count(qp->rumem); + qp->npages_recv = ib_umem_num_dma_blocks( + qp->rumem, PAGE_SIZE); else qp->npages_recv = 0; qp->npages = qp->npages_send + qp->npages_recv; diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c index d330decfb80a..082208f9aa90 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c @@ -90,7 +90,7 @@ int pvrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) /** * pvrdma_create_srq - create shared receive queue - * @pd: protection domain + * @ibsrq: the IB shared receive queue * @init_attr: shared receive queue attributes * @udata: user data * @@ -152,7 +152,7 @@ int pvrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, goto err_srq; } - srq->npages = ib_umem_page_count(srq->umem); + srq->npages = ib_umem_num_dma_blocks(srq->umem, PAGE_SIZE); if (srq->npages < 0 || srq->npages > PVRDMA_PAGE_DIR_MAX_PAGES) { dev_warn(&dev->pdev->dev, @@ -240,7 +240,7 @@ static void pvrdma_free_srq(struct pvrdma_dev *dev, struct pvrdma_srq *srq) * * @return: 0 for success. */ -void pvrdma_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) +int pvrdma_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) { struct pvrdma_srq *vsrq = to_vsrq(srq); union pvrdma_cmd_req req; @@ -259,6 +259,7 @@ void pvrdma_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) ret); pvrdma_free_srq(dev, vsrq); + return 0; } /** diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c index ccbded2d26ce..fc412cbfd042 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c @@ -479,9 +479,9 @@ err: * @pd: the protection domain to be released * @udata: user data or null for kernel object * - * @return: 0 on success, otherwise errno. + * @return: Always 0 */ -void pvrdma_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) +int pvrdma_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) { struct pvrdma_dev *dev = to_vdev(pd->device); union pvrdma_cmd_req req = {}; @@ -498,14 +498,14 @@ void pvrdma_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) ret); atomic_dec(&dev->num_pds); + return 0; } /** * pvrdma_create_ah - create an address handle - * @pd: the protection domain - * @ah_attr: the attributes of the AH - * @udata: user data blob - * @flags: create address handle flags (see enum rdma_create_ah_flags) + * @ibah: the IB address handle + * @init_attr: the attributes of the AH + * @udata: pointer to user data * * @return: 0 on success, otherwise errno. */ @@ -548,9 +548,10 @@ int pvrdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, * @flags: destroy address handle flags (see enum rdma_destroy_ah_flags) * */ -void pvrdma_destroy_ah(struct ib_ah *ah, u32 flags) +int pvrdma_destroy_ah(struct ib_ah *ah, u32 flags) { struct pvrdma_dev *dev = to_vdev(ah->device); atomic_dec(&dev->num_ahs); + return 0; } diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h index 699b20849a7e..f0e5ffba2d51 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h @@ -176,7 +176,7 @@ struct pvrdma_port_attr { u8 subnet_timeout; u8 init_type_reply; u8 active_width; - u8 active_speed; + u16 active_speed; u8 phys_state; u8 reserved[2]; }; @@ -399,7 +399,7 @@ int pvrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma); int pvrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata); void pvrdma_dealloc_ucontext(struct ib_ucontext *context); int pvrdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata); -void pvrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); +int pvrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); struct ib_mr *pvrdma_get_dma_mr(struct ib_pd *pd, int acc); struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt_addr, int access_flags, @@ -411,19 +411,19 @@ int pvrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset); int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata); -void pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +int pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int pvrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int pvrdma_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); int pvrdma_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata); -void pvrdma_destroy_ah(struct ib_ah *ah, u32 flags); +int pvrdma_destroy_ah(struct ib_ah *ah, u32 flags); int pvrdma_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *init_attr, struct ib_udata *udata); int pvrdma_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); int pvrdma_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr); -void pvrdma_destroy_srq(struct ib_srq *srq, struct ib_udata *udata); +int pvrdma_destroy_srq(struct ib_srq *srq, struct ib_udata *udata); struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, diff --git a/drivers/infiniband/sw/rdmavt/ah.c b/drivers/infiniband/sw/rdmavt/ah.c index 75a04b1497c4..b938c4ffa99a 100644 --- a/drivers/infiniband/sw/rdmavt/ah.c +++ b/drivers/infiniband/sw/rdmavt/ah.c @@ -132,7 +132,7 @@ int rvt_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, * * Return: 0 on success */ -void rvt_destroy_ah(struct ib_ah *ibah, u32 destroy_flags) +int rvt_destroy_ah(struct ib_ah *ibah, u32 destroy_flags) { struct rvt_dev_info *dev = ib_to_rvt(ibah->device); struct rvt_ah *ah = ibah_to_rvtah(ibah); @@ -143,6 +143,7 @@ void rvt_destroy_ah(struct ib_ah *ibah, u32 destroy_flags) spin_unlock_irqrestore(&dev->n_ahs_lock, flags); rdma_destroy_ah_attr(&ah->attr); + return 0; } /** diff --git a/drivers/infiniband/sw/rdmavt/ah.h b/drivers/infiniband/sw/rdmavt/ah.h index 40b7123fec76..5a85edd06491 100644 --- a/drivers/infiniband/sw/rdmavt/ah.h +++ b/drivers/infiniband/sw/rdmavt/ah.h @@ -52,7 +52,7 @@ int rvt_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata); -void rvt_destroy_ah(struct ib_ah *ibah, u32 destroy_flags); +int rvt_destroy_ah(struct ib_ah *ibah, u32 destroy_flags); int rvt_modify_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); int rvt_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); diff --git a/drivers/infiniband/sw/rdmavt/cq.c b/drivers/infiniband/sw/rdmavt/cq.c index 04d2e72017fe..19248be14093 100644 --- a/drivers/infiniband/sw/rdmavt/cq.c +++ b/drivers/infiniband/sw/rdmavt/cq.c @@ -315,7 +315,7 @@ bail_wc: * * Called by ib_destroy_cq() in the generic verbs code. */ -void rvt_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +int rvt_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct rvt_cq *cq = ibcq_to_rvtcq(ibcq); struct rvt_dev_info *rdi = cq->rdi; @@ -328,6 +328,7 @@ void rvt_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) kref_put(&cq->ip->ref, rvt_release_mmap_info); else vfree(cq->kqueue); + return 0; } /** diff --git a/drivers/infiniband/sw/rdmavt/cq.h b/drivers/infiniband/sw/rdmavt/cq.h index 5e26a2eb19a4..feb01e7ee004 100644 --- a/drivers/infiniband/sw/rdmavt/cq.h +++ b/drivers/infiniband/sw/rdmavt/cq.h @@ -53,7 +53,7 @@ int rvt_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata); -void rvt_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); +int rvt_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); int rvt_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags); int rvt_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata); int rvt_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry); diff --git a/drivers/infiniband/sw/rdmavt/pd.c b/drivers/infiniband/sw/rdmavt/pd.c index a403718f0b5e..01b7abf91520 100644 --- a/drivers/infiniband/sw/rdmavt/pd.c +++ b/drivers/infiniband/sw/rdmavt/pd.c @@ -95,11 +95,12 @@ bail: * * Return: always 0 */ -void rvt_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +int rvt_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct rvt_dev_info *dev = ib_to_rvt(ibpd->device); spin_lock(&dev->n_pds_lock); dev->n_pds_allocated--; spin_unlock(&dev->n_pds_lock); + return 0; } diff --git a/drivers/infiniband/sw/rdmavt/pd.h b/drivers/infiniband/sw/rdmavt/pd.h index 71ba76d72b1d..06a6a38beedc 100644 --- a/drivers/infiniband/sw/rdmavt/pd.h +++ b/drivers/infiniband/sw/rdmavt/pd.h @@ -51,6 +51,6 @@ #include <rdma/rdma_vt.h> int rvt_alloc_pd(struct ib_pd *pd, struct ib_udata *udata); -void rvt_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); +int rvt_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); #endif /* DEF_RDMAVTPD_H */ diff --git a/drivers/infiniband/sw/rdmavt/srq.c b/drivers/infiniband/sw/rdmavt/srq.c index f547c115af03..64d98bf238ab 100644 --- a/drivers/infiniband/sw/rdmavt/srq.c +++ b/drivers/infiniband/sw/rdmavt/srq.c @@ -332,7 +332,7 @@ int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) * @ibsrq: srq object to destroy * */ -void rvt_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) +int rvt_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) { struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device); @@ -343,4 +343,5 @@ void rvt_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) if (srq->ip) kref_put(&srq->ip->ref, rvt_release_mmap_info); kvfree(srq->rq.kwq); + return 0; } diff --git a/drivers/infiniband/sw/rdmavt/srq.h b/drivers/infiniband/sw/rdmavt/srq.h index 6427d7d62a9a..d5a1a053b1b9 100644 --- a/drivers/infiniband/sw/rdmavt/srq.h +++ b/drivers/infiniband/sw/rdmavt/srq.h @@ -56,6 +56,6 @@ int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr); -void rvt_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); +int rvt_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); #endif /* DEF_RVTSRQ_H */ diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c index f904bb34477a..52218684ad4a 100644 --- a/drivers/infiniband/sw/rdmavt/vt.c +++ b/drivers/infiniband/sw/rdmavt/vt.c @@ -95,9 +95,7 @@ struct rvt_dev_info *rvt_alloc_device(size_t size, int nports) if (!rdi) return rdi; - rdi->ports = kcalloc(nports, - sizeof(struct rvt_ibport **), - GFP_KERNEL); + rdi->ports = kcalloc(nports, sizeof(*rdi->ports), GFP_KERNEL); if (!rdi->ports) ib_dealloc_device(&rdi->ibdev); @@ -581,7 +579,9 @@ int rvt_register_device(struct rvt_dev_info *rdi) spin_lock_init(&rdi->n_cqs_lock); /* DMA Operations */ - rdi->ibdev.dev.dma_ops = rdi->ibdev.dev.dma_ops ? : &dma_virt_ops; + rdi->ibdev.dev.dma_parms = rdi->ibdev.dev.parent->dma_parms; + dma_set_coherent_mask(&rdi->ibdev.dev, + rdi->ibdev.dev.parent->coherent_dma_mask); /* Protection Domain */ spin_lock_init(&rdi->n_pds_lock); @@ -629,7 +629,7 @@ int rvt_register_device(struct rvt_dev_info *rdi) rdi->ibdev.num_comp_vectors = 1; /* We are now good to announce we exist */ - ret = ib_register_device(&rdi->ibdev, dev_name(&rdi->ibdev.dev)); + ret = ib_register_device(&rdi->ibdev, dev_name(&rdi->ibdev.dev), NULL); if (ret) { rvt_pr_err(rdi, "Failed to register driver with ib core.\n"); goto bail_wss; diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c index 77f2c7cd1216..95f0de0c8b49 100644 --- a/drivers/infiniband/sw/rxe/rxe.c +++ b/drivers/infiniband/sw/rxe/rxe.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include <rdma/rdma_netlink.h> @@ -279,6 +252,12 @@ static int rxe_newlink(const char *ibdev_name, struct net_device *ndev) struct rxe_dev *exists; int err = 0; + if (is_vlan_dev(ndev)) { + pr_err("rxe creation allowed on top of a real device only\n"); + err = -EPERM; + goto err; + } + exists = rxe_get_dev_from_net(ndev); if (exists) { ib_device_put(&exists->ib_dev); @@ -305,13 +284,6 @@ static int __init rxe_module_init(void) { int err; - /* initialize slab caches for managed objects */ - err = rxe_cache_init(); - if (err) { - pr_err("unable to init object pools\n"); - return err; - } - err = rxe_net_init(); if (err) return err; @@ -327,7 +299,6 @@ static void __exit rxe_module_exit(void) rdma_link_unregister(&rxe_link_ops); ib_unregister_driver(RDMA_DRIVER_RXE); rxe_net_exit(); - rxe_cache_exit(); rxe_initialized = false; pr_info("unloaded\n"); diff --git a/drivers/infiniband/sw/rxe/rxe.h b/drivers/infiniband/sw/rxe/rxe.h index cae1b0a24c85..623fd17df02d 100644 --- a/drivers/infiniband/sw/rxe/rxe.h +++ b/drivers/infiniband/sw/rxe/rxe.h @@ -1,34 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #ifndef RXE_H diff --git a/drivers/infiniband/sw/rxe/rxe_av.c b/drivers/infiniband/sw/rxe/rxe_av.c index 81ee756c19b8..38021e2c8688 100644 --- a/drivers/infiniband/sw/rxe/rxe_av.c +++ b/drivers/infiniband/sw/rxe/rxe_av.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include "rxe.h" diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c index 7b4df0028388..0a1e6393250b 100644 --- a/drivers/infiniband/sw/rxe/rxe_comp.c +++ b/drivers/infiniband/sw/rxe/rxe_comp.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include <linux/skbuff.h> @@ -690,9 +663,8 @@ int rxe_completer(void *arg) */ /* there is nothing to retry in this case */ - if (!wqe || (wqe->state == wqe_state_posted)) { + if (!wqe || (wqe->state == wqe_state_posted)) goto exit; - } /* if we've started a retry, don't start another * retry sequence, unless this is a timeout. diff --git a/drivers/infiniband/sw/rxe/rxe_cq.c b/drivers/infiniband/sw/rxe/rxe_cq.c index ad3090131126..43394c3f29d4 100644 --- a/drivers/infiniband/sw/rxe/rxe_cq.c +++ b/drivers/infiniband/sw/rxe/rxe_cq.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include <linux/vmalloc.h> #include "rxe.h" @@ -66,9 +39,9 @@ err1: return -EINVAL; } -static void rxe_send_complete(unsigned long data) +static void rxe_send_complete(struct tasklet_struct *t) { - struct rxe_cq *cq = (struct rxe_cq *)data; + struct rxe_cq *cq = from_tasklet(cq, t, comp_task); unsigned long flags; spin_lock_irqsave(&cq->cq_lock, flags); @@ -107,7 +80,7 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe, cq->is_dying = false; - tasklet_init(&cq->comp_task, rxe_send_complete, (unsigned long)cq); + tasklet_setup(&cq->comp_task, rxe_send_complete); spin_lock_init(&cq->cq_lock); cq->ibcq.cqe = cqe; diff --git a/drivers/infiniband/sw/rxe/rxe_hdr.h b/drivers/infiniband/sw/rxe/rxe_hdr.h index ce003666b800..3b483b75dfe3 100644 --- a/drivers/infiniband/sw/rxe/rxe_hdr.h +++ b/drivers/infiniband/sw/rxe/rxe_hdr.h @@ -1,34 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #ifndef RXE_HDR_H diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.c b/drivers/infiniband/sw/rxe/rxe_hw_counters.c index 636edb5f4cf4..ac9154f0593d 100644 --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.c +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.c @@ -1,33 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2017 Mellanox Technologies Ltd. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include "rxe.h" diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.h b/drivers/infiniband/sw/rxe/rxe_hw_counters.h index 72c0d63c79e0..49ee6f96656d 100644 --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.h +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.h @@ -1,33 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ /* * Copyright (c) 2017 Mellanox Technologies Ltd. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #ifndef RXE_HW_COUNTERS_H diff --git a/drivers/infiniband/sw/rxe/rxe_icrc.c b/drivers/infiniband/sw/rxe/rxe_icrc.c index 39e0be31aab1..66b2aad54bb7 100644 --- a/drivers/infiniband/sw/rxe/rxe_icrc.c +++ b/drivers/infiniband/sw/rxe/rxe_icrc.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include "rxe.h" diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h index 39dc3bfa5d5d..0d758760b9ae 100644 --- a/drivers/infiniband/sw/rxe/rxe_loc.h +++ b/drivers/infiniband/sw/rxe/rxe_loc.h @@ -1,34 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #ifndef RXE_LOC_H diff --git a/drivers/infiniband/sw/rxe/rxe_mcast.c b/drivers/infiniband/sw/rxe/rxe_mcast.c index 522a7942c56c..c02315aed8d1 100644 --- a/drivers/infiniband/sw/rxe/rxe_mcast.c +++ b/drivers/infiniband/sw/rxe/rxe_mcast.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include "rxe.h" diff --git a/drivers/infiniband/sw/rxe/rxe_mmap.c b/drivers/infiniband/sw/rxe/rxe_mmap.c index 7887f623f62c..035f226af133 100644 --- a/drivers/infiniband/sw/rxe/rxe_mmap.c +++ b/drivers/infiniband/sw/rxe/rxe_mmap.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include <linux/module.h> diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index ce24144de16a..d2ce852447c1 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include "rxe.h" @@ -79,13 +52,8 @@ static void rxe_mem_init(int access, struct rxe_mem *mem) u32 lkey = mem->pelem.index << 8 | rxe_get_key(); u32 rkey = (access & IB_ACCESS_REMOTE) ? lkey : 0; - if (mem->pelem.pool->type == RXE_TYPE_MR) { - mem->ibmr.lkey = lkey; - mem->ibmr.rkey = rkey; - } - - mem->lkey = lkey; - mem->rkey = rkey; + mem->ibmr.lkey = lkey; + mem->ibmr.rkey = rkey; mem->state = RXE_MEM_STATE_INVALID; mem->type = RXE_MEM_TYPE_NONE; mem->map_shift = ilog2(RXE_BUF_PER_MAP); @@ -149,7 +117,7 @@ void rxe_mem_init_dma(struct rxe_pd *pd, { rxe_mem_init(access, mem); - mem->pd = pd; + mem->ibmr.pd = &pd->ibpd; mem->access = access; mem->state = RXE_MEM_STATE_VALID; mem->type = RXE_MEM_TYPE_DMA; @@ -218,7 +186,7 @@ int rxe_mem_init_user(struct rxe_pd *pd, u64 start, } } - mem->pd = pd; + mem->ibmr.pd = &pd->ibpd; mem->umem = umem; mem->access = access; mem->length = length; @@ -248,7 +216,7 @@ int rxe_mem_init_fast(struct rxe_pd *pd, if (err) goto err1; - mem->pd = pd; + mem->ibmr.pd = &pd->ibpd; mem->max_buf = max_pages; mem->state = RXE_MEM_STATE_FREE; mem->type = RXE_MEM_TYPE_MR; @@ -368,7 +336,7 @@ int rxe_mem_copy(struct rxe_mem *mem, u64 iova, void *addr, int length, memcpy(dest, src, length); if (crcp) - *crcp = rxe_crc32(to_rdev(mem->pd->ibpd.device), + *crcp = rxe_crc32(to_rdev(mem->ibmr.device), *crcp, dest, length); return 0; @@ -402,7 +370,7 @@ int rxe_mem_copy(struct rxe_mem *mem, u64 iova, void *addr, int length, memcpy(dest, src, bytes); if (crcp) - crc = rxe_crc32(to_rdev(mem->pd->ibpd.device), + crc = rxe_crc32(to_rdev(mem->ibmr.device), crc, dest, bytes); length -= bytes; @@ -575,9 +543,9 @@ struct rxe_mem *lookup_mem(struct rxe_pd *pd, int access, u32 key, if (!mem) return NULL; - if (unlikely((type == lookup_local && mem->lkey != key) || - (type == lookup_remote && mem->rkey != key) || - mem->pd != pd || + if (unlikely((type == lookup_local && mr_lkey(mem) != key) || + (type == lookup_remote && mr_rkey(mem) != key) || + mr_pd(mem) != pd || (access && !(access & mem->access)) || mem->state != RXE_MEM_STATE_VALID)) { rxe_drop_ref(mem); diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index 0c3808611f95..575e1a4ec821 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include <linux/skbuff.h> @@ -120,7 +93,7 @@ static struct dst_entry *rxe_find_route6(struct net_device *ndev, ndst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(recv_sockets.sk6->sk), recv_sockets.sk6->sk, &fl6, NULL); - if (unlikely(IS_ERR(ndst))) { + if (IS_ERR(ndst)) { pr_err_ratelimited("no route to %pI6\n", daddr); return NULL; } @@ -160,14 +133,14 @@ static struct dst_entry *rxe_find_route(struct net_device *ndev, if (dst) dst_release(dst); - if (av->network_type == RDMA_NETWORK_IPV4) { + if (av->network_type == RXE_NETWORK_TYPE_IPV4) { struct in_addr *saddr; struct in_addr *daddr; saddr = &av->sgid_addr._sockaddr_in.sin_addr; daddr = &av->dgid_addr._sockaddr_in.sin_addr; dst = rxe_find_route4(ndev, saddr, daddr); - } else if (av->network_type == RDMA_NETWORK_IPV6) { + } else if (av->network_type == RXE_NETWORK_TYPE_IPV6) { struct in6_addr *saddr6; struct in6_addr *daddr6; @@ -469,7 +442,7 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, if (IS_ERR(attr)) return NULL; - if (av->network_type == RDMA_NETWORK_IPV4) + if (av->network_type == RXE_NETWORK_TYPE_IPV6) hdr_len = ETH_HLEN + sizeof(struct udphdr) + sizeof(struct iphdr); else @@ -496,7 +469,7 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, skb->dev = ndev; rcu_read_unlock(); - if (av->network_type == RDMA_NETWORK_IPV4) + if (av->network_type == RXE_NETWORK_TYPE_IPV4) skb->protocol = htons(ETH_P_IP); else skb->protocol = htons(ETH_P_IPV6); diff --git a/drivers/infiniband/sw/rxe/rxe_net.h b/drivers/infiniband/sw/rxe/rxe_net.h index 2ca71d3d245c..45d80d00f86b 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.h +++ b/drivers/infiniband/sw/rxe/rxe_net.h @@ -1,34 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #ifndef RXE_NET_H diff --git a/drivers/infiniband/sw/rxe/rxe_opcode.c b/drivers/infiniband/sw/rxe/rxe_opcode.c index 4cf11063e0b5..0cb4b01fd910 100644 --- a/drivers/infiniband/sw/rxe/rxe_opcode.c +++ b/drivers/infiniband/sw/rxe/rxe_opcode.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include <rdma/ib_pack.h> diff --git a/drivers/infiniband/sw/rxe/rxe_opcode.h b/drivers/infiniband/sw/rxe/rxe_opcode.h index 307604e9c78d..1041ac9a9233 100644 --- a/drivers/infiniband/sw/rxe/rxe_opcode.h +++ b/drivers/infiniband/sw/rxe/rxe_opcode.h @@ -1,34 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #ifndef RXE_OPCODE_H diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h index 2f381aeafcb5..25ab50d9b7c2 100644 --- a/drivers/infiniband/sw/rxe/rxe_param.h +++ b/drivers/infiniband/sw/rxe/rxe_param.h @@ -1,34 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #ifndef RXE_PARAM_H diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c index fbcbac52290b..b374eb53e2fe 100644 --- a/drivers/infiniband/sw/rxe/rxe_pool.c +++ b/drivers/infiniband/sw/rxe/rxe_pool.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include "rxe.h" @@ -110,62 +83,6 @@ static inline const char *pool_name(struct rxe_pool *pool) return rxe_type_info[pool->type].name; } -static inline struct kmem_cache *pool_cache(struct rxe_pool *pool) -{ - return rxe_type_info[pool->type].cache; -} - -static void rxe_cache_clean(size_t cnt) -{ - int i; - struct rxe_type_info *type; - - for (i = 0; i < cnt; i++) { - type = &rxe_type_info[i]; - if (!(type->flags & RXE_POOL_NO_ALLOC)) { - kmem_cache_destroy(type->cache); - type->cache = NULL; - } - } -} - -int rxe_cache_init(void) -{ - int err; - int i; - size_t size; - struct rxe_type_info *type; - - for (i = 0; i < RXE_NUM_TYPES; i++) { - type = &rxe_type_info[i]; - size = ALIGN(type->size, RXE_POOL_ALIGN); - if (!(type->flags & RXE_POOL_NO_ALLOC)) { - type->cache = - kmem_cache_create(type->name, size, - RXE_POOL_ALIGN, - RXE_POOL_CACHE_FLAGS, NULL); - if (!type->cache) { - pr_err("Unable to init kmem cache for %s\n", - type->name); - err = -ENOMEM; - goto err1; - } - } - } - - return 0; - -err1: - rxe_cache_clean(i); - - return err; -} - -void rxe_cache_exit(void) -{ - rxe_cache_clean(RXE_NUM_TYPES); -} - static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min) { int err = 0; @@ -406,7 +323,7 @@ void *rxe_alloc(struct rxe_pool *pool) if (atomic_inc_return(&pool->num_elem) > pool->max_elem) goto out_cnt; - elem = kmem_cache_zalloc(pool_cache(pool), + elem = kzalloc(rxe_type_info[pool->type].size, (pool->flags & RXE_POOL_ATOMIC) ? GFP_ATOMIC : GFP_KERNEL); if (!elem) @@ -468,7 +385,7 @@ void rxe_elem_release(struct kref *kref) pool->cleanup(elem); if (!(pool->flags & RXE_POOL_NO_ALLOC)) - kmem_cache_free(pool_cache(pool), elem); + kfree(elem); atomic_dec(&pool->num_elem); ib_device_put(&pool->rxe->ib_dev); rxe_pool_put(pool); diff --git a/drivers/infiniband/sw/rxe/rxe_pool.h b/drivers/infiniband/sw/rxe/rxe_pool.h index 2f2cff1cbe43..432745ffc8d4 100644 --- a/drivers/infiniband/sw/rxe/rxe_pool.h +++ b/drivers/infiniband/sw/rxe/rxe_pool.h @@ -1,34 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #ifndef RXE_POOL_H @@ -69,7 +42,6 @@ struct rxe_type_info { u32 min_index; size_t key_offset; size_t key_size; - struct kmem_cache *cache; }; extern struct rxe_type_info rxe_type_info[]; @@ -113,12 +85,6 @@ struct rxe_pool { size_t key_size; }; -/* initialize slab caches for managed objects */ -int rxe_cache_init(void); - -/* cleanup slab caches for managed objects */ -void rxe_cache_exit(void); - /* initialize a pool of objects with given limit on * number of elements. gets parameters from rxe_type_info * pool elements will be allocated out of a slab cache diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index 6c11c3aeeca6..656a5b4be847 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include <linux/skbuff.h> @@ -628,9 +601,8 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask, if (mask & IB_QP_QKEY) qp->attr.qkey = attr->qkey; - if (mask & IB_QP_AV) { + if (mask & IB_QP_AV) rxe_init_av(&attr->ah_attr, &qp->pri_av); - } if (mask & IB_QP_ALT_PATH) { rxe_init_av(&attr->alt_ah_attr, &qp->alt_av); diff --git a/drivers/infiniband/sw/rxe/rxe_queue.c b/drivers/infiniband/sw/rxe/rxe_queue.c index 245040c3a35d..fa69241b1187 100644 --- a/drivers/infiniband/sw/rxe/rxe_queue.c +++ b/drivers/infiniband/sw/rxe/rxe_queue.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must retailuce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include <linux/vmalloc.h> diff --git a/drivers/infiniband/sw/rxe/rxe_queue.h b/drivers/infiniband/sw/rxe/rxe_queue.h index 8ef17d617022..7d434a6837a7 100644 --- a/drivers/infiniband/sw/rxe/rxe_queue.h +++ b/drivers/infiniband/sw/rxe/rxe_queue.h @@ -1,34 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #ifndef RXE_QUEUE_H diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c index 7e123d3c4d09..c9984a28eecc 100644 --- a/drivers/infiniband/sw/rxe/rxe_recv.c +++ b/drivers/infiniband/sw/rxe/rxe_recv.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include <linux/skbuff.h> @@ -260,6 +233,8 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb) struct rxe_mc_elem *mce; struct rxe_qp *qp; union ib_gid dgid; + struct sk_buff *per_qp_skb; + struct rxe_pkt_info *per_qp_pkt; int err; if (skb->protocol == htons(ETH_P_IP)) @@ -288,26 +263,44 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb) if (err) continue; - /* if *not* the last qp in the list - * increase the users of the skb then post to the next qp + /* for all but the last qp create a new clone of the + * skb and pass to the qp. */ if (mce->qp_list.next != &mcg->qp_list) - skb_get(skb); + per_qp_skb = skb_clone(skb, GFP_ATOMIC); + else + per_qp_skb = skb; + + if (unlikely(!per_qp_skb)) + continue; - pkt->qp = qp; + per_qp_pkt = SKB_TO_PKT(per_qp_skb); + per_qp_pkt->qp = qp; rxe_add_ref(qp); - rxe_rcv_pkt(pkt, skb); + rxe_rcv_pkt(per_qp_pkt, per_qp_skb); } spin_unlock_bh(&mcg->mcg_lock); rxe_drop_ref(mcg); /* drop ref from rxe_pool_get_key. */ + return; + err1: kfree_skb(skb); } -static int rxe_match_dgid(struct rxe_dev *rxe, struct sk_buff *skb) +/** + * rxe_chk_dgid - validate destination IP address + * @rxe: rxe device that received packet + * @skb: the received packet buffer + * + * Accept any loopback packets + * Extract IP address from packet and + * Accept if multicast packet + * Accept if matches an SGID table entry + */ +static int rxe_chk_dgid(struct rxe_dev *rxe, struct sk_buff *skb) { struct rxe_pkt_info *pkt = SKB_TO_PKT(skb); const struct ib_gid_attr *gid_attr; @@ -325,6 +318,9 @@ static int rxe_match_dgid(struct rxe_dev *rxe, struct sk_buff *skb) pdgid = (union ib_gid *)&ipv6_hdr(skb)->daddr; } + if (rdma_is_multicast_addr((struct in6_addr *)pdgid)) + return 0; + gid_attr = rdma_find_gid_by_port(&rxe->ib_dev, pdgid, IB_GID_TYPE_ROCE_UDP_ENCAP, 1, skb->dev); @@ -349,8 +345,8 @@ void rxe_rcv(struct sk_buff *skb) if (unlikely(skb->len < pkt->offset + RXE_BTH_BYTES)) goto drop; - if (rxe_match_dgid(rxe, skb) < 0) { - pr_warn_ratelimited("failed matching dgid\n"); + if (rxe_chk_dgid(rxe, skb) < 0) { + pr_warn_ratelimited("failed checking dgid\n"); goto drop; } diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 34df2b55e650..af3923bf0a36 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include <linux/skbuff.h> @@ -644,8 +617,8 @@ next_wqe: rmr->state = RXE_MEM_STATE_VALID; rmr->access = wqe->wr.wr.reg.access; - rmr->lkey = wqe->wr.wr.reg.key; - rmr->rkey = wqe->wr.wr.reg.key; + rmr->ibmr.lkey = wqe->wr.wr.reg.key; + rmr->ibmr.rkey = wqe->wr.wr.reg.key; rmr->iova = wqe->wr.wr.reg.mr->iova; wqe->state = wqe_state_done; wqe->status = IB_WC_SUCCESS; diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index c4a8195bf670..c7e3b6a4af38 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include <linux/skbuff.h> diff --git a/drivers/infiniband/sw/rxe/rxe_srq.c b/drivers/infiniband/sw/rxe/rxe_srq.c index d8459431534e..41b0d1e11baf 100644 --- a/drivers/infiniband/sw/rxe/rxe_srq.c +++ b/drivers/infiniband/sw/rxe/rxe_srq.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include <linux/vmalloc.h> diff --git a/drivers/infiniband/sw/rxe/rxe_sysfs.c b/drivers/infiniband/sw/rxe/rxe_sysfs.c index 2af31d421bfc..666202ddff48 100644 --- a/drivers/infiniband/sw/rxe/rxe_sysfs.c +++ b/drivers/infiniband/sw/rxe/rxe_sysfs.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include "rxe.h" @@ -78,6 +51,12 @@ static int rxe_param_set_add(const char *val, const struct kernel_param *kp) return -EINVAL; } + if (is_vlan_dev(ndev)) { + pr_err("rxe creation allowed on top of a real device only\n"); + err = -EPERM; + goto err; + } + exists = rxe_get_dev_from_net(ndev); if (exists) { ib_device_put(&exists->ib_dev); diff --git a/drivers/infiniband/sw/rxe/rxe_task.c b/drivers/infiniband/sw/rxe/rxe_task.c index ecdac3f8fcc9..6951fdcb31bf 100644 --- a/drivers/infiniband/sw/rxe/rxe_task.c +++ b/drivers/infiniband/sw/rxe/rxe_task.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include <linux/kernel.h> @@ -55,12 +28,12 @@ int __rxe_do_task(struct rxe_task *task) * a second caller finds the task already running * but looks just after the last call to func */ -void rxe_do_task(unsigned long data) +void rxe_do_task(struct tasklet_struct *t) { int cont; int ret; unsigned long flags; - struct rxe_task *task = (struct rxe_task *)data; + struct rxe_task *task = from_tasklet(task, t, tasklet); spin_lock_irqsave(&task->state_lock, flags); switch (task->state) { @@ -123,7 +96,7 @@ int rxe_init_task(void *obj, struct rxe_task *task, snprintf(task->name, sizeof(task->name), "%s", name); task->destroyed = false; - tasklet_init(&task->tasklet, rxe_do_task, (unsigned long)task); + tasklet_setup(&task->tasklet, rxe_do_task); task->state = TASK_STATE_START; spin_lock_init(&task->state_lock); @@ -159,7 +132,7 @@ void rxe_run_task(struct rxe_task *task, int sched) if (sched) tasklet_schedule(&task->tasklet); else - rxe_do_task((unsigned long)task); + rxe_do_task(&task->tasklet); } void rxe_disable_task(struct rxe_task *task) diff --git a/drivers/infiniband/sw/rxe/rxe_task.h b/drivers/infiniband/sw/rxe/rxe_task.h index 08ff42d451c6..11d183fd3338 100644 --- a/drivers/infiniband/sw/rxe/rxe_task.h +++ b/drivers/infiniband/sw/rxe/rxe_task.h @@ -1,34 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #ifndef RXE_TASK_H @@ -60,7 +33,7 @@ struct rxe_task { /* * init rxe_task structure * arg => parameter to pass to fcn - * fcn => function to call until it returns != 0 + * func => function to call until it returns != 0 */ int rxe_init_task(void *obj, struct rxe_task *task, void *arg, int (*func)(void *), char *name); @@ -80,7 +53,7 @@ int __rxe_do_task(struct rxe_task *task); * work to do someone must reschedule the task before * leaving */ -void rxe_do_task(unsigned long data); +void rxe_do_task(struct tasklet_struct *t); /* run a task, else schedule it to run as a tasklet, The decision * to run or schedule tasklet is based on the parameter sched. diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index 8522e9a3e914..1fc022362fbe 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include <linux/dma-mapping.h> @@ -175,11 +148,12 @@ static int rxe_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) return rxe_add_to_pool(&rxe->pd_pool, &pd->pelem); } -static void rxe_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +static int rxe_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct rxe_pd *pd = to_rpd(ibpd); rxe_drop_ref(pd); + return 0; } static int rxe_create_ah(struct ib_ah *ibah, @@ -227,11 +201,12 @@ static int rxe_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr) return 0; } -static void rxe_destroy_ah(struct ib_ah *ibah, u32 flags) +static int rxe_destroy_ah(struct ib_ah *ibah, u32 flags) { struct rxe_ah *ah = to_rah(ibah); rxe_drop_ref(ah); + return 0; } static int post_one_recv(struct rxe_rq *rq, const struct ib_recv_wr *ibwr) @@ -365,7 +340,7 @@ static int rxe_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) return 0; } -static void rxe_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) +static int rxe_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) { struct rxe_srq *srq = to_rsrq(ibsrq); @@ -374,6 +349,7 @@ static void rxe_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) rxe_drop_ref(srq->pd); rxe_drop_ref(srq); + return 0; } static int rxe_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, @@ -803,13 +779,14 @@ static int rxe_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, return rxe_add_to_pool(&rxe->cq_pool, &cq->pelem); } -static void rxe_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +static int rxe_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct rxe_cq *cq = to_rcq(ibcq); rxe_cq_disable(cq); rxe_drop_ref(cq); + return 0; } static int rxe_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata) @@ -944,7 +921,7 @@ static int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) struct rxe_mem *mr = to_rmr(ibmr); mr->state = RXE_MEM_STATE_ZOMBIE; - rxe_drop_ref(mr->pd); + rxe_drop_ref(mr_pd(mr)); rxe_drop_index(mr); rxe_drop_ref(mr); return 0; @@ -1151,12 +1128,9 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name) dev->local_dma_lkey = 0; addrconf_addr_eui48((unsigned char *)&dev->node_guid, rxe->ndev->dev_addr); - dev->dev.dma_ops = &dma_virt_ops; dev->dev.dma_parms = &rxe->dma_parms; - rxe->dma_parms = (struct device_dma_parameters) - { .max_segment_size = SZ_2G }; - dma_coerce_mask_and_coherent(&dev->dev, - dma_get_required_mask(&dev->dev)); + dma_set_max_seg_size(&dev->dev, UINT_MAX); + dma_set_coherent_mask(&dev->dev, dma_get_required_mask(&dev->dev)); dev->uverbs_cmd_mask = BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT) | BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) @@ -1205,7 +1179,7 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name) rxe->tfm = tfm; rdma_set_device_sysfs_group(dev, &rxe_attr_group); - err = ib_register_device(dev, ibdev_name); + err = ib_register_device(dev, ibdev_name, NULL); if (err) pr_warn("%s failed with error %d\n", __func__, err); diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index c664c7f36ab5..3414b341b709 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -1,34 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ /* * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #ifndef RXE_VERBS_H @@ -322,12 +295,8 @@ struct rxe_mem { struct ib_mw ibmw; }; - struct rxe_pd *pd; struct ib_umem *umem; - u32 lkey; - u32 rkey; - enum rxe_mem_state state; enum rxe_mem_type type; u64 va; @@ -465,6 +434,21 @@ static inline struct rxe_mem *to_rmw(struct ib_mw *mw) return mw ? container_of(mw, struct rxe_mem, ibmw) : NULL; } +static inline struct rxe_pd *mr_pd(struct rxe_mem *mr) +{ + return to_rpd(mr->ibmr.pd); +} + +static inline u32 mr_lkey(struct rxe_mem *mr) +{ + return mr->ibmr.lkey; +} + +static inline u32 mr_rkey(struct rxe_mem *mr) +{ + return mr->ibmr.rkey; +} + int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name); void rxe_mc_cleanup(struct rxe_pool_entry *arg); diff --git a/drivers/infiniband/sw/siw/siw_main.c b/drivers/infiniband/sw/siw/siw_main.c index d862bec84376..ca8bc7296867 100644 --- a/drivers/infiniband/sw/siw/siw_main.c +++ b/drivers/infiniband/sw/siw/siw_main.c @@ -69,7 +69,7 @@ static int siw_device_register(struct siw_device *sdev, const char *name) sdev->vendor_part_id = dev_id++; - rv = ib_register_device(base_dev, name); + rv = ib_register_device(base_dev, name, NULL); if (rv) { pr_warn("siw: device registration error %d\n", rv); return rv; @@ -382,10 +382,10 @@ static struct siw_device *siw_device_create(struct net_device *netdev) */ base_dev->phys_port_cnt = 1; base_dev->dev.parent = parent; - base_dev->dev.dma_ops = &dma_virt_ops; base_dev->dev.dma_parms = &sdev->dma_parms; - sdev->dma_parms = (struct device_dma_parameters) - { .max_segment_size = SZ_2G }; + dma_set_max_seg_size(&base_dev->dev, UINT_MAX); + dma_set_coherent_mask(&base_dev->dev, + dma_get_required_mask(&base_dev->dev)); base_dev->num_comp_vectors = num_possible_cpus(); xa_init_flags(&sdev->qp_xa, XA_FLAGS_ALLOC1); diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c index adafa1b8bebe..7cf3242ffb41 100644 --- a/drivers/infiniband/sw/siw/siw_verbs.c +++ b/drivers/infiniband/sw/siw/siw_verbs.c @@ -234,12 +234,13 @@ int siw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) return 0; } -void siw_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) +int siw_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) { struct siw_device *sdev = to_siw_dev(pd->device); siw_dbg_pd(pd, "free PD\n"); atomic_dec(&sdev->num_pd); + return 0; } void siw_qp_get_ref(struct ib_qp *base_qp) @@ -1055,7 +1056,7 @@ int siw_post_receive(struct ib_qp *base_qp, const struct ib_recv_wr *wr, return rv > 0 ? 0 : rv; } -void siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata) +int siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata) { struct siw_cq *cq = to_siw_cq(base_cq); struct siw_device *sdev = to_siw_dev(base_cq->device); @@ -1073,6 +1074,7 @@ void siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata) atomic_dec(&sdev->num_cq); vfree(cq->queue); + return 0; } /* @@ -1690,7 +1692,7 @@ int siw_query_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs) * QP anymore - the code trusts the RDMA core environment to keep track * of QP references. */ -void siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata) +int siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata) { struct siw_srq *srq = to_siw_srq(base_srq); struct siw_device *sdev = to_siw_dev(base_srq->device); @@ -1702,6 +1704,7 @@ void siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata) rdma_user_mmap_entry_remove(srq->srq_entry); vfree(srq->recvq); atomic_dec(&sdev->num_srq); + return 0; } /* diff --git a/drivers/infiniband/sw/siw/siw_verbs.h b/drivers/infiniband/sw/siw/siw_verbs.h index d9572275a6b6..637454529357 100644 --- a/drivers/infiniband/sw/siw/siw_verbs.h +++ b/drivers/infiniband/sw/siw/siw_verbs.h @@ -49,7 +49,7 @@ int siw_query_port(struct ib_device *base_dev, u8 port, int siw_query_gid(struct ib_device *base_dev, u8 port, int idx, union ib_gid *gid); int siw_alloc_pd(struct ib_pd *base_pd, struct ib_udata *udata); -void siw_dealloc_pd(struct ib_pd *base_pd, struct ib_udata *udata); +int siw_dealloc_pd(struct ib_pd *base_pd, struct ib_udata *udata); struct ib_qp *siw_create_qp(struct ib_pd *base_pd, struct ib_qp_init_attr *attr, struct ib_udata *udata); @@ -62,7 +62,7 @@ int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr); int siw_post_receive(struct ib_qp *base_qp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr); -void siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata); +int siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata); int siw_poll_cq(struct ib_cq *base_cq, int num_entries, struct ib_wc *wc); int siw_req_notify_cq(struct ib_cq *base_cq, enum ib_cq_notify_flags flags); struct ib_mr *siw_reg_user_mr(struct ib_pd *base_pd, u64 start, u64 len, @@ -78,7 +78,7 @@ int siw_create_srq(struct ib_srq *base_srq, struct ib_srq_init_attr *attr, int siw_modify_srq(struct ib_srq *base_srq, struct ib_srq_attr *attr, enum ib_srq_attr_mask mask, struct ib_udata *udata); int siw_query_srq(struct ib_srq *base_srq, struct ib_srq_attr *attr); -void siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata); +int siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata); int siw_post_srq_recv(struct ib_srq *base_srq, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr); int siw_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma); diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c index 7c41fb040f7c..8f0b598a46ec 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c @@ -1647,17 +1647,13 @@ int ipoib_cm_dev_init(struct net_device *dev) void ipoib_cm_dev_cleanup(struct net_device *dev) { struct ipoib_dev_priv *priv = ipoib_priv(dev); - int ret; if (!priv->cm.srq) return; ipoib_dbg(priv, "Cleanup ipoib connected mode.\n"); - ret = ib_destroy_srq(priv->cm.srq); - if (ret) - ipoib_warn(priv, "ib_destroy_srq failed: %d\n", ret); - + ib_destroy_srq(priv->cm.srq); priv->cm.srq = NULL; if (!priv->cm.srq_ring) return; diff --git a/drivers/infiniband/ulp/ipoib/ipoib_fs.c b/drivers/infiniband/ulp/ipoib/ipoib_fs.c index 64c19f6fa931..12ba7a0fe0b5 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_fs.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_fs.c @@ -124,35 +124,14 @@ static int ipoib_mcg_seq_show(struct seq_file *file, void *iter_ptr) return 0; } -static const struct seq_operations ipoib_mcg_seq_ops = { +static const struct seq_operations ipoib_mcg_sops = { .start = ipoib_mcg_seq_start, .next = ipoib_mcg_seq_next, .stop = ipoib_mcg_seq_stop, .show = ipoib_mcg_seq_show, }; -static int ipoib_mcg_open(struct inode *inode, struct file *file) -{ - struct seq_file *seq; - int ret; - - ret = seq_open(file, &ipoib_mcg_seq_ops); - if (ret) - return ret; - - seq = file->private_data; - seq->private = inode->i_private; - - return 0; -} - -static const struct file_operations ipoib_mcg_fops = { - .owner = THIS_MODULE, - .open = ipoib_mcg_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release -}; +DEFINE_SEQ_ATTRIBUTE(ipoib_mcg); static void *ipoib_path_seq_start(struct seq_file *file, loff_t *pos) { @@ -229,35 +208,14 @@ static int ipoib_path_seq_show(struct seq_file *file, void *iter_ptr) return 0; } -static const struct seq_operations ipoib_path_seq_ops = { +static const struct seq_operations ipoib_path_sops = { .start = ipoib_path_seq_start, .next = ipoib_path_seq_next, .stop = ipoib_path_seq_stop, .show = ipoib_path_seq_show, }; -static int ipoib_path_open(struct inode *inode, struct file *file) -{ - struct seq_file *seq; - int ret; - - ret = seq_open(file, &ipoib_path_seq_ops); - if (ret) - return ret; - - seq = file->private_data; - seq->private = inode->i_private; - - return 0; -} - -static const struct file_operations ipoib_path_fops = { - .owner = THIS_MODULE, - .open = ipoib_path_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release -}; +DEFINE_SEQ_ATTRIBUTE(ipoib_path); void ipoib_create_debug_files(struct net_device *dev) { diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index f772fe8c5b66..abfab89423f4 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -2480,6 +2480,8 @@ static struct net_device *ipoib_add_port(const char *format, /* call event handler to ensure pkey in sync */ queue_work(ipoib_workqueue, &priv->flush_heavy); + ndev->rtnl_link_ops = ipoib_get_link_ops(); + result = register_netdev(ndev); if (result) { pr_warn("%s: couldn't register ipoib port %d; error %d\n", diff --git a/drivers/infiniband/ulp/ipoib/ipoib_netlink.c b/drivers/infiniband/ulp/ipoib/ipoib_netlink.c index 38c984d16996..d5a90a66b45c 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_netlink.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_netlink.c @@ -144,6 +144,16 @@ static int ipoib_new_child_link(struct net *src_net, struct net_device *dev, return 0; } +static void ipoib_del_child_link(struct net_device *dev, struct list_head *head) +{ + struct ipoib_dev_priv *priv = ipoib_priv(dev); + + if (!priv->parent) + return; + + unregister_netdevice_queue(dev, head); +} + static size_t ipoib_get_size(const struct net_device *dev) { return nla_total_size(2) + /* IFLA_IPOIB_PKEY */ @@ -158,6 +168,7 @@ static struct rtnl_link_ops ipoib_link_ops __read_mostly = { .priv_size = sizeof(struct ipoib_dev_priv), .setup = ipoib_setup_common, .newlink = ipoib_new_child_link, + .dellink = ipoib_del_child_link, .changelink = ipoib_changelink, .get_size = ipoib_get_size, .fill_info = ipoib_fill_info, diff --git a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c index 30865605e098..4c50a87ed7cc 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c @@ -195,6 +195,8 @@ int ipoib_vlan_add(struct net_device *pdev, unsigned short pkey) } priv = ipoib_priv(ndev); + ndev->rtnl_link_ops = ipoib_get_link_ops(); + result = __ipoib_vlan_add(ppriv, priv, pkey, IPOIB_LEGACY_CHILD); if (result && ndev->reg_state == NETREG_UNINITIALIZED) diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c index 695f701dc43d..436e17f1d0e5 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.c +++ b/drivers/infiniband/ulp/isert/ib_isert.c @@ -1141,12 +1141,7 @@ isert_handle_iscsi_dataout(struct isert_conn *isert_conn, * multiple data-outs on the same command can arrive - * so post the buffer before hand */ - rc = isert_post_recv(isert_conn, rx_desc); - if (rc) { - isert_err("ib_post_recv failed with %d\n", rc); - return rc; - } - return 0; + return isert_post_recv(isert_conn, rx_desc); } static int @@ -1723,10 +1718,8 @@ isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) int ret; ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); - if (ret) { - isert_err("ib_post_recv failed with %d\n", ret); + if (ret) return ret; - } ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL); if (ret) { @@ -2098,10 +2091,8 @@ isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) &isert_cmd->tx_desc.send_wr); rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); - if (rc) { - isert_err("ib_post_recv failed with %d\n", rc); + if (rc) return rc; - } chain_wr = &isert_cmd->tx_desc.send_wr; } diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c b/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c index 298b747d0330..ac4c49cbf153 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c @@ -312,7 +312,7 @@ static struct attribute *rtrs_clt_stats_attrs[] = { NULL }; -static struct attribute_group rtrs_clt_stats_attr_group = { +static const struct attribute_group rtrs_clt_stats_attr_group = { .attrs = rtrs_clt_stats_attrs, }; @@ -388,7 +388,7 @@ static struct attribute *rtrs_clt_sess_attrs[] = { NULL, }; -static struct attribute_group rtrs_clt_sess_attr_group = { +static const struct attribute_group rtrs_clt_sess_attr_group = { .attrs = rtrs_clt_sess_attrs, }; @@ -460,7 +460,7 @@ static struct attribute *rtrs_clt_attrs[] = { NULL, }; -static struct attribute_group rtrs_clt_attr_group = { +static const struct attribute_group rtrs_clt_attr_group = { .attrs = rtrs_clt_attrs, }; diff --git a/drivers/infiniband/ulp/rtrs/rtrs-pri.h b/drivers/infiniband/ulp/rtrs/rtrs-pri.h index 0a93c87ef92b..b8e43dc4d95a 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-pri.h +++ b/drivers/infiniband/ulp/rtrs/rtrs-pri.h @@ -115,7 +115,6 @@ struct rtrs_sess { /* rtrs information unit */ struct rtrs_iu { - struct list_head list; struct ib_cqe cqe; dma_addr_t dma_addr; void *buf; diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c b/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c index cf6a2be61695..07fbb063555d 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c @@ -135,7 +135,7 @@ static struct attribute *rtrs_srv_sess_attrs[] = { NULL, }; -static struct attribute_group rtrs_srv_sess_attr_group = { +static const struct attribute_group rtrs_srv_sess_attr_group = { .attrs = rtrs_srv_sess_attrs, }; @@ -148,7 +148,7 @@ static struct attribute *rtrs_srv_stats_attrs[] = { NULL, }; -static struct attribute_group rtrs_srv_stats_attr_group = { +static const struct attribute_group rtrs_srv_stats_attr_group = { .attrs = rtrs_srv_stats_attrs, }; diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c index 28f6414dfa3d..d6f93601712e 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c @@ -16,6 +16,7 @@ #include "rtrs-srv.h" #include "rtrs-log.h" #include <rdma/ib_cm.h> +#include <rdma/ib_verbs.h> MODULE_DESCRIPTION("RDMA Transport Server"); MODULE_LICENSE("GPL"); @@ -31,6 +32,7 @@ MODULE_LICENSE("GPL"); static struct rtrs_rdma_dev_pd dev_pd; static mempool_t *chunk_pool; struct class *rtrs_dev_class; +static struct rtrs_srv_ib_ctx ib_ctx; static int __read_mostly max_chunk_size = DEFAULT_MAX_CHUNK_SIZE; static int __read_mostly sess_queue_depth = DEFAULT_SESS_QUEUE_DEPTH; @@ -2042,6 +2044,70 @@ static void free_srv_ctx(struct rtrs_srv_ctx *ctx) kfree(ctx); } +static int rtrs_srv_add_one(struct ib_device *device) +{ + struct rtrs_srv_ctx *ctx; + int ret = 0; + + mutex_lock(&ib_ctx.ib_dev_mutex); + if (ib_ctx.ib_dev_count) + goto out; + + /* + * Since our CM IDs are NOT bound to any ib device we will create them + * only once + */ + ctx = ib_ctx.srv_ctx; + ret = rtrs_srv_rdma_init(ctx, ib_ctx.port); + if (ret) { + /* + * We errored out here. + * According to the ib code, if we encounter an error here then the + * error code is ignored, and no more calls to our ops are made. + */ + pr_err("Failed to initialize RDMA connection"); + goto err_out; + } + +out: + /* + * Keep a track on the number of ib devices added + */ + ib_ctx.ib_dev_count++; + +err_out: + mutex_unlock(&ib_ctx.ib_dev_mutex); + return ret; +} + +static void rtrs_srv_remove_one(struct ib_device *device, void *client_data) +{ + struct rtrs_srv_ctx *ctx; + + mutex_lock(&ib_ctx.ib_dev_mutex); + ib_ctx.ib_dev_count--; + + if (ib_ctx.ib_dev_count) + goto out; + + /* + * Since our CM IDs are NOT bound to any ib device we will remove them + * only once, when the last device is removed + */ + ctx = ib_ctx.srv_ctx; + rdma_destroy_id(ctx->cm_id_ip); + rdma_destroy_id(ctx->cm_id_ib); + +out: + mutex_unlock(&ib_ctx.ib_dev_mutex); +} + +static struct ib_client rtrs_srv_client = { + .name = "rtrs_server", + .add = rtrs_srv_add_one, + .remove = rtrs_srv_remove_one +}; + /** * rtrs_srv_open() - open RTRS server context * @ops: callback functions @@ -2060,7 +2126,11 @@ struct rtrs_srv_ctx *rtrs_srv_open(struct rtrs_srv_ops *ops, u16 port) if (!ctx) return ERR_PTR(-ENOMEM); - err = rtrs_srv_rdma_init(ctx, port); + mutex_init(&ib_ctx.ib_dev_mutex); + ib_ctx.srv_ctx = ctx; + ib_ctx.port = port; + + err = ib_register_client(&rtrs_srv_client); if (err) { free_srv_ctx(ctx); return ERR_PTR(err); @@ -2099,8 +2169,8 @@ static void close_ctx(struct rtrs_srv_ctx *ctx) */ void rtrs_srv_close(struct rtrs_srv_ctx *ctx) { - rdma_destroy_id(ctx->cm_id_ip); - rdma_destroy_id(ctx->cm_id_ib); + ib_unregister_client(&rtrs_srv_client); + mutex_destroy(&ib_ctx.ib_dev_mutex); close_ctx(ctx); free_srv_ctx(ctx); } diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.h b/drivers/infiniband/ulp/rtrs/rtrs-srv.h index dc95b0932f0d..08b0b8a6eebe 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.h +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.h @@ -118,6 +118,13 @@ struct rtrs_srv_ctx { struct list_head srv_list; }; +struct rtrs_srv_ib_ctx { + struct rtrs_srv_ctx *srv_ctx; + u16 port; + struct mutex ib_dev_mutex; + int ib_dev_count; +}; + extern struct class *rtrs_dev_class; void close_sess(struct rtrs_srv_sess *sess); diff --git a/drivers/input/evdev.c b/drivers/input/evdev.c index e494295d1c7b..95f90699d2b1 100644 --- a/drivers/input/evdev.c +++ b/drivers/input/evdev.c @@ -28,7 +28,6 @@ struct evdev { int open; struct input_handle handle; - wait_queue_head_t wait; struct evdev_client __rcu *grab; struct list_head client_list; spinlock_t client_lock; /* protects client_list */ @@ -43,6 +42,7 @@ struct evdev_client { unsigned int tail; unsigned int packet_head; /* [future] position of the first element of next packet */ spinlock_t buffer_lock; /* protects access to buffer, head and tail */ + wait_queue_head_t wait; struct fasync_struct *fasync; struct evdev *evdev; struct list_head node; @@ -245,7 +245,6 @@ static void evdev_pass_values(struct evdev_client *client, const struct input_value *vals, unsigned int count, ktime_t *ev_time) { - struct evdev *evdev = client->evdev; const struct input_value *v; struct input_event event; struct timespec64 ts; @@ -282,7 +281,7 @@ static void evdev_pass_values(struct evdev_client *client, spin_unlock(&client->buffer_lock); if (wakeup) - wake_up_interruptible_poll(&evdev->wait, + wake_up_interruptible_poll(&client->wait, EPOLLIN | EPOLLOUT | EPOLLRDNORM | EPOLLWRNORM); } @@ -426,11 +425,11 @@ static void evdev_hangup(struct evdev *evdev) struct evdev_client *client; spin_lock(&evdev->client_lock); - list_for_each_entry(client, &evdev->client_list, node) + list_for_each_entry(client, &evdev->client_list, node) { kill_fasync(&client->fasync, SIGIO, POLL_HUP); + wake_up_interruptible_poll(&client->wait, EPOLLHUP | EPOLLERR); + } spin_unlock(&evdev->client_lock); - - wake_up_interruptible_poll(&evdev->wait, EPOLLHUP | EPOLLERR); } static int evdev_release(struct inode *inode, struct file *file) @@ -479,6 +478,7 @@ static int evdev_open(struct inode *inode, struct file *file) if (!client) return -ENOMEM; + init_waitqueue_head(&client->wait); client->bufsize = bufsize; spin_lock_init(&client->buffer_lock); client->evdev = evdev; @@ -595,7 +595,7 @@ static ssize_t evdev_read(struct file *file, char __user *buffer, break; if (!(file->f_flags & O_NONBLOCK)) { - error = wait_event_interruptible(evdev->wait, + error = wait_event_interruptible(client->wait, client->packet_head != client->tail || !evdev->exist || client->revoked); if (error) @@ -613,7 +613,7 @@ static __poll_t evdev_poll(struct file *file, poll_table *wait) struct evdev *evdev = client->evdev; __poll_t mask; - poll_wait(file, &evdev->wait, wait); + poll_wait(file, &client->wait, wait); if (evdev->exist && !client->revoked) mask = EPOLLOUT | EPOLLWRNORM; @@ -946,7 +946,7 @@ static int evdev_revoke(struct evdev *evdev, struct evdev_client *client, client->revoked = true; evdev_ungrab(evdev, client); input_flush_device(&evdev->handle, file); - wake_up_interruptible_poll(&evdev->wait, EPOLLHUP | EPOLLERR); + wake_up_interruptible_poll(&client->wait, EPOLLHUP | EPOLLERR); return 0; } @@ -1358,7 +1358,6 @@ static int evdev_connect(struct input_handler *handler, struct input_dev *dev, INIT_LIST_HEAD(&evdev->client_list); spin_lock_init(&evdev->client_lock); mutex_init(&evdev->mutex); - init_waitqueue_head(&evdev->wait); evdev->exist = true; dev_no = minor; diff --git a/drivers/input/input-mt.c b/drivers/input/input-mt.c index f699538bdac4..44fe6f2f063c 100644 --- a/drivers/input/input-mt.c +++ b/drivers/input/input-mt.c @@ -323,11 +323,14 @@ static int adjust_dual(int *begin, int step, int *end, int eq, int mu) p = begin + step; s = p == end ? f + 1 : *p; - for (; p != end; p += step) - if (*p < f) - s = f, f = *p; - else if (*p < s) + for (; p != end; p += step) { + if (*p < f) { + s = f; + f = *p; + } else if (*p < s) { s = *p; + } + } c = (f + s + 1) / 2; if (c == 0 || (c > mu && (!eq || mu > 0))) diff --git a/drivers/input/joystick/Kconfig b/drivers/input/joystick/Kconfig index eb031b7a4866..b080f0cfb068 100644 --- a/drivers/input/joystick/Kconfig +++ b/drivers/input/joystick/Kconfig @@ -42,6 +42,16 @@ config JOYSTICK_A3D To compile this driver as a module, choose M here: the module will be called a3d. +config JOYSTICK_ADC + tristate "Simple joystick connected over ADC" + depends on IIO + select IIO_BUFFER_CB + help + Say Y here if you have a simple joystick connected over ADC. + + To compile this driver as a module, choose M here: the + module will be called adc-joystick. + config JOYSTICK_ADI tristate "Logitech ADI digital joysticks and gamepads" select GAMEPORT diff --git a/drivers/input/joystick/Makefile b/drivers/input/joystick/Makefile index 8656023f6ef5..58232b3057d3 100644 --- a/drivers/input/joystick/Makefile +++ b/drivers/input/joystick/Makefile @@ -6,6 +6,7 @@ # Each configuration option enables a list of files. obj-$(CONFIG_JOYSTICK_A3D) += a3d.o +obj-$(CONFIG_JOYSTICK_ADC) += adc-joystick.o obj-$(CONFIG_JOYSTICK_ADI) += adi.o obj-$(CONFIG_JOYSTICK_AMIGA) += amijoy.o obj-$(CONFIG_JOYSTICK_AS5011) += as5011.o diff --git a/drivers/input/joystick/adc-joystick.c b/drivers/input/joystick/adc-joystick.c new file mode 100644 index 000000000000..78ebca7d400a --- /dev/null +++ b/drivers/input/joystick/adc-joystick.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Input driver for joysticks connected over ADC. + * Copyright (c) 2019-2020 Artur Rojek <contact@artur-rojek.eu> + */ +#include <linux/ctype.h> +#include <linux/input.h> +#include <linux/iio/iio.h> +#include <linux/iio/consumer.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/property.h> + +#include <asm/unaligned.h> + +struct adc_joystick_axis { + u32 code; + s32 range[2]; + s32 fuzz; + s32 flat; +}; + +struct adc_joystick { + struct input_dev *input; + struct iio_cb_buffer *buffer; + struct adc_joystick_axis *axes; + struct iio_channel *chans; + int num_chans; +}; + +static int adc_joystick_handle(const void *data, void *private) +{ + struct adc_joystick *joy = private; + enum iio_endian endianness; + int bytes, msb, val, idx, i; + const u16 *data_u16; + bool sign; + + bytes = joy->chans[0].channel->scan_type.storagebits >> 3; + + for (i = 0; i < joy->num_chans; ++i) { + idx = joy->chans[i].channel->scan_index; + endianness = joy->chans[i].channel->scan_type.endianness; + msb = joy->chans[i].channel->scan_type.realbits - 1; + sign = tolower(joy->chans[i].channel->scan_type.sign) == 's'; + + switch (bytes) { + case 1: + val = ((const u8 *)data)[idx]; + break; + case 2: + data_u16 = (const u16 *)data + idx; + + /* + * Data is aligned to the sample size by IIO core. + * Call `get_unaligned_xe16` to hide type casting. + */ + if (endianness == IIO_BE) + val = get_unaligned_be16(data_u16); + else if (endianness == IIO_LE) + val = get_unaligned_le16(data_u16); + else /* IIO_CPU */ + val = *data_u16; + break; + default: + return -EINVAL; + } + + val >>= joy->chans[i].channel->scan_type.shift; + if (sign) + val = sign_extend32(val, msb); + else + val &= GENMASK(msb, 0); + input_report_abs(joy->input, joy->axes[i].code, val); + } + + input_sync(joy->input); + + return 0; +} + +static int adc_joystick_open(struct input_dev *dev) +{ + struct adc_joystick *joy = input_get_drvdata(dev); + struct device *devp = &dev->dev; + int ret; + + ret = iio_channel_start_all_cb(joy->buffer); + if (ret) + dev_err(devp, "Unable to start callback buffer: %d\n", ret); + + return ret; +} + +static void adc_joystick_close(struct input_dev *dev) +{ + struct adc_joystick *joy = input_get_drvdata(dev); + + iio_channel_stop_all_cb(joy->buffer); +} + +static void adc_joystick_cleanup(void *data) +{ + iio_channel_release_all_cb(data); +} + +static int adc_joystick_set_axes(struct device *dev, struct adc_joystick *joy) +{ + struct adc_joystick_axis *axes; + struct fwnode_handle *child; + int num_axes, error, i; + + num_axes = device_get_child_node_count(dev); + if (!num_axes) { + dev_err(dev, "Unable to find child nodes\n"); + return -EINVAL; + } + + if (num_axes != joy->num_chans) { + dev_err(dev, "Got %d child nodes for %d channels\n", + num_axes, joy->num_chans); + return -EINVAL; + } + + axes = devm_kmalloc_array(dev, num_axes, sizeof(*axes), GFP_KERNEL); + if (!axes) + return -ENOMEM; + + device_for_each_child_node(dev, child) { + error = fwnode_property_read_u32(child, "reg", &i); + if (error) { + dev_err(dev, "reg invalid or missing\n"); + goto err_fwnode_put; + } + + if (i >= num_axes) { + error = -EINVAL; + dev_err(dev, "No matching axis for reg %d\n", i); + goto err_fwnode_put; + } + + error = fwnode_property_read_u32(child, "linux,code", + &axes[i].code); + if (error) { + dev_err(dev, "linux,code invalid or missing\n"); + goto err_fwnode_put; + } + + error = fwnode_property_read_u32_array(child, "abs-range", + axes[i].range, 2); + if (error) { + dev_err(dev, "abs-range invalid or missing\n"); + goto err_fwnode_put; + } + + fwnode_property_read_u32(child, "abs-fuzz", &axes[i].fuzz); + fwnode_property_read_u32(child, "abs-flat", &axes[i].flat); + + input_set_abs_params(joy->input, axes[i].code, + axes[i].range[0], axes[i].range[1], + axes[i].fuzz, axes[i].flat); + input_set_capability(joy->input, EV_ABS, axes[i].code); + } + + joy->axes = axes; + + return 0; + +err_fwnode_put: + fwnode_handle_put(child); + return error; +} + +static int adc_joystick_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct adc_joystick *joy; + struct input_dev *input; + int error; + int bits; + int i; + + joy = devm_kzalloc(dev, sizeof(*joy), GFP_KERNEL); + if (!joy) + return -ENOMEM; + + joy->chans = devm_iio_channel_get_all(dev); + if (IS_ERR(joy->chans)) { + error = PTR_ERR(joy->chans); + if (error != -EPROBE_DEFER) + dev_err(dev, "Unable to get IIO channels"); + return error; + } + + /* Count how many channels we got. NULL terminated. */ + for (i = 0; joy->chans[i].indio_dev; i++) { + bits = joy->chans[i].channel->scan_type.storagebits; + if (!bits || bits > 16) { + dev_err(dev, "Unsupported channel storage size\n"); + return -EINVAL; + } + if (bits != joy->chans[0].channel->scan_type.storagebits) { + dev_err(dev, "Channels must have equal storage size\n"); + return -EINVAL; + } + } + joy->num_chans = i; + + input = devm_input_allocate_device(dev); + if (!input) { + dev_err(dev, "Unable to allocate input device\n"); + return -ENOMEM; + } + + joy->input = input; + input->name = pdev->name; + input->id.bustype = BUS_HOST; + input->open = adc_joystick_open; + input->close = adc_joystick_close; + + error = adc_joystick_set_axes(dev, joy); + if (error) + return error; + + input_set_drvdata(input, joy); + error = input_register_device(input); + if (error) { + dev_err(dev, "Unable to register input device\n"); + return error; + } + + joy->buffer = iio_channel_get_all_cb(dev, adc_joystick_handle, joy); + if (IS_ERR(joy->buffer)) { + dev_err(dev, "Unable to allocate callback buffer\n"); + return PTR_ERR(joy->buffer); + } + + error = devm_add_action_or_reset(dev, adc_joystick_cleanup, joy->buffer); + if (error) { + dev_err(dev, "Unable to add action\n"); + return error; + } + + return 0; +} + +static const struct of_device_id adc_joystick_of_match[] = { + { .compatible = "adc-joystick", }, + { } +}; +MODULE_DEVICE_TABLE(of, adc_joystick_of_match); + +static struct platform_driver adc_joystick_driver = { + .driver = { + .name = "adc-joystick", + .of_match_table = adc_joystick_of_match, + }, + .probe = adc_joystick_probe, +}; +module_platform_driver(adc_joystick_driver); + +MODULE_DESCRIPTION("Input driver for joysticks connected over ADC"); +MODULE_AUTHOR("Artur Rojek <contact@artur-rojek.eu>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/input/keyboard/cros_ec_keyb.c b/drivers/input/keyboard/cros_ec_keyb.c index fc1793ca2f17..15d17c717081 100644 --- a/drivers/input/keyboard/cros_ec_keyb.c +++ b/drivers/input/keyboard/cros_ec_keyb.c @@ -348,7 +348,7 @@ static int cros_ec_keyb_info(struct cros_ec_device *ec_dev, params->event_type = event_type; ret = cros_ec_cmd_xfer_status(ec_dev, msg); - if (ret == -ENOTSUPP) { + if (ret == -ENOPROTOOPT) { /* With older ECs we just return 0 for everything */ memset(result, 0, result_size); ret = 0; diff --git a/drivers/input/keyboard/ep93xx_keypad.c b/drivers/input/keyboard/ep93xx_keypad.c index 7c70492d9d6b..f831f01501d5 100644 --- a/drivers/input/keyboard/ep93xx_keypad.c +++ b/drivers/input/keyboard/ep93xx_keypad.c @@ -250,8 +250,8 @@ static int ep93xx_keypad_probe(struct platform_device *pdev) } keypad->irq = platform_get_irq(pdev, 0); - if (!keypad->irq) { - err = -ENXIO; + if (keypad->irq < 0) { + err = keypad->irq; goto failed_free; } diff --git a/drivers/input/keyboard/omap4-keypad.c b/drivers/input/keyboard/omap4-keypad.c index 94c94d7f5155..d6c924032aaa 100644 --- a/drivers/input/keyboard/omap4-keypad.c +++ b/drivers/input/keyboard/omap4-keypad.c @@ -240,10 +240,8 @@ static int omap4_keypad_probe(struct platform_device *pdev) } irq = platform_get_irq(pdev, 0); - if (!irq) { - dev_err(&pdev->dev, "no keyboard irq assigned\n"); - return -EINVAL; - } + if (irq < 0) + return irq; keypad_data = kzalloc(sizeof(struct omap4_keypad), GFP_KERNEL); if (!keypad_data) { diff --git a/drivers/input/keyboard/twl4030_keypad.c b/drivers/input/keyboard/twl4030_keypad.c index af3a6824f1a4..77e0743a3cf8 100644 --- a/drivers/input/keyboard/twl4030_keypad.c +++ b/drivers/input/keyboard/twl4030_keypad.c @@ -50,7 +50,7 @@ struct twl4030_keypad { bool autorepeat; unsigned int n_rows; unsigned int n_cols; - unsigned int irq; + int irq; struct device *dbg_dev; struct input_dev *input; @@ -376,10 +376,8 @@ static int twl4030_kp_probe(struct platform_device *pdev) } kp->irq = platform_get_irq(pdev, 0); - if (!kp->irq) { - dev_err(&pdev->dev, "no keyboard irq assigned\n"); - return -EINVAL; - } + if (kp->irq < 0) + return kp->irq; error = matrix_keypad_build_keymap(keymap_data, NULL, TWL4030_MAX_ROWS, diff --git a/drivers/input/misc/soc_button_array.c b/drivers/input/misc/soc_button_array.c index 08520b3a18b8..cae1a3fae83a 100644 --- a/drivers/input/misc/soc_button_array.c +++ b/drivers/input/misc/soc_button_array.c @@ -11,6 +11,7 @@ #include <linux/init.h> #include <linux/kernel.h> #include <linux/acpi.h> +#include <linux/dmi.h> #include <linux/gpio/consumer.h> #include <linux/gpio_keys.h> #include <linux/gpio.h> @@ -23,6 +24,7 @@ struct soc_button_info { unsigned int event_code; bool autorepeat; bool wakeup; + bool active_low; }; struct soc_device_data { @@ -42,22 +44,65 @@ struct soc_button_data { }; /* + * Some 2-in-1s which use the soc_button_array driver have this ugly issue in + * their DSDT where the _LID method modifies the irq-type settings of the GPIOs + * used for the power and home buttons. The intend of this AML code is to + * disable these buttons when the lid is closed. + * The AML does this by directly poking the GPIO controllers registers. This is + * problematic because when re-enabling the irq, which happens whenever _LID + * gets called with the lid open (e.g. on boot and on resume), it sets the + * irq-type to IRQ_TYPE_LEVEL_LOW. Where as the gpio-keys driver programs the + * type to, and expects it to be, IRQ_TYPE_EDGE_BOTH. + * To work around this we don't set gpio_keys_button.gpio on these 2-in-1s, + * instead we get the irq for the GPIO ourselves, configure it as + * IRQ_TYPE_LEVEL_LOW (to match how the _LID AML code configures it) and pass + * the irq in gpio_keys_button.irq. Below is a list of affected devices. + */ +static const struct dmi_system_id dmi_use_low_level_irq[] = { + { + /* + * Acer Switch 10 SW5-012. _LID method messes with home- and + * power-button GPIO IRQ settings. When (re-)enabling the irq + * it ors in its own flags without clearing the previous set + * ones, leading to an irq-type of IRQ_TYPE_LEVEL_LOW | + * IRQ_TYPE_LEVEL_HIGH causing a continuous interrupt storm. + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Acer"), + DMI_MATCH(DMI_PRODUCT_NAME, "Aspire SW5-012"), + }, + }, + { + /* + * Acer One S1003. _LID method messes with power-button GPIO + * IRQ settings, leading to a non working power-button. + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Acer"), + DMI_MATCH(DMI_PRODUCT_NAME, "One S1003"), + }, + }, + {} /* Terminating entry */ +}; + +/* * Get the Nth GPIO number from the ACPI object. */ -static int soc_button_lookup_gpio(struct device *dev, int acpi_index) +static int soc_button_lookup_gpio(struct device *dev, int acpi_index, + int *gpio_ret, int *irq_ret) { struct gpio_desc *desc; - int gpio; desc = gpiod_get_index(dev, NULL, acpi_index, GPIOD_ASIS); if (IS_ERR(desc)) return PTR_ERR(desc); - gpio = desc_to_gpio(desc); + *gpio_ret = desc_to_gpio(desc); + *irq_ret = gpiod_to_irq(desc); gpiod_put(desc); - return gpio; + return 0; } static struct platform_device * @@ -69,9 +114,8 @@ soc_button_device_create(struct platform_device *pdev, struct platform_device *pd; struct gpio_keys_button *gpio_keys; struct gpio_keys_platform_data *gpio_keys_pdata; + int error, gpio, irq; int n_buttons = 0; - int gpio; - int error; for (info = button_info; info->name; info++) if (info->autorepeat == autorepeat) @@ -91,8 +135,8 @@ soc_button_device_create(struct platform_device *pdev, if (info->autorepeat != autorepeat) continue; - gpio = soc_button_lookup_gpio(&pdev->dev, info->acpi_index); - if (!gpio_is_valid(gpio)) { + error = soc_button_lookup_gpio(&pdev->dev, info->acpi_index, &gpio, &irq); + if (error || irq < 0) { /* * Skip GPIO if not present. Note we deliberately * ignore -EPROBE_DEFER errors here. On some devices @@ -107,10 +151,18 @@ soc_button_device_create(struct platform_device *pdev, continue; } + /* See dmi_use_low_level_irq[] comment */ + if (!autorepeat && dmi_check_system(dmi_use_low_level_irq)) { + irq_set_irq_type(irq, IRQ_TYPE_LEVEL_LOW); + gpio_keys[n_buttons].irq = irq; + gpio_keys[n_buttons].gpio = -ENOENT; + } else { + gpio_keys[n_buttons].gpio = gpio; + } + gpio_keys[n_buttons].type = info->event_type; gpio_keys[n_buttons].code = info->event_code; - gpio_keys[n_buttons].gpio = gpio; - gpio_keys[n_buttons].active_low = 1; + gpio_keys[n_buttons].active_low = info->active_low; gpio_keys[n_buttons].desc = info->name; gpio_keys[n_buttons].wakeup = info->wakeup; /* These devices often use cheap buttons, use 50 ms debounce */ @@ -173,6 +225,7 @@ static int soc_button_parse_btn_desc(struct device *dev, } info->event_type = EV_KEY; + info->active_low = true; info->acpi_index = soc_button_get_acpi_object_int(&desc->package.elements[1]); upage = soc_button_get_acpi_object_int(&desc->package.elements[3]); @@ -383,11 +436,11 @@ static int soc_button_probe(struct platform_device *pdev) * Platforms" */ static const struct soc_button_info soc_button_PNP0C40[] = { - { "power", 0, EV_KEY, KEY_POWER, false, true }, - { "home", 1, EV_KEY, KEY_LEFTMETA, false, true }, - { "volume_up", 2, EV_KEY, KEY_VOLUMEUP, true, false }, - { "volume_down", 3, EV_KEY, KEY_VOLUMEDOWN, true, false }, - { "rotation_lock", 4, EV_KEY, KEY_ROTATE_LOCK_TOGGLE, false, false }, + { "power", 0, EV_KEY, KEY_POWER, false, true, true }, + { "home", 1, EV_KEY, KEY_LEFTMETA, false, true, true }, + { "volume_up", 2, EV_KEY, KEY_VOLUMEUP, true, false, true }, + { "volume_down", 3, EV_KEY, KEY_VOLUMEDOWN, true, false, true }, + { "rotation_lock", 4, EV_KEY, KEY_ROTATE_LOCK_TOGGLE, false, false, true }, { } }; @@ -395,6 +448,15 @@ static const struct soc_device_data soc_device_PNP0C40 = { .button_info = soc_button_PNP0C40, }; +static const struct soc_button_info soc_button_INT33D3[] = { + { "tablet_mode", 0, EV_SW, SW_TABLET_MODE, false, false, false }, + { } +}; + +static const struct soc_device_data soc_device_INT33D3 = { + .button_info = soc_button_INT33D3, +}; + /* * Special device check for Surface Book 2 and Surface Pro (2017). * Both, the Surface Pro 4 (surfacepro3_button.c) and the above mentioned @@ -444,9 +506,9 @@ static int soc_device_check_MSHW0040(struct device *dev) * Obtained from DSDT/testing. */ static const struct soc_button_info soc_button_MSHW0040[] = { - { "power", 0, EV_KEY, KEY_POWER, false, true }, - { "volume_up", 2, EV_KEY, KEY_VOLUMEUP, true, false }, - { "volume_down", 4, EV_KEY, KEY_VOLUMEDOWN, true, false }, + { "power", 0, EV_KEY, KEY_POWER, false, true, true }, + { "volume_up", 2, EV_KEY, KEY_VOLUMEUP, true, false, true }, + { "volume_down", 4, EV_KEY, KEY_VOLUMEDOWN, true, false, true }, { } }; @@ -457,6 +519,8 @@ static const struct soc_device_data soc_device_MSHW0040 = { static const struct acpi_device_id soc_button_acpi_match[] = { { "PNP0C40", (unsigned long)&soc_device_PNP0C40 }, + { "INT33D3", (unsigned long)&soc_device_INT33D3 }, + { "ID9001", (unsigned long)&soc_device_INT33D3 }, { "ACPI0011", 0 }, /* Microsoft Surface Devices (5th and 6th generation) */ diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c index 4b81b2d0fe06..82577095e175 100644 --- a/drivers/input/mouse/synaptics.c +++ b/drivers/input/mouse/synaptics.c @@ -179,12 +179,14 @@ static const char * const smbus_pnp_ids[] = { "LEN0093", /* T480 */ "LEN0096", /* X280 */ "LEN0097", /* X280 -> ALPS trackpoint */ - "LEN0099", /* X1 Extreme 1st */ + "LEN0099", /* X1 Extreme Gen 1 / P1 Gen 1 */ "LEN009b", /* T580 */ + "LEN0402", /* X1 Extreme Gen 2 / P1 Gen 2 */ "LEN200f", /* T450s */ "LEN2044", /* L470 */ "LEN2054", /* E480 */ "LEN2055", /* E580 */ + "LEN2068", /* T14 Gen 1 */ "SYN3052", /* HP EliteBook 840 G4 */ "SYN3221", /* HP 15-ay000 */ "SYN323d", /* HP Spectre X360 13-w013dx */ @@ -1752,7 +1754,7 @@ static int synaptics_create_intertouch(struct psmouse *psmouse, .kernel_tracking = false, .topbuttonpad = topbuttonpad, }, - .f30_data = { + .gpio_data = { .buttonpad = SYN_CAP_CLICKPAD(info->ext_cap_0c), .trackstick_buttons = !!SYN_CAP_EXT_BUTTONS_STICK(info->ext_cap_10), diff --git a/drivers/input/rmi4/Kconfig b/drivers/input/rmi4/Kconfig index a212ff706f74..16119f760d11 100644 --- a/drivers/input/rmi4/Kconfig +++ b/drivers/input/rmi4/Kconfig @@ -100,6 +100,14 @@ config RMI4_F34 device via the firmware loader interface. This is triggered using a sysfs attribute. +config RMI4_F3A + bool "RMI4 Function 3A (GPIO)" + help + Say Y here if you want to add support for RMI4 function 3A. + + Function 3A provides GPIO support for RMI4 devices. This includes + support for buttons on TouchPads and ClickPads. + config RMI4_F54 bool "RMI4 Function 54 (Analog diagnostics)" depends on VIDEO_V4L2=y || (RMI4_CORE=m && VIDEO_V4L2=m) diff --git a/drivers/input/rmi4/Makefile b/drivers/input/rmi4/Makefile index f17631656987..02f14c846861 100644 --- a/drivers/input/rmi4/Makefile +++ b/drivers/input/rmi4/Makefile @@ -10,6 +10,7 @@ rmi_core-$(CONFIG_RMI4_F11) += rmi_f11.o rmi_core-$(CONFIG_RMI4_F12) += rmi_f12.o rmi_core-$(CONFIG_RMI4_F30) += rmi_f30.o rmi_core-$(CONFIG_RMI4_F34) += rmi_f34.o rmi_f34v7.o +rmi_core-$(CONFIG_RMI4_F3A) += rmi_f3a.o rmi_core-$(CONFIG_RMI4_F54) += rmi_f54.o rmi_core-$(CONFIG_RMI4_F55) += rmi_f55.o diff --git a/drivers/input/rmi4/rmi_bus.c b/drivers/input/rmi4/rmi_bus.c index af706a583656..47d1b97ed6cf 100644 --- a/drivers/input/rmi4/rmi_bus.c +++ b/drivers/input/rmi4/rmi_bus.c @@ -365,6 +365,9 @@ static struct rmi_function_handler *fn_handlers[] = { #ifdef CONFIG_RMI4_F34 &rmi_f34_handler, #endif +#ifdef CONFIG_RMI4_F3A + &rmi_f3a_handler, +#endif #ifdef CONFIG_RMI4_F54 &rmi_f54_handler, #endif diff --git a/drivers/input/rmi4/rmi_driver.h b/drivers/input/rmi4/rmi_driver.h index 65bfaa95e193..1c6c6086c0e5 100644 --- a/drivers/input/rmi4/rmi_driver.h +++ b/drivers/input/rmi4/rmi_driver.h @@ -135,6 +135,7 @@ extern struct rmi_function_handler rmi_f11_handler; extern struct rmi_function_handler rmi_f12_handler; extern struct rmi_function_handler rmi_f30_handler; extern struct rmi_function_handler rmi_f34_handler; +extern struct rmi_function_handler rmi_f3a_handler; extern struct rmi_function_handler rmi_f54_handler; extern struct rmi_function_handler rmi_f55_handler; #endif diff --git a/drivers/input/rmi4/rmi_f30.c b/drivers/input/rmi4/rmi_f30.c index a90dad1d9ac7..35045f161dc2 100644 --- a/drivers/input/rmi4/rmi_f30.c +++ b/drivers/input/rmi4/rmi_f30.c @@ -168,17 +168,17 @@ static int rmi_f30_config(struct rmi_function *fn) rmi_get_platform_data(fn->rmi_dev); int error; - /* can happen if f30_data.disable is set */ + /* can happen if gpio_data.disable is set */ if (!f30) return 0; - if (pdata->f30_data.trackstick_buttons) { + if (pdata->gpio_data.trackstick_buttons) { /* Try [re-]establish link to F03. */ f30->f03 = rmi_find_function(fn->rmi_dev, 0x03); f30->trackstick_buttons = f30->f03 != NULL; } - if (pdata->f30_data.disable) { + if (pdata->gpio_data.disable) { drv->clear_irq_bits(fn->rmi_dev, fn->irq_mask); } else { /* Write Control Register values back to device */ @@ -245,10 +245,10 @@ static int rmi_f30_map_gpios(struct rmi_function *fn, if (!rmi_f30_is_valid_button(i, f30->ctrl)) continue; - if (pdata->f30_data.trackstick_buttons && + if (pdata->gpio_data.trackstick_buttons && i >= TRACKSTICK_RANGE_START && i < TRACKSTICK_RANGE_END) { f30->gpioled_key_map[i] = trackstick_button++; - } else if (!pdata->f30_data.buttonpad || !button_mapped) { + } else if (!pdata->gpio_data.buttonpad || !button_mapped) { f30->gpioled_key_map[i] = button; input_set_capability(input, EV_KEY, button++); button_mapped = true; @@ -264,7 +264,7 @@ static int rmi_f30_map_gpios(struct rmi_function *fn, * but I am not sure, so use only the pdata info and the number of * mapped buttons. */ - if (pdata->f30_data.buttonpad || (button - BTN_LEFT == 1)) + if (pdata->gpio_data.buttonpad || (button - BTN_LEFT == 1)) __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); return 0; @@ -372,7 +372,7 @@ static int rmi_f30_probe(struct rmi_function *fn) struct f30_data *f30; int error; - if (pdata->f30_data.disable) + if (pdata->gpio_data.disable) return 0; if (!drv_data->input) { diff --git a/drivers/input/rmi4/rmi_f34v7.c b/drivers/input/rmi4/rmi_f34v7.c index 74f7c6f214ff..8d7ec9d89b18 100644 --- a/drivers/input/rmi4/rmi_f34v7.c +++ b/drivers/input/rmi4/rmi_f34v7.c @@ -1364,9 +1364,14 @@ int rmi_f34v7_probe(struct f34_data *f34) f34->bl_version = 6; } else if (f34->bootloader_id[1] == 7) { f34->bl_version = 7; + } else if (f34->bootloader_id[1] == 8) { + f34->bl_version = 8; } else { - dev_err(&f34->fn->dev, "%s: Unrecognized bootloader version\n", - __func__); + dev_err(&f34->fn->dev, + "%s: Unrecognized bootloader version: %d (%c) %d (%c)\n", + __func__, + f34->bootloader_id[0], f34->bootloader_id[0], + f34->bootloader_id[1], f34->bootloader_id[1]); return -EINVAL; } diff --git a/drivers/input/rmi4/rmi_f3a.c b/drivers/input/rmi4/rmi_f3a.c new file mode 100644 index 000000000000..0e8baed84dbb --- /dev/null +++ b/drivers/input/rmi4/rmi_f3a.c @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2012-2020 Synaptics Incorporated + */ + +#include <linux/kernel.h> +#include <linux/rmi.h> +#include <linux/input.h> +#include <linux/slab.h> +#include "rmi_driver.h" + +#define RMI_F3A_MAX_GPIO_COUNT 128 +#define RMI_F3A_MAX_REG_SIZE DIV_ROUND_UP(RMI_F3A_MAX_GPIO_COUNT, 8) + +/* Defs for Query 0 */ +#define RMI_F3A_GPIO_COUNT 0x7F + +#define RMI_F3A_DATA_REGS_MAX_SIZE RMI_F3A_MAX_REG_SIZE + +#define TRACKSTICK_RANGE_START 3 +#define TRACKSTICK_RANGE_END 6 + +struct f3a_data { + /* Query Data */ + u8 gpio_count; + + u8 register_count; + + u8 data_regs[RMI_F3A_DATA_REGS_MAX_SIZE]; + u16 *gpio_key_map; + + struct input_dev *input; + + struct rmi_function *f03; + bool trackstick_buttons; +}; + +static void rmi_f3a_report_button(struct rmi_function *fn, + struct f3a_data *f3a, unsigned int button) +{ + u16 key_code = f3a->gpio_key_map[button]; + bool key_down = !(f3a->data_regs[0] & BIT(button)); + + if (f3a->trackstick_buttons && + button >= TRACKSTICK_RANGE_START && + button <= TRACKSTICK_RANGE_END) { + rmi_f03_overwrite_button(f3a->f03, key_code, key_down); + } else { + rmi_dbg(RMI_DEBUG_FN, &fn->dev, + "%s: call input report key (0x%04x) value (0x%02x)", + __func__, key_code, key_down); + input_report_key(f3a->input, key_code, key_down); + } +} + +static irqreturn_t rmi_f3a_attention(int irq, void *ctx) +{ + struct rmi_function *fn = ctx; + struct f3a_data *f3a = dev_get_drvdata(&fn->dev); + struct rmi_driver_data *drvdata = dev_get_drvdata(&fn->rmi_dev->dev); + int error; + int i; + + if (drvdata->attn_data.data) { + if (drvdata->attn_data.size < f3a->register_count) { + dev_warn(&fn->dev, + "F3A interrupted, but data is missing\n"); + return IRQ_HANDLED; + } + memcpy(f3a->data_regs, drvdata->attn_data.data, + f3a->register_count); + drvdata->attn_data.data += f3a->register_count; + drvdata->attn_data.size -= f3a->register_count; + } else { + error = rmi_read_block(fn->rmi_dev, fn->fd.data_base_addr, + f3a->data_regs, f3a->register_count); + if (error) { + dev_err(&fn->dev, + "%s: Failed to read F3a data registers: %d\n", + __func__, error); + return IRQ_RETVAL(error); + } + } + + for (i = 0; i < f3a->gpio_count; i++) + if (f3a->gpio_key_map[i] != KEY_RESERVED) + rmi_f3a_report_button(fn, f3a, i); + if (f3a->trackstick_buttons) + rmi_f03_commit_buttons(f3a->f03); + + return IRQ_HANDLED; +} + +static int rmi_f3a_config(struct rmi_function *fn) +{ + struct f3a_data *f3a = dev_get_drvdata(&fn->dev); + struct rmi_driver *drv = fn->rmi_dev->driver; + const struct rmi_device_platform_data *pdata = + rmi_get_platform_data(fn->rmi_dev); + + if (!f3a) + return 0; + + if (pdata->gpio_data.trackstick_buttons) { + /* Try [re-]establish link to F03. */ + f3a->f03 = rmi_find_function(fn->rmi_dev, 0x03); + f3a->trackstick_buttons = f3a->f03 != NULL; + } + + drv->set_irq_bits(fn->rmi_dev, fn->irq_mask); + + return 0; +} + +static bool rmi_f3a_is_valid_button(int button, struct f3a_data *f3a, + u8 *query1_regs, u8 *ctrl1_regs) +{ + /* gpio exist && direction input */ + return (query1_regs[0] & BIT(button)) && !(ctrl1_regs[0] & BIT(button)); +} + +static int rmi_f3a_map_gpios(struct rmi_function *fn, struct f3a_data *f3a, + u8 *query1_regs, u8 *ctrl1_regs) +{ + const struct rmi_device_platform_data *pdata = + rmi_get_platform_data(fn->rmi_dev); + struct input_dev *input = f3a->input; + unsigned int button = BTN_LEFT; + unsigned int trackstick_button = BTN_LEFT; + bool button_mapped = false; + int i; + int button_count = min_t(u8, f3a->gpio_count, TRACKSTICK_RANGE_END); + + f3a->gpio_key_map = devm_kcalloc(&fn->dev, + button_count, + sizeof(f3a->gpio_key_map[0]), + GFP_KERNEL); + if (!f3a->gpio_key_map) { + dev_err(&fn->dev, "Failed to allocate gpio map memory.\n"); + return -ENOMEM; + } + + for (i = 0; i < button_count; i++) { + if (!rmi_f3a_is_valid_button(i, f3a, query1_regs, ctrl1_regs)) + continue; + + if (pdata->gpio_data.trackstick_buttons && + i >= TRACKSTICK_RANGE_START && + i < TRACKSTICK_RANGE_END) { + f3a->gpio_key_map[i] = trackstick_button++; + } else if (!pdata->gpio_data.buttonpad || !button_mapped) { + f3a->gpio_key_map[i] = button; + input_set_capability(input, EV_KEY, button++); + button_mapped = true; + } + } + input->keycode = f3a->gpio_key_map; + input->keycodesize = sizeof(f3a->gpio_key_map[0]); + input->keycodemax = f3a->gpio_count; + + if (pdata->gpio_data.buttonpad || (button - BTN_LEFT == 1)) + __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); + + return 0; +} + +static int rmi_f3a_initialize(struct rmi_function *fn, struct f3a_data *f3a) +{ + u8 query1[RMI_F3A_MAX_REG_SIZE]; + u8 ctrl1[RMI_F3A_MAX_REG_SIZE]; + u8 buf; + int error; + + error = rmi_read(fn->rmi_dev, fn->fd.query_base_addr, &buf); + if (error < 0) { + dev_err(&fn->dev, "Failed to read general info register: %d\n", + error); + return -ENODEV; + } + + f3a->gpio_count = buf & RMI_F3A_GPIO_COUNT; + f3a->register_count = DIV_ROUND_UP(f3a->gpio_count, 8); + + /* Query1 -> gpio exist */ + error = rmi_read_block(fn->rmi_dev, fn->fd.query_base_addr + 1, + query1, f3a->register_count); + if (error) { + dev_err(&fn->dev, "Failed to read query1 register\n"); + return error; + } + + /* Ctrl1 -> gpio direction */ + error = rmi_read_block(fn->rmi_dev, fn->fd.control_base_addr + 1, + ctrl1, f3a->register_count); + if (error) { + dev_err(&fn->dev, "Failed to read control1 register\n"); + return error; + } + + error = rmi_f3a_map_gpios(fn, f3a, query1, ctrl1); + if (error) + return error; + + return 0; +} + +static int rmi_f3a_probe(struct rmi_function *fn) +{ + struct rmi_device *rmi_dev = fn->rmi_dev; + struct rmi_driver_data *drv_data = dev_get_drvdata(&rmi_dev->dev); + struct f3a_data *f3a; + int error; + + if (!drv_data->input) { + dev_info(&fn->dev, "F3A: no input device found, ignoring\n"); + return -ENXIO; + } + + f3a = devm_kzalloc(&fn->dev, sizeof(*f3a), GFP_KERNEL); + if (!f3a) + return -ENOMEM; + + f3a->input = drv_data->input; + + error = rmi_f3a_initialize(fn, f3a); + if (error) + return error; + + dev_set_drvdata(&fn->dev, f3a); + return 0; +} + +struct rmi_function_handler rmi_f3a_handler = { + .driver = { + .name = "rmi4_f3a", + }, + .func = 0x3a, + .probe = rmi_f3a_probe, + .config = rmi_f3a_config, + .attention = rmi_f3a_attention, +}; diff --git a/drivers/input/serio/sun4i-ps2.c b/drivers/input/serio/sun4i-ps2.c index a681a2c04e39..f15ed3dcdb9b 100644 --- a/drivers/input/serio/sun4i-ps2.c +++ b/drivers/input/serio/sun4i-ps2.c @@ -211,7 +211,6 @@ static int sun4i_ps2_probe(struct platform_device *pdev) struct sun4i_ps2data *drvdata; struct serio *serio; struct device *dev = &pdev->dev; - unsigned int irq; int error; drvdata = kzalloc(sizeof(struct sun4i_ps2data), GFP_KERNEL); @@ -264,14 +263,12 @@ static int sun4i_ps2_probe(struct platform_device *pdev) writel(0, drvdata->reg_base + PS2_REG_GCTL); /* Get IRQ for the device */ - irq = platform_get_irq(pdev, 0); - if (!irq) { - dev_err(dev, "no IRQ found\n"); - error = -ENXIO; + drvdata->irq = platform_get_irq(pdev, 0); + if (drvdata->irq < 0) { + error = drvdata->irq; goto err_disable_clk; } - drvdata->irq = irq; drvdata->serio = serio; drvdata->dev = dev; diff --git a/drivers/input/touchscreen/Kconfig b/drivers/input/touchscreen/Kconfig index 35c867b2d9a7..f012fe746df0 100644 --- a/drivers/input/touchscreen/Kconfig +++ b/drivers/input/touchscreen/Kconfig @@ -1322,4 +1322,16 @@ config TOUCHSCREEN_IQS5XX To compile this driver as a module, choose M here: the module will be called iqs5xx. +config TOUCHSCREEN_ZINITIX + tristate "Zinitix touchscreen support" + depends on I2C + help + Say Y here if you have a touchscreen using Zinitix bt541, + or something similar enough. + + If unsure, say N. + + To compile this driver as a module, choose M here: the + module will be called zinitix. + endif diff --git a/drivers/input/touchscreen/Makefile b/drivers/input/touchscreen/Makefile index 30d1e1b42492..6233541e9173 100644 --- a/drivers/input/touchscreen/Makefile +++ b/drivers/input/touchscreen/Makefile @@ -111,3 +111,4 @@ obj-$(CONFIG_TOUCHSCREEN_COLIBRI_VF50) += colibri-vf50-ts.o obj-$(CONFIG_TOUCHSCREEN_ROHM_BU21023) += rohm_bu21023.o obj-$(CONFIG_TOUCHSCREEN_RASPBERRYPI_FW) += raspberrypi-ts.o obj-$(CONFIG_TOUCHSCREEN_IQS5XX) += iqs5xx.o +obj-$(CONFIG_TOUCHSCREEN_ZINITIX) += zinitix.o diff --git a/drivers/input/touchscreen/elants_i2c.c b/drivers/input/touchscreen/elants_i2c.c index b0bd5bb079be..50c348297e38 100644 --- a/drivers/input/touchscreen/elants_i2c.c +++ b/drivers/input/touchscreen/elants_i2c.c @@ -90,7 +90,7 @@ /* FW read command, 0x53 0x?? 0x0, 0x01 */ #define E_ELAN_INFO_FW_VER 0x00 #define E_ELAN_INFO_BC_VER 0x10 -#define E_ELAN_INFO_REK 0xE0 +#define E_ELAN_INFO_REK 0xD0 #define E_ELAN_INFO_TEST_VER 0xE0 #define E_ELAN_INFO_FW_ID 0xF0 #define E_INFO_OSR 0xD6 @@ -134,6 +134,7 @@ struct elants_data { u8 bc_version; u8 iap_version; u16 hw_version; + u8 major_res; unsigned int x_res; /* resolution in units/mm */ unsigned int y_res; unsigned int x_max; @@ -459,6 +460,9 @@ static int elants_i2c_query_ts_info(struct elants_data *ts) rows = resp[2] + resp[6] + resp[10]; cols = resp[3] + resp[7] + resp[11]; + /* Get report resolution value of ABS_MT_TOUCH_MAJOR */ + ts->major_res = resp[16]; + /* Process mm_to_pixel information */ error = elants_i2c_execute_command(client, get_osr_cmd, sizeof(get_osr_cmd), @@ -1325,6 +1329,8 @@ static int elants_i2c_probe(struct i2c_client *client, 0, MT_TOOL_PALM, 0, 0); input_abs_set_res(ts->input, ABS_MT_POSITION_X, ts->x_res); input_abs_set_res(ts->input, ABS_MT_POSITION_Y, ts->y_res); + if (ts->major_res > 0) + input_abs_set_res(ts->input, ABS_MT_TOUCH_MAJOR, ts->major_res); touchscreen_parse_properties(ts->input, true, &ts->prop); diff --git a/drivers/input/touchscreen/imx6ul_tsc.c b/drivers/input/touchscreen/imx6ul_tsc.c index 9ed258854349..cd369f9ac5e6 100644 --- a/drivers/input/touchscreen/imx6ul_tsc.c +++ b/drivers/input/touchscreen/imx6ul_tsc.c @@ -315,9 +315,8 @@ static irqreturn_t adc_irq_fn(int irq, void *dev_id) return IRQ_HANDLED; } -static int imx6ul_tsc_open(struct input_dev *input_dev) +static int imx6ul_tsc_start(struct imx6ul_tsc *tsc) { - struct imx6ul_tsc *tsc = input_get_drvdata(input_dev); int err; err = clk_prepare_enable(tsc->adc_clk); @@ -349,16 +348,29 @@ disable_adc_clk: return err; } -static void imx6ul_tsc_close(struct input_dev *input_dev) +static void imx6ul_tsc_stop(struct imx6ul_tsc *tsc) { - struct imx6ul_tsc *tsc = input_get_drvdata(input_dev); - imx6ul_tsc_disable(tsc); clk_disable_unprepare(tsc->tsc_clk); clk_disable_unprepare(tsc->adc_clk); } + +static int imx6ul_tsc_open(struct input_dev *input_dev) +{ + struct imx6ul_tsc *tsc = input_get_drvdata(input_dev); + + return imx6ul_tsc_start(tsc); +} + +static void imx6ul_tsc_close(struct input_dev *input_dev) +{ + struct imx6ul_tsc *tsc = input_get_drvdata(input_dev); + + imx6ul_tsc_stop(tsc); +} + static int imx6ul_tsc_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; @@ -509,12 +521,8 @@ static int __maybe_unused imx6ul_tsc_suspend(struct device *dev) mutex_lock(&input_dev->mutex); - if (input_dev->users) { - imx6ul_tsc_disable(tsc); - - clk_disable_unprepare(tsc->tsc_clk); - clk_disable_unprepare(tsc->adc_clk); - } + if (input_dev->users) + imx6ul_tsc_stop(tsc); mutex_unlock(&input_dev->mutex); @@ -530,22 +538,11 @@ static int __maybe_unused imx6ul_tsc_resume(struct device *dev) mutex_lock(&input_dev->mutex); - if (input_dev->users) { - retval = clk_prepare_enable(tsc->adc_clk); - if (retval) - goto out; - - retval = clk_prepare_enable(tsc->tsc_clk); - if (retval) { - clk_disable_unprepare(tsc->adc_clk); - goto out; - } - - retval = imx6ul_tsc_init(tsc); - } + if (input_dev->users) + retval = imx6ul_tsc_start(tsc); -out: mutex_unlock(&input_dev->mutex); + return retval; } diff --git a/drivers/input/touchscreen/raydium_i2c_ts.c b/drivers/input/touchscreen/raydium_i2c_ts.c index fe245439adee..e694a9b2b1e5 100644 --- a/drivers/input/touchscreen/raydium_i2c_ts.c +++ b/drivers/input/touchscreen/raydium_i2c_ts.c @@ -51,6 +51,7 @@ /* Touch relative info */ #define RM_MAX_RETRIES 3 +#define RM_RETRY_DELAY_MS 20 #define RM_MAX_TOUCH_NUM 10 #define RM_BOOT_DELAY_MS 100 @@ -136,83 +137,82 @@ struct raydium_data { bool wake_irq_enabled; }; -static int raydium_i2c_send(struct i2c_client *client, - u8 addr, const void *data, size_t len) +static int raydium_i2c_xfer(struct i2c_client *client, + u32 addr, void *data, size_t len, bool is_read) { - u8 *buf; - int tries = 0; - int ret; - - buf = kmalloc(len + 1, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - buf[0] = addr; - memcpy(buf + 1, data, len); - - do { - ret = i2c_master_send(client, buf, len + 1); - if (likely(ret == len + 1)) - break; - - msleep(20); - } while (++tries < RM_MAX_RETRIES); - - kfree(buf); - - if (unlikely(ret != len + 1)) { - if (ret >= 0) - ret = -EIO; - dev_err(&client->dev, "%s failed: %d\n", __func__, ret); - return ret; - } + struct raydium_bank_switch_header { + u8 cmd; + __be32 be_addr; + } __packed header = { + .cmd = RM_CMD_BANK_SWITCH, + .be_addr = cpu_to_be32(addr), + }; - return 0; -} + u8 reg_addr = addr & 0xff; -static int raydium_i2c_read(struct i2c_client *client, - u8 addr, void *data, size_t len) -{ struct i2c_msg xfer[] = { { .addr = client->addr, + .len = sizeof(header), + .buf = (u8 *)&header, + }, + { + .addr = client->addr, .len = 1, - .buf = &addr, + .buf = ®_addr, }, { .addr = client->addr, - .flags = I2C_M_RD, .len = len, .buf = data, + .flags = is_read ? I2C_M_RD : 0, } }; + + /* + * If address is greater than 255, then RM_CMD_BANK_SWITCH needs to be + * sent first. Else, skip the header i.e. xfer[0]. + */ + int xfer_start_idx = (addr > 0xff) ? 0 : 1; + size_t xfer_count = ARRAY_SIZE(xfer) - xfer_start_idx; int ret; - ret = i2c_transfer(client->adapter, xfer, ARRAY_SIZE(xfer)); - if (unlikely(ret != ARRAY_SIZE(xfer))) - return ret < 0 ? ret : -EIO; + ret = i2c_transfer(client->adapter, &xfer[xfer_start_idx], xfer_count); + if (likely(ret == xfer_count)) + return 0; + + return ret < 0 ? ret : -EIO; +} - return 0; +static int raydium_i2c_send(struct i2c_client *client, + u32 addr, const void *data, size_t len) +{ + int tries = 0; + int error; + + do { + error = raydium_i2c_xfer(client, addr, (void *)data, len, + false); + if (likely(!error)) + return 0; + + msleep(RM_RETRY_DELAY_MS); + } while (++tries < RM_MAX_RETRIES); + + dev_err(&client->dev, "%s failed: %d\n", __func__, error); + return error; } -static int raydium_i2c_read_message(struct i2c_client *client, - u32 addr, void *data, size_t len) +static int raydium_i2c_read(struct i2c_client *client, + u32 addr, void *data, size_t len) { - __be32 be_addr; size_t xfer_len; int error; while (len) { xfer_len = min_t(size_t, len, RM_MAX_READ_SIZE); - - be_addr = cpu_to_be32(addr); - - error = raydium_i2c_send(client, RM_CMD_BANK_SWITCH, - &be_addr, sizeof(be_addr)); - if (!error) - error = raydium_i2c_read(client, addr & 0xff, - data, xfer_len); - if (error) + error = raydium_i2c_xfer(client, addr, data, xfer_len, true); + if (unlikely(error)) return error; len -= xfer_len; @@ -223,27 +223,13 @@ static int raydium_i2c_read_message(struct i2c_client *client, return 0; } -static int raydium_i2c_send_message(struct i2c_client *client, - u32 addr, const void *data, size_t len) -{ - __be32 be_addr = cpu_to_be32(addr); - int error; - - error = raydium_i2c_send(client, RM_CMD_BANK_SWITCH, - &be_addr, sizeof(be_addr)); - if (!error) - error = raydium_i2c_send(client, addr & 0xff, data, len); - - return error; -} - static int raydium_i2c_sw_reset(struct i2c_client *client) { const u8 soft_rst_cmd = 0x01; int error; - error = raydium_i2c_send_message(client, RM_RESET_MSG_ADDR, - &soft_rst_cmd, sizeof(soft_rst_cmd)); + error = raydium_i2c_send(client, RM_RESET_MSG_ADDR, &soft_rst_cmd, + sizeof(soft_rst_cmd)); if (error) { dev_err(&client->dev, "software reset failed: %d\n", error); return error; @@ -295,9 +281,8 @@ static int raydium_i2c_query_ts_info(struct raydium_data *ts) if (error) continue; - error = raydium_i2c_read_message(client, - le32_to_cpu(query_bank_addr), - &ts->info, sizeof(ts->info)); + error = raydium_i2c_read(client, le32_to_cpu(query_bank_addr), + &ts->info, sizeof(ts->info)); if (error) continue; @@ -834,8 +819,8 @@ static irqreturn_t raydium_i2c_irq(int irq, void *_dev) if (ts->boot_mode != RAYDIUM_TS_MAIN) goto out; - error = raydium_i2c_read_message(ts->client, ts->data_bank_addr, - ts->report_data, ts->pkg_size); + error = raydium_i2c_read(ts->client, ts->data_bank_addr, + ts->report_data, ts->pkg_size); if (error) goto out; diff --git a/drivers/input/touchscreen/s3c2410_ts.c b/drivers/input/touchscreen/s3c2410_ts.c index 82920ff46f72..2e70c0b79444 100644 --- a/drivers/input/touchscreen/s3c2410_ts.c +++ b/drivers/input/touchscreen/s3c2410_ts.c @@ -20,10 +20,43 @@ #include <linux/clk.h> #include <linux/io.h> -#include <plat/adc.h> -#include <plat/regs-adc.h> +#include <linux/soc/samsung/s3c-adc.h> #include <linux/platform_data/touchscreen-s3c2410.h> +#define S3C2410_ADCCON (0x00) +#define S3C2410_ADCTSC (0x04) +#define S3C2410_ADCDLY (0x08) +#define S3C2410_ADCDAT0 (0x0C) +#define S3C2410_ADCDAT1 (0x10) +#define S3C64XX_ADCUPDN (0x14) +#define S3C2443_ADCMUX (0x18) +#define S3C64XX_ADCCLRINT (0x18) +#define S5P_ADCMUX (0x1C) +#define S3C64XX_ADCCLRINTPNDNUP (0x20) + +/* ADCTSC Register Bits */ +#define S3C2443_ADCTSC_UD_SEN (1 << 8) +#define S3C2410_ADCTSC_YM_SEN (1<<7) +#define S3C2410_ADCTSC_YP_SEN (1<<6) +#define S3C2410_ADCTSC_XM_SEN (1<<5) +#define S3C2410_ADCTSC_XP_SEN (1<<4) +#define S3C2410_ADCTSC_PULL_UP_DISABLE (1<<3) +#define S3C2410_ADCTSC_AUTO_PST (1<<2) +#define S3C2410_ADCTSC_XY_PST(x) (((x)&0x3)<<0) + +/* ADCDAT0 Bits */ +#define S3C2410_ADCDAT0_UPDOWN (1<<15) +#define S3C2410_ADCDAT0_AUTO_PST (1<<14) +#define S3C2410_ADCDAT0_XY_PST (0x3<<12) +#define S3C2410_ADCDAT0_XPDATA_MASK (0x03FF) + +/* ADCDAT1 Bits */ +#define S3C2410_ADCDAT1_UPDOWN (1<<15) +#define S3C2410_ADCDAT1_AUTO_PST (1<<14) +#define S3C2410_ADCDAT1_XY_PST (0x3<<12) +#define S3C2410_ADCDAT1_YPDATA_MASK (0x03FF) + + #define TSC_SLEEP (S3C2410_ADCTSC_PULL_UP_DISABLE | S3C2410_ADCTSC_XY_PST(0)) #define INT_DOWN (0) diff --git a/drivers/input/touchscreen/stmfts.c b/drivers/input/touchscreen/stmfts.c index df946869d4cd..9a64e1dbc04a 100644 --- a/drivers/input/touchscreen/stmfts.c +++ b/drivers/input/touchscreen/stmfts.c @@ -479,7 +479,7 @@ static ssize_t stmfts_sysfs_hover_enable_write(struct device *dev, mutex_lock(&sdata->mutex); - if (value & sdata->hover_enabled) + if (value && sdata->hover_enabled) goto out; if (sdata->running) diff --git a/drivers/input/touchscreen/zinitix.c b/drivers/input/touchscreen/zinitix.c new file mode 100644 index 000000000000..1acc2eb2bcb3 --- /dev/null +++ b/drivers/input/touchscreen/zinitix.c @@ -0,0 +1,581 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/delay.h> +#include <linux/i2c.h> +#include <linux/input.h> +#include <linux/input/mt.h> +#include <linux/input/touchscreen.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/regulator/consumer.h> +#include <linux/slab.h> + +/* Register Map */ + +#define BT541_SWRESET_CMD 0x0000 +#define BT541_WAKEUP_CMD 0x0001 + +#define BT541_IDLE_CMD 0x0004 +#define BT541_SLEEP_CMD 0x0005 + +#define BT541_CLEAR_INT_STATUS_CMD 0x0003 +#define BT541_CALIBRATE_CMD 0x0006 +#define BT541_SAVE_STATUS_CMD 0x0007 +#define BT541_SAVE_CALIBRATION_CMD 0x0008 +#define BT541_RECALL_FACTORY_CMD 0x000f + +#define BT541_THRESHOLD 0x0020 + +#define BT541_LARGE_PALM_REJECT_AREA_TH 0x003F + +#define BT541_DEBUG_REG 0x0115 /* 0~7 */ + +#define BT541_TOUCH_MODE 0x0010 +#define BT541_CHIP_REVISION 0x0011 +#define BT541_FIRMWARE_VERSION 0x0012 + +#define ZINITIX_USB_DETECT 0x116 + +#define BT541_MINOR_FW_VERSION 0x0121 + +#define BT541_VENDOR_ID 0x001C +#define BT541_HW_ID 0x0014 + +#define BT541_DATA_VERSION_REG 0x0013 +#define BT541_SUPPORTED_FINGER_NUM 0x0015 +#define BT541_EEPROM_INFO 0x0018 +#define BT541_INITIAL_TOUCH_MODE 0x0019 + +#define BT541_TOTAL_NUMBER_OF_X 0x0060 +#define BT541_TOTAL_NUMBER_OF_Y 0x0061 + +#define BT541_DELAY_RAW_FOR_HOST 0x007f + +#define BT541_BUTTON_SUPPORTED_NUM 0x00B0 +#define BT541_BUTTON_SENSITIVITY 0x00B2 +#define BT541_DUMMY_BUTTON_SENSITIVITY 0X00C8 + +#define BT541_X_RESOLUTION 0x00C0 +#define BT541_Y_RESOLUTION 0x00C1 + +#define BT541_POINT_STATUS_REG 0x0080 +#define BT541_ICON_STATUS_REG 0x00AA + +#define BT541_POINT_COORD_REG (BT541_POINT_STATUS_REG + 2) + +#define BT541_AFE_FREQUENCY 0x0100 +#define BT541_DND_N_COUNT 0x0122 +#define BT541_DND_U_COUNT 0x0135 + +#define BT541_RAWDATA_REG 0x0200 + +#define BT541_EEPROM_INFO_REG 0x0018 + +#define BT541_INT_ENABLE_FLAG 0x00f0 +#define BT541_PERIODICAL_INTERRUPT_INTERVAL 0x00f1 + +#define BT541_BTN_WIDTH 0x016d + +#define BT541_CHECKSUM_RESULT 0x012c + +#define BT541_INIT_FLASH 0x01d0 +#define BT541_WRITE_FLASH 0x01d1 +#define BT541_READ_FLASH 0x01d2 + +#define ZINITIX_INTERNAL_FLAG_02 0x011e +#define ZINITIX_INTERNAL_FLAG_03 0x011f + +#define ZINITIX_I2C_CHECKSUM_WCNT 0x016a +#define ZINITIX_I2C_CHECKSUM_RESULT 0x016c + +/* Interrupt & status register flags */ + +#define BIT_PT_CNT_CHANGE BIT(0) +#define BIT_DOWN BIT(1) +#define BIT_MOVE BIT(2) +#define BIT_UP BIT(3) +#define BIT_PALM BIT(4) +#define BIT_PALM_REJECT BIT(5) +#define BIT_RESERVED_0 BIT(6) +#define BIT_RESERVED_1 BIT(7) +#define BIT_WEIGHT_CHANGE BIT(8) +#define BIT_PT_NO_CHANGE BIT(9) +#define BIT_REJECT BIT(10) +#define BIT_PT_EXIST BIT(11) +#define BIT_RESERVED_2 BIT(12) +#define BIT_ERROR BIT(13) +#define BIT_DEBUG BIT(14) +#define BIT_ICON_EVENT BIT(15) + +#define SUB_BIT_EXIST BIT(0) +#define SUB_BIT_DOWN BIT(1) +#define SUB_BIT_MOVE BIT(2) +#define SUB_BIT_UP BIT(3) +#define SUB_BIT_UPDATE BIT(4) +#define SUB_BIT_WAIT BIT(5) + +#define DEFAULT_TOUCH_POINT_MODE 2 +#define MAX_SUPPORTED_FINGER_NUM 5 + +#define CHIP_ON_DELAY 15 // ms +#define FIRMWARE_ON_DELAY 40 // ms + +struct point_coord { + __le16 x; + __le16 y; + u8 width; + u8 sub_status; + // currently unused, but needed as padding: + u8 minor_width; + u8 angle; +}; + +struct touch_event { + __le16 status; + u8 finger_cnt; + u8 time_stamp; + struct point_coord point_coord[MAX_SUPPORTED_FINGER_NUM]; +}; + +struct bt541_ts_data { + struct i2c_client *client; + struct input_dev *input_dev; + struct touchscreen_properties prop; + struct regulator_bulk_data supplies[2]; + u32 zinitix_mode; +}; + +static int zinitix_read_data(struct i2c_client *client, + u16 reg, void *values, size_t length) +{ + __le16 reg_le = cpu_to_le16(reg); + int ret; + + /* A single i2c_transfer() transaction does not work here. */ + ret = i2c_master_send(client, (u8 *)®_le, sizeof(reg_le)); + if (ret != sizeof(reg_le)) + return ret < 0 ? ret : -EIO; + + ret = i2c_master_recv(client, (u8 *)values, length); + if (ret != length) + return ret < 0 ? ret : -EIO; ; + + return 0; +} + +static int zinitix_write_u16(struct i2c_client *client, u16 reg, u16 value) +{ + __le16 packet[2] = {cpu_to_le16(reg), cpu_to_le16(value)}; + int ret; + + ret = i2c_master_send(client, (u8 *)packet, sizeof(packet)); + if (ret != sizeof(packet)) + return ret < 0 ? ret : -EIO; + + return 0; +} + +static int zinitix_write_cmd(struct i2c_client *client, u16 reg) +{ + __le16 reg_le = cpu_to_le16(reg); + int ret; + + ret = i2c_master_send(client, (u8 *)®_le, sizeof(reg_le)); + if (ret != sizeof(reg_le)) + return ret < 0 ? ret : -EIO; + + return 0; +} + +static bool zinitix_init_touch(struct bt541_ts_data *bt541) +{ + struct i2c_client *client = bt541->client; + int i; + int error; + + error = zinitix_write_cmd(client, BT541_SWRESET_CMD); + if (error) { + dev_err(&client->dev, "Failed to write reset command\n"); + return error; + } + + error = zinitix_write_u16(client, BT541_INT_ENABLE_FLAG, 0x0); + if (error) { + dev_err(&client->dev, + "Failed to reset interrupt enable flag\n"); + return error; + } + + /* initialize */ + error = zinitix_write_u16(client, BT541_X_RESOLUTION, + bt541->prop.max_x); + if (error) + return error; + + error = zinitix_write_u16(client, BT541_Y_RESOLUTION, + bt541->prop.max_y); + if (error) + return error; + + error = zinitix_write_u16(client, BT541_SUPPORTED_FINGER_NUM, + MAX_SUPPORTED_FINGER_NUM); + if (error) + return error; + + error = zinitix_write_u16(client, BT541_INITIAL_TOUCH_MODE, + bt541->zinitix_mode); + if (error) + return error; + + error = zinitix_write_u16(client, BT541_TOUCH_MODE, + bt541->zinitix_mode); + if (error) + return error; + + error = zinitix_write_u16(client, BT541_INT_ENABLE_FLAG, + BIT_PT_CNT_CHANGE | BIT_DOWN | BIT_MOVE | + BIT_UP); + if (error) + return error; + + /* clear queue */ + for (i = 0; i < 10; i++) { + zinitix_write_cmd(client, BT541_CLEAR_INT_STATUS_CMD); + udelay(10); + } + + return 0; +} + +static int zinitix_init_regulators(struct bt541_ts_data *bt541) +{ + struct i2c_client *client = bt541->client; + int error; + + bt541->supplies[0].supply = "vdd"; + bt541->supplies[1].supply = "vddo"; + error = devm_regulator_bulk_get(&client->dev, + ARRAY_SIZE(bt541->supplies), + bt541->supplies); + if (error < 0) { + dev_err(&client->dev, "Failed to get regulators: %d\n", error); + return error; + } + + return 0; +} + +static int zinitix_send_power_on_sequence(struct bt541_ts_data *bt541) +{ + int error; + struct i2c_client *client = bt541->client; + + error = zinitix_write_u16(client, 0xc000, 0x0001); + if (error) { + dev_err(&client->dev, + "Failed to send power sequence(vendor cmd enable)\n"); + return error; + } + udelay(10); + + error = zinitix_write_cmd(client, 0xc004); + if (error) { + dev_err(&client->dev, + "Failed to send power sequence (intn clear)\n"); + return error; + } + udelay(10); + + error = zinitix_write_u16(client, 0xc002, 0x0001); + if (error) { + dev_err(&client->dev, + "Failed to send power sequence (nvm init)\n"); + return error; + } + mdelay(2); + + error = zinitix_write_u16(client, 0xc001, 0x0001); + if (error) { + dev_err(&client->dev, + "Failed to send power sequence (program start)\n"); + return error; + } + msleep(FIRMWARE_ON_DELAY); + + return 0; +} + +static void zinitix_report_finger(struct bt541_ts_data *bt541, int slot, + const struct point_coord *p) +{ + input_mt_slot(bt541->input_dev, slot); + input_mt_report_slot_state(bt541->input_dev, MT_TOOL_FINGER, true); + touchscreen_report_pos(bt541->input_dev, &bt541->prop, + le16_to_cpu(p->x), le16_to_cpu(p->y), true); + input_report_abs(bt541->input_dev, ABS_MT_TOUCH_MAJOR, p->width); +} + +static irqreturn_t zinitix_ts_irq_handler(int irq, void *bt541_handler) +{ + struct bt541_ts_data *bt541 = bt541_handler; + struct i2c_client *client = bt541->client; + struct touch_event touch_event; + int error; + int i; + + memset(&touch_event, 0, sizeof(struct touch_event)); + + error = zinitix_read_data(bt541->client, BT541_POINT_STATUS_REG, + &touch_event, sizeof(struct touch_event)); + if (error) { + dev_err(&client->dev, "Failed to read in touchpoint struct\n"); + goto out; + } + + for (i = 0; i < MAX_SUPPORTED_FINGER_NUM; i++) + if (touch_event.point_coord[i].sub_status & SUB_BIT_EXIST) + zinitix_report_finger(bt541, i, + &touch_event.point_coord[i]); + + input_mt_sync_frame(bt541->input_dev); + input_sync(bt541->input_dev); + +out: + zinitix_write_cmd(bt541->client, BT541_CLEAR_INT_STATUS_CMD); + return IRQ_HANDLED; +} + +static int zinitix_start(struct bt541_ts_data *bt541) +{ + int error; + + error = regulator_bulk_enable(ARRAY_SIZE(bt541->supplies), + bt541->supplies); + if (error) { + dev_err(&bt541->client->dev, + "Failed to enable regulators: %d\n", error); + return error; + } + + msleep(CHIP_ON_DELAY); + + error = zinitix_send_power_on_sequence(bt541); + if (error) { + dev_err(&bt541->client->dev, + "Error while sending power-on sequence: %d\n", error); + return error; + } + + error = zinitix_init_touch(bt541); + if (error) { + dev_err(&bt541->client->dev, + "Error while configuring touch IC\n"); + return error; + } + + enable_irq(bt541->client->irq); + + return 0; +} + +static int zinitix_stop(struct bt541_ts_data *bt541) +{ + int error; + + disable_irq(bt541->client->irq); + + error = regulator_bulk_disable(ARRAY_SIZE(bt541->supplies), + bt541->supplies); + if (error) { + dev_err(&bt541->client->dev, + "Failed to disable regulators: %d\n", error); + return error; + } + + return 0; +} + +static int zinitix_input_open(struct input_dev *dev) +{ + struct bt541_ts_data *bt541 = input_get_drvdata(dev); + + return zinitix_start(bt541); +} + +static void zinitix_input_close(struct input_dev *dev) +{ + struct bt541_ts_data *bt541 = input_get_drvdata(dev); + + zinitix_stop(bt541); +} + +static int zinitix_init_input_dev(struct bt541_ts_data *bt541) +{ + struct input_dev *input_dev; + int error; + + input_dev = devm_input_allocate_device(&bt541->client->dev); + if (!input_dev) { + dev_err(&bt541->client->dev, + "Failed to allocate input device."); + return -ENOMEM; + } + + input_set_drvdata(input_dev, bt541); + bt541->input_dev = input_dev; + + input_dev->name = "Zinitix Capacitive TouchScreen"; + input_dev->phys = "input/ts"; + input_dev->id.bustype = BUS_I2C; + input_dev->open = zinitix_input_open; + input_dev->close = zinitix_input_close; + + input_set_capability(input_dev, EV_ABS, ABS_MT_POSITION_X); + input_set_capability(input_dev, EV_ABS, ABS_MT_POSITION_Y); + input_set_abs_params(input_dev, ABS_MT_WIDTH_MAJOR, 0, 255, 0, 0); + input_set_abs_params(input_dev, ABS_MT_TOUCH_MAJOR, 0, 255, 0, 0); + + touchscreen_parse_properties(input_dev, true, &bt541->prop); + if (!bt541->prop.max_x || !bt541->prop.max_y) { + dev_err(&bt541->client->dev, + "Touchscreen-size-x and/or touchscreen-size-y not set in dts\n"); + return -EINVAL; + } + + error = input_mt_init_slots(input_dev, MAX_SUPPORTED_FINGER_NUM, + INPUT_MT_DIRECT | INPUT_MT_DROP_UNUSED); + if (error) { + dev_err(&bt541->client->dev, + "Failed to initialize MT slots: %d", error); + return error; + } + + error = input_register_device(input_dev); + if (error) { + dev_err(&bt541->client->dev, + "Failed to register input device: %d", error); + return error; + } + + return 0; +} + +static int zinitix_ts_probe(struct i2c_client *client) +{ + struct bt541_ts_data *bt541; + int error; + + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) { + dev_err(&client->dev, + "Failed to assert adapter's support for plain I2C.\n"); + return -ENXIO; + } + + bt541 = devm_kzalloc(&client->dev, sizeof(*bt541), GFP_KERNEL); + if (!bt541) + return -ENOMEM; + + bt541->client = client; + i2c_set_clientdata(client, bt541); + + error = zinitix_init_regulators(bt541); + if (error) { + dev_err(&client->dev, + "Failed to initialize regulators: %d\n", error); + return error; + } + + error = zinitix_init_input_dev(bt541); + if (error) { + dev_err(&client->dev, + "Failed to initialize input device: %d\n", error); + return error; + } + + error = device_property_read_u32(&client->dev, "zinitix,mode", + &bt541->zinitix_mode); + if (error < 0) { + /* fall back to mode 2 */ + bt541->zinitix_mode = DEFAULT_TOUCH_POINT_MODE; + } + + if (bt541->zinitix_mode != 2) { + /* + * If there are devices that don't support mode 2, support + * for other modes (0, 1) will be needed. + */ + dev_err(&client->dev, + "Malformed zinitix,mode property, must be 2 (supplied: %d)\n", + bt541->zinitix_mode); + return -EINVAL; + } + + irq_set_status_flags(client->irq, IRQ_NOAUTOEN); + error = devm_request_threaded_irq(&client->dev, client->irq, + NULL, zinitix_ts_irq_handler, + IRQF_ONESHOT, client->name, bt541); + if (error) { + dev_err(&client->dev, "Failed to request IRQ: %d\n", error); + return error; + } + + return 0; +} + +static int __maybe_unused zinitix_suspend(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + struct bt541_ts_data *bt541 = i2c_get_clientdata(client); + + mutex_lock(&bt541->input_dev->mutex); + + if (bt541->input_dev->users) + zinitix_stop(bt541); + + mutex_unlock(&bt541->input_dev->mutex); + + return 0; +} + +static int __maybe_unused zinitix_resume(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + struct bt541_ts_data *bt541 = i2c_get_clientdata(client); + int ret = 0; + + mutex_lock(&bt541->input_dev->mutex); + + if (bt541->input_dev->users) + ret = zinitix_start(bt541); + + mutex_unlock(&bt541->input_dev->mutex); + + return ret; +} + +static SIMPLE_DEV_PM_OPS(zinitix_pm_ops, zinitix_suspend, zinitix_resume); + +#ifdef CONFIG_OF +static const struct of_device_id zinitix_of_match[] = { + { .compatible = "zinitix,bt541" }, + { } +}; +MODULE_DEVICE_TABLE(of, zinitix_of_match); +#endif + +static struct i2c_driver zinitix_ts_driver = { + .probe_new = zinitix_ts_probe, + .driver = { + .name = "Zinitix-TS", + .pm = &zinitix_pm_ops, + .of_match_table = of_match_ptr(zinitix_of_match), + }, +}; +module_i2c_driver(zinitix_ts_driver); + +MODULE_AUTHOR("Michael Srba <Michael.Srba@seznam.cz>"); +MODULE_DESCRIPTION("Zinitix touchscreen driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 2d70d56d8e0d..404b40af31cb 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1136,7 +1136,7 @@ error: static void free_iommu(struct intel_iommu *iommu) { - if (intel_iommu_enabled && iommu->iommu.ops) { + if (intel_iommu_enabled && !iommu->drhd->ignored) { iommu_device_unregister(&iommu->iommu); iommu_device_sysfs_remove(&iommu->iommu); } diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 80e3a84709ee..94c2885882ee 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -17,7 +17,6 @@ obj-$(CONFIG_ARCH_LPC32XX) += irq-lpc32xx.o obj-$(CONFIG_ARCH_MMP) += irq-mmp.o obj-$(CONFIG_IRQ_MXS) += irq-mxs.o obj-$(CONFIG_ARCH_TEGRA) += irq-tegra.o -obj-$(CONFIG_ARCH_S3C24XX) += irq-s3c24xx.o obj-$(CONFIG_DW_APB_ICTL) += irq-dw-apb-ictl.o obj-$(CONFIG_CLPS711X_IRQCHIP) += irq-clps711x.o obj-$(CONFIG_OMPIC) += irq-ompic.o diff --git a/drivers/irqchip/irq-s3c24xx.c b/drivers/irqchip/irq-s3c24xx.c deleted file mode 100644 index d2031fecc386..000000000000 --- a/drivers/irqchip/irq-s3c24xx.c +++ /dev/null @@ -1,1330 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * S3C24XX IRQ handling - * - * Copyright (c) 2003-2004 Simtec Electronics - * Ben Dooks <ben@simtec.co.uk> - * Copyright (c) 2012 Heiko Stuebner <heiko@sntech.de> -*/ - -#include <linux/init.h> -#include <linux/slab.h> -#include <linux/module.h> -#include <linux/io.h> -#include <linux/err.h> -#include <linux/interrupt.h> -#include <linux/ioport.h> -#include <linux/device.h> -#include <linux/irqdomain.h> -#include <linux/irqchip.h> -#include <linux/irqchip/chained_irq.h> -#include <linux/of.h> -#include <linux/of_irq.h> -#include <linux/of_address.h> - -#include <asm/exception.h> -#include <asm/mach/irq.h> - -#include <mach/regs-irq.h> -#include <mach/regs-gpio.h> - -#include <plat/cpu.h> -#include <plat/regs-irqtype.h> -#include <plat/pm.h> - -#define S3C_IRQTYPE_NONE 0 -#define S3C_IRQTYPE_EINT 1 -#define S3C_IRQTYPE_EDGE 2 -#define S3C_IRQTYPE_LEVEL 3 - -struct s3c_irq_data { - unsigned int type; - unsigned long offset; - unsigned long parent_irq; - - /* data gets filled during init */ - struct s3c_irq_intc *intc; - unsigned long sub_bits; - struct s3c_irq_intc *sub_intc; -}; - -/* - * Structure holding the controller data - * @reg_pending register holding pending irqs - * @reg_intpnd special register intpnd in main intc - * @reg_mask mask register - * @domain irq_domain of the controller - * @parent parent controller for ext and sub irqs - * @irqs irq-data, always s3c_irq_data[32] - */ -struct s3c_irq_intc { - void __iomem *reg_pending; - void __iomem *reg_intpnd; - void __iomem *reg_mask; - struct irq_domain *domain; - struct s3c_irq_intc *parent; - struct s3c_irq_data *irqs; -}; - -/* - * Array holding pointers to the global controller structs - * [0] ... main_intc - * [1] ... sub_intc - * [2] ... main_intc2 on s3c2416 - */ -static struct s3c_irq_intc *s3c_intc[3]; - -static void s3c_irq_mask(struct irq_data *data) -{ - struct s3c_irq_data *irq_data = irq_data_get_irq_chip_data(data); - struct s3c_irq_intc *intc = irq_data->intc; - struct s3c_irq_intc *parent_intc = intc->parent; - struct s3c_irq_data *parent_data; - unsigned long mask; - unsigned int irqno; - - mask = readl_relaxed(intc->reg_mask); - mask |= (1UL << irq_data->offset); - writel_relaxed(mask, intc->reg_mask); - - if (parent_intc) { - parent_data = &parent_intc->irqs[irq_data->parent_irq]; - - /* check to see if we need to mask the parent IRQ - * The parent_irq is always in main_intc, so the hwirq - * for find_mapping does not need an offset in any case. - */ - if ((mask & parent_data->sub_bits) == parent_data->sub_bits) { - irqno = irq_find_mapping(parent_intc->domain, - irq_data->parent_irq); - s3c_irq_mask(irq_get_irq_data(irqno)); - } - } -} - -static void s3c_irq_unmask(struct irq_data *data) -{ - struct s3c_irq_data *irq_data = irq_data_get_irq_chip_data(data); - struct s3c_irq_intc *intc = irq_data->intc; - struct s3c_irq_intc *parent_intc = intc->parent; - unsigned long mask; - unsigned int irqno; - - mask = readl_relaxed(intc->reg_mask); - mask &= ~(1UL << irq_data->offset); - writel_relaxed(mask, intc->reg_mask); - - if (parent_intc) { - irqno = irq_find_mapping(parent_intc->domain, - irq_data->parent_irq); - s3c_irq_unmask(irq_get_irq_data(irqno)); - } -} - -static inline void s3c_irq_ack(struct irq_data *data) -{ - struct s3c_irq_data *irq_data = irq_data_get_irq_chip_data(data); - struct s3c_irq_intc *intc = irq_data->intc; - unsigned long bitval = 1UL << irq_data->offset; - - writel_relaxed(bitval, intc->reg_pending); - if (intc->reg_intpnd) - writel_relaxed(bitval, intc->reg_intpnd); -} - -static int s3c_irq_type(struct irq_data *data, unsigned int type) -{ - switch (type) { - case IRQ_TYPE_NONE: - break; - case IRQ_TYPE_EDGE_RISING: - case IRQ_TYPE_EDGE_FALLING: - case IRQ_TYPE_EDGE_BOTH: - irq_set_handler(data->irq, handle_edge_irq); - break; - case IRQ_TYPE_LEVEL_LOW: - case IRQ_TYPE_LEVEL_HIGH: - irq_set_handler(data->irq, handle_level_irq); - break; - default: - pr_err("No such irq type %d\n", type); - return -EINVAL; - } - - return 0; -} - -static int s3c_irqext_type_set(void __iomem *gpcon_reg, - void __iomem *extint_reg, - unsigned long gpcon_offset, - unsigned long extint_offset, - unsigned int type) -{ - unsigned long newvalue = 0, value; - - /* Set the GPIO to external interrupt mode */ - value = readl_relaxed(gpcon_reg); - value = (value & ~(3 << gpcon_offset)) | (0x02 << gpcon_offset); - writel_relaxed(value, gpcon_reg); - - /* Set the external interrupt to pointed trigger type */ - switch (type) - { - case IRQ_TYPE_NONE: - pr_warn("No edge setting!\n"); - break; - - case IRQ_TYPE_EDGE_RISING: - newvalue = S3C2410_EXTINT_RISEEDGE; - break; - - case IRQ_TYPE_EDGE_FALLING: - newvalue = S3C2410_EXTINT_FALLEDGE; - break; - - case IRQ_TYPE_EDGE_BOTH: - newvalue = S3C2410_EXTINT_BOTHEDGE; - break; - - case IRQ_TYPE_LEVEL_LOW: - newvalue = S3C2410_EXTINT_LOWLEV; - break; - - case IRQ_TYPE_LEVEL_HIGH: - newvalue = S3C2410_EXTINT_HILEV; - break; - - default: - pr_err("No such irq type %d\n", type); - return -EINVAL; - } - - value = readl_relaxed(extint_reg); - value = (value & ~(7 << extint_offset)) | (newvalue << extint_offset); - writel_relaxed(value, extint_reg); - - return 0; -} - -static int s3c_irqext_type(struct irq_data *data, unsigned int type) -{ - void __iomem *extint_reg; - void __iomem *gpcon_reg; - unsigned long gpcon_offset, extint_offset; - - if ((data->hwirq >= 4) && (data->hwirq <= 7)) { - gpcon_reg = S3C2410_GPFCON; - extint_reg = S3C24XX_EXTINT0; - gpcon_offset = (data->hwirq) * 2; - extint_offset = (data->hwirq) * 4; - } else if ((data->hwirq >= 8) && (data->hwirq <= 15)) { - gpcon_reg = S3C2410_GPGCON; - extint_reg = S3C24XX_EXTINT1; - gpcon_offset = (data->hwirq - 8) * 2; - extint_offset = (data->hwirq - 8) * 4; - } else if ((data->hwirq >= 16) && (data->hwirq <= 23)) { - gpcon_reg = S3C2410_GPGCON; - extint_reg = S3C24XX_EXTINT2; - gpcon_offset = (data->hwirq - 8) * 2; - extint_offset = (data->hwirq - 16) * 4; - } else { - return -EINVAL; - } - - return s3c_irqext_type_set(gpcon_reg, extint_reg, gpcon_offset, - extint_offset, type); -} - -static int s3c_irqext0_type(struct irq_data *data, unsigned int type) -{ - void __iomem *extint_reg; - void __iomem *gpcon_reg; - unsigned long gpcon_offset, extint_offset; - - if (data->hwirq <= 3) { - gpcon_reg = S3C2410_GPFCON; - extint_reg = S3C24XX_EXTINT0; - gpcon_offset = (data->hwirq) * 2; - extint_offset = (data->hwirq) * 4; - } else { - return -EINVAL; - } - - return s3c_irqext_type_set(gpcon_reg, extint_reg, gpcon_offset, - extint_offset, type); -} - -static struct irq_chip s3c_irq_chip = { - .name = "s3c", - .irq_ack = s3c_irq_ack, - .irq_mask = s3c_irq_mask, - .irq_unmask = s3c_irq_unmask, - .irq_set_type = s3c_irq_type, - .irq_set_wake = s3c_irq_wake -}; - -static struct irq_chip s3c_irq_level_chip = { - .name = "s3c-level", - .irq_mask = s3c_irq_mask, - .irq_unmask = s3c_irq_unmask, - .irq_ack = s3c_irq_ack, - .irq_set_type = s3c_irq_type, -}; - -static struct irq_chip s3c_irqext_chip = { - .name = "s3c-ext", - .irq_mask = s3c_irq_mask, - .irq_unmask = s3c_irq_unmask, - .irq_ack = s3c_irq_ack, - .irq_set_type = s3c_irqext_type, - .irq_set_wake = s3c_irqext_wake -}; - -static struct irq_chip s3c_irq_eint0t4 = { - .name = "s3c-ext0", - .irq_ack = s3c_irq_ack, - .irq_mask = s3c_irq_mask, - .irq_unmask = s3c_irq_unmask, - .irq_set_wake = s3c_irq_wake, - .irq_set_type = s3c_irqext0_type, -}; - -static void s3c_irq_demux(struct irq_desc *desc) -{ - struct irq_chip *chip = irq_desc_get_chip(desc); - struct s3c_irq_data *irq_data = irq_desc_get_chip_data(desc); - struct s3c_irq_intc *intc = irq_data->intc; - struct s3c_irq_intc *sub_intc = irq_data->sub_intc; - unsigned int n, offset, irq; - unsigned long src, msk; - - /* we're using individual domains for the non-dt case - * and one big domain for the dt case where the subintc - * starts at hwirq number 32. - */ - offset = irq_domain_get_of_node(intc->domain) ? 32 : 0; - - chained_irq_enter(chip, desc); - - src = readl_relaxed(sub_intc->reg_pending); - msk = readl_relaxed(sub_intc->reg_mask); - - src &= ~msk; - src &= irq_data->sub_bits; - - while (src) { - n = __ffs(src); - src &= ~(1 << n); - irq = irq_find_mapping(sub_intc->domain, offset + n); - generic_handle_irq(irq); - } - - chained_irq_exit(chip, desc); -} - -static inline int s3c24xx_handle_intc(struct s3c_irq_intc *intc, - struct pt_regs *regs, int intc_offset) -{ - int pnd; - int offset; - - pnd = readl_relaxed(intc->reg_intpnd); - if (!pnd) - return false; - - /* non-dt machines use individual domains */ - if (!irq_domain_get_of_node(intc->domain)) - intc_offset = 0; - - /* We have a problem that the INTOFFSET register does not always - * show one interrupt. Occasionally we get two interrupts through - * the prioritiser, and this causes the INTOFFSET register to show - * what looks like the logical-or of the two interrupt numbers. - * - * Thanks to Klaus, Shannon, et al for helping to debug this problem - */ - offset = readl_relaxed(intc->reg_intpnd + 4); - - /* Find the bit manually, when the offset is wrong. - * The pending register only ever contains the one bit of the next - * interrupt to handle. - */ - if (!(pnd & (1 << offset))) - offset = __ffs(pnd); - - handle_domain_irq(intc->domain, intc_offset + offset, regs); - return true; -} - -asmlinkage void __exception_irq_entry s3c24xx_handle_irq(struct pt_regs *regs) -{ - do { - if (likely(s3c_intc[0])) - if (s3c24xx_handle_intc(s3c_intc[0], regs, 0)) - continue; - - if (s3c_intc[2]) - if (s3c24xx_handle_intc(s3c_intc[2], regs, 64)) - continue; - - break; - } while (1); -} - -#ifdef CONFIG_FIQ -/** - * s3c24xx_set_fiq - set the FIQ routing - * @irq: IRQ number to route to FIQ on processor. - * @on: Whether to route @irq to the FIQ, or to remove the FIQ routing. - * - * Change the state of the IRQ to FIQ routing depending on @irq and @on. If - * @on is true, the @irq is checked to see if it can be routed and the - * interrupt controller updated to route the IRQ. If @on is false, the FIQ - * routing is cleared, regardless of which @irq is specified. - */ -int s3c24xx_set_fiq(unsigned int irq, bool on) -{ - u32 intmod; - unsigned offs; - - if (on) { - offs = irq - FIQ_START; - if (offs > 31) - return -EINVAL; - - intmod = 1 << offs; - } else { - intmod = 0; - } - - writel_relaxed(intmod, S3C2410_INTMOD); - return 0; -} - -EXPORT_SYMBOL_GPL(s3c24xx_set_fiq); -#endif - -static int s3c24xx_irq_map(struct irq_domain *h, unsigned int virq, - irq_hw_number_t hw) -{ - struct s3c_irq_intc *intc = h->host_data; - struct s3c_irq_data *irq_data = &intc->irqs[hw]; - struct s3c_irq_intc *parent_intc; - struct s3c_irq_data *parent_irq_data; - unsigned int irqno; - - /* attach controller pointer to irq_data */ - irq_data->intc = intc; - irq_data->offset = hw; - - parent_intc = intc->parent; - - /* set handler and flags */ - switch (irq_data->type) { - case S3C_IRQTYPE_NONE: - return 0; - case S3C_IRQTYPE_EINT: - /* On the S3C2412, the EINT0to3 have a parent irq - * but need the s3c_irq_eint0t4 chip - */ - if (parent_intc && (!soc_is_s3c2412() || hw >= 4)) - irq_set_chip_and_handler(virq, &s3c_irqext_chip, - handle_edge_irq); - else - irq_set_chip_and_handler(virq, &s3c_irq_eint0t4, - handle_edge_irq); - break; - case S3C_IRQTYPE_EDGE: - if (parent_intc || intc->reg_pending == S3C2416_SRCPND2) - irq_set_chip_and_handler(virq, &s3c_irq_level_chip, - handle_edge_irq); - else - irq_set_chip_and_handler(virq, &s3c_irq_chip, - handle_edge_irq); - break; - case S3C_IRQTYPE_LEVEL: - if (parent_intc) - irq_set_chip_and_handler(virq, &s3c_irq_level_chip, - handle_level_irq); - else - irq_set_chip_and_handler(virq, &s3c_irq_chip, - handle_level_irq); - break; - default: - pr_err("irq-s3c24xx: unsupported irqtype %d\n", irq_data->type); - return -EINVAL; - } - - irq_set_chip_data(virq, irq_data); - - if (parent_intc && irq_data->type != S3C_IRQTYPE_NONE) { - if (irq_data->parent_irq > 31) { - pr_err("irq-s3c24xx: parent irq %lu is out of range\n", - irq_data->parent_irq); - return -EINVAL; - } - - parent_irq_data = &parent_intc->irqs[irq_data->parent_irq]; - parent_irq_data->sub_intc = intc; - parent_irq_data->sub_bits |= (1UL << hw); - - /* attach the demuxer to the parent irq */ - irqno = irq_find_mapping(parent_intc->domain, - irq_data->parent_irq); - if (!irqno) { - pr_err("irq-s3c24xx: could not find mapping for parent irq %lu\n", - irq_data->parent_irq); - return -EINVAL; - } - irq_set_chained_handler(irqno, s3c_irq_demux); - } - - return 0; -} - -static const struct irq_domain_ops s3c24xx_irq_ops = { - .map = s3c24xx_irq_map, - .xlate = irq_domain_xlate_twocell, -}; - -static void s3c24xx_clear_intc(struct s3c_irq_intc *intc) -{ - void __iomem *reg_source; - unsigned long pend; - unsigned long last; - int i; - - /* if intpnd is set, read the next pending irq from there */ - reg_source = intc->reg_intpnd ? intc->reg_intpnd : intc->reg_pending; - - last = 0; - for (i = 0; i < 4; i++) { - pend = readl_relaxed(reg_source); - - if (pend == 0 || pend == last) - break; - - writel_relaxed(pend, intc->reg_pending); - if (intc->reg_intpnd) - writel_relaxed(pend, intc->reg_intpnd); - - pr_info("irq: clearing pending status %08x\n", (int)pend); - last = pend; - } -} - -static struct s3c_irq_intc * __init s3c24xx_init_intc(struct device_node *np, - struct s3c_irq_data *irq_data, - struct s3c_irq_intc *parent, - unsigned long address) -{ - struct s3c_irq_intc *intc; - void __iomem *base = (void *)0xf6000000; /* static mapping */ - int irq_num; - int irq_start; - int ret; - - intc = kzalloc(sizeof(struct s3c_irq_intc), GFP_KERNEL); - if (!intc) - return ERR_PTR(-ENOMEM); - - intc->irqs = irq_data; - - if (parent) - intc->parent = parent; - - /* select the correct data for the controller. - * Need to hard code the irq num start and offset - * to preserve the static mapping for now - */ - switch (address) { - case 0x4a000000: - pr_debug("irq: found main intc\n"); - intc->reg_pending = base; - intc->reg_mask = base + 0x08; - intc->reg_intpnd = base + 0x10; - irq_num = 32; - irq_start = S3C2410_IRQ(0); - break; - case 0x4a000018: - pr_debug("irq: found subintc\n"); - intc->reg_pending = base + 0x18; - intc->reg_mask = base + 0x1c; - irq_num = 29; - irq_start = S3C2410_IRQSUB(0); - break; - case 0x4a000040: - pr_debug("irq: found intc2\n"); - intc->reg_pending = base + 0x40; - intc->reg_mask = base + 0x48; - intc->reg_intpnd = base + 0x50; - irq_num = 8; - irq_start = S3C2416_IRQ(0); - break; - case 0x560000a4: - pr_debug("irq: found eintc\n"); - base = (void *)0xfd000000; - - intc->reg_mask = base + 0xa4; - intc->reg_pending = base + 0xa8; - irq_num = 24; - irq_start = S3C2410_IRQ(32); - break; - default: - pr_err("irq: unsupported controller address\n"); - ret = -EINVAL; - goto err; - } - - /* now that all the data is complete, init the irq-domain */ - s3c24xx_clear_intc(intc); - intc->domain = irq_domain_add_legacy(np, irq_num, irq_start, - 0, &s3c24xx_irq_ops, - intc); - if (!intc->domain) { - pr_err("irq: could not create irq-domain\n"); - ret = -EINVAL; - goto err; - } - - set_handle_irq(s3c24xx_handle_irq); - - return intc; - -err: - kfree(intc); - return ERR_PTR(ret); -} - -static struct s3c_irq_data __maybe_unused init_eint[32] = { - { .type = S3C_IRQTYPE_NONE, }, /* reserved */ - { .type = S3C_IRQTYPE_NONE, }, /* reserved */ - { .type = S3C_IRQTYPE_NONE, }, /* reserved */ - { .type = S3C_IRQTYPE_NONE, }, /* reserved */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 4 }, /* EINT4 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 4 }, /* EINT5 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 4 }, /* EINT6 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 4 }, /* EINT7 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT8 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT9 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT10 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT11 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT12 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT13 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT14 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT15 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT16 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT17 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT18 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT19 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT20 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT21 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT22 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT23 */ -}; - -#ifdef CONFIG_CPU_S3C2410 -static struct s3c_irq_data init_s3c2410base[32] = { - { .type = S3C_IRQTYPE_EINT, }, /* EINT0 */ - { .type = S3C_IRQTYPE_EINT, }, /* EINT1 */ - { .type = S3C_IRQTYPE_EINT, }, /* EINT2 */ - { .type = S3C_IRQTYPE_EINT, }, /* EINT3 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT4to7 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT8to23 */ - { .type = S3C_IRQTYPE_NONE, }, /* reserved */ - { .type = S3C_IRQTYPE_EDGE, }, /* nBATT_FLT */ - { .type = S3C_IRQTYPE_EDGE, }, /* TICK */ - { .type = S3C_IRQTYPE_EDGE, }, /* WDT */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER2 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER3 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER4 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART2 */ - { .type = S3C_IRQTYPE_EDGE, }, /* LCD */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA2 */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA3 */ - { .type = S3C_IRQTYPE_EDGE, }, /* SDI */ - { .type = S3C_IRQTYPE_EDGE, }, /* SPI0 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART1 */ - { .type = S3C_IRQTYPE_NONE, }, /* reserved */ - { .type = S3C_IRQTYPE_EDGE, }, /* USBD */ - { .type = S3C_IRQTYPE_EDGE, }, /* USBH */ - { .type = S3C_IRQTYPE_EDGE, }, /* IIC */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* SPI1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* RTC */ - { .type = S3C_IRQTYPE_LEVEL, }, /* ADCPARENT */ -}; - -static struct s3c_irq_data init_s3c2410subint[32] = { - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-ERR */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-ERR */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-ERR */ - { .type = S3C_IRQTYPE_EDGE, .parent_irq = 31 }, /* TC */ - { .type = S3C_IRQTYPE_EDGE, .parent_irq = 31 }, /* ADC */ -}; - -void __init s3c2410_init_irq(void) -{ -#ifdef CONFIG_FIQ - init_FIQ(FIQ_START); -#endif - - s3c_intc[0] = s3c24xx_init_intc(NULL, &init_s3c2410base[0], NULL, - 0x4a000000); - if (IS_ERR(s3c_intc[0])) { - pr_err("irq: could not create main interrupt controller\n"); - return; - } - - s3c_intc[1] = s3c24xx_init_intc(NULL, &init_s3c2410subint[0], - s3c_intc[0], 0x4a000018); - s3c24xx_init_intc(NULL, &init_eint[0], s3c_intc[0], 0x560000a4); -} -#endif - -#ifdef CONFIG_CPU_S3C2412 -static struct s3c_irq_data init_s3c2412base[32] = { - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT0 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT1 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT2 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT3 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT4to7 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT8to23 */ - { .type = S3C_IRQTYPE_NONE, }, /* reserved */ - { .type = S3C_IRQTYPE_EDGE, }, /* nBATT_FLT */ - { .type = S3C_IRQTYPE_EDGE, }, /* TICK */ - { .type = S3C_IRQTYPE_EDGE, }, /* WDT */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER2 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER3 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER4 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART2 */ - { .type = S3C_IRQTYPE_EDGE, }, /* LCD */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA2 */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA3 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* SDI/CF */ - { .type = S3C_IRQTYPE_EDGE, }, /* SPI0 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART1 */ - { .type = S3C_IRQTYPE_NONE, }, /* reserved */ - { .type = S3C_IRQTYPE_EDGE, }, /* USBD */ - { .type = S3C_IRQTYPE_EDGE, }, /* USBH */ - { .type = S3C_IRQTYPE_EDGE, }, /* IIC */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* SPI1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* RTC */ - { .type = S3C_IRQTYPE_LEVEL, }, /* ADCPARENT */ -}; - -static struct s3c_irq_data init_s3c2412eint[32] = { - { .type = S3C_IRQTYPE_EINT, .parent_irq = 0 }, /* EINT0 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 1 }, /* EINT1 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 2 }, /* EINT2 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 3 }, /* EINT3 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 4 }, /* EINT4 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 4 }, /* EINT5 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 4 }, /* EINT6 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 4 }, /* EINT7 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT8 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT9 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT10 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT11 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT12 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT13 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT14 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT15 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT16 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT17 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT18 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT19 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT20 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT21 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT22 */ - { .type = S3C_IRQTYPE_EINT, .parent_irq = 5 }, /* EINT23 */ -}; - -static struct s3c_irq_data init_s3c2412subint[32] = { - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-ERR */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-ERR */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-ERR */ - { .type = S3C_IRQTYPE_EDGE, .parent_irq = 31 }, /* TC */ - { .type = S3C_IRQTYPE_EDGE, .parent_irq = 31 }, /* ADC */ - { .type = S3C_IRQTYPE_NONE, }, - { .type = S3C_IRQTYPE_NONE, }, - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 21 }, /* SDI */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 21 }, /* CF */ -}; - -void __init s3c2412_init_irq(void) -{ - pr_info("S3C2412: IRQ Support\n"); - -#ifdef CONFIG_FIQ - init_FIQ(FIQ_START); -#endif - - s3c_intc[0] = s3c24xx_init_intc(NULL, &init_s3c2412base[0], NULL, - 0x4a000000); - if (IS_ERR(s3c_intc[0])) { - pr_err("irq: could not create main interrupt controller\n"); - return; - } - - s3c24xx_init_intc(NULL, &init_s3c2412eint[0], s3c_intc[0], 0x560000a4); - s3c_intc[1] = s3c24xx_init_intc(NULL, &init_s3c2412subint[0], - s3c_intc[0], 0x4a000018); -} -#endif - -#ifdef CONFIG_CPU_S3C2416 -static struct s3c_irq_data init_s3c2416base[32] = { - { .type = S3C_IRQTYPE_EINT, }, /* EINT0 */ - { .type = S3C_IRQTYPE_EINT, }, /* EINT1 */ - { .type = S3C_IRQTYPE_EINT, }, /* EINT2 */ - { .type = S3C_IRQTYPE_EINT, }, /* EINT3 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT4to7 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT8to23 */ - { .type = S3C_IRQTYPE_NONE, }, /* reserved */ - { .type = S3C_IRQTYPE_EDGE, }, /* nBATT_FLT */ - { .type = S3C_IRQTYPE_EDGE, }, /* TICK */ - { .type = S3C_IRQTYPE_LEVEL, }, /* WDT/AC97 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER2 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER3 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER4 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART2 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* LCD */ - { .type = S3C_IRQTYPE_LEVEL, }, /* DMA */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART3 */ - { .type = S3C_IRQTYPE_NONE, }, /* reserved */ - { .type = S3C_IRQTYPE_EDGE, }, /* SDI1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* SDI0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* SPI0 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* NAND */ - { .type = S3C_IRQTYPE_EDGE, }, /* USBD */ - { .type = S3C_IRQTYPE_EDGE, }, /* USBH */ - { .type = S3C_IRQTYPE_EDGE, }, /* IIC */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART0 */ - { .type = S3C_IRQTYPE_NONE, }, - { .type = S3C_IRQTYPE_EDGE, }, /* RTC */ - { .type = S3C_IRQTYPE_LEVEL, }, /* ADCPARENT */ -}; - -static struct s3c_irq_data init_s3c2416subint[32] = { - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-ERR */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-ERR */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-ERR */ - { .type = S3C_IRQTYPE_EDGE, .parent_irq = 31 }, /* TC */ - { .type = S3C_IRQTYPE_EDGE, .parent_irq = 31 }, /* ADC */ - { .type = S3C_IRQTYPE_NONE }, /* reserved */ - { .type = S3C_IRQTYPE_NONE }, /* reserved */ - { .type = S3C_IRQTYPE_NONE }, /* reserved */ - { .type = S3C_IRQTYPE_NONE }, /* reserved */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 16 }, /* LCD2 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 16 }, /* LCD3 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 16 }, /* LCD4 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 17 }, /* DMA0 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 17 }, /* DMA1 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 17 }, /* DMA2 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 17 }, /* DMA3 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 17 }, /* DMA4 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 17 }, /* DMA5 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 18 }, /* UART3-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 18 }, /* UART3-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 18 }, /* UART3-ERR */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 9 }, /* WDT */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 9 }, /* AC97 */ -}; - -static struct s3c_irq_data init_s3c2416_second[32] = { - { .type = S3C_IRQTYPE_EDGE }, /* 2D */ - { .type = S3C_IRQTYPE_NONE }, /* reserved */ - { .type = S3C_IRQTYPE_NONE }, /* reserved */ - { .type = S3C_IRQTYPE_NONE }, /* reserved */ - { .type = S3C_IRQTYPE_EDGE }, /* PCM0 */ - { .type = S3C_IRQTYPE_NONE }, /* reserved */ - { .type = S3C_IRQTYPE_EDGE }, /* I2S0 */ -}; - -void __init s3c2416_init_irq(void) -{ - pr_info("S3C2416: IRQ Support\n"); - -#ifdef CONFIG_FIQ - init_FIQ(FIQ_START); -#endif - - s3c_intc[0] = s3c24xx_init_intc(NULL, &init_s3c2416base[0], NULL, - 0x4a000000); - if (IS_ERR(s3c_intc[0])) { - pr_err("irq: could not create main interrupt controller\n"); - return; - } - - s3c24xx_init_intc(NULL, &init_eint[0], s3c_intc[0], 0x560000a4); - s3c_intc[1] = s3c24xx_init_intc(NULL, &init_s3c2416subint[0], - s3c_intc[0], 0x4a000018); - - s3c_intc[2] = s3c24xx_init_intc(NULL, &init_s3c2416_second[0], - NULL, 0x4a000040); -} - -#endif - -#ifdef CONFIG_CPU_S3C2440 -static struct s3c_irq_data init_s3c2440base[32] = { - { .type = S3C_IRQTYPE_EINT, }, /* EINT0 */ - { .type = S3C_IRQTYPE_EINT, }, /* EINT1 */ - { .type = S3C_IRQTYPE_EINT, }, /* EINT2 */ - { .type = S3C_IRQTYPE_EINT, }, /* EINT3 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT4to7 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT8to23 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* CAM */ - { .type = S3C_IRQTYPE_EDGE, }, /* nBATT_FLT */ - { .type = S3C_IRQTYPE_EDGE, }, /* TICK */ - { .type = S3C_IRQTYPE_LEVEL, }, /* WDT/AC97 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER2 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER3 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER4 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART2 */ - { .type = S3C_IRQTYPE_EDGE, }, /* LCD */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA2 */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA3 */ - { .type = S3C_IRQTYPE_EDGE, }, /* SDI */ - { .type = S3C_IRQTYPE_EDGE, }, /* SPI0 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART1 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* NFCON */ - { .type = S3C_IRQTYPE_EDGE, }, /* USBD */ - { .type = S3C_IRQTYPE_EDGE, }, /* USBH */ - { .type = S3C_IRQTYPE_EDGE, }, /* IIC */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* SPI1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* RTC */ - { .type = S3C_IRQTYPE_LEVEL, }, /* ADCPARENT */ -}; - -static struct s3c_irq_data init_s3c2440subint[32] = { - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-ERR */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-ERR */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-ERR */ - { .type = S3C_IRQTYPE_EDGE, .parent_irq = 31 }, /* TC */ - { .type = S3C_IRQTYPE_EDGE, .parent_irq = 31 }, /* ADC */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 6 }, /* CAM_C */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 6 }, /* CAM_P */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 9 }, /* WDT */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 9 }, /* AC97 */ -}; - -void __init s3c2440_init_irq(void) -{ - pr_info("S3C2440: IRQ Support\n"); - -#ifdef CONFIG_FIQ - init_FIQ(FIQ_START); -#endif - - s3c_intc[0] = s3c24xx_init_intc(NULL, &init_s3c2440base[0], NULL, - 0x4a000000); - if (IS_ERR(s3c_intc[0])) { - pr_err("irq: could not create main interrupt controller\n"); - return; - } - - s3c24xx_init_intc(NULL, &init_eint[0], s3c_intc[0], 0x560000a4); - s3c_intc[1] = s3c24xx_init_intc(NULL, &init_s3c2440subint[0], - s3c_intc[0], 0x4a000018); -} -#endif - -#ifdef CONFIG_CPU_S3C2442 -static struct s3c_irq_data init_s3c2442base[32] = { - { .type = S3C_IRQTYPE_EINT, }, /* EINT0 */ - { .type = S3C_IRQTYPE_EINT, }, /* EINT1 */ - { .type = S3C_IRQTYPE_EINT, }, /* EINT2 */ - { .type = S3C_IRQTYPE_EINT, }, /* EINT3 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT4to7 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT8to23 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* CAM */ - { .type = S3C_IRQTYPE_EDGE, }, /* nBATT_FLT */ - { .type = S3C_IRQTYPE_EDGE, }, /* TICK */ - { .type = S3C_IRQTYPE_EDGE, }, /* WDT */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER2 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER3 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER4 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART2 */ - { .type = S3C_IRQTYPE_EDGE, }, /* LCD */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA2 */ - { .type = S3C_IRQTYPE_EDGE, }, /* DMA3 */ - { .type = S3C_IRQTYPE_EDGE, }, /* SDI */ - { .type = S3C_IRQTYPE_EDGE, }, /* SPI0 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART1 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* NFCON */ - { .type = S3C_IRQTYPE_EDGE, }, /* USBD */ - { .type = S3C_IRQTYPE_EDGE, }, /* USBH */ - { .type = S3C_IRQTYPE_EDGE, }, /* IIC */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* SPI1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* RTC */ - { .type = S3C_IRQTYPE_LEVEL, }, /* ADCPARENT */ -}; - -static struct s3c_irq_data init_s3c2442subint[32] = { - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-ERR */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-ERR */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-ERR */ - { .type = S3C_IRQTYPE_EDGE, .parent_irq = 31 }, /* TC */ - { .type = S3C_IRQTYPE_EDGE, .parent_irq = 31 }, /* ADC */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 6 }, /* CAM_C */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 6 }, /* CAM_P */ -}; - -void __init s3c2442_init_irq(void) -{ - pr_info("S3C2442: IRQ Support\n"); - -#ifdef CONFIG_FIQ - init_FIQ(FIQ_START); -#endif - - s3c_intc[0] = s3c24xx_init_intc(NULL, &init_s3c2442base[0], NULL, - 0x4a000000); - if (IS_ERR(s3c_intc[0])) { - pr_err("irq: could not create main interrupt controller\n"); - return; - } - - s3c24xx_init_intc(NULL, &init_eint[0], s3c_intc[0], 0x560000a4); - s3c_intc[1] = s3c24xx_init_intc(NULL, &init_s3c2442subint[0], - s3c_intc[0], 0x4a000018); -} -#endif - -#ifdef CONFIG_CPU_S3C2443 -static struct s3c_irq_data init_s3c2443base[32] = { - { .type = S3C_IRQTYPE_EINT, }, /* EINT0 */ - { .type = S3C_IRQTYPE_EINT, }, /* EINT1 */ - { .type = S3C_IRQTYPE_EINT, }, /* EINT2 */ - { .type = S3C_IRQTYPE_EINT, }, /* EINT3 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT4to7 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* EINT8to23 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* CAM */ - { .type = S3C_IRQTYPE_EDGE, }, /* nBATT_FLT */ - { .type = S3C_IRQTYPE_EDGE, }, /* TICK */ - { .type = S3C_IRQTYPE_LEVEL, }, /* WDT/AC97 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER2 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER3 */ - { .type = S3C_IRQTYPE_EDGE, }, /* TIMER4 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART2 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* LCD */ - { .type = S3C_IRQTYPE_LEVEL, }, /* DMA */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART3 */ - { .type = S3C_IRQTYPE_EDGE, }, /* CFON */ - { .type = S3C_IRQTYPE_EDGE, }, /* SDI1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* SDI0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* SPI0 */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* NAND */ - { .type = S3C_IRQTYPE_EDGE, }, /* USBD */ - { .type = S3C_IRQTYPE_EDGE, }, /* USBH */ - { .type = S3C_IRQTYPE_EDGE, }, /* IIC */ - { .type = S3C_IRQTYPE_LEVEL, }, /* UART0 */ - { .type = S3C_IRQTYPE_EDGE, }, /* SPI1 */ - { .type = S3C_IRQTYPE_EDGE, }, /* RTC */ - { .type = S3C_IRQTYPE_LEVEL, }, /* ADCPARENT */ -}; - - -static struct s3c_irq_data init_s3c2443subint[32] = { - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 28 }, /* UART0-ERR */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 23 }, /* UART1-ERR */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 15 }, /* UART2-ERR */ - { .type = S3C_IRQTYPE_EDGE, .parent_irq = 31 }, /* TC */ - { .type = S3C_IRQTYPE_EDGE, .parent_irq = 31 }, /* ADC */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 6 }, /* CAM_C */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 6 }, /* CAM_P */ - { .type = S3C_IRQTYPE_NONE }, /* reserved */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 16 }, /* LCD1 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 16 }, /* LCD2 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 16 }, /* LCD3 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 16 }, /* LCD4 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 17 }, /* DMA0 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 17 }, /* DMA1 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 17 }, /* DMA2 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 17 }, /* DMA3 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 17 }, /* DMA4 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 17 }, /* DMA5 */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 18 }, /* UART3-RX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 18 }, /* UART3-TX */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 18 }, /* UART3-ERR */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 9 }, /* WDT */ - { .type = S3C_IRQTYPE_LEVEL, .parent_irq = 9 }, /* AC97 */ -}; - -void __init s3c2443_init_irq(void) -{ - pr_info("S3C2443: IRQ Support\n"); - -#ifdef CONFIG_FIQ - init_FIQ(FIQ_START); -#endif - - s3c_intc[0] = s3c24xx_init_intc(NULL, &init_s3c2443base[0], NULL, - 0x4a000000); - if (IS_ERR(s3c_intc[0])) { - pr_err("irq: could not create main interrupt controller\n"); - return; - } - - s3c24xx_init_intc(NULL, &init_eint[0], s3c_intc[0], 0x560000a4); - s3c_intc[1] = s3c24xx_init_intc(NULL, &init_s3c2443subint[0], - s3c_intc[0], 0x4a000018); -} -#endif - -#ifdef CONFIG_OF -static int s3c24xx_irq_map_of(struct irq_domain *h, unsigned int virq, - irq_hw_number_t hw) -{ - unsigned int ctrl_num = hw / 32; - unsigned int intc_hw = hw % 32; - struct s3c_irq_intc *intc = s3c_intc[ctrl_num]; - struct s3c_irq_intc *parent_intc = intc->parent; - struct s3c_irq_data *irq_data = &intc->irqs[intc_hw]; - - /* attach controller pointer to irq_data */ - irq_data->intc = intc; - irq_data->offset = intc_hw; - - if (!parent_intc) - irq_set_chip_and_handler(virq, &s3c_irq_chip, handle_edge_irq); - else - irq_set_chip_and_handler(virq, &s3c_irq_level_chip, - handle_edge_irq); - - irq_set_chip_data(virq, irq_data); - - return 0; -} - -/* Translate our of irq notation - * format: <ctrl_num ctrl_irq parent_irq type> - */ -static int s3c24xx_irq_xlate_of(struct irq_domain *d, struct device_node *n, - const u32 *intspec, unsigned int intsize, - irq_hw_number_t *out_hwirq, unsigned int *out_type) -{ - struct s3c_irq_intc *intc; - struct s3c_irq_intc *parent_intc; - struct s3c_irq_data *irq_data; - struct s3c_irq_data *parent_irq_data; - int irqno; - - if (WARN_ON(intsize < 4)) - return -EINVAL; - - if (intspec[0] > 2 || !s3c_intc[intspec[0]]) { - pr_err("controller number %d invalid\n", intspec[0]); - return -EINVAL; - } - intc = s3c_intc[intspec[0]]; - - *out_hwirq = intspec[0] * 32 + intspec[2]; - *out_type = intspec[3] & IRQ_TYPE_SENSE_MASK; - - parent_intc = intc->parent; - if (parent_intc) { - irq_data = &intc->irqs[intspec[2]]; - irq_data->parent_irq = intspec[1]; - parent_irq_data = &parent_intc->irqs[irq_data->parent_irq]; - parent_irq_data->sub_intc = intc; - parent_irq_data->sub_bits |= (1UL << intspec[2]); - - /* parent_intc is always s3c_intc[0], so no offset */ - irqno = irq_create_mapping(parent_intc->domain, intspec[1]); - if (irqno < 0) { - pr_err("irq: could not map parent interrupt\n"); - return irqno; - } - - irq_set_chained_handler(irqno, s3c_irq_demux); - } - - return 0; -} - -static const struct irq_domain_ops s3c24xx_irq_ops_of = { - .map = s3c24xx_irq_map_of, - .xlate = s3c24xx_irq_xlate_of, -}; - -struct s3c24xx_irq_of_ctrl { - char *name; - unsigned long offset; - struct s3c_irq_intc **handle; - struct s3c_irq_intc **parent; - struct irq_domain_ops *ops; -}; - -static int __init s3c_init_intc_of(struct device_node *np, - struct device_node *interrupt_parent, - struct s3c24xx_irq_of_ctrl *s3c_ctrl, int num_ctrl) -{ - struct s3c_irq_intc *intc; - struct s3c24xx_irq_of_ctrl *ctrl; - struct irq_domain *domain; - void __iomem *reg_base; - int i; - - reg_base = of_iomap(np, 0); - if (!reg_base) { - pr_err("irq-s3c24xx: could not map irq registers\n"); - return -EINVAL; - } - - domain = irq_domain_add_linear(np, num_ctrl * 32, - &s3c24xx_irq_ops_of, NULL); - if (!domain) { - pr_err("irq: could not create irq-domain\n"); - return -EINVAL; - } - - for (i = 0; i < num_ctrl; i++) { - ctrl = &s3c_ctrl[i]; - - pr_debug("irq: found controller %s\n", ctrl->name); - - intc = kzalloc(sizeof(struct s3c_irq_intc), GFP_KERNEL); - if (!intc) - return -ENOMEM; - - intc->domain = domain; - intc->irqs = kcalloc(32, sizeof(struct s3c_irq_data), - GFP_KERNEL); - if (!intc->irqs) { - kfree(intc); - return -ENOMEM; - } - - if (ctrl->parent) { - intc->reg_pending = reg_base + ctrl->offset; - intc->reg_mask = reg_base + ctrl->offset + 0x4; - - if (*(ctrl->parent)) { - intc->parent = *(ctrl->parent); - } else { - pr_warn("irq: parent of %s missing\n", - ctrl->name); - kfree(intc->irqs); - kfree(intc); - continue; - } - } else { - intc->reg_pending = reg_base + ctrl->offset; - intc->reg_mask = reg_base + ctrl->offset + 0x08; - intc->reg_intpnd = reg_base + ctrl->offset + 0x10; - } - - s3c24xx_clear_intc(intc); - s3c_intc[i] = intc; - } - - set_handle_irq(s3c24xx_handle_irq); - - return 0; -} - -static struct s3c24xx_irq_of_ctrl s3c2410_ctrl[] = { - { - .name = "intc", - .offset = 0, - }, { - .name = "subintc", - .offset = 0x18, - .parent = &s3c_intc[0], - } -}; - -int __init s3c2410_init_intc_of(struct device_node *np, - struct device_node *interrupt_parent) -{ - return s3c_init_intc_of(np, interrupt_parent, - s3c2410_ctrl, ARRAY_SIZE(s3c2410_ctrl)); -} -IRQCHIP_DECLARE(s3c2410_irq, "samsung,s3c2410-irq", s3c2410_init_intc_of); - -static struct s3c24xx_irq_of_ctrl s3c2416_ctrl[] = { - { - .name = "intc", - .offset = 0, - }, { - .name = "subintc", - .offset = 0x18, - .parent = &s3c_intc[0], - }, { - .name = "intc2", - .offset = 0x40, - } -}; - -int __init s3c2416_init_intc_of(struct device_node *np, - struct device_node *interrupt_parent) -{ - return s3c_init_intc_of(np, interrupt_parent, - s3c2416_ctrl, ARRAY_SIZE(s3c2416_ctrl)); -} -IRQCHIP_DECLARE(s3c2416_irq, "samsung,s3c2416-irq", s3c2416_init_intc_of); -#endif diff --git a/drivers/macintosh/smu.c b/drivers/macintosh/smu.c index 96684581a25d..94fb63a7b357 100644 --- a/drivers/macintosh/smu.c +++ b/drivers/macintosh/smu.c @@ -638,7 +638,7 @@ static void smu_expose_childs(struct work_struct *unused) { struct device_node *np; - for (np = NULL; (np = of_get_next_child(smu->of_node, np)) != NULL;) + for_each_child_of_node(smu->of_node, np) if (of_device_is_compatible(np, "smu-sensors")) of_platform_device_create(np, "smu-sensors", &smu->of_dev->dev); @@ -1015,7 +1015,7 @@ static struct smu_sdbp_header *smu_create_sdb_partition(int id) /* Note: Only allowed to return error code in pointers (using ERR_PTR) * when interruptible is 1 */ -const struct smu_sdbp_header *__smu_get_sdb_partition(int id, +static const struct smu_sdbp_header *__smu_get_sdb_partition(int id, unsigned int *size, int interruptible) { char pname[32]; diff --git a/drivers/macintosh/windfarm_lm75_sensor.c b/drivers/macintosh/windfarm_lm75_sensor.c index 1e5fa09845e7..29f48c2028b6 100644 --- a/drivers/macintosh/windfarm_lm75_sensor.c +++ b/drivers/macintosh/windfarm_lm75_sensor.c @@ -152,8 +152,6 @@ static int wf_lm75_remove(struct i2c_client *client) { struct wf_lm75_sensor *lm = i2c_get_clientdata(client); - DBG("wf_lm75: i2c detatch called for %s\n", lm->sens.name); - /* Mark client detached */ lm->i2c = NULL; diff --git a/drivers/macintosh/windfarm_lm87_sensor.c b/drivers/macintosh/windfarm_lm87_sensor.c index d011899c0a8a..9fab0b47cd3d 100644 --- a/drivers/macintosh/windfarm_lm87_sensor.c +++ b/drivers/macintosh/windfarm_lm87_sensor.c @@ -149,8 +149,6 @@ static int wf_lm87_remove(struct i2c_client *client) { struct wf_lm87_sensor *lm = i2c_get_clientdata(client); - DBG("wf_lm87: i2c detatch called for %s\n", lm->sens.name); - /* Mark client detached */ lm->i2c = NULL; diff --git a/drivers/macintosh/windfarm_smu_sat.c b/drivers/macintosh/windfarm_smu_sat.c index cb75dc035616..e46e1153a0b4 100644 --- a/drivers/macintosh/windfarm_smu_sat.c +++ b/drivers/macintosh/windfarm_smu_sat.c @@ -216,8 +216,7 @@ static int wf_sat_probe(struct i2c_client *client, vsens[0] = vsens[1] = -1; isens[0] = isens[1] = -1; - child = NULL; - while ((child = of_get_next_child(dev, child)) != NULL) { + for_each_child_of_node(dev, child) { reg = of_get_property(child, "reg", NULL); loc = of_get_property(child, "location", NULL); if (reg == NULL || loc == NULL) diff --git a/drivers/macintosh/windfarm_smu_sensors.c b/drivers/macintosh/windfarm_smu_sensors.c index 3e6059eaa138..c8706cfb83fd 100644 --- a/drivers/macintosh/windfarm_smu_sensors.c +++ b/drivers/macintosh/windfarm_smu_sensors.c @@ -421,8 +421,7 @@ static int __init smu_sensors_init(void) return -ENODEV; /* Look for sensors subdir */ - for (sensors = NULL; - (sensors = of_get_next_child(smu, sensors)) != NULL;) + for_each_child_of_node(smu, sensors) if (of_node_name_eq(sensors, "sensors")) break; diff --git a/drivers/mailbox/Makefile b/drivers/mailbox/Makefile index 60d224b723a1..2e06e02b2e03 100644 --- a/drivers/mailbox/Makefile +++ b/drivers/mailbox/Makefile @@ -5,7 +5,7 @@ obj-$(CONFIG_MAILBOX) += mailbox.o obj-$(CONFIG_MAILBOX_TEST) += mailbox-test.o -obj-$(CONFIG_ARM_MHU) += arm_mhu.o +obj-$(CONFIG_ARM_MHU) += arm_mhu.o arm_mhu_db.o obj-$(CONFIG_IMX_MBOX) += imx-mailbox.o diff --git a/drivers/mailbox/arm_mhu.c b/drivers/mailbox/arm_mhu.c index 9da236552bd7..b7fbf276eb62 100644 --- a/drivers/mailbox/arm_mhu.c +++ b/drivers/mailbox/arm_mhu.c @@ -113,6 +113,9 @@ static int mhu_probe(struct amba_device *adev, const struct amba_id *id) struct device *dev = &adev->dev; int mhu_reg[MHU_CHANS] = {MHU_LP_OFFSET, MHU_HP_OFFSET, MHU_SEC_OFFSET}; + if (!of_device_is_compatible(dev->of_node, "arm,mhu")) + return -ENODEV; + /* Allocate memory for device */ mhu = devm_kzalloc(dev, sizeof(*mhu), GFP_KERNEL); if (!mhu) diff --git a/drivers/mailbox/arm_mhu_db.c b/drivers/mailbox/arm_mhu_db.c new file mode 100644 index 000000000000..275efe4cca0c --- /dev/null +++ b/drivers/mailbox/arm_mhu_db.c @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2013-2015 Fujitsu Semiconductor Ltd. + * Copyright (C) 2015 Linaro Ltd. + * Based on ARM MHU driver by Jassi Brar <jaswinder.singh@linaro.org> + * Copyright (C) 2020 ARM Ltd. + */ + +#include <linux/amba/bus.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/mailbox_controller.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> + +#define INTR_STAT_OFS 0x0 +#define INTR_SET_OFS 0x8 +#define INTR_CLR_OFS 0x10 + +#define MHU_LP_OFFSET 0x0 +#define MHU_HP_OFFSET 0x20 +#define MHU_SEC_OFFSET 0x200 +#define TX_REG_OFFSET 0x100 + +#define MHU_CHANS 3 /* Secure, Non-Secure High and Low Priority */ +#define MHU_CHAN_MAX 20 /* Max channels to save on unused RAM */ +#define MHU_NUM_DOORBELLS 32 + +struct mhu_db_link { + unsigned int irq; + void __iomem *tx_reg; + void __iomem *rx_reg; +}; + +struct arm_mhu { + void __iomem *base; + struct mhu_db_link mlink[MHU_CHANS]; + struct mbox_controller mbox; + struct device *dev; +}; + +/** + * ARM MHU Mailbox allocated channel information + * + * @mhu: Pointer to parent mailbox device + * @pchan: Physical channel within which this doorbell resides in + * @doorbell: doorbell number pertaining to this channel + */ +struct mhu_db_channel { + struct arm_mhu *mhu; + unsigned int pchan; + unsigned int doorbell; +}; + +static inline struct mbox_chan * +mhu_db_mbox_to_channel(struct mbox_controller *mbox, unsigned int pchan, + unsigned int doorbell) +{ + int i; + struct mhu_db_channel *chan_info; + + for (i = 0; i < mbox->num_chans; i++) { + chan_info = mbox->chans[i].con_priv; + if (chan_info && chan_info->pchan == pchan && + chan_info->doorbell == doorbell) + return &mbox->chans[i]; + } + + return NULL; +} + +static void mhu_db_mbox_clear_irq(struct mbox_chan *chan) +{ + struct mhu_db_channel *chan_info = chan->con_priv; + void __iomem *base = chan_info->mhu->mlink[chan_info->pchan].rx_reg; + + writel_relaxed(BIT(chan_info->doorbell), base + INTR_CLR_OFS); +} + +static unsigned int mhu_db_mbox_irq_to_pchan_num(struct arm_mhu *mhu, int irq) +{ + unsigned int pchan; + + for (pchan = 0; pchan < MHU_CHANS; pchan++) + if (mhu->mlink[pchan].irq == irq) + break; + return pchan; +} + +static struct mbox_chan * +mhu_db_mbox_irq_to_channel(struct arm_mhu *mhu, unsigned int pchan) +{ + unsigned long bits; + unsigned int doorbell; + struct mbox_chan *chan = NULL; + struct mbox_controller *mbox = &mhu->mbox; + void __iomem *base = mhu->mlink[pchan].rx_reg; + + bits = readl_relaxed(base + INTR_STAT_OFS); + if (!bits) + /* No IRQs fired in specified physical channel */ + return NULL; + + /* An IRQ has fired, find the associated channel */ + for (doorbell = 0; bits; doorbell++) { + if (!test_and_clear_bit(doorbell, &bits)) + continue; + + chan = mhu_db_mbox_to_channel(mbox, pchan, doorbell); + if (chan) + break; + dev_err(mbox->dev, + "Channel not registered: pchan: %d doorbell: %d\n", + pchan, doorbell); + } + + return chan; +} + +static irqreturn_t mhu_db_mbox_rx_handler(int irq, void *data) +{ + struct mbox_chan *chan; + struct arm_mhu *mhu = data; + unsigned int pchan = mhu_db_mbox_irq_to_pchan_num(mhu, irq); + + while (NULL != (chan = mhu_db_mbox_irq_to_channel(mhu, pchan))) { + mbox_chan_received_data(chan, NULL); + mhu_db_mbox_clear_irq(chan); + } + + return IRQ_HANDLED; +} + +static bool mhu_db_last_tx_done(struct mbox_chan *chan) +{ + struct mhu_db_channel *chan_info = chan->con_priv; + void __iomem *base = chan_info->mhu->mlink[chan_info->pchan].tx_reg; + + if (readl_relaxed(base + INTR_STAT_OFS) & BIT(chan_info->doorbell)) + return false; + + return true; +} + +static int mhu_db_send_data(struct mbox_chan *chan, void *data) +{ + struct mhu_db_channel *chan_info = chan->con_priv; + void __iomem *base = chan_info->mhu->mlink[chan_info->pchan].tx_reg; + + /* Send event to co-processor */ + writel_relaxed(BIT(chan_info->doorbell), base + INTR_SET_OFS); + + return 0; +} + +static int mhu_db_startup(struct mbox_chan *chan) +{ + mhu_db_mbox_clear_irq(chan); + return 0; +} + +static void mhu_db_shutdown(struct mbox_chan *chan) +{ + struct mhu_db_channel *chan_info = chan->con_priv; + struct mbox_controller *mbox = &chan_info->mhu->mbox; + int i; + + for (i = 0; i < mbox->num_chans; i++) + if (chan == &mbox->chans[i]) + break; + + if (mbox->num_chans == i) { + dev_warn(mbox->dev, "Request to free non-existent channel\n"); + return; + } + + /* Reset channel */ + mhu_db_mbox_clear_irq(chan); + kfree(chan->con_priv); + chan->con_priv = NULL; +} + +static struct mbox_chan *mhu_db_mbox_xlate(struct mbox_controller *mbox, + const struct of_phandle_args *spec) +{ + struct arm_mhu *mhu = dev_get_drvdata(mbox->dev); + struct mhu_db_channel *chan_info; + struct mbox_chan *chan; + unsigned int pchan = spec->args[0]; + unsigned int doorbell = spec->args[1]; + int i; + + /* Bounds checking */ + if (pchan >= MHU_CHANS || doorbell >= MHU_NUM_DOORBELLS) { + dev_err(mbox->dev, + "Invalid channel requested pchan: %d doorbell: %d\n", + pchan, doorbell); + return ERR_PTR(-EINVAL); + } + + /* Is requested channel free? */ + chan = mhu_db_mbox_to_channel(mbox, pchan, doorbell); + if (chan) { + dev_err(mbox->dev, "Channel in use: pchan: %d doorbell: %d\n", + pchan, doorbell); + return ERR_PTR(-EBUSY); + } + + /* Find the first free slot */ + for (i = 0; i < mbox->num_chans; i++) + if (!mbox->chans[i].con_priv) + break; + + if (mbox->num_chans == i) { + dev_err(mbox->dev, "No free channels left\n"); + return ERR_PTR(-EBUSY); + } + + chan = &mbox->chans[i]; + + chan_info = devm_kzalloc(mbox->dev, sizeof(*chan_info), GFP_KERNEL); + if (!chan_info) + return ERR_PTR(-ENOMEM); + + chan_info->mhu = mhu; + chan_info->pchan = pchan; + chan_info->doorbell = doorbell; + + chan->con_priv = chan_info; + + dev_dbg(mbox->dev, "mbox: created channel phys: %d doorbell: %d\n", + pchan, doorbell); + + return chan; +} + +static const struct mbox_chan_ops mhu_db_ops = { + .send_data = mhu_db_send_data, + .startup = mhu_db_startup, + .shutdown = mhu_db_shutdown, + .last_tx_done = mhu_db_last_tx_done, +}; + +static int mhu_db_probe(struct amba_device *adev, const struct amba_id *id) +{ + u32 cell_count; + int i, err, max_chans; + struct arm_mhu *mhu; + struct mbox_chan *chans; + struct device *dev = &adev->dev; + struct device_node *np = dev->of_node; + int mhu_reg[MHU_CHANS] = { + MHU_LP_OFFSET, MHU_HP_OFFSET, MHU_SEC_OFFSET, + }; + + if (!of_device_is_compatible(np, "arm,mhu-doorbell")) + return -ENODEV; + + err = of_property_read_u32(np, "#mbox-cells", &cell_count); + if (err) { + dev_err(dev, "failed to read #mbox-cells in '%pOF'\n", np); + return err; + } + + if (cell_count == 2) { + max_chans = MHU_CHAN_MAX; + } else { + dev_err(dev, "incorrect value of #mbox-cells in '%pOF'\n", np); + return -EINVAL; + } + + mhu = devm_kzalloc(dev, sizeof(*mhu), GFP_KERNEL); + if (!mhu) + return -ENOMEM; + + mhu->base = devm_ioremap_resource(dev, &adev->res); + if (IS_ERR(mhu->base)) { + dev_err(dev, "ioremap failed\n"); + return PTR_ERR(mhu->base); + } + + chans = devm_kcalloc(dev, max_chans, sizeof(*chans), GFP_KERNEL); + if (!chans) + return -ENOMEM; + + mhu->dev = dev; + mhu->mbox.dev = dev; + mhu->mbox.chans = chans; + mhu->mbox.num_chans = max_chans; + mhu->mbox.txdone_irq = false; + mhu->mbox.txdone_poll = true; + mhu->mbox.txpoll_period = 1; + + mhu->mbox.of_xlate = mhu_db_mbox_xlate; + amba_set_drvdata(adev, mhu); + + mhu->mbox.ops = &mhu_db_ops; + + err = devm_mbox_controller_register(dev, &mhu->mbox); + if (err) { + dev_err(dev, "Failed to register mailboxes %d\n", err); + return err; + } + + for (i = 0; i < MHU_CHANS; i++) { + int irq = mhu->mlink[i].irq = adev->irq[i]; + + if (irq <= 0) { + dev_dbg(dev, "No IRQ found for Channel %d\n", i); + continue; + } + + mhu->mlink[i].rx_reg = mhu->base + mhu_reg[i]; + mhu->mlink[i].tx_reg = mhu->mlink[i].rx_reg + TX_REG_OFFSET; + + err = devm_request_threaded_irq(dev, irq, NULL, + mhu_db_mbox_rx_handler, + IRQF_ONESHOT, "mhu_db_link", mhu); + if (err) { + dev_err(dev, "Can't claim IRQ %d\n", irq); + mbox_controller_unregister(&mhu->mbox); + return err; + } + } + + dev_info(dev, "ARM MHU Doorbell mailbox registered\n"); + return 0; +} + +static struct amba_id mhu_ids[] = { + { + .id = 0x1bb098, + .mask = 0xffffff, + }, + { 0, 0 }, +}; +MODULE_DEVICE_TABLE(amba, mhu_ids); + +static struct amba_driver arm_mhu_db_driver = { + .drv = { + .name = "mhu-doorbell", + }, + .id_table = mhu_ids, + .probe = mhu_db_probe, +}; +module_amba_driver(arm_mhu_db_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("ARM MHU Doorbell Driver"); +MODULE_AUTHOR("Sudeep Holla <sudeep.holla@arm.com>"); diff --git a/drivers/mailbox/bcm-pdc-mailbox.c b/drivers/mailbox/bcm-pdc-mailbox.c index 53945ca5d785..5b375985f7b8 100644 --- a/drivers/mailbox/bcm-pdc-mailbox.c +++ b/drivers/mailbox/bcm-pdc-mailbox.c @@ -962,9 +962,9 @@ static irqreturn_t pdc_irq_handler(int irq, void *data) * a DMA receive interrupt. Reenables the receive interrupt. * @data: PDC state structure */ -static void pdc_tasklet_cb(unsigned long data) +static void pdc_tasklet_cb(struct tasklet_struct *t) { - struct pdc_state *pdcs = (struct pdc_state *)data; + struct pdc_state *pdcs = from_tasklet(pdcs, t, rx_tasklet); pdc_receive(pdcs); @@ -1589,7 +1589,7 @@ static int pdc_probe(struct platform_device *pdev) pdc_hw_init(pdcs); /* Init tasklet for deferred DMA rx processing */ - tasklet_init(&pdcs->rx_tasklet, pdc_tasklet_cb, (unsigned long)pdcs); + tasklet_setup(&pdcs->rx_tasklet, pdc_tasklet_cb); err = pdc_interrupts_init(pdcs); if (err) diff --git a/drivers/mailbox/mailbox.c b/drivers/mailbox/mailbox.c index 0b821a5b2db8..3e7d4b20ab34 100644 --- a/drivers/mailbox/mailbox.c +++ b/drivers/mailbox/mailbox.c @@ -82,9 +82,12 @@ static void msg_submit(struct mbox_chan *chan) exit: spin_unlock_irqrestore(&chan->lock, flags); - if (!err && (chan->txdone_method & TXDONE_BY_POLL)) - /* kick start the timer immediately to avoid delays */ - hrtimer_start(&chan->mbox->poll_hrt, 0, HRTIMER_MODE_REL); + /* kick start the timer immediately to avoid delays */ + if (!err && (chan->txdone_method & TXDONE_BY_POLL)) { + /* but only if not already active */ + if (!hrtimer_active(&chan->mbox->poll_hrt)) + hrtimer_start(&chan->mbox->poll_hrt, 0, HRTIMER_MODE_REL); + } } static void tx_tick(struct mbox_chan *chan, int r) @@ -122,11 +125,10 @@ static enum hrtimer_restart txdone_hrtimer(struct hrtimer *hrtimer) struct mbox_chan *chan = &mbox->chans[i]; if (chan->active_req && chan->cl) { + resched = true; txdone = chan->mbox->ops->last_tx_done(chan); if (txdone) tx_tick(chan, 0); - else - resched = true; } } diff --git a/drivers/mailbox/mtk-cmdq-mailbox.c b/drivers/mailbox/mtk-cmdq-mailbox.c index 484d4438cd83..5665b6ea8119 100644 --- a/drivers/mailbox/mtk-cmdq-mailbox.c +++ b/drivers/mailbox/mtk-cmdq-mailbox.c @@ -69,7 +69,7 @@ struct cmdq_task { struct cmdq { struct mbox_controller mbox; void __iomem *base; - u32 irq; + int irq; u32 thread_nr; u32 irq_mask; struct cmdq_thread *thread; @@ -525,10 +525,8 @@ static int cmdq_probe(struct platform_device *pdev) } cmdq->irq = platform_get_irq(pdev, 0); - if (!cmdq->irq) { - dev_err(dev, "failed to get irq\n"); - return -EINVAL; - } + if (cmdq->irq < 0) + return cmdq->irq; plat_data = (struct gce_plat *)of_device_get_match_data(dev); if (!plat_data) { diff --git a/drivers/mailbox/tegra-hsp.c b/drivers/mailbox/tegra-hsp.c index 834b35dc3b13..e07091d71986 100644 --- a/drivers/mailbox/tegra-hsp.c +++ b/drivers/mailbox/tegra-hsp.c @@ -13,6 +13,8 @@ #include <linux/pm.h> #include <linux/slab.h> +#include <soc/tegra/fuse.h> + #include <dt-bindings/mailbox/tegra186-hsp.h> #include "mailbox.h" @@ -322,7 +324,12 @@ static int tegra_hsp_doorbell_startup(struct mbox_chan *chan) if (!ccplex) return -ENODEV; - if (!tegra_hsp_doorbell_can_ring(db)) + /* + * On simulation platforms the BPMP hasn't had a chance yet to mark + * the doorbell as ringable by the CCPLEX, so we want to skip extra + * checks here. + */ + if (tegra_is_silicon() && !tegra_hsp_doorbell_can_ring(db)) return -ENODEV; spin_lock_irqsave(&hsp->lock, flags); diff --git a/drivers/memory/Kconfig b/drivers/memory/Kconfig index 2c79e95dd486..00e013b14703 100644 --- a/drivers/memory/Kconfig +++ b/drivers/memory/Kconfig @@ -32,8 +32,9 @@ config ARM_PL172_MPMC config ATMEL_SDRAMC bool "Atmel (Multi-port DDR-)SDRAM Controller" - default y - depends on ARCH_AT91 && OF + default y if ARCH_AT91 + depends on ARCH_AT91 || COMPILE_TEST + depends on OF help This driver is for Atmel SDRAM Controller or Atmel Multi-port DDR-SDRAM Controller available on Atmel AT91SAM9 and SAMA5 SoCs. @@ -42,8 +43,9 @@ config ATMEL_SDRAMC config ATMEL_EBI bool "Atmel EBI driver" - default y - depends on ARCH_AT91 && OF + default y if ARCH_AT91 + depends on ARCH_AT91 || COMPILE_TEST + depends on OF select MFD_SYSCON select MFD_ATMEL_SMC help @@ -52,6 +54,18 @@ config ATMEL_EBI tree is used. This bus supports NANDs, external ethernet controller, SRAMs, ATA devices, etc. +config BRCMSTB_DPFE + bool "Broadcom STB DPFE driver" if COMPILE_TEST + default y if ARCH_BRCMSTB + depends on ARCH_BRCMSTB || COMPILE_TEST + help + This driver provides access to the DPFE interface of Broadcom + STB SoCs. The firmware running on the DCPU inside the DDR PHY can + provide current information about the system's RAM, for instance + the DRAM refresh rate. This can be used as an indirect indicator + for the DRAM's temperature. Slower refresh rate means cooler RAM, + higher refresh rate means hotter RAM. + config BT1_L2_CTL bool "Baikal-T1 CM2 L2-RAM Cache Control Block" depends on MIPS_BAIKAL_T1 || COMPILE_TEST @@ -65,7 +79,8 @@ config BT1_L2_CTL config TI_AEMIF tristate "Texas Instruments AEMIF driver" - depends on (ARCH_DAVINCI || ARCH_KEYSTONE) && OF + depends on ARCH_DAVINCI || ARCH_KEYSTONE || COMPILE_TEST + depends on OF help This driver is for the AEMIF module available in Texas Instruments SoCs. AEMIF stands for Asynchronous External Memory Interface and @@ -76,7 +91,7 @@ config TI_AEMIF config TI_EMIF tristate "Texas Instruments EMIF driver" - depends on ARCH_OMAP2PLUS + depends on ARCH_OMAP2PLUS || COMPILE_TEST select DDR help This driver is for the EMIF module available in Texas Instruments @@ -88,7 +103,8 @@ config TI_EMIF temperature changes config OMAP_GPMC - bool + bool "Texas Instruments OMAP SoC GPMC driver" if COMPILE_TEST + depends on OF_ADDRESS select GPIOLIB help This driver is for the General Purpose Memory Controller (GPMC) @@ -112,7 +128,8 @@ config OMAP_GPMC_DEBUG config TI_EMIF_SRAM tristate "Texas Instruments EMIF SRAM driver" - depends on (SOC_AM33XX || SOC_AM43XX) && SRAM + depends on SOC_AM33XX || SOC_AM43XX || (ARM && COMPILE_TEST) + depends on SRAM help This driver is for the EMIF module available on Texas Instruments AM33XX and AM43XX SoCs and is required for PM. Certain parts of @@ -122,8 +139,9 @@ config TI_EMIF_SRAM config MVEBU_DEVBUS bool "Marvell EBU Device Bus Controller" - default y - depends on PLAT_ORION && OF + default y if PLAT_ORION + depends on PLAT_ORION || COMPILE_TEST + depends on OF help This driver is for the Device Bus controller available in some Marvell EBU SoCs such as Discovery (mv78xx0), Orion (88f5xxx) and @@ -132,7 +150,7 @@ config MVEBU_DEVBUS config FSL_CORENET_CF tristate "Freescale CoreNet Error Reporting" - depends on FSL_SOC_BOOKE + depends on FSL_SOC_BOOKE || COMPILE_TEST help Say Y for reporting of errors from the Freescale CoreNet Coherency Fabric. Errors reported include accesses to @@ -141,7 +159,7 @@ config FSL_CORENET_CF represents a coherency violation. config FSL_IFC - bool + bool "Freescale IFC driver" if COMPILE_TEST depends on FSL_SOC || ARCH_LAYERSCAPE || SOC_LS1021A || COMPILE_TEST depends on HAS_IOMEM @@ -155,7 +173,7 @@ config JZ4780_NEMC memory devices such as NAND and SRAM. config MTK_SMI - bool + bool "Mediatek SoC Memory Controller driver" if COMPILE_TEST depends on ARCH_MEDIATEK || COMPILE_TEST help This driver is for the Memory Controller module in MediaTek SoCs, @@ -164,7 +182,7 @@ config MTK_SMI config DA8XX_DDRCTL bool "Texas Instruments da8xx DDR2/mDDR driver" - depends on ARCH_DAVINCI_DA8XX + depends on ARCH_DAVINCI_DA8XX || COMPILE_TEST help This driver is for the DDR2/mDDR Memory Controller present on Texas Instruments da8xx SoCs. It's used to tweak various memory @@ -172,16 +190,16 @@ config DA8XX_DDRCTL config PL353_SMC tristate "ARM PL35X Static Memory Controller(SMC) driver" - default y + default y if ARM depends on ARM - depends on ARM_AMBA + depends on ARM_AMBA || COMPILE_TEST help This driver is for the ARM PL351/PL353 Static Memory Controller(SMC) module. config RENESAS_RPCIF tristate "Renesas RPC-IF driver" - depends on ARCH_RENESAS + depends on ARCH_RENESAS || COMPILE_TEST select REGMAP_MMIO help This supports Renesas R-Car Gen3 RPC-IF which provides either SPI diff --git a/drivers/memory/Makefile b/drivers/memory/Makefile index b4533ffff2bc..e71cf7b99641 100644 --- a/drivers/memory/Makefile +++ b/drivers/memory/Makefile @@ -10,7 +10,7 @@ endif obj-$(CONFIG_ARM_PL172_MPMC) += pl172.o obj-$(CONFIG_ATMEL_SDRAMC) += atmel-sdramc.o obj-$(CONFIG_ATMEL_EBI) += atmel-ebi.o -obj-$(CONFIG_ARCH_BRCMSTB) += brcmstb_dpfe.o +obj-$(CONFIG_BRCMSTB_DPFE) += brcmstb_dpfe.o obj-$(CONFIG_BT1_L2_CTL) += bt1-l2-ctl.o obj-$(CONFIG_TI_AEMIF) += ti-aemif.o obj-$(CONFIG_TI_EMIF) += emif.o diff --git a/drivers/memory/brcmstb_dpfe.c b/drivers/memory/brcmstb_dpfe.c index 60e8633b1175..f43ba69fbb3e 100644 --- a/drivers/memory/brcmstb_dpfe.c +++ b/drivers/memory/brcmstb_dpfe.c @@ -188,11 +188,6 @@ struct brcmstb_dpfe_priv { struct mutex lock; }; -static const char * const error_text[] = { - "Success", "Header code incorrect", "Unknown command or argument", - "Incorrect checksum", "Malformed command", "Timed out", -}; - /* * Forward declaration of our sysfs attribute functions, so we can declare the * attribute data structures early. @@ -307,6 +302,20 @@ static const struct dpfe_api dpfe_api_v3 = { }, }; +static const char *get_error_text(unsigned int i) +{ + static const char * const error_text[] = { + "Success", "Header code incorrect", + "Unknown command or argument", "Incorrect checksum", + "Malformed command", "Timed out", "Unknown error", + }; + + if (unlikely(i >= ARRAY_SIZE(error_text))) + i = ARRAY_SIZE(error_text) - 1; + + return error_text[i]; +} + static bool is_dcpu_enabled(struct brcmstb_dpfe_priv *priv) { u32 val; @@ -445,7 +454,7 @@ static int __send_command(struct brcmstb_dpfe_priv *priv, unsigned int cmd, } if (resp != 0) { mutex_unlock(&priv->lock); - return -ETIMEDOUT; + return -ffs(DCPU_RET_ERR_TIMEDOUT); } /* Compute checksum over the message */ @@ -647,8 +656,10 @@ static int brcmstb_dpfe_download_firmware(struct brcmstb_dpfe_priv *priv) return (ret == -ENOENT) ? -EPROBE_DEFER : ret; ret = __verify_firmware(&init, fw); - if (ret) - return -EFAULT; + if (ret) { + ret = -EFAULT; + goto release_fw; + } __disable_dcpu(priv); @@ -667,18 +678,20 @@ static int brcmstb_dpfe_download_firmware(struct brcmstb_dpfe_priv *priv) ret = __write_firmware(priv->dmem, dmem, dmem_size, is_big_endian); if (ret) - return ret; + goto release_fw; ret = __write_firmware(priv->imem, imem, imem_size, is_big_endian); if (ret) - return ret; + goto release_fw; ret = __verify_fw_checksum(&init, priv, header, init.chksum); if (ret) - return ret; + goto release_fw; __enable_dcpu(priv); - return 0; +release_fw: + release_firmware(fw); + return ret; } static ssize_t generic_show(unsigned int command, u32 response[], @@ -691,7 +704,7 @@ static ssize_t generic_show(unsigned int command, u32 response[], ret = __send_command(priv, command, response); if (ret < 0) - return sprintf(buf, "ERROR: %s\n", error_text[-ret]); + return sprintf(buf, "ERROR: %s\n", get_error_text(-ret)); return 0; } @@ -888,11 +901,8 @@ static int brcmstb_dpfe_probe(struct platform_device *pdev) } ret = brcmstb_dpfe_download_firmware(priv); - if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(dev, "Couldn't download firmware -- %d\n", ret); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "Couldn't download firmware\n"); ret = sysfs_create_groups(&pdev->dev.kobj, priv->dpfe_api->sysfs_attrs); if (!ret) diff --git a/drivers/memory/emif.c b/drivers/memory/emif.c index bb6a71d26798..ddb1879f07d3 100644 --- a/drivers/memory/emif.c +++ b/drivers/memory/emif.c @@ -131,16 +131,7 @@ static int emif_regdump_show(struct seq_file *s, void *unused) return 0; } -static int emif_regdump_open(struct inode *inode, struct file *file) -{ - return single_open(file, emif_regdump_show, inode->i_private); -} - -static const struct file_operations emif_regdump_fops = { - .open = emif_regdump_open, - .read = seq_read, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(emif_regdump); static int emif_mr4_show(struct seq_file *s, void *unused) { @@ -150,48 +141,16 @@ static int emif_mr4_show(struct seq_file *s, void *unused) return 0; } -static int emif_mr4_open(struct inode *inode, struct file *file) -{ - return single_open(file, emif_mr4_show, inode->i_private); -} - -static const struct file_operations emif_mr4_fops = { - .open = emif_mr4_open, - .read = seq_read, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(emif_mr4); static int __init_or_module emif_debugfs_init(struct emif_data *emif) { - struct dentry *dentry; - int ret; - - dentry = debugfs_create_dir(dev_name(emif->dev), NULL); - if (!dentry) { - ret = -ENOMEM; - goto err0; - } - emif->debugfs_root = dentry; - - dentry = debugfs_create_file("regcache_dump", S_IRUGO, - emif->debugfs_root, emif, &emif_regdump_fops); - if (!dentry) { - ret = -ENOMEM; - goto err1; - } - - dentry = debugfs_create_file("mr4", S_IRUGO, - emif->debugfs_root, emif, &emif_mr4_fops); - if (!dentry) { - ret = -ENOMEM; - goto err1; - } - + emif->debugfs_root = debugfs_create_dir(dev_name(emif->dev), NULL); + debugfs_create_file("regcache_dump", S_IRUGO, emif->debugfs_root, emif, + &emif_regdump_fops); + debugfs_create_file("mr4", S_IRUGO, emif->debugfs_root, emif, + &emif_mr4_fops); return 0; -err1: - debugfs_remove_recursive(emif->debugfs_root); -err0: - return ret; } static void __exit emif_debugfs_exit(struct emif_data *emif) diff --git a/drivers/memory/fsl-corenet-cf.c b/drivers/memory/fsl-corenet-cf.c index 0b0ed72016da..0309bd5a1800 100644 --- a/drivers/memory/fsl-corenet-cf.c +++ b/drivers/memory/fsl-corenet-cf.c @@ -211,10 +211,8 @@ static int ccf_probe(struct platform_device *pdev) dev_set_drvdata(&pdev->dev, ccf); irq = platform_get_irq(pdev, 0); - if (!irq) { - dev_err(&pdev->dev, "%s: no irq\n", __func__); - return -ENXIO; - } + if (irq < 0) + return irq; ret = devm_request_irq(&pdev->dev, irq, ccf_irq, 0, pdev->name, ccf); if (ret) { diff --git a/drivers/memory/mtk-smi.c b/drivers/memory/mtk-smi.c index c21262502581..691e4c344cf8 100644 --- a/drivers/memory/mtk-smi.c +++ b/drivers/memory/mtk-smi.c @@ -19,6 +19,9 @@ /* mt8173 */ #define SMI_LARB_MMU_EN 0xf00 +/* mt8167 */ +#define MT8167_SMI_LARB_MMU_EN 0xfc0 + /* mt2701 */ #define REG_SMI_SECUR_CON_BASE 0x5c0 @@ -179,6 +182,13 @@ static void mtk_smi_larb_config_port_mt8173(struct device *dev) writel(*larb->mmu, larb->base + SMI_LARB_MMU_EN); } +static void mtk_smi_larb_config_port_mt8167(struct device *dev) +{ + struct mtk_smi_larb *larb = dev_get_drvdata(dev); + + writel(*larb->mmu, larb->base + MT8167_SMI_LARB_MMU_EN); +} + static void mtk_smi_larb_config_port_gen1(struct device *dev) { struct mtk_smi_larb *larb = dev_get_drvdata(dev); @@ -226,6 +236,11 @@ static const struct mtk_smi_larb_gen mtk_smi_larb_mt8173 = { .config_port = mtk_smi_larb_config_port_mt8173, }; +static const struct mtk_smi_larb_gen mtk_smi_larb_mt8167 = { + /* mt8167 do not need the port in larb */ + .config_port = mtk_smi_larb_config_port_mt8167, +}; + static const struct mtk_smi_larb_gen mtk_smi_larb_mt2701 = { .port_in_larb = { LARB0_PORT_OFFSET, LARB1_PORT_OFFSET, @@ -255,6 +270,10 @@ static const struct mtk_smi_larb_gen mtk_smi_larb_mt8183 = { static const struct of_device_id mtk_smi_larb_of_ids[] = { { + .compatible = "mediatek,mt8167-smi-larb", + .data = &mtk_smi_larb_mt8167 + }, + { .compatible = "mediatek,mt8173-smi-larb", .data = &mtk_smi_larb_mt8173 }, @@ -419,6 +438,10 @@ static const struct of_device_id mtk_smi_common_of_ids[] = { .data = &mtk_smi_common_gen2, }, { + .compatible = "mediatek,mt8167-smi-common", + .data = &mtk_smi_common_gen2, + }, + { .compatible = "mediatek,mt2701-smi-common", .data = &mtk_smi_common_gen1, }, diff --git a/drivers/memory/omap-gpmc.c b/drivers/memory/omap-gpmc.c index ca0097664b12..cfa730cfd145 100644 --- a/drivers/memory/omap-gpmc.c +++ b/drivers/memory/omap-gpmc.c @@ -33,8 +33,6 @@ #include <linux/platform_data/mtd-nand-omap2.h> -#include <asm/mach-types.h> - #define DEVICE_NAME "omap-gpmc" /* GPMC register offsets */ @@ -245,7 +243,6 @@ static DEFINE_SPINLOCK(gpmc_mem_lock); /* Define chip-selects as reserved by default until probe completes */ static unsigned int gpmc_cs_num = GPMC_CS_NUM; static unsigned int gpmc_nr_waitpins; -static resource_size_t phys_base, mem_size; static unsigned int gpmc_capability; static void __iomem *gpmc_base; @@ -634,14 +631,6 @@ static int set_gpmc_timing_reg(int cs, int reg, int st_bit, int end_bit, int max return 0; } -#define GPMC_SET_ONE_CD_MAX(reg, st, end, max, field, cd) \ - if (set_gpmc_timing_reg(cs, (reg), (st), (end), (max), \ - t->field, (cd), #field) < 0) \ - return -1 - -#define GPMC_SET_ONE(reg, st, end, field) \ - GPMC_SET_ONE_CD_MAX(reg, st, end, 0, field, GPMC_CD_FCLK) - /** * gpmc_calc_waitmonitoring_divider - calculate proper GPMCFCLKDIVIDER based on WAITMONITORINGTIME * WAITMONITORINGTIME will be _at least_ as long as desired, i.e. @@ -700,12 +689,12 @@ int gpmc_calc_divider(unsigned int sync_clk) int gpmc_cs_set_timings(int cs, const struct gpmc_timings *t, const struct gpmc_settings *s) { - int div; + int div, ret; u32 l; div = gpmc_calc_divider(t->sync_clk); if (div < 0) - return div; + return -EINVAL; /* * See if we need to change the divider for waitmonitoringtime. @@ -729,57 +718,114 @@ int gpmc_cs_set_timings(int cs, const struct gpmc_timings *t, __func__, t->wait_monitoring ); - return -1; + return -ENXIO; } } - GPMC_SET_ONE(GPMC_CS_CONFIG2, 0, 3, cs_on); - GPMC_SET_ONE(GPMC_CS_CONFIG2, 8, 12, cs_rd_off); - GPMC_SET_ONE(GPMC_CS_CONFIG2, 16, 20, cs_wr_off); + ret = 0; + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG2, 0, 3, 0, t->cs_on, + GPMC_CD_FCLK, "cs_on"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG2, 8, 12, 0, t->cs_rd_off, + GPMC_CD_FCLK, "cs_rd_off"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG2, 16, 20, 0, t->cs_wr_off, + GPMC_CD_FCLK, "cs_wr_off"); + if (ret) + return -ENXIO; + + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG3, 0, 3, 0, t->adv_on, + GPMC_CD_FCLK, "adv_on"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG3, 8, 12, 0, t->adv_rd_off, + GPMC_CD_FCLK, "adv_rd_off"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG3, 16, 20, 0, t->adv_wr_off, + GPMC_CD_FCLK, "adv_wr_off"); + if (ret) + return -ENXIO; - GPMC_SET_ONE(GPMC_CS_CONFIG3, 0, 3, adv_on); - GPMC_SET_ONE(GPMC_CS_CONFIG3, 8, 12, adv_rd_off); - GPMC_SET_ONE(GPMC_CS_CONFIG3, 16, 20, adv_wr_off); if (gpmc_capability & GPMC_HAS_MUX_AAD) { - GPMC_SET_ONE(GPMC_CS_CONFIG3, 4, 6, adv_aad_mux_on); - GPMC_SET_ONE(GPMC_CS_CONFIG3, 24, 26, adv_aad_mux_rd_off); - GPMC_SET_ONE(GPMC_CS_CONFIG3, 28, 30, adv_aad_mux_wr_off); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG3, 4, 6, 0, + t->adv_aad_mux_on, GPMC_CD_FCLK, + "adv_aad_mux_on"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG3, 24, 26, 0, + t->adv_aad_mux_rd_off, GPMC_CD_FCLK, + "adv_aad_mux_rd_off"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG3, 28, 30, 0, + t->adv_aad_mux_wr_off, GPMC_CD_FCLK, + "adv_aad_mux_wr_off"); + if (ret) + return -ENXIO; } - GPMC_SET_ONE(GPMC_CS_CONFIG4, 0, 3, oe_on); - GPMC_SET_ONE(GPMC_CS_CONFIG4, 8, 12, oe_off); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG4, 0, 3, 0, t->oe_on, + GPMC_CD_FCLK, "oe_on"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG4, 8, 12, 0, t->oe_off, + GPMC_CD_FCLK, "oe_off"); if (gpmc_capability & GPMC_HAS_MUX_AAD) { - GPMC_SET_ONE(GPMC_CS_CONFIG4, 4, 6, oe_aad_mux_on); - GPMC_SET_ONE(GPMC_CS_CONFIG4, 13, 15, oe_aad_mux_off); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG4, 4, 6, 0, + t->oe_aad_mux_on, GPMC_CD_FCLK, + "oe_aad_mux_on"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG4, 13, 15, 0, + t->oe_aad_mux_off, GPMC_CD_FCLK, + "oe_aad_mux_off"); + } + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG4, 16, 19, 0, t->we_on, + GPMC_CD_FCLK, "we_on"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG4, 24, 28, 0, t->we_off, + GPMC_CD_FCLK, "we_off"); + if (ret) + return -ENXIO; + + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG5, 0, 4, 0, t->rd_cycle, + GPMC_CD_FCLK, "rd_cycle"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG5, 8, 12, 0, t->wr_cycle, + GPMC_CD_FCLK, "wr_cycle"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG5, 16, 20, 0, t->access, + GPMC_CD_FCLK, "access"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG5, 24, 27, 0, + t->page_burst_access, GPMC_CD_FCLK, + "page_burst_access"); + if (ret) + return -ENXIO; + + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG6, 0, 3, 0, + t->bus_turnaround, GPMC_CD_FCLK, + "bus_turnaround"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG6, 8, 11, 0, + t->cycle2cycle_delay, GPMC_CD_FCLK, + "cycle2cycle_delay"); + if (ret) + return -ENXIO; + + if (gpmc_capability & GPMC_HAS_WR_DATA_MUX_BUS) { + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG6, 16, 19, 0, + t->wr_data_mux_bus, GPMC_CD_FCLK, + "wr_data_mux_bus"); + if (ret) + return -ENXIO; + } + if (gpmc_capability & GPMC_HAS_WR_ACCESS) { + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG6, 24, 28, 0, + t->wr_access, GPMC_CD_FCLK, + "wr_access"); + if (ret) + return -ENXIO; } - GPMC_SET_ONE(GPMC_CS_CONFIG4, 16, 19, we_on); - GPMC_SET_ONE(GPMC_CS_CONFIG4, 24, 28, we_off); - - GPMC_SET_ONE(GPMC_CS_CONFIG5, 0, 4, rd_cycle); - GPMC_SET_ONE(GPMC_CS_CONFIG5, 8, 12, wr_cycle); - GPMC_SET_ONE(GPMC_CS_CONFIG5, 16, 20, access); - - GPMC_SET_ONE(GPMC_CS_CONFIG5, 24, 27, page_burst_access); - - GPMC_SET_ONE(GPMC_CS_CONFIG6, 0, 3, bus_turnaround); - GPMC_SET_ONE(GPMC_CS_CONFIG6, 8, 11, cycle2cycle_delay); - - if (gpmc_capability & GPMC_HAS_WR_DATA_MUX_BUS) - GPMC_SET_ONE(GPMC_CS_CONFIG6, 16, 19, wr_data_mux_bus); - if (gpmc_capability & GPMC_HAS_WR_ACCESS) - GPMC_SET_ONE(GPMC_CS_CONFIG6, 24, 28, wr_access); l = gpmc_cs_read_reg(cs, GPMC_CS_CONFIG1); l &= ~0x03; l |= (div - 1); gpmc_cs_write_reg(cs, GPMC_CS_CONFIG1, l); - GPMC_SET_ONE_CD_MAX(GPMC_CS_CONFIG1, 18, 19, - GPMC_CONFIG1_WAITMONITORINGTIME_MAX, - wait_monitoring, GPMC_CD_CLK); - GPMC_SET_ONE_CD_MAX(GPMC_CS_CONFIG1, 25, 26, - GPMC_CONFIG1_CLKACTIVATIONTIME_MAX, - clk_activation, GPMC_CD_FCLK); + ret = 0; + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG1, 18, 19, + GPMC_CONFIG1_WAITMONITORINGTIME_MAX, + t->wait_monitoring, GPMC_CD_CLK, + "wait_monitoring"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG1, 25, 26, + GPMC_CONFIG1_CLKACTIVATIONTIME_MAX, + t->clk_activation, GPMC_CD_FCLK, + "clk_activation"); + if (ret) + return -ENXIO; #ifdef CONFIG_OMAP_GPMC_DEBUG pr_info("GPMC CS%d CLK period is %lu ns (div %d)\n", @@ -870,20 +916,6 @@ static bool gpmc_cs_reserved(int cs) return gpmc->flags & GPMC_CS_RESERVED; } -static void gpmc_cs_set_name(int cs, const char *name) -{ - struct gpmc_cs_data *gpmc = &gpmc_cs[cs]; - - gpmc->name = name; -} - -static const char *gpmc_cs_get_name(int cs) -{ - struct gpmc_cs_data *gpmc = &gpmc_cs[cs]; - - return gpmc->name; -} - static unsigned long gpmc_mem_align(unsigned long size) { int order; @@ -929,56 +961,13 @@ static int gpmc_cs_delete_mem(int cs) return r; } -/** - * gpmc_cs_remap - remaps a chip-select physical base address - * @cs: chip-select to remap - * @base: physical base address to re-map chip-select to - * - * Re-maps a chip-select to a new physical base address specified by - * "base". Returns 0 on success and appropriate negative error code - * on failure. - */ -static int gpmc_cs_remap(int cs, u32 base) -{ - int ret; - u32 old_base, size; - - if (cs > gpmc_cs_num) { - pr_err("%s: requested chip-select is disabled\n", __func__); - return -ENODEV; - } - - /* - * Make sure we ignore any device offsets from the GPMC partition - * allocated for the chip select and that the new base confirms - * to the GPMC 16MB minimum granularity. - */ - base &= ~(SZ_16M - 1); - - gpmc_cs_get_memconf(cs, &old_base, &size); - if (base == old_base) - return 0; - - ret = gpmc_cs_delete_mem(cs); - if (ret < 0) - return ret; - - ret = gpmc_cs_insert_mem(cs, base, size); - if (ret < 0) - return ret; - - ret = gpmc_cs_set_memconf(cs, base, size); - - return ret; -} - int gpmc_cs_request(int cs, unsigned long size, unsigned long *base) { struct gpmc_cs_data *gpmc = &gpmc_cs[cs]; struct resource *res = &gpmc->mem; int r = -1; - if (cs > gpmc_cs_num) { + if (cs >= gpmc_cs_num) { pr_err("%s: requested chip-select is disabled\n", __func__); return -ENODEV; } @@ -1025,8 +1014,7 @@ void gpmc_cs_free(int cs) spin_lock(&gpmc_mem_lock); if (cs >= gpmc_cs_num || cs < 0 || !gpmc_cs_reserved(cs)) { - printk(KERN_ERR "Trying to free non-reserved GPMC CS%d\n", cs); - BUG(); + WARN(1, "Trying to free non-reserved GPMC CS%d\n", cs); spin_unlock(&gpmc_mem_lock); return; } @@ -1896,6 +1884,63 @@ static const struct of_device_id gpmc_dt_ids[] = { { } }; +static void gpmc_cs_set_name(int cs, const char *name) +{ + struct gpmc_cs_data *gpmc = &gpmc_cs[cs]; + + gpmc->name = name; +} + +static const char *gpmc_cs_get_name(int cs) +{ + struct gpmc_cs_data *gpmc = &gpmc_cs[cs]; + + return gpmc->name; +} + +/** + * gpmc_cs_remap - remaps a chip-select physical base address + * @cs: chip-select to remap + * @base: physical base address to re-map chip-select to + * + * Re-maps a chip-select to a new physical base address specified by + * "base". Returns 0 on success and appropriate negative error code + * on failure. + */ +static int gpmc_cs_remap(int cs, u32 base) +{ + int ret; + u32 old_base, size; + + if (cs >= gpmc_cs_num) { + pr_err("%s: requested chip-select is disabled\n", __func__); + return -ENODEV; + } + + /* + * Make sure we ignore any device offsets from the GPMC partition + * allocated for the chip select and that the new base confirms + * to the GPMC 16MB minimum granularity. + */ + base &= ~(SZ_16M - 1); + + gpmc_cs_get_memconf(cs, &old_base, &size); + if (base == old_base) + return 0; + + ret = gpmc_cs_delete_mem(cs); + if (ret < 0) + return ret; + + ret = gpmc_cs_insert_mem(cs, base, size); + if (ret < 0) + return ret; + + ret = gpmc_cs_set_memconf(cs, base, size); + + return ret; +} + /** * gpmc_read_settings_dt - read gpmc settings from device-tree * @np: pointer to device-tree node for a gpmc child device @@ -2265,6 +2310,10 @@ static void gpmc_probe_dt_children(struct platform_device *pdev) } } #else +void gpmc_read_settings_dt(struct device_node *np, struct gpmc_settings *p) +{ + memset(p, 0, sizeof(*p)); +} static int gpmc_probe_dt(struct platform_device *pdev) { return 0; @@ -2347,12 +2396,9 @@ static int gpmc_probe(struct platform_device *pdev) platform_set_drvdata(pdev, gpmc); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (res == NULL) + if (!res) return -ENOENT; - phys_base = res->start; - mem_size = resource_size(res); - gpmc_base = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(gpmc_base)) return PTR_ERR(gpmc_base); diff --git a/drivers/memory/renesas-rpc-if.c b/drivers/memory/renesas-rpc-if.c index 88f51ec8f1d1..f2a33a1af836 100644 --- a/drivers/memory/renesas-rpc-if.c +++ b/drivers/memory/renesas-rpc-if.c @@ -199,10 +199,8 @@ int rpcif_sw_init(struct rpcif *rpc, struct device *dev) rpc->dirmap = NULL; rpc->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL); - if (IS_ERR(rpc->rstc)) - return PTR_ERR(rpc->rstc); - return 0; + return PTR_ERR_OR_ZERO(rpc->rstc); } EXPORT_SYMBOL(rpcif_sw_init); diff --git a/drivers/memory/samsung/exynos5422-dmc.c b/drivers/memory/samsung/exynos5422-dmc.c index 714d1f6f077c..c5ee4121a4d2 100644 --- a/drivers/memory/samsung/exynos5422-dmc.c +++ b/drivers/memory/samsung/exynos5422-dmc.c @@ -98,6 +98,8 @@ MODULE_PARM_DESC(irqmode, "Enable IRQ mode (0=off [default], 1=on)"); /** * struct dmc_opp_table - Operating level desciption + * @freq_hz: target frequency in Hz + * @volt_uv: target voltage in uV * * Covers frequency and voltage settings of the DMC operating mode. */ @@ -108,6 +110,41 @@ struct dmc_opp_table { /** * struct exynos5_dmc - main structure describing DMC device + * @dev: DMC device + * @df: devfreq device structure returned by devfreq framework + * @gov_data: configuration of devfreq governor + * @base_drexi0: DREX0 registers mapping + * @base_drexi1: DREX1 registers mapping + * @clk_regmap: regmap for clock controller registers + * @lock: protects curr_rate and frequency/voltage setting section + * @curr_rate: current frequency + * @curr_volt: current voltage + * @opp: OPP table + * @opp_count: number of 'opp' elements + * @timings_arr_size: number of 'timings' elements + * @timing_row: values for timing row register, for each OPP + * @timing_data: values for timing data register, for each OPP + * @timing_power: balues for timing power register, for each OPP + * @timings: DDR memory timings, from device tree + * @min_tck: DDR memory minimum timing values, from device tree + * @bypass_timing_row: value for timing row register for bypass timings + * @bypass_timing_data: value for timing data register for bypass timings + * @bypass_timing_power: value for timing power register for bypass + * timings + * @vdd_mif: Memory interface regulator + * @fout_spll: clock: SPLL + * @fout_bpll: clock: BPLL + * @mout_spll: clock: mux SPLL + * @mout_bpll: clock: mux BPLL + * @mout_mclk_cdrex: clock: mux mclk_cdrex + * @mout_mx_mspll_ccore: clock: mux mx_mspll_ccore + * @counter: devfreq events + * @num_counters: number of 'counter' elements + * @last_overflow_ts: time (in ns) of last overflow of each DREX + * @load: utilization in percents + * @total: total time between devfreq events + * @in_irq_mode: whether running in interrupt mode (true) + * or polling (false) * * The main structure for the Dynamic Memory Controller which covers clocks, * memory regions, HW information, parameters and current operating mode. @@ -119,12 +156,11 @@ struct exynos5_dmc { void __iomem *base_drexi0; void __iomem *base_drexi1; struct regmap *clk_regmap; + /* Protects curr_rate and frequency/voltage setting section */ struct mutex lock; unsigned long curr_rate; unsigned long curr_volt; - unsigned long bypass_rate; struct dmc_opp_table *opp; - struct dmc_opp_table opp_bypass; int opp_count; u32 timings_arr_size; u32 *timing_row; @@ -142,8 +178,6 @@ struct exynos5_dmc { struct clk *mout_bpll; struct clk *mout_mclk_cdrex; struct clk *mout_mx_mspll_ccore; - struct clk *mx_mspll_ccore_phy; - struct clk *mout_mx_mspll_ccore_phy; struct devfreq_event_dev **counter; int num_counters; u64 last_overflow_ts[2]; @@ -169,7 +203,7 @@ struct timing_reg { unsigned int val; }; -static const struct timing_reg timing_row[] = { +static const struct timing_reg timing_row_reg_fields[] = { TIMING_FIELD("tRFC", 24, 31), TIMING_FIELD("tRRD", 20, 23), TIMING_FIELD("tRP", 16, 19), @@ -178,7 +212,7 @@ static const struct timing_reg timing_row[] = { TIMING_FIELD("tRAS", 0, 5), }; -static const struct timing_reg timing_data[] = { +static const struct timing_reg timing_data_reg_fields[] = { TIMING_FIELD("tWTR", 28, 31), TIMING_FIELD("tWR", 24, 27), TIMING_FIELD("tRTP", 20, 23), @@ -189,7 +223,7 @@ static const struct timing_reg timing_data[] = { TIMING_FIELD("RL", 0, 3), }; -static const struct timing_reg timing_power[] = { +static const struct timing_reg timing_power_reg_fields[] = { TIMING_FIELD("tFAW", 26, 31), TIMING_FIELD("tXSR", 16, 25), TIMING_FIELD("tXP", 8, 15), @@ -197,8 +231,9 @@ static const struct timing_reg timing_power[] = { TIMING_FIELD("tMRD", 0, 3), }; -#define TIMING_COUNT (ARRAY_SIZE(timing_row) + ARRAY_SIZE(timing_data) + \ - ARRAY_SIZE(timing_power)) +#define TIMING_COUNT (ARRAY_SIZE(timing_row_reg_fields) + \ + ARRAY_SIZE(timing_data_reg_fields) + \ + ARRAY_SIZE(timing_power_reg_fields)) static int exynos5_counters_set_event(struct exynos5_dmc *dmc) { @@ -346,7 +381,6 @@ err_opp: /** * exynos5_set_bypass_dram_timings() - Low-level changes of the DRAM timings * @dmc: device for which the new settings is going to be applied - * @param: DRAM parameters which passes timing data * * Low-level function for changing timings for DRAM memory clocking from * 'bypass' clock source (fixed frequency @400MHz). @@ -453,9 +487,6 @@ static int exynos5_dmc_align_bypass_voltage(struct exynos5_dmc *dmc, unsigned long target_volt) { int ret = 0; - unsigned long bypass_volt = dmc->opp_bypass.volt_uv; - - target_volt = max(bypass_volt, target_volt); if (dmc->curr_volt >= target_volt) return 0; @@ -617,6 +648,7 @@ disable_clocks: * requested * @target_volt: returned voltage which corresponds to the returned * frequency + * @flags: devfreq flags provided for this frequency change request * * Function gets requested frequency and checks OPP framework for needed * frequency and voltage. It populates the values 'target_rate' and @@ -908,7 +940,10 @@ static int exynos5_dmc_get_status(struct device *dev, int ret; if (dmc->in_irq_mode) { + mutex_lock(&dmc->lock); stat->current_frequency = dmc->curr_rate; + mutex_unlock(&dmc->lock); + stat->busy_time = dmc->load; stat->total_time = dmc->total; } else { @@ -950,7 +985,7 @@ static int exynos5_dmc_get_cur_freq(struct device *dev, unsigned long *freq) return 0; } -/** +/* * exynos5_dmc_df_profile - Devfreq governor's profile structure * * It provides to the devfreq framework needed functions and polling period. @@ -993,7 +1028,9 @@ exynos5_dmc_align_init_freq(struct exynos5_dmc *dmc, /** * create_timings_aligned() - Create register values and align with standard * @dmc: device for which the frequency is going to be set - * @idx: speed bin in the OPP table + * @reg_timing_row: array to fill with values for timing row register + * @reg_timing_data: array to fill with values for timing data register + * @reg_timing_power: array to fill with values for timing power register * @clk_period_ps: the period of the clock, known as tCK * * The function calculates timings and creates a register value ready for @@ -1018,117 +1055,117 @@ static int create_timings_aligned(struct exynos5_dmc *dmc, u32 *reg_timing_row, val = dmc->timings->tRFC / clk_period_ps; val += dmc->timings->tRFC % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRFC); - reg = &timing_row[0]; + reg = &timing_row_reg_fields[0]; *reg_timing_row |= TIMING_VAL2REG(reg, val); val = dmc->timings->tRRD / clk_period_ps; val += dmc->timings->tRRD % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRRD); - reg = &timing_row[1]; + reg = &timing_row_reg_fields[1]; *reg_timing_row |= TIMING_VAL2REG(reg, val); val = dmc->timings->tRPab / clk_period_ps; val += dmc->timings->tRPab % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRPab); - reg = &timing_row[2]; + reg = &timing_row_reg_fields[2]; *reg_timing_row |= TIMING_VAL2REG(reg, val); val = dmc->timings->tRCD / clk_period_ps; val += dmc->timings->tRCD % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRCD); - reg = &timing_row[3]; + reg = &timing_row_reg_fields[3]; *reg_timing_row |= TIMING_VAL2REG(reg, val); val = dmc->timings->tRC / clk_period_ps; val += dmc->timings->tRC % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRC); - reg = &timing_row[4]; + reg = &timing_row_reg_fields[4]; *reg_timing_row |= TIMING_VAL2REG(reg, val); val = dmc->timings->tRAS / clk_period_ps; val += dmc->timings->tRAS % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRAS); - reg = &timing_row[5]; + reg = &timing_row_reg_fields[5]; *reg_timing_row |= TIMING_VAL2REG(reg, val); /* data related timings */ val = dmc->timings->tWTR / clk_period_ps; val += dmc->timings->tWTR % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tWTR); - reg = &timing_data[0]; + reg = &timing_data_reg_fields[0]; *reg_timing_data |= TIMING_VAL2REG(reg, val); val = dmc->timings->tWR / clk_period_ps; val += dmc->timings->tWR % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tWR); - reg = &timing_data[1]; + reg = &timing_data_reg_fields[1]; *reg_timing_data |= TIMING_VAL2REG(reg, val); val = dmc->timings->tRTP / clk_period_ps; val += dmc->timings->tRTP % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRTP); - reg = &timing_data[2]; + reg = &timing_data_reg_fields[2]; *reg_timing_data |= TIMING_VAL2REG(reg, val); val = dmc->timings->tW2W_C2C / clk_period_ps; val += dmc->timings->tW2W_C2C % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tW2W_C2C); - reg = &timing_data[3]; + reg = &timing_data_reg_fields[3]; *reg_timing_data |= TIMING_VAL2REG(reg, val); val = dmc->timings->tR2R_C2C / clk_period_ps; val += dmc->timings->tR2R_C2C % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tR2R_C2C); - reg = &timing_data[4]; + reg = &timing_data_reg_fields[4]; *reg_timing_data |= TIMING_VAL2REG(reg, val); val = dmc->timings->tWL / clk_period_ps; val += dmc->timings->tWL % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tWL); - reg = &timing_data[5]; + reg = &timing_data_reg_fields[5]; *reg_timing_data |= TIMING_VAL2REG(reg, val); val = dmc->timings->tDQSCK / clk_period_ps; val += dmc->timings->tDQSCK % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tDQSCK); - reg = &timing_data[6]; + reg = &timing_data_reg_fields[6]; *reg_timing_data |= TIMING_VAL2REG(reg, val); val = dmc->timings->tRL / clk_period_ps; val += dmc->timings->tRL % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRL); - reg = &timing_data[7]; + reg = &timing_data_reg_fields[7]; *reg_timing_data |= TIMING_VAL2REG(reg, val); /* power related timings */ val = dmc->timings->tFAW / clk_period_ps; val += dmc->timings->tFAW % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tFAW); - reg = &timing_power[0]; + reg = &timing_power_reg_fields[0]; *reg_timing_power |= TIMING_VAL2REG(reg, val); val = dmc->timings->tXSR / clk_period_ps; val += dmc->timings->tXSR % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tXSR); - reg = &timing_power[1]; + reg = &timing_power_reg_fields[1]; *reg_timing_power |= TIMING_VAL2REG(reg, val); val = dmc->timings->tXP / clk_period_ps; val += dmc->timings->tXP % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tXP); - reg = &timing_power[2]; + reg = &timing_power_reg_fields[2]; *reg_timing_power |= TIMING_VAL2REG(reg, val); val = dmc->timings->tCKE / clk_period_ps; val += dmc->timings->tCKE % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tCKE); - reg = &timing_power[3]; + reg = &timing_power_reg_fields[3]; *reg_timing_power |= TIMING_VAL2REG(reg, val); val = dmc->timings->tMRD / clk_period_ps; val += dmc->timings->tMRD % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tMRD); - reg = &timing_power[4]; + reg = &timing_power_reg_fields[4]; *reg_timing_power |= TIMING_VAL2REG(reg, val); return 0; @@ -1263,8 +1300,6 @@ static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc) clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll); - dmc->bypass_rate = clk_get_rate(dmc->mout_mx_mspll_ccore); - clk_prepare_enable(dmc->fout_bpll); clk_prepare_enable(dmc->mout_bpll); @@ -1332,7 +1367,6 @@ static int exynos5_performance_counters_init(struct exynos5_dmc *dmc) /** * exynos5_dmc_set_pause_on_switching() - Controls a pause feature in DMC * @dmc: device which is used for changing this feature - * @set: a boolean state passing enable/disable request * * There is a need of pausing DREX DMC when divider or MUX in clock tree * changes its configuration. In such situation access to the memory is blocked diff --git a/drivers/memory/tegra/tegra124-emc.c b/drivers/memory/tegra/tegra124-emc.c index ba5cb1f4dfc2..76ace42a688a 100644 --- a/drivers/memory/tegra/tegra124-emc.c +++ b/drivers/memory/tegra/tegra124-emc.c @@ -1060,19 +1060,7 @@ static int tegra_emc_debug_available_rates_show(struct seq_file *s, return 0; } -static int tegra_emc_debug_available_rates_open(struct inode *inode, - struct file *file) -{ - return single_open(file, tegra_emc_debug_available_rates_show, - inode->i_private); -} - -static const struct file_operations tegra_emc_debug_available_rates_fops = { - .open = tegra_emc_debug_available_rates_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(tegra_emc_debug_available_rates); static int tegra_emc_debug_min_rate_get(void *data, u64 *rate) { diff --git a/drivers/memory/tegra/tegra124.c b/drivers/memory/tegra/tegra124.c index 493b5dc3a4b3..0cede24479bf 100644 --- a/drivers/memory/tegra/tegra124.c +++ b/drivers/memory/tegra/tegra124.c @@ -957,7 +957,6 @@ static const struct tegra_smmu_swgroup tegra124_swgroups[] = { static const unsigned int tegra124_group_drm[] = { TEGRA_SWGROUP_DC, TEGRA_SWGROUP_DCB, - TEGRA_SWGROUP_GPU, TEGRA_SWGROUP_VIC, }; diff --git a/drivers/memory/tegra/tegra186-emc.c b/drivers/memory/tegra/tegra186-emc.c index 8478f59db432..fa8af17b0e2d 100644 --- a/drivers/memory/tegra/tegra186-emc.c +++ b/drivers/memory/tegra/tegra186-emc.c @@ -172,14 +172,8 @@ static int tegra186_emc_probe(struct platform_device *pdev) return -ENOMEM; emc->bpmp = tegra_bpmp_get(&pdev->dev); - if (IS_ERR(emc->bpmp)) { - err = PTR_ERR(emc->bpmp); - - if (err != -EPROBE_DEFER) - dev_err(&pdev->dev, "failed to get BPMP: %d\n", err); - - return err; - } + if (IS_ERR(emc->bpmp)) + return dev_err_probe(&pdev->dev, PTR_ERR(emc->bpmp), "failed to get BPMP\n"); emc->clk = devm_clk_get(&pdev->dev, "emc"); if (IS_ERR(emc->clk)) { diff --git a/drivers/memory/tegra/tegra210-emc-cc-r21021.c b/drivers/memory/tegra/tegra210-emc-cc-r21021.c index ff55a17896fa..0ebfa8eccf0c 100644 --- a/drivers/memory/tegra/tegra210-emc-cc-r21021.c +++ b/drivers/memory/tegra/tegra210-emc-cc-r21021.c @@ -501,7 +501,6 @@ static u32 tegra210_emc_r21021_periodic_compensation(struct tegra210_emc *emc) emc_cfg_o = emc_readl(emc, EMC_CFG); emc_cfg = emc_cfg_o & ~(EMC_CFG_DYN_SELF_REF | EMC_CFG_DRAM_ACPD | - EMC_CFG_DRAM_CLKSTOP_PD | EMC_CFG_DRAM_CLKSTOP_PD); @@ -1044,7 +1043,7 @@ static void tegra210_emc_r21021_set_clock(struct tegra210_emc *emc, u32 clksrc) !opt_cc_short_zcal && opt_short_zcal) { value = (value & ~(EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK << EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_SHIFT)) | - ((zq_wait_long & EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK) << + ((zq_wait_long & EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK) << EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT); } else if (offset == EMC_ZCAL_INTERVAL && opt_zcal_en_cc) { value = 0; /* EMC_ZCAL_INTERVAL reset value. */ diff --git a/drivers/memory/tegra/tegra210.c b/drivers/memory/tegra/tegra210.c index 7212d1d7b348..7fb8b5438bf4 100644 --- a/drivers/memory/tegra/tegra210.c +++ b/drivers/memory/tegra/tegra210.c @@ -842,7 +842,7 @@ static const struct tegra_mc_client tegra210_mc_clients[] = { }, .la = { .reg = 0x3dc, - .shift = 0, + .shift = 16, .mask = 0xff, .def = 0x80, }, diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c index 25a9dd9c0c1b..2ba899f5659f 100644 --- a/drivers/misc/cxl/pci.c +++ b/drivers/misc/cxl/pci.c @@ -393,8 +393,8 @@ int cxl_calc_capp_routing(struct pci_dev *dev, u64 *chipid, *capp_unit_id = get_capp_unit_id(np, *phb_index); of_node_put(np); if (!*capp_unit_id) { - pr_err("cxl: invalid capp unit id (phb_index: %d)\n", - *phb_index); + pr_err("cxl: No capp unit found for PHB[%lld,%d]. Make sure the adapter is on a capi-compatible slot\n", + *chipid, *phb_index); return -ENODEV; } diff --git a/drivers/misc/eeprom/at24.c b/drivers/misc/eeprom/at24.c index 26a23abc053d..1c0a41803bb6 100644 --- a/drivers/misc/eeprom/at24.c +++ b/drivers/misc/eeprom/at24.c @@ -8,6 +8,7 @@ #include <linux/acpi.h> #include <linux/bitops.h> +#include <linux/capability.h> #include <linux/delay.h> #include <linux/i2c.h> #include <linux/init.h> @@ -89,6 +90,7 @@ struct at24_data { struct nvmem_device *nvmem; struct regulator *vcc_reg; + void (*read_post)(unsigned int off, char *buf, size_t count); /* * Some chips tie up multiple I2C addresses; dummy devices reserve @@ -121,6 +123,7 @@ MODULE_PARM_DESC(at24_write_timeout, "Time (in ms) to try writes (default 25)"); struct at24_chip_data { u32 byte_len; u8 flags; + void (*read_post)(unsigned int off, char *buf, size_t count); }; #define AT24_CHIP_DATA(_name, _len, _flags) \ @@ -128,6 +131,32 @@ struct at24_chip_data { .byte_len = _len, .flags = _flags, \ } +#define AT24_CHIP_DATA_CB(_name, _len, _flags, _read_post) \ + static const struct at24_chip_data _name = { \ + .byte_len = _len, .flags = _flags, \ + .read_post = _read_post, \ + } + +static void at24_read_post_vaio(unsigned int off, char *buf, size_t count) +{ + int i; + + if (capable(CAP_SYS_ADMIN)) + return; + + /* + * Hide VAIO private settings to regular users: + * - BIOS passwords: bytes 0x00 to 0x0f + * - UUID: bytes 0x10 to 0x1f + * - Serial number: 0xc0 to 0xdf + */ + for (i = 0; i < count; i++) { + if ((off + i <= 0x1f) || + (off + i >= 0xc0 && off + i <= 0xdf)) + buf[i] = 0; + } +} + /* needs 8 addresses as A0-A2 are ignored */ AT24_CHIP_DATA(at24_data_24c00, 128 / 8, AT24_FLAG_TAKE8ADDR); /* old variants can't be handled with this generic entry! */ @@ -144,6 +173,10 @@ AT24_CHIP_DATA(at24_data_24mac602, 64 / 8, /* spd is a 24c02 in memory DIMMs */ AT24_CHIP_DATA(at24_data_spd, 2048 / 8, AT24_FLAG_READONLY | AT24_FLAG_IRUGO); +/* 24c02_vaio is a 24c02 on some Sony laptops */ +AT24_CHIP_DATA_CB(at24_data_24c02_vaio, 2048 / 8, + AT24_FLAG_READONLY | AT24_FLAG_IRUGO, + at24_read_post_vaio); AT24_CHIP_DATA(at24_data_24c04, 4096 / 8, 0); AT24_CHIP_DATA(at24_data_24cs04, 16, AT24_FLAG_SERIAL | AT24_FLAG_READONLY); @@ -177,6 +210,7 @@ static const struct i2c_device_id at24_ids[] = { { "24mac402", (kernel_ulong_t)&at24_data_24mac402 }, { "24mac602", (kernel_ulong_t)&at24_data_24mac602 }, { "spd", (kernel_ulong_t)&at24_data_spd }, + { "24c02-vaio", (kernel_ulong_t)&at24_data_24c02_vaio }, { "24c04", (kernel_ulong_t)&at24_data_24c04 }, { "24cs04", (kernel_ulong_t)&at24_data_24cs04 }, { "24c08", (kernel_ulong_t)&at24_data_24c08 }, @@ -388,7 +422,7 @@ static int at24_read(void *priv, unsigned int off, void *val, size_t count) struct at24_data *at24; struct device *dev; char *buf = val; - int ret; + int i, ret; at24 = priv; dev = at24_base_client_dev(at24); @@ -411,22 +445,22 @@ static int at24_read(void *priv, unsigned int off, void *val, size_t count) */ mutex_lock(&at24->lock); - while (count) { - ret = at24_regmap_read(at24, buf, off, count); + for (i = 0; count; i += ret, count -= ret) { + ret = at24_regmap_read(at24, buf + i, off + i, count); if (ret < 0) { mutex_unlock(&at24->lock); pm_runtime_put(dev); return ret; } - buf += ret; - off += ret; - count -= ret; } mutex_unlock(&at24->lock); pm_runtime_put(dev); + if (unlikely(at24->read_post)) + at24->read_post(off, buf, i); + return 0; } @@ -654,6 +688,7 @@ static int at24_probe(struct i2c_client *client) at24->byte_len = byte_len; at24->page_size = page_size; at24->flags = flags; + at24->read_post = cdata->read_post; at24->num_addresses = num_addresses; at24->offset_adj = at24_get_offset_adj(flags, byte_len); at24->client[0].client = client; @@ -678,8 +713,30 @@ static int at24_probe(struct i2c_client *client) return err; } - nvmem_config.name = dev_name(dev); + /* + * If the 'label' property is not present for the AT24 EEPROM, + * then nvmem_config.id is initialised to NVMEM_DEVID_AUTO, + * and this will append the 'devid' to the name of the NVMEM + * device. This is purely legacy and the AT24 driver has always + * defaulted to this. However, if the 'label' property is + * present then this means that the name is specified by the + * firmware and this name should be used verbatim and so it is + * not necessary to append the 'devid'. + */ + if (device_property_present(dev, "label")) { + nvmem_config.id = NVMEM_DEVID_NONE; + err = device_property_read_string(dev, "label", + &nvmem_config.name); + if (err) + return err; + } else { + nvmem_config.id = NVMEM_DEVID_AUTO; + nvmem_config.name = dev_name(dev); + } + + nvmem_config.type = NVMEM_TYPE_EEPROM; nvmem_config.dev = dev; + nvmem_config.id = NVMEM_DEVID_AUTO; nvmem_config.read_only = !writable; nvmem_config.root_only = !(flags & AT24_FLAG_IRUGO); nvmem_config.owner = THIS_MODULE; diff --git a/drivers/misc/eeprom/eeprom.c b/drivers/misc/eeprom/eeprom.c index 226b5efa6a77..34fa385dfd4b 100644 --- a/drivers/misc/eeprom/eeprom.c +++ b/drivers/misc/eeprom/eeprom.c @@ -76,7 +76,7 @@ static ssize_t eeprom_read(struct file *filp, struct kobject *kobj, struct bin_attribute *bin_attr, char *buf, loff_t off, size_t count) { - struct i2c_client *client = to_i2c_client(kobj_to_dev(kobj)); + struct i2c_client *client = kobj_to_i2c_client(kobj); struct eeprom_data *data = i2c_get_clientdata(client); u8 slice; diff --git a/drivers/misc/lkdtm/bugs.c b/drivers/misc/lkdtm/bugs.c index 4dfbfd51bdf7..a0675d4154d2 100644 --- a/drivers/misc/lkdtm/bugs.c +++ b/drivers/misc/lkdtm/bugs.c @@ -312,16 +312,6 @@ void lkdtm_CORRUPT_LIST_DEL(void) pr_err("list_del() corruption not detected!\n"); } -/* Test if unbalanced set_fs(KERNEL_DS)/set_fs(USER_DS) check exists. */ -void lkdtm_CORRUPT_USER_DS(void) -{ - pr_info("setting bad task size limit\n"); - set_fs(KERNEL_DS); - - /* Make sure we do not keep running with a KERNEL_DS! */ - force_sig(SIGKILL); -} - /* Test that VMAP_STACK is actually allocating with a leading guard page */ void lkdtm_STACK_GUARD_PAGE_LEADING(void) { diff --git a/drivers/misc/lkdtm/core.c b/drivers/misc/lkdtm/core.c index a5e344df9166..97803f213d9d 100644 --- a/drivers/misc/lkdtm/core.c +++ b/drivers/misc/lkdtm/core.c @@ -112,7 +112,6 @@ static const struct crashtype crashtypes[] = { CRASHTYPE(CORRUPT_STACK_STRONG), CRASHTYPE(CORRUPT_LIST_ADD), CRASHTYPE(CORRUPT_LIST_DEL), - CRASHTYPE(CORRUPT_USER_DS), CRASHTYPE(STACK_GUARD_PAGE_LEADING), CRASHTYPE(STACK_GUARD_PAGE_TRAILING), CRASHTYPE(UNSET_SMEP), @@ -172,7 +171,6 @@ static const struct crashtype crashtypes[] = { CRASHTYPE(USERCOPY_STACK_FRAME_FROM), CRASHTYPE(USERCOPY_STACK_BEYOND), CRASHTYPE(USERCOPY_KERNEL), - CRASHTYPE(USERCOPY_KERNEL_DS), CRASHTYPE(STACKLEAK_ERASING), CRASHTYPE(CFI_FORWARD_PROTO), #ifdef CONFIG_X86_32 diff --git a/drivers/misc/lkdtm/lkdtm.h b/drivers/misc/lkdtm/lkdtm.h index 8878538b2c13..6dec4c9b442f 100644 --- a/drivers/misc/lkdtm/lkdtm.h +++ b/drivers/misc/lkdtm/lkdtm.h @@ -27,7 +27,6 @@ void lkdtm_OVERFLOW_UNSIGNED(void); void lkdtm_ARRAY_BOUNDS(void); void lkdtm_CORRUPT_LIST_ADD(void); void lkdtm_CORRUPT_LIST_DEL(void); -void lkdtm_CORRUPT_USER_DS(void); void lkdtm_STACK_GUARD_PAGE_LEADING(void); void lkdtm_STACK_GUARD_PAGE_TRAILING(void); void lkdtm_UNSET_SMEP(void); @@ -96,7 +95,6 @@ void lkdtm_USERCOPY_STACK_FRAME_TO(void); void lkdtm_USERCOPY_STACK_FRAME_FROM(void); void lkdtm_USERCOPY_STACK_BEYOND(void); void lkdtm_USERCOPY_KERNEL(void); -void lkdtm_USERCOPY_KERNEL_DS(void); /* lkdtm_stackleak.c */ void lkdtm_STACKLEAK_ERASING(void); diff --git a/drivers/misc/lkdtm/usercopy.c b/drivers/misc/lkdtm/usercopy.c index b833367a45d0..109e8d4302c1 100644 --- a/drivers/misc/lkdtm/usercopy.c +++ b/drivers/misc/lkdtm/usercopy.c @@ -325,21 +325,6 @@ free_user: vm_munmap(user_addr, PAGE_SIZE); } -void lkdtm_USERCOPY_KERNEL_DS(void) -{ - char __user *user_ptr = - (char __user *)(0xFUL << (sizeof(unsigned long) * 8 - 4)); - mm_segment_t old_fs = get_fs(); - char buf[10] = {0}; - - pr_info("attempting copy_to_user() to noncanonical address: %px\n", - user_ptr); - set_fs(KERNEL_DS); - if (copy_to_user(user_ptr, buf, sizeof(buf)) == 0) - pr_err("copy_to_user() to noncanonical address succeeded!?\n"); - set_fs(old_fs); -} - void __init lkdtm_usercopy_init(void) { /* Prepare cache that lacks SLAB_USERCOPY flag. */ diff --git a/drivers/misc/ocxl/Kconfig b/drivers/misc/ocxl/Kconfig index 947294f6d7f4..c9b0a27caf64 100644 --- a/drivers/misc/ocxl/Kconfig +++ b/drivers/misc/ocxl/Kconfig @@ -9,7 +9,7 @@ config OCXL_BASE config OCXL tristate "OpenCAPI coherent accelerator support" - depends on PPC_POWERNV && PCI && EEH && HOTPLUG_PCI_POWERNV + depends on HOTPLUG_PCI_POWERNV select OCXL_BASE default m help diff --git a/drivers/misc/ocxl/afu_irq.c b/drivers/misc/ocxl/afu_irq.c index 70f8f1c3929d..ecdcfae025b7 100644 --- a/drivers/misc/ocxl/afu_irq.c +++ b/drivers/misc/ocxl/afu_irq.c @@ -2,6 +2,7 @@ // Copyright 2017 IBM Corp. #include <linux/interrupt.h> #include <asm/pnv-ocxl.h> +#include <asm/xive.h> #include "ocxl_internal.h" #include "trace.h" @@ -10,7 +11,6 @@ struct afu_irq { int hw_irq; unsigned int virq; char *name; - u64 trigger_page; irqreturn_t (*handler)(void *private); void (*free_private)(void *private); void *private; @@ -124,8 +124,7 @@ int ocxl_afu_irq_alloc(struct ocxl_context *ctx, int *irq_id) goto err_unlock; } - rc = ocxl_link_irq_alloc(ctx->afu->fn->link, &irq->hw_irq, - &irq->trigger_page); + rc = ocxl_link_irq_alloc(ctx->afu->fn->link, &irq->hw_irq); if (rc) goto err_idr; @@ -196,13 +195,16 @@ void ocxl_afu_irq_free_all(struct ocxl_context *ctx) u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, int irq_id) { + struct xive_irq_data *xd; struct afu_irq *irq; u64 addr = 0; mutex_lock(&ctx->irq_lock); irq = idr_find(&ctx->irq_idr, irq_id); - if (irq) - addr = irq->trigger_page; + if (irq) { + xd = irq_get_handler_data(irq->virq); + addr = xd ? xd->trig_page : 0; + } mutex_unlock(&ctx->irq_lock); return addr; } diff --git a/drivers/misc/ocxl/link.c b/drivers/misc/ocxl/link.c index 58d111afd9f6..fd73d3bc0eb6 100644 --- a/drivers/misc/ocxl/link.c +++ b/drivers/misc/ocxl/link.c @@ -6,6 +6,7 @@ #include <linux/mmu_context.h> #include <asm/copro.h> #include <asm/pnv-ocxl.h> +#include <asm/xive.h> #include <misc/ocxl.h> #include "ocxl_internal.h" #include "trace.h" @@ -682,23 +683,21 @@ unlock: } EXPORT_SYMBOL_GPL(ocxl_link_remove_pe); -int ocxl_link_irq_alloc(void *link_handle, int *hw_irq, u64 *trigger_addr) +int ocxl_link_irq_alloc(void *link_handle, int *hw_irq) { struct ocxl_link *link = (struct ocxl_link *) link_handle; - int rc, irq; - u64 addr; + int irq; if (atomic_dec_if_positive(&link->irq_available) < 0) return -ENOSPC; - rc = pnv_ocxl_alloc_xive_irq(&irq, &addr); - if (rc) { + irq = xive_native_alloc_irq(); + if (!irq) { atomic_inc(&link->irq_available); - return rc; + return -ENXIO; } *hw_irq = irq; - *trigger_addr = addr; return 0; } EXPORT_SYMBOL_GPL(ocxl_link_irq_alloc); @@ -707,7 +706,7 @@ void ocxl_link_free_irq(void *link_handle, int hw_irq) { struct ocxl_link *link = (struct ocxl_link *) link_handle; - pnv_ocxl_free_xive_irq(hw_irq); + xive_native_free_irq(hw_irq); atomic_inc(&link->irq_available); } EXPORT_SYMBOL_GPL(ocxl_link_free_irq); diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c index e060796f9caa..146ca6fb3260 100644 --- a/drivers/misc/pci_endpoint_test.c +++ b/drivers/misc/pci_endpoint_test.c @@ -70,11 +70,15 @@ #define PCI_DEVICE_ID_TI_J721E 0xb00d #define PCI_DEVICE_ID_TI_AM654 0xb00c +#define PCI_DEVICE_ID_LS1088A 0x80c0 #define is_am654_pci_dev(pdev) \ ((pdev)->device == PCI_DEVICE_ID_TI_AM654) +#define PCI_DEVICE_ID_RENESAS_R8A774A1 0x0028 +#define PCI_DEVICE_ID_RENESAS_R8A774B1 0x002b #define PCI_DEVICE_ID_RENESAS_R8A774C0 0x002d +#define PCI_DEVICE_ID_RENESAS_R8A774E1 0x0025 static DEFINE_IDA(pci_endpoint_test_ida); @@ -945,13 +949,20 @@ static const struct pci_device_id pci_endpoint_test_tbl[] = { { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA72x), .driver_data = (kernel_ulong_t)&default_data, }, - { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x81c0) }, + { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x81c0), + .driver_data = (kernel_ulong_t)&default_data, + }, + { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, PCI_DEVICE_ID_LS1088A), + .driver_data = (kernel_ulong_t)&default_data, + }, { PCI_DEVICE_DATA(SYNOPSYS, EDDA, NULL) }, { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM654), .driver_data = (kernel_ulong_t)&am654_data }, - { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774C0), - }, + { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774A1),}, + { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774B1),}, + { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774C0),}, + { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774E1),}, { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721E), .driver_data = (kernel_ulong_t)&j721e_data, }, diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig index f0cb7aeabbc4..31481c9fcc2e 100644 --- a/drivers/mmc/host/Kconfig +++ b/drivers/mmc/host/Kconfig @@ -302,7 +302,7 @@ config MMC_SDHCI_TEGRA config MMC_SDHCI_S3C tristate "SDHCI support on Samsung S3C SoC" depends on MMC_SDHCI - depends on PLAT_SAMSUNG || COMPILE_TEST + depends on PLAT_SAMSUNG || ARCH_S5PV210 || ARCH_EXYNOS || COMPILE_TEST help This selects the Secure Digital Host Controller Interface (SDHCI) often referrered to as the HSMMC block in some of the Samsung S3C diff --git a/drivers/mmc/host/s3cmci.c b/drivers/mmc/host/s3cmci.c index e3698aba8dd3..643d54eceef6 100644 --- a/drivers/mmc/host/s3cmci.c +++ b/drivers/mmc/host/s3cmci.c @@ -24,11 +24,6 @@ #include <linux/of.h> #include <linux/of_device.h> #include <linux/mmc/slot-gpio.h> - -#include <plat/gpio-cfg.h> -#include <mach/dma.h> -#include <mach/gpio-samsung.h> - #include <linux/platform_data/mmc-s3cmci.h> #include "s3cmci.h" @@ -305,7 +300,8 @@ static inline void clear_imask(struct s3cmci_host *host) static void s3cmci_check_sdio_irq(struct s3cmci_host *host) { if (host->sdio_irqen) { - if (gpio_get_value(S3C2410_GPE(8)) == 0) { + if (host->pdata->bus[3] && + gpiod_get_value(host->pdata->bus[3]) == 0) { pr_debug("%s: signalling irq\n", __func__); mmc_signal_sdio_irq(host->mmc); } @@ -1201,33 +1197,20 @@ static void s3cmci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) switch (ios->power_mode) { case MMC_POWER_ON: case MMC_POWER_UP: - /* Configure GPE5...GPE10 pins in SD mode */ - if (!host->pdev->dev.of_node) - s3c_gpio_cfgall_range(S3C2410_GPE(5), 6, S3C_GPIO_SFN(2), - S3C_GPIO_PULL_NONE); - - if (host->pdata->set_power) - host->pdata->set_power(ios->power_mode, ios->vdd); - if (!host->is2440) mci_con |= S3C2410_SDICON_FIFORESET; - break; case MMC_POWER_OFF: default: - if (!host->pdev->dev.of_node) - gpio_direction_output(S3C2410_GPE(5), 0); - if (host->is2440) mci_con |= S3C2440_SDICON_SDRESET; - - if (host->pdata->set_power) - host->pdata->set_power(ios->power_mode, ios->vdd); - break; } + if (host->pdata->set_power) + host->pdata->set_power(ios->power_mode, ios->vdd); + s3cmci_set_clk(host, ios); /* Set CLOCK_ENABLE */ @@ -1305,13 +1288,6 @@ static const struct mmc_host_ops s3cmci_ops = { .enable_sdio_irq = s3cmci_enable_sdio_irq, }; -static struct s3c24xx_mci_pdata s3cmci_def_pdata = { - /* This is currently here to avoid a number of if (host->pdata) - * checks. Any zero fields to ensure reasonable defaults are picked. */ - .no_wprotect = 1, - .no_detect = 1, -}; - #ifdef CONFIG_ARM_S3C24XX_CPUFREQ static int s3cmci_cpufreq_transition(struct notifier_block *nb, @@ -1465,24 +1441,21 @@ static int s3cmci_probe_pdata(struct s3cmci_host *host) int i, ret; host->is2440 = platform_get_device_id(pdev)->driver_data; + pdata = pdev->dev.platform_data; + if (!pdata) { + dev_err(&pdev->dev, "need platform data"); + return -ENXIO; + } - for (i = S3C2410_GPE(5); i <= S3C2410_GPE(10); i++) { - ret = gpio_request(i, dev_name(&pdev->dev)); - if (ret) { + for (i = 0; i < 6; i++) { + pdata->bus[i] = devm_gpiod_get_index(&pdev->dev, "bus", i, + GPIOD_OUT_LOW); + if (IS_ERR(pdata->bus[i])) { dev_err(&pdev->dev, "failed to get gpio %d\n", i); - - for (i--; i >= S3C2410_GPE(5); i--) - gpio_free(i); - - return ret; + return PTR_ERR(pdata->bus[i]); } } - if (!pdev->dev.platform_data) - pdev->dev.platform_data = &s3cmci_def_pdata; - - pdata = pdev->dev.platform_data; - if (pdata->no_wprotect) mmc->caps2 |= MMC_CAP2_NO_WRITE_PROTECT; @@ -1537,7 +1510,6 @@ static int s3cmci_probe(struct platform_device *pdev) struct s3cmci_host *host; struct mmc_host *mmc; int ret; - int i; mmc = mmc_alloc_host(sizeof(struct s3cmci_host), &pdev->dev); if (!mmc) { @@ -1581,7 +1553,7 @@ static int s3cmci_probe(struct platform_device *pdev) "failed to get io memory region resource.\n"); ret = -ENOENT; - goto probe_free_gpio; + goto probe_free_host; } host->mem = request_mem_region(host->mem->start, @@ -1590,7 +1562,7 @@ static int s3cmci_probe(struct platform_device *pdev) if (!host->mem) { dev_err(&pdev->dev, "failed to request io memory region.\n"); ret = -ENOENT; - goto probe_free_gpio; + goto probe_free_host; } host->base = ioremap(host->mem->start, resource_size(host->mem)); @@ -1714,11 +1686,6 @@ static int s3cmci_probe(struct platform_device *pdev) probe_free_mem_region: release_mem_region(host->mem->start, resource_size(host->mem)); - probe_free_gpio: - if (!pdev->dev.of_node) - for (i = S3C2410_GPE(5); i <= S3C2410_GPE(10); i++) - gpio_free(i); - probe_free_host: mmc_free_host(mmc); @@ -1744,7 +1711,6 @@ static int s3cmci_remove(struct platform_device *pdev) { struct mmc_host *mmc = platform_get_drvdata(pdev); struct s3cmci_host *host = mmc_priv(mmc); - int i; s3cmci_shutdown(pdev); @@ -1757,10 +1723,6 @@ static int s3cmci_remove(struct platform_device *pdev) free_irq(host->irq, host); - if (!pdev->dev.of_node) - for (i = S3C2410_GPE(5); i <= S3C2410_GPE(10); i++) - gpio_free(i); - iounmap(host->base); release_mem_region(host->mem->start, resource_size(host->mem)); diff --git a/drivers/mtd/devices/Kconfig b/drivers/mtd/devices/Kconfig index f96287c4b789..0f4c2d823de8 100644 --- a/drivers/mtd/devices/Kconfig +++ b/drivers/mtd/devices/Kconfig @@ -91,7 +91,7 @@ config MTD_MCHP23K256 config MTD_SPEAR_SMI tristate "SPEAR MTD NOR Support through SMI controller" - depends on PLAT_SPEAR + depends on PLAT_SPEAR || COMPILE_TEST default y help This enable SNOR support on SPEAR platforms using SMI controller diff --git a/drivers/mtd/devices/lart.c b/drivers/mtd/devices/lart.c index 56f50d27b7fd..aecd441e4183 100644 --- a/drivers/mtd/devices/lart.c +++ b/drivers/mtd/devices/lart.c @@ -436,7 +436,10 @@ static int flash_read (struct mtd_info *mtd,loff_t from,size_t len,size_t *retle { int gap = BUSWIDTH - (from & (BUSWIDTH - 1)); - while (len && gap--) *buf++ = read8 (from++), len--; + while (len && gap--) { + *buf++ = read8 (from++); + len--; + } } /* now we read dwords until we reach a non-dword boundary */ @@ -518,7 +521,10 @@ static int flash_write (struct mtd_info *mtd,loff_t to,size_t len,size_t *retlen i = n = 0; while (gap--) tmp[i++] = 0xFF; - while (len && i < BUSWIDTH) tmp[i++] = buf[n++], len--; + while (len && i < BUSWIDTH) { + tmp[i++] = buf[n++]; + len--; + } while (i < BUSWIDTH) tmp[i++] = 0xFF; if (!write_dword (aligned,*((__u32 *) tmp))) return (-EIO); diff --git a/drivers/mtd/devices/spear_smi.c b/drivers/mtd/devices/spear_smi.c index 79dcca16481d..2e00862389dd 100644 --- a/drivers/mtd/devices/spear_smi.c +++ b/drivers/mtd/devices/spear_smi.c @@ -793,7 +793,7 @@ static int spear_smi_probe_config_dt(struct platform_device *pdev, struct device_node *np) { struct spear_smi_plat_data *pdata = dev_get_platdata(&pdev->dev); - struct device_node *pp = NULL; + struct device_node *pp; const __be32 *addr; u32 val; int len; @@ -812,7 +812,7 @@ static int spear_smi_probe_config_dt(struct platform_device *pdev, return -ENOMEM; /* Fill structs for each subnode (flash device) */ - while ((pp = of_get_next_child(np, pp))) { + for_each_child_of_node(np, pp) { pdata->np[i] = pp; /* Read base-addr and size from DT */ diff --git a/drivers/mtd/hyperbus/Kconfig b/drivers/mtd/hyperbus/Kconfig index a4d8968d133d..46c7e407e378 100644 --- a/drivers/mtd/hyperbus/Kconfig +++ b/drivers/mtd/hyperbus/Kconfig @@ -22,4 +22,11 @@ config HBMC_AM654 This is the driver for HyperBus controller on TI's AM65x and other SoCs +config RPCIF_HYPERBUS + tristate "Renesas RPC-IF HyperBus driver" + depends on RENESAS_RPCIF + depends on MTD_CFI_BE_BYTE_SWAP + help + This option includes Renesas RPC-IF HyperBus support. + endif # MTD_HYPERBUS diff --git a/drivers/mtd/hyperbus/Makefile b/drivers/mtd/hyperbus/Makefile index 8a936e066f48..5fc2b5124542 100644 --- a/drivers/mtd/hyperbus/Makefile +++ b/drivers/mtd/hyperbus/Makefile @@ -2,3 +2,4 @@ obj-$(CONFIG_MTD_HYPERBUS) += hyperbus-core.o obj-$(CONFIG_HBMC_AM654) += hbmc-am654.o +obj-$(CONFIG_RPCIF_HYPERBUS) += rpc-if.o diff --git a/drivers/mtd/hyperbus/hbmc-am654.c b/drivers/mtd/hyperbus/hbmc-am654.c index e0e33f6bf513..a3439b791eeb 100644 --- a/drivers/mtd/hyperbus/hbmc-am654.c +++ b/drivers/mtd/hyperbus/hbmc-am654.c @@ -3,6 +3,10 @@ // Copyright (C) 2019 Texas Instruments Incorporated - https://www.ti.com/ // Author: Vignesh Raghavendra <vigneshr@ti.com> +#include <linux/completion.h> +#include <linux/dma-direction.h> +#include <linux/dma-mapping.h> +#include <linux/dmaengine.h> #include <linux/err.h> #include <linux/kernel.h> #include <linux/module.h> @@ -13,11 +17,18 @@ #include <linux/of.h> #include <linux/of_address.h> #include <linux/platform_device.h> -#include <linux/pm_runtime.h> +#include <linux/sched/task_stack.h> #include <linux/types.h> #define AM654_HBMC_CALIB_COUNT 25 +struct am654_hbmc_device_priv { + struct completion rx_dma_complete; + phys_addr_t device_base; + struct hyperbus_ctlr *ctlr; + struct dma_chan *rx_chan; +}; + struct am654_hbmc_priv { struct hyperbus_ctlr ctlr; struct hyperbus_device hbdev; @@ -52,13 +63,103 @@ static int am654_hbmc_calibrate(struct hyperbus_device *hbdev) return ret; } +static void am654_hbmc_dma_callback(void *param) +{ + struct am654_hbmc_device_priv *priv = param; + + complete(&priv->rx_dma_complete); +} + +static int am654_hbmc_dma_read(struct am654_hbmc_device_priv *priv, void *to, + unsigned long from, ssize_t len) + +{ + enum dma_ctrl_flags flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT; + struct dma_chan *rx_chan = priv->rx_chan; + struct dma_async_tx_descriptor *tx; + dma_addr_t dma_dst, dma_src; + dma_cookie_t cookie; + int ret; + + if (!priv->rx_chan || !virt_addr_valid(to) || object_is_on_stack(to)) + return -EINVAL; + + dma_dst = dma_map_single(rx_chan->device->dev, to, len, DMA_FROM_DEVICE); + if (dma_mapping_error(rx_chan->device->dev, dma_dst)) { + dev_dbg(priv->ctlr->dev, "DMA mapping failed\n"); + return -EIO; + } + + dma_src = priv->device_base + from; + tx = dmaengine_prep_dma_memcpy(rx_chan, dma_dst, dma_src, len, flags); + if (!tx) { + dev_err(priv->ctlr->dev, "device_prep_dma_memcpy error\n"); + ret = -EIO; + goto unmap_dma; + } + + reinit_completion(&priv->rx_dma_complete); + tx->callback = am654_hbmc_dma_callback; + tx->callback_param = priv; + cookie = dmaengine_submit(tx); + + ret = dma_submit_error(cookie); + if (ret) { + dev_err(priv->ctlr->dev, "dma_submit_error %d\n", cookie); + goto unmap_dma; + } + + dma_async_issue_pending(rx_chan); + if (!wait_for_completion_timeout(&priv->rx_dma_complete, msecs_to_jiffies(len + 1000))) { + dmaengine_terminate_sync(rx_chan); + dev_err(priv->ctlr->dev, "DMA wait_for_completion_timeout\n"); + ret = -ETIMEDOUT; + } + +unmap_dma: + dma_unmap_single(rx_chan->device->dev, dma_dst, len, DMA_FROM_DEVICE); + return ret; +} + +static void am654_hbmc_read(struct hyperbus_device *hbdev, void *to, + unsigned long from, ssize_t len) +{ + struct am654_hbmc_device_priv *priv = hbdev->priv; + + if (len < SZ_1K || am654_hbmc_dma_read(priv, to, from, len)) + memcpy_fromio(to, hbdev->map.virt + from, len); +} + static const struct hyperbus_ops am654_hbmc_ops = { .calibrate = am654_hbmc_calibrate, + .copy_from = am654_hbmc_read, }; +static int am654_hbmc_request_mmap_dma(struct am654_hbmc_device_priv *priv) +{ + struct dma_chan *rx_chan; + dma_cap_mask_t mask; + + dma_cap_zero(mask); + dma_cap_set(DMA_MEMCPY, mask); + + rx_chan = dma_request_chan_by_mask(&mask); + if (IS_ERR(rx_chan)) { + if (PTR_ERR(rx_chan) == -EPROBE_DEFER) + return -EPROBE_DEFER; + dev_dbg(priv->ctlr->dev, "No DMA channel available\n"); + return 0; + } + priv->rx_chan = rx_chan; + init_completion(&priv->rx_dma_complete); + + return 0; +} + static int am654_hbmc_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; + struct am654_hbmc_device_priv *dev_priv; struct device *dev = &pdev->dev; struct am654_hbmc_priv *priv; struct resource res; @@ -70,7 +171,8 @@ static int am654_hbmc_probe(struct platform_device *pdev) platform_set_drvdata(pdev, priv); - ret = of_address_to_resource(np, 0, &res); + priv->hbdev.np = of_get_next_child(np, NULL); + ret = of_address_to_resource(priv->hbdev.np, 0, &res); if (ret) return ret; @@ -88,13 +190,6 @@ static int am654_hbmc_probe(struct platform_device *pdev) priv->mux_ctrl = control; } - pm_runtime_enable(dev); - ret = pm_runtime_get_sync(dev); - if (ret < 0) { - pm_runtime_put_noidle(dev); - goto disable_pm; - } - priv->hbdev.map.size = resource_size(&res); priv->hbdev.map.virt = devm_ioremap_resource(dev, &res); if (IS_ERR(priv->hbdev.map.virt)) @@ -103,17 +198,32 @@ static int am654_hbmc_probe(struct platform_device *pdev) priv->ctlr.dev = dev; priv->ctlr.ops = &am654_hbmc_ops; priv->hbdev.ctlr = &priv->ctlr; - priv->hbdev.np = of_get_next_child(dev->of_node, NULL); + + dev_priv = devm_kzalloc(dev, sizeof(*dev_priv), GFP_KERNEL); + if (!dev_priv) { + ret = -ENOMEM; + goto disable_mux; + } + + priv->hbdev.priv = dev_priv; + dev_priv->device_base = res.start; + dev_priv->ctlr = &priv->ctlr; + + ret = am654_hbmc_request_mmap_dma(dev_priv); + if (ret) + goto disable_mux; + ret = hyperbus_register_device(&priv->hbdev); if (ret) { dev_err(dev, "failed to register controller\n"); - pm_runtime_put_sync(&pdev->dev); - goto disable_pm; + goto release_dma; } return 0; -disable_pm: - pm_runtime_disable(dev); +release_dma: + if (dev_priv->rx_chan) + dma_release_channel(dev_priv->rx_chan); +disable_mux: if (priv->mux_ctrl) mux_control_deselect(priv->mux_ctrl); return ret; @@ -122,13 +232,15 @@ disable_pm: static int am654_hbmc_remove(struct platform_device *pdev) { struct am654_hbmc_priv *priv = platform_get_drvdata(pdev); + struct am654_hbmc_device_priv *dev_priv = priv->hbdev.priv; int ret; ret = hyperbus_unregister_device(&priv->hbdev); if (priv->mux_ctrl) mux_control_deselect(priv->mux_ctrl); - pm_runtime_put_sync(&pdev->dev); - pm_runtime_disable(&pdev->dev); + + if (dev_priv->rx_chan) + dma_release_channel(dev_priv->rx_chan); return ret; } diff --git a/drivers/mtd/hyperbus/rpc-if.c b/drivers/mtd/hyperbus/rpc-if.c new file mode 100644 index 000000000000..ecb050ba95cd --- /dev/null +++ b/drivers/mtd/hyperbus/rpc-if.c @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Linux driver for RPC-IF HyperFlash + * + * Copyright (C) 2019-2020 Cogent Embedded, Inc. + */ + +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mtd/hyperbus.h> +#include <linux/mtd/mtd.h> +#include <linux/mux/consumer.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/types.h> + +#include <memory/renesas-rpc-if.h> + +struct rpcif_hyperbus { + struct rpcif rpc; + struct hyperbus_ctlr ctlr; + struct hyperbus_device hbdev; +}; + +static const struct rpcif_op rpcif_op_tmpl = { + .cmd = { + .buswidth = 8, + .ddr = true, + }, + .ocmd = { + .buswidth = 8, + .ddr = true, + }, + .addr = { + .nbytes = 1, + .buswidth = 8, + .ddr = true, + }, + .data = { + .buswidth = 8, + .ddr = true, + }, +}; + +static void rpcif_hb_prepare_read(struct rpcif *rpc, void *to, + unsigned long from, ssize_t len) +{ + struct rpcif_op op = rpcif_op_tmpl; + + op.cmd.opcode = HYPERBUS_RW_READ | HYPERBUS_AS_MEM; + op.addr.val = from >> 1; + op.dummy.buswidth = 1; + op.dummy.ncycles = 15; + op.data.dir = RPCIF_DATA_IN; + op.data.nbytes = len; + op.data.buf.in = to; + + rpcif_prepare(rpc, &op, NULL, NULL); +} + +static void rpcif_hb_prepare_write(struct rpcif *rpc, unsigned long to, + void *from, ssize_t len) +{ + struct rpcif_op op = rpcif_op_tmpl; + + op.cmd.opcode = HYPERBUS_RW_WRITE | HYPERBUS_AS_MEM; + op.addr.val = to >> 1; + op.data.dir = RPCIF_DATA_OUT; + op.data.nbytes = len; + op.data.buf.out = from; + + rpcif_prepare(rpc, &op, NULL, NULL); +} + +static u16 rpcif_hb_read16(struct hyperbus_device *hbdev, unsigned long addr) +{ + struct rpcif_hyperbus *hyperbus = + container_of(hbdev, struct rpcif_hyperbus, hbdev); + map_word data; + + rpcif_hb_prepare_read(&hyperbus->rpc, &data, addr, 2); + + rpcif_manual_xfer(&hyperbus->rpc); + + return data.x[0]; +} + +static void rpcif_hb_write16(struct hyperbus_device *hbdev, unsigned long addr, + u16 data) +{ + struct rpcif_hyperbus *hyperbus = + container_of(hbdev, struct rpcif_hyperbus, hbdev); + + rpcif_hb_prepare_write(&hyperbus->rpc, addr, &data, 2); + + rpcif_manual_xfer(&hyperbus->rpc); +} + +static void rpcif_hb_copy_from(struct hyperbus_device *hbdev, void *to, + unsigned long from, ssize_t len) +{ + struct rpcif_hyperbus *hyperbus = + container_of(hbdev, struct rpcif_hyperbus, hbdev); + + rpcif_hb_prepare_read(&hyperbus->rpc, to, from, len); + + rpcif_dirmap_read(&hyperbus->rpc, from, len, to); +} + +static const struct hyperbus_ops rpcif_hb_ops = { + .read16 = rpcif_hb_read16, + .write16 = rpcif_hb_write16, + .copy_from = rpcif_hb_copy_from, +}; + +static int rpcif_hb_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rpcif_hyperbus *hyperbus; + int error; + + hyperbus = devm_kzalloc(dev, sizeof(*hyperbus), GFP_KERNEL); + if (!hyperbus) + return -ENOMEM; + + rpcif_sw_init(&hyperbus->rpc, pdev->dev.parent); + + platform_set_drvdata(pdev, hyperbus); + + rpcif_enable_rpm(&hyperbus->rpc); + + rpcif_hw_init(&hyperbus->rpc, true); + + hyperbus->hbdev.map.size = hyperbus->rpc.size; + hyperbus->hbdev.map.virt = hyperbus->rpc.dirmap; + + hyperbus->ctlr.dev = dev; + hyperbus->ctlr.ops = &rpcif_hb_ops; + hyperbus->hbdev.ctlr = &hyperbus->ctlr; + hyperbus->hbdev.np = of_get_next_child(pdev->dev.parent->of_node, NULL); + error = hyperbus_register_device(&hyperbus->hbdev); + if (error) + rpcif_disable_rpm(&hyperbus->rpc); + + return error; +} + +static int rpcif_hb_remove(struct platform_device *pdev) +{ + struct rpcif_hyperbus *hyperbus = platform_get_drvdata(pdev); + int error = hyperbus_unregister_device(&hyperbus->hbdev); + struct rpcif *rpc = dev_get_drvdata(pdev->dev.parent); + + rpcif_disable_rpm(rpc); + return error; +} + +static struct platform_driver rpcif_platform_driver = { + .probe = rpcif_hb_probe, + .remove = rpcif_hb_remove, + .driver = { + .name = "rpc-if-hyperflash", + }, +}; + +module_platform_driver(rpcif_platform_driver); + +MODULE_DESCRIPTION("Renesas RPC-IF HyperFlash driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/mtd/lpddr/lpddr2_nvm.c b/drivers/mtd/lpddr/lpddr2_nvm.c index 0f1547f09d08..72f5c7b30079 100644 --- a/drivers/mtd/lpddr/lpddr2_nvm.c +++ b/drivers/mtd/lpddr/lpddr2_nvm.c @@ -393,6 +393,17 @@ static int lpddr2_nvm_lock(struct mtd_info *mtd, loff_t start_add, return lpddr2_nvm_do_block_op(mtd, start_add, len, LPDDR2_NVM_LOCK); } +static const struct mtd_info lpddr2_nvm_mtd_info = { + .type = MTD_RAM, + .writesize = 1, + .flags = (MTD_CAP_NVRAM | MTD_POWERUP_LOCK), + ._read = lpddr2_nvm_read, + ._write = lpddr2_nvm_write, + ._erase = lpddr2_nvm_erase, + ._unlock = lpddr2_nvm_unlock, + ._lock = lpddr2_nvm_lock, +}; + /* * lpddr2_nvm driver probe method */ @@ -433,6 +444,7 @@ static int lpddr2_nvm_probe(struct platform_device *pdev) .pfow_base = OW_BASE_ADDRESS, .fldrv_priv = pcm_data, }; + if (IS_ERR(map->virt)) return PTR_ERR(map->virt); @@ -444,22 +456,13 @@ static int lpddr2_nvm_probe(struct platform_device *pdev) return PTR_ERR(pcm_data->ctl_regs); /* Populate mtd_info data structure */ - *mtd = (struct mtd_info) { - .dev = { .parent = &pdev->dev }, - .name = pdev->dev.init_name, - .type = MTD_RAM, - .priv = map, - .size = resource_size(add_range), - .erasesize = ERASE_BLOCKSIZE * pcm_data->bus_width, - .writesize = 1, - .writebufsize = WRITE_BUFFSIZE * pcm_data->bus_width, - .flags = (MTD_CAP_NVRAM | MTD_POWERUP_LOCK), - ._read = lpddr2_nvm_read, - ._write = lpddr2_nvm_write, - ._erase = lpddr2_nvm_erase, - ._unlock = lpddr2_nvm_unlock, - ._lock = lpddr2_nvm_lock, - }; + *mtd = lpddr2_nvm_mtd_info; + mtd->dev.parent = &pdev->dev; + mtd->name = pdev->dev.init_name; + mtd->priv = map; + mtd->size = resource_size(add_range); + mtd->erasesize = ERASE_BLOCKSIZE * pcm_data->bus_width; + mtd->writebufsize = WRITE_BUFFSIZE * pcm_data->bus_width; /* Verify the presence of the device looking for PFOW string */ if (!lpddr2_nvm_pfow_present(map)) { diff --git a/drivers/mtd/lpddr/lpddr_cmds.c b/drivers/mtd/lpddr/lpddr_cmds.c index fb1cbc9a2870..ee063baed136 100644 --- a/drivers/mtd/lpddr/lpddr_cmds.c +++ b/drivers/mtd/lpddr/lpddr_cmds.c @@ -94,6 +94,34 @@ struct mtd_info *lpddr_cmdset(struct map_info *map) } EXPORT_SYMBOL(lpddr_cmdset); +static void print_drs_error(unsigned int dsr) +{ + int prog_status = (dsr & DSR_RPS) >> 8; + + if (!(dsr & DSR_AVAILABLE)) + pr_notice("DSR.15: (0) Device not Available\n"); + if ((prog_status & 0x03) == 0x03) + pr_notice("DSR.9,8: (11) Attempt to program invalid half with 41h command\n"); + else if (prog_status & 0x02) + pr_notice("DSR.9,8: (10) Object Mode Program attempt in region with Control Mode data\n"); + else if (prog_status & 0x01) + pr_notice("DSR.9,8: (01) Program attempt in region with Object Mode data\n"); + if (!(dsr & DSR_READY_STATUS)) + pr_notice("DSR.7: (0) Device is Busy\n"); + if (dsr & DSR_ESS) + pr_notice("DSR.6: (1) Erase Suspended\n"); + if (dsr & DSR_ERASE_STATUS) + pr_notice("DSR.5: (1) Erase/Blank check error\n"); + if (dsr & DSR_PROGRAM_STATUS) + pr_notice("DSR.4: (1) Program Error\n"); + if (dsr & DSR_VPPS) + pr_notice("DSR.3: (1) Vpp low detect, operation aborted\n"); + if (dsr & DSR_PSS) + pr_notice("DSR.2: (1) Program suspended\n"); + if (dsr & DSR_DPS) + pr_notice("DSR.1: (1) Aborted Erase/Program attempt on locked block\n"); +} + static int wait_for_ready(struct map_info *map, struct flchip *chip, unsigned int chip_op_time) { diff --git a/drivers/mtd/maps/Kconfig b/drivers/mtd/maps/Kconfig index fd37553f1b07..6650acbc961e 100644 --- a/drivers/mtd/maps/Kconfig +++ b/drivers/mtd/maps/Kconfig @@ -75,6 +75,17 @@ config MTD_PHYSMAP_OF physically into the CPU's memory. The mapping description here is taken from OF device tree. +config MTD_PHYSMAP_BT1_ROM + bool "Baikal-T1 Boot ROMs OF-based physical memory map handling" + depends on MTD_PHYSMAP_OF + depends on MIPS_BAIKAL_T1 || COMPILE_TEST + select MTD_COMPLEX_MAPPINGS + select MULTIPLEXER + select MUX_MMIO + help + This provides some extra DT physmap parsing for the Baikal-T1 + platforms, some detection and setting up ROMs-specific accessors. + config MTD_PHYSMAP_VERSATILE bool "ARM Versatile OF-based physical memory map handling" depends on MTD_PHYSMAP_OF diff --git a/drivers/mtd/maps/Makefile b/drivers/mtd/maps/Makefile index c0da86a5d26f..79f018cf412f 100644 --- a/drivers/mtd/maps/Makefile +++ b/drivers/mtd/maps/Makefile @@ -18,6 +18,7 @@ obj-$(CONFIG_MTD_CK804XROM) += ck804xrom.o obj-$(CONFIG_MTD_TSUNAMI) += tsunami_flash.o obj-$(CONFIG_MTD_PXA2XX) += pxa2xx-flash.o physmap-objs-y += physmap-core.o +physmap-objs-$(CONFIG_MTD_PHYSMAP_BT1_ROM) += physmap-bt1-rom.o physmap-objs-$(CONFIG_MTD_PHYSMAP_VERSATILE) += physmap-versatile.o physmap-objs-$(CONFIG_MTD_PHYSMAP_GEMINI) += physmap-gemini.o physmap-objs-$(CONFIG_MTD_PHYSMAP_IXP4XX) += physmap-ixp4xx.o diff --git a/drivers/mtd/maps/physmap-bt1-rom.c b/drivers/mtd/maps/physmap-bt1-rom.c new file mode 100644 index 000000000000..27cfe1c63a2e --- /dev/null +++ b/drivers/mtd/maps/physmap-bt1-rom.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC + * + * Authors: + * Serge Semin <Sergey.Semin@baikalelectronics.ru> + * + * Baikal-T1 Physically Mapped Internal ROM driver + */ +#include <linux/bits.h> +#include <linux/device.h> +#include <linux/kernel.h> +#include <linux/mtd/map.h> +#include <linux/mtd/xip.h> +#include <linux/mux/consumer.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/string.h> +#include <linux/types.h> + +#include "physmap-bt1-rom.h" + +/* + * Baikal-T1 SoC ROMs are only accessible by the dword-aligned instructions. + * We have to take this into account when implementing the data read-methods. + * Note there is no need in bothering with endianness, since both Baikal-T1 + * CPU and MMIO are LE. + */ +static map_word __xipram bt1_rom_map_read(struct map_info *map, + unsigned long ofs) +{ + void __iomem *src = map->virt + ofs; + unsigned long shift; + map_word ret; + u32 data; + + /* Read data within offset dword. */ + shift = (unsigned long)src & 0x3; + data = readl_relaxed(src - shift); + if (!shift) { + ret.x[0] = data; + return ret; + } + ret.x[0] = data >> (shift * BITS_PER_BYTE); + + /* Read data from the next dword. */ + shift = 4 - shift; + if (ofs + shift >= map->size) + return ret; + + data = readl_relaxed(src + shift); + ret.x[0] |= data << (shift * BITS_PER_BYTE); + + return ret; +} + +static void __xipram bt1_rom_map_copy_from(struct map_info *map, + void *to, unsigned long from, + ssize_t len) +{ + void __iomem *src = map->virt + from; + ssize_t shift, chunk; + u32 data; + + if (len <= 0 || from >= map->size) + return; + + /* Make sure we don't go over the map limit. */ + len = min_t(ssize_t, map->size - from, len); + + /* + * Since requested data size can be pretty big we have to implement + * the copy procedure as optimal as possible. That's why it's split + * up into the next three stages: unaligned head, aligned body, + * unaligned tail. + */ + shift = (ssize_t)src & 0x3; + if (shift) { + chunk = min_t(ssize_t, 4 - shift, len); + data = readl_relaxed(src - shift); + memcpy(to, &data + shift, chunk); + src += chunk; + to += chunk; + len -= chunk; + } + + while (len >= 4) { + data = readl_relaxed(src); + memcpy(to, &data, 4); + src += 4; + to += 4; + len -= 4; + } + + if (len) { + data = readl_relaxed(src); + memcpy(to, &data, len); + } +} + +int of_flash_probe_bt1_rom(struct platform_device *pdev, + struct device_node *np, + struct map_info *map) +{ + struct device *dev = &pdev->dev; + + /* It's supposed to be read-only MTD. */ + if (!of_device_is_compatible(np, "mtd-rom")) { + dev_info(dev, "No mtd-rom compatible string\n"); + return 0; + } + + /* Multiplatform guard. */ + if (!of_device_is_compatible(np, "baikal,bt1-int-rom")) + return 0; + + /* Sanity check the device parameters retrieved from DTB. */ + if (map->bankwidth != 4) + dev_warn(dev, "Bank width is supposed to be 32 bits wide\n"); + + map->read = bt1_rom_map_read; + map->copy_from = bt1_rom_map_copy_from; + + return 0; +} diff --git a/drivers/mtd/maps/physmap-bt1-rom.h b/drivers/mtd/maps/physmap-bt1-rom.h new file mode 100644 index 000000000000..6782899598a4 --- /dev/null +++ b/drivers/mtd/maps/physmap-bt1-rom.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include <linux/mtd/map.h> +#include <linux/of.h> + +#ifdef CONFIG_MTD_PHYSMAP_BT1_ROM +int of_flash_probe_bt1_rom(struct platform_device *pdev, + struct device_node *np, + struct map_info *map); +#else +static inline +int of_flash_probe_bt1_rom(struct platform_device *pdev, + struct device_node *np, + struct map_info *map) +{ + return 0; +} +#endif diff --git a/drivers/mtd/maps/physmap-core.c b/drivers/mtd/maps/physmap-core.c index 8f7f966fa9a7..001ed5deb622 100644 --- a/drivers/mtd/maps/physmap-core.c +++ b/drivers/mtd/maps/physmap-core.c @@ -41,6 +41,7 @@ #include <linux/pm_runtime.h> #include <linux/gpio/consumer.h> +#include "physmap-bt1-rom.h" #include "physmap-gemini.h" #include "physmap-ixp4xx.h" #include "physmap-versatile.h" @@ -371,6 +372,10 @@ static int physmap_flash_of_init(struct platform_device *dev) info->maps[i].bankwidth = bankwidth; info->maps[i].device_node = dp; + err = of_flash_probe_bt1_rom(dev, dp, &info->maps[i]); + if (err) + return err; + err = of_flash_probe_gemini(dev, dp, &info->maps[i]); if (err) return err; @@ -515,7 +520,8 @@ static int physmap_flash_probe(struct platform_device *dev) dev_notice(&dev->dev, "physmap platform flash device: %pR\n", res); - info->maps[i].name = dev_name(&dev->dev); + if (!info->maps[i].name) + info->maps[i].name = dev_name(&dev->dev); if (!info->maps[i].phys) info->maps[i].phys = res->start; diff --git a/drivers/mtd/maps/vmu-flash.c b/drivers/mtd/maps/vmu-flash.c index 177bf134e189..a7ec947a3ebb 100644 --- a/drivers/mtd/maps/vmu-flash.c +++ b/drivers/mtd/maps/vmu-flash.c @@ -40,7 +40,7 @@ struct memcard { u32 blocklen; u32 writecnt; u32 readcnt; - u32 removeable; + u32 removable; int partition; int read; unsigned char *blockread; @@ -619,7 +619,7 @@ static int vmu_connect(struct maple_device *mdev) card->blocklen = ((basic_flash_data >> 16 & 0xFF) + 1) << 5; card->writecnt = basic_flash_data >> 12 & 0xF; card->readcnt = basic_flash_data >> 8 & 0xF; - card->removeable = basic_flash_data >> 7 & 1; + card->removable = basic_flash_data >> 7 & 1; card->partition = 0; @@ -772,7 +772,6 @@ static void vmu_file_error(struct maple_device *mdev, void *recvbuf) static int probe_maple_vmu(struct device *dev) { - int error; struct maple_device *mdev = to_maple_dev(dev); struct maple_driver *mdrv = to_maple_driver(dev->driver); @@ -780,11 +779,7 @@ static int probe_maple_vmu(struct device *dev) mdev->fileerr_handler = vmu_file_error; mdev->driver = mdrv; - error = vmu_connect(mdev); - if (error) - return error; - - return 0; + return vmu_connect(mdev); } static int remove_maple_vmu(struct device *dev) diff --git a/drivers/mtd/mtdconcat.c b/drivers/mtd/mtdconcat.c index 1d6c9e7e7b7d..6e4d0017c0bd 100644 --- a/drivers/mtd/mtdconcat.c +++ b/drivers/mtd/mtdconcat.c @@ -103,6 +103,47 @@ concat_read(struct mtd_info *mtd, loff_t from, size_t len, } static int +concat_panic_write(struct mtd_info *mtd, loff_t to, size_t len, + size_t * retlen, const u_char * buf) +{ + struct mtd_concat *concat = CONCAT(mtd); + int err = -EINVAL; + int i; + for (i = 0; i < concat->num_subdev; i++) { + struct mtd_info *subdev = concat->subdev[i]; + size_t size, retsize; + + if (to >= subdev->size) { + to -= subdev->size; + continue; + } + if (to + len > subdev->size) + size = subdev->size - to; + else + size = len; + + err = mtd_panic_write(subdev, to, size, &retsize, buf); + if (err == -EOPNOTSUPP) { + printk(KERN_ERR "mtdconcat: Cannot write from panic without panic_write\n"); + return err; + } + if (err) + break; + + *retlen += retsize; + len -= size; + if (len == 0) + break; + + err = -EINVAL; + buf += size; + to = 0; + } + return err; +} + + +static int concat_write(struct mtd_info *mtd, loff_t to, size_t len, size_t * retlen, const u_char * buf) { @@ -648,6 +689,8 @@ struct mtd_info *mtd_concat_create(struct mtd_info *subdev[], /* subdevices to c concat->mtd._block_isbad = concat_block_isbad; if (subdev[0]->_block_markbad) concat->mtd._block_markbad = concat_block_markbad; + if (subdev[0]->_panic_write) + concat->mtd._panic_write = concat_panic_write; concat->mtd.ecc_stats.badblocks = subdev[0]->ecc_stats.badblocks; diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c index b5e5d3140f57..e9e163ae9d86 100644 --- a/drivers/mtd/mtdcore.c +++ b/drivers/mtd/mtdcore.c @@ -335,7 +335,7 @@ static const struct device_type mtd_devtype = { .release = mtd_release, }; -static int mtd_partid_show(struct seq_file *s, void *p) +static int mtd_partid_debug_show(struct seq_file *s, void *p) { struct mtd_info *mtd = s->private; @@ -344,19 +344,9 @@ static int mtd_partid_show(struct seq_file *s, void *p) return 0; } -static int mtd_partid_debugfs_open(struct inode *inode, struct file *file) -{ - return single_open(file, mtd_partid_show, inode->i_private); -} - -static const struct file_operations mtd_partid_debug_fops = { - .open = mtd_partid_debugfs_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(mtd_partid_debug); -static int mtd_partname_show(struct seq_file *s, void *p) +static int mtd_partname_debug_show(struct seq_file *s, void *p) { struct mtd_info *mtd = s->private; @@ -365,17 +355,7 @@ static int mtd_partname_show(struct seq_file *s, void *p) return 0; } -static int mtd_partname_debugfs_open(struct inode *inode, struct file *file) -{ - return single_open(file, mtd_partname_show, inode->i_private); -} - -static const struct file_operations mtd_partname_debug_fops = { - .open = mtd_partname_debugfs_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(mtd_partname_debug); static struct dentry *dfs_dir_mtd; diff --git a/drivers/mtd/mtdoops.c b/drivers/mtd/mtdoops.c index 4ced68be7ed7..774970bfcf85 100644 --- a/drivers/mtd/mtdoops.c +++ b/drivers/mtd/mtdoops.c @@ -279,12 +279,13 @@ static void mtdoops_do_dump(struct kmsg_dumper *dumper, kmsg_dump_get_buffer(dumper, true, cxt->oops_buf + MTDOOPS_HEADER_SIZE, record_size - MTDOOPS_HEADER_SIZE, NULL); - /* Panics must be written immediately */ - if (reason != KMSG_DUMP_OOPS) + if (reason != KMSG_DUMP_OOPS) { + /* Panics must be written immediately */ mtdoops_write(cxt, 1); - - /* For other cases, schedule work to write it "nicely" */ - schedule_work(&cxt->work_write); + } else { + /* For other cases, schedule work to write it "nicely" */ + schedule_work(&cxt->work_write); + } } static void mtdoops_notify_add(struct mtd_info *mtd) diff --git a/drivers/mtd/nand/Kconfig b/drivers/mtd/nand/Kconfig index c1a45b071165..4a9aed4f0104 100644 --- a/drivers/mtd/nand/Kconfig +++ b/drivers/mtd/nand/Kconfig @@ -9,4 +9,12 @@ source "drivers/mtd/nand/onenand/Kconfig" source "drivers/mtd/nand/raw/Kconfig" source "drivers/mtd/nand/spi/Kconfig" +menu "ECC engine support" + +config MTD_NAND_ECC + bool + depends on MTD_NAND_CORE + +endmenu + endmenu diff --git a/drivers/mtd/nand/Makefile b/drivers/mtd/nand/Makefile index 7ecd80c0a66e..981372953b56 100644 --- a/drivers/mtd/nand/Makefile +++ b/drivers/mtd/nand/Makefile @@ -6,3 +6,5 @@ obj-$(CONFIG_MTD_NAND_CORE) += nandcore.o obj-y += onenand/ obj-y += raw/ obj-y += spi/ + +nandcore-$(CONFIG_MTD_NAND_ECC) += ecc.o diff --git a/drivers/mtd/nand/ecc.c b/drivers/mtd/nand/ecc.c new file mode 100644 index 000000000000..4a56e6c0da67 --- /dev/null +++ b/drivers/mtd/nand/ecc.c @@ -0,0 +1,484 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Generic Error-Correcting Code (ECC) engine + * + * Copyright (C) 2019 Macronix + * Author: + * Miquèl RAYNAL <miquel.raynal@bootlin.com> + * + * + * This file describes the abstraction of any NAND ECC engine. It has been + * designed to fit most cases, including parallel NANDs and SPI-NANDs. + * + * There are three main situations where instantiating this ECC engine makes + * sense: + * - external: The ECC engine is outside the NAND pipeline, typically this + * is a software ECC engine, or an hardware engine that is + * outside the NAND controller pipeline. + * - pipelined: The ECC engine is inside the NAND pipeline, ie. on the + * controller's side. This is the case of most of the raw NAND + * controllers. In the pipeline case, the ECC bytes are + * generated/data corrected on the fly when a page is + * written/read. + * - ondie: The ECC engine is inside the NAND pipeline, on the chip's side. + * Some NAND chips can correct themselves the data. + * + * Besides the initial setup and final cleanups, the interfaces are rather + * simple: + * - prepare: Prepare an I/O request. Enable/disable the ECC engine based on + * the I/O request type. In case of software correction or external + * engine, this step may involve to derive the ECC bytes and place + * them in the OOB area before a write. + * - finish: Finish an I/O request. Correct the data in case of a read + * request and report the number of corrected bits/uncorrectable + * errors. Most likely empty for write operations, unless you have + * hardware specific stuff to do, like shutting down the engine to + * save power. + * + * The I/O request should be enclosed in a prepare()/finish() pair of calls + * and will behave differently depending on the requested I/O type: + * - raw: Correction disabled + * - ecc: Correction enabled + * + * The request direction is impacting the logic as well: + * - read: Load data from the NAND chip + * - write: Store data in the NAND chip + * + * Mixing all this combinations together gives the following behavior. + * Those are just examples, drivers are free to add custom steps in their + * prepare/finish hook. + * + * [external ECC engine] + * - external + prepare + raw + read: do nothing + * - external + finish + raw + read: do nothing + * - external + prepare + raw + write: do nothing + * - external + finish + raw + write: do nothing + * - external + prepare + ecc + read: do nothing + * - external + finish + ecc + read: calculate expected ECC bytes, extract + * ECC bytes from OOB buffer, correct + * and report any bitflip/error + * - external + prepare + ecc + write: calculate ECC bytes and store them at + * the right place in the OOB buffer based + * on the OOB layout + * - external + finish + ecc + write: do nothing + * + * [pipelined ECC engine] + * - pipelined + prepare + raw + read: disable the controller's ECC engine if + * activated + * - pipelined + finish + raw + read: do nothing + * - pipelined + prepare + raw + write: disable the controller's ECC engine if + * activated + * - pipelined + finish + raw + write: do nothing + * - pipelined + prepare + ecc + read: enable the controller's ECC engine if + * deactivated + * - pipelined + finish + ecc + read: check the status, report any + * error/bitflip + * - pipelined + prepare + ecc + write: enable the controller's ECC engine if + * deactivated + * - pipelined + finish + ecc + write: do nothing + * + * [ondie ECC engine] + * - ondie + prepare + raw + read: send commands to disable the on-chip ECC + * engine if activated + * - ondie + finish + raw + read: do nothing + * - ondie + prepare + raw + write: send commands to disable the on-chip ECC + * engine if activated + * - ondie + finish + raw + write: do nothing + * - ondie + prepare + ecc + read: send commands to enable the on-chip ECC + * engine if deactivated + * - ondie + finish + ecc + read: send commands to check the status, report + * any error/bitflip + * - ondie + prepare + ecc + write: send commands to enable the on-chip ECC + * engine if deactivated + * - ondie + finish + ecc + write: do nothing + */ + +#include <linux/module.h> +#include <linux/mtd/nand.h> + +/** + * nand_ecc_init_ctx - Init the ECC engine context + * @nand: the NAND device + * + * On success, the caller is responsible of calling @nand_ecc_cleanup_ctx(). + */ +int nand_ecc_init_ctx(struct nand_device *nand) +{ + if (!nand->ecc.engine->ops->init_ctx) + return 0; + + return nand->ecc.engine->ops->init_ctx(nand); +} +EXPORT_SYMBOL(nand_ecc_init_ctx); + +/** + * nand_ecc_cleanup_ctx - Cleanup the ECC engine context + * @nand: the NAND device + */ +void nand_ecc_cleanup_ctx(struct nand_device *nand) +{ + if (nand->ecc.engine->ops->cleanup_ctx) + nand->ecc.engine->ops->cleanup_ctx(nand); +} +EXPORT_SYMBOL(nand_ecc_cleanup_ctx); + +/** + * nand_ecc_prepare_io_req - Prepare an I/O request + * @nand: the NAND device + * @req: the I/O request + */ +int nand_ecc_prepare_io_req(struct nand_device *nand, + struct nand_page_io_req *req) +{ + if (!nand->ecc.engine->ops->prepare_io_req) + return 0; + + return nand->ecc.engine->ops->prepare_io_req(nand, req); +} +EXPORT_SYMBOL(nand_ecc_prepare_io_req); + +/** + * nand_ecc_finish_io_req - Finish an I/O request + * @nand: the NAND device + * @req: the I/O request + */ +int nand_ecc_finish_io_req(struct nand_device *nand, + struct nand_page_io_req *req) +{ + if (!nand->ecc.engine->ops->finish_io_req) + return 0; + + return nand->ecc.engine->ops->finish_io_req(nand, req); +} +EXPORT_SYMBOL(nand_ecc_finish_io_req); + +/* Define default OOB placement schemes for large and small page devices */ +static int nand_ooblayout_ecc_sp(struct mtd_info *mtd, int section, + struct mtd_oob_region *oobregion) +{ + struct nand_device *nand = mtd_to_nanddev(mtd); + unsigned int total_ecc_bytes = nand->ecc.ctx.total; + + if (section > 1) + return -ERANGE; + + if (!section) { + oobregion->offset = 0; + if (mtd->oobsize == 16) + oobregion->length = 4; + else + oobregion->length = 3; + } else { + if (mtd->oobsize == 8) + return -ERANGE; + + oobregion->offset = 6; + oobregion->length = total_ecc_bytes - 4; + } + + return 0; +} + +static int nand_ooblayout_free_sp(struct mtd_info *mtd, int section, + struct mtd_oob_region *oobregion) +{ + if (section > 1) + return -ERANGE; + + if (mtd->oobsize == 16) { + if (section) + return -ERANGE; + + oobregion->length = 8; + oobregion->offset = 8; + } else { + oobregion->length = 2; + if (!section) + oobregion->offset = 3; + else + oobregion->offset = 6; + } + + return 0; +} + +static const struct mtd_ooblayout_ops nand_ooblayout_sp_ops = { + .ecc = nand_ooblayout_ecc_sp, + .free = nand_ooblayout_free_sp, +}; + +const struct mtd_ooblayout_ops *nand_get_small_page_ooblayout(void) +{ + return &nand_ooblayout_sp_ops; +} +EXPORT_SYMBOL_GPL(nand_get_small_page_ooblayout); + +static int nand_ooblayout_ecc_lp(struct mtd_info *mtd, int section, + struct mtd_oob_region *oobregion) +{ + struct nand_device *nand = mtd_to_nanddev(mtd); + unsigned int total_ecc_bytes = nand->ecc.ctx.total; + + if (section || !total_ecc_bytes) + return -ERANGE; + + oobregion->length = total_ecc_bytes; + oobregion->offset = mtd->oobsize - oobregion->length; + + return 0; +} + +static int nand_ooblayout_free_lp(struct mtd_info *mtd, int section, + struct mtd_oob_region *oobregion) +{ + struct nand_device *nand = mtd_to_nanddev(mtd); + unsigned int total_ecc_bytes = nand->ecc.ctx.total; + + if (section) + return -ERANGE; + + oobregion->length = mtd->oobsize - total_ecc_bytes - 2; + oobregion->offset = 2; + + return 0; +} + +static const struct mtd_ooblayout_ops nand_ooblayout_lp_ops = { + .ecc = nand_ooblayout_ecc_lp, + .free = nand_ooblayout_free_lp, +}; + +const struct mtd_ooblayout_ops *nand_get_large_page_ooblayout(void) +{ + return &nand_ooblayout_lp_ops; +} +EXPORT_SYMBOL_GPL(nand_get_large_page_ooblayout); + +/* + * Support the old "large page" layout used for 1-bit Hamming ECC where ECC + * are placed at a fixed offset. + */ +static int nand_ooblayout_ecc_lp_hamming(struct mtd_info *mtd, int section, + struct mtd_oob_region *oobregion) +{ + struct nand_device *nand = mtd_to_nanddev(mtd); + unsigned int total_ecc_bytes = nand->ecc.ctx.total; + + if (section) + return -ERANGE; + + switch (mtd->oobsize) { + case 64: + oobregion->offset = 40; + break; + case 128: + oobregion->offset = 80; + break; + default: + return -EINVAL; + } + + oobregion->length = total_ecc_bytes; + if (oobregion->offset + oobregion->length > mtd->oobsize) + return -ERANGE; + + return 0; +} + +static int nand_ooblayout_free_lp_hamming(struct mtd_info *mtd, int section, + struct mtd_oob_region *oobregion) +{ + struct nand_device *nand = mtd_to_nanddev(mtd); + unsigned int total_ecc_bytes = nand->ecc.ctx.total; + int ecc_offset = 0; + + if (section < 0 || section > 1) + return -ERANGE; + + switch (mtd->oobsize) { + case 64: + ecc_offset = 40; + break; + case 128: + ecc_offset = 80; + break; + default: + return -EINVAL; + } + + if (section == 0) { + oobregion->offset = 2; + oobregion->length = ecc_offset - 2; + } else { + oobregion->offset = ecc_offset + total_ecc_bytes; + oobregion->length = mtd->oobsize - oobregion->offset; + } + + return 0; +} + +static const struct mtd_ooblayout_ops nand_ooblayout_lp_hamming_ops = { + .ecc = nand_ooblayout_ecc_lp_hamming, + .free = nand_ooblayout_free_lp_hamming, +}; + +const struct mtd_ooblayout_ops *nand_get_large_page_hamming_ooblayout(void) +{ + return &nand_ooblayout_lp_hamming_ops; +} +EXPORT_SYMBOL_GPL(nand_get_large_page_hamming_ooblayout); + +static enum nand_ecc_engine_type +of_get_nand_ecc_engine_type(struct device_node *np) +{ + struct device_node *eng_np; + + if (of_property_read_bool(np, "nand-no-ecc-engine")) + return NAND_ECC_ENGINE_TYPE_NONE; + + if (of_property_read_bool(np, "nand-use-soft-ecc-engine")) + return NAND_ECC_ENGINE_TYPE_SOFT; + + eng_np = of_parse_phandle(np, "nand-ecc-engine", 0); + of_node_put(eng_np); + + if (eng_np) { + if (eng_np == np) + return NAND_ECC_ENGINE_TYPE_ON_DIE; + else + return NAND_ECC_ENGINE_TYPE_ON_HOST; + } + + return NAND_ECC_ENGINE_TYPE_INVALID; +} + +static const char * const nand_ecc_placement[] = { + [NAND_ECC_PLACEMENT_OOB] = "oob", + [NAND_ECC_PLACEMENT_INTERLEAVED] = "interleaved", +}; + +static enum nand_ecc_placement of_get_nand_ecc_placement(struct device_node *np) +{ + enum nand_ecc_placement placement; + const char *pm; + int err; + + err = of_property_read_string(np, "nand-ecc-placement", &pm); + if (!err) { + for (placement = NAND_ECC_PLACEMENT_OOB; + placement < ARRAY_SIZE(nand_ecc_placement); placement++) { + if (!strcasecmp(pm, nand_ecc_placement[placement])) + return placement; + } + } + + return NAND_ECC_PLACEMENT_UNKNOWN; +} + +static const char * const nand_ecc_algos[] = { + [NAND_ECC_ALGO_HAMMING] = "hamming", + [NAND_ECC_ALGO_BCH] = "bch", + [NAND_ECC_ALGO_RS] = "rs", +}; + +static enum nand_ecc_algo of_get_nand_ecc_algo(struct device_node *np) +{ + enum nand_ecc_algo ecc_algo; + const char *pm; + int err; + + err = of_property_read_string(np, "nand-ecc-algo", &pm); + if (!err) { + for (ecc_algo = NAND_ECC_ALGO_HAMMING; + ecc_algo < ARRAY_SIZE(nand_ecc_algos); + ecc_algo++) { + if (!strcasecmp(pm, nand_ecc_algos[ecc_algo])) + return ecc_algo; + } + } + + return NAND_ECC_ALGO_UNKNOWN; +} + +static int of_get_nand_ecc_step_size(struct device_node *np) +{ + int ret; + u32 val; + + ret = of_property_read_u32(np, "nand-ecc-step-size", &val); + return ret ? ret : val; +} + +static int of_get_nand_ecc_strength(struct device_node *np) +{ + int ret; + u32 val; + + ret = of_property_read_u32(np, "nand-ecc-strength", &val); + return ret ? ret : val; +} + +void of_get_nand_ecc_user_config(struct nand_device *nand) +{ + struct device_node *dn = nanddev_get_of_node(nand); + int strength, size; + + nand->ecc.user_conf.engine_type = of_get_nand_ecc_engine_type(dn); + nand->ecc.user_conf.algo = of_get_nand_ecc_algo(dn); + nand->ecc.user_conf.placement = of_get_nand_ecc_placement(dn); + + strength = of_get_nand_ecc_strength(dn); + if (strength >= 0) + nand->ecc.user_conf.strength = strength; + + size = of_get_nand_ecc_step_size(dn); + if (size >= 0) + nand->ecc.user_conf.step_size = size; + + if (of_property_read_bool(dn, "nand-ecc-maximize")) + nand->ecc.user_conf.flags |= NAND_ECC_MAXIMIZE_STRENGTH; +} +EXPORT_SYMBOL(of_get_nand_ecc_user_config); + +/** + * nand_ecc_is_strong_enough - Check if the chip configuration meets the + * datasheet requirements. + * + * @nand: Device to check + * + * If our configuration corrects A bits per B bytes and the minimum + * required correction level is X bits per Y bytes, then we must ensure + * both of the following are true: + * + * (1) A / B >= X / Y + * (2) A >= X + * + * Requirement (1) ensures we can correct for the required bitflip density. + * Requirement (2) ensures we can correct even when all bitflips are clumped + * in the same sector. + */ +bool nand_ecc_is_strong_enough(struct nand_device *nand) +{ + const struct nand_ecc_props *reqs = nanddev_get_ecc_requirements(nand); + const struct nand_ecc_props *conf = nanddev_get_ecc_conf(nand); + struct mtd_info *mtd = nanddev_to_mtd(nand); + int corr, ds_corr; + + if (conf->step_size == 0 || reqs->step_size == 0) + /* Not enough information */ + return true; + + /* + * We get the number of corrected bits per page to compare + * the correction density. + */ + corr = (mtd->writesize * conf->strength) / conf->step_size; + ds_corr = (mtd->writesize * reqs->strength) / reqs->step_size; + + return corr >= ds_corr && conf->strength >= reqs->strength; +} +EXPORT_SYMBOL(nand_ecc_is_strong_enough); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>"); +MODULE_DESCRIPTION("Generic ECC engine"); diff --git a/drivers/mtd/nand/onenand/onenand_base.c b/drivers/mtd/nand/onenand/onenand_base.c index ec18ade33262..188b8061e1f7 100644 --- a/drivers/mtd/nand/onenand/onenand_base.c +++ b/drivers/mtd/nand/onenand/onenand_base.c @@ -1052,16 +1052,11 @@ static int onenand_transfer_auto_oob(struct mtd_info *mtd, uint8_t *buf, int col int thislen) { struct onenand_chip *this = mtd->priv; - int ret; this->read_bufferram(mtd, ONENAND_SPARERAM, this->oob_buf, 0, mtd->oobsize); - ret = mtd_ooblayout_get_databytes(mtd, buf, this->oob_buf, - column, thislen); - if (ret) - return ret; - - return 0; + return mtd_ooblayout_get_databytes(mtd, buf, this->oob_buf, + column, thislen); } /** diff --git a/drivers/mtd/nand/onenand/onenand_omap2.c b/drivers/mtd/nand/onenand/onenand_omap2.c index aa9368bf7a0c..d8c0bd002c2b 100644 --- a/drivers/mtd/nand/onenand/onenand_omap2.c +++ b/drivers/mtd/nand/onenand/onenand_omap2.c @@ -494,11 +494,8 @@ static int omap2_onenand_probe(struct platform_device *pdev) c->int_gpiod = devm_gpiod_get_optional(dev, "int", GPIOD_IN); if (IS_ERR(c->int_gpiod)) { - r = PTR_ERR(c->int_gpiod); /* Just try again if this happens */ - if (r != -EPROBE_DEFER) - dev_err(dev, "error getting gpio: %d\n", r); - return r; + return dev_err_probe(dev, PTR_ERR(c->int_gpiod), "error getting gpio\n"); } if (c->int_gpiod) { diff --git a/drivers/mtd/nand/raw/Kconfig b/drivers/mtd/nand/raw/Kconfig index 1203775023ad..6c46f25b57e2 100644 --- a/drivers/mtd/nand/raw/Kconfig +++ b/drivers/mtd/nand/raw/Kconfig @@ -13,6 +13,7 @@ config MTD_NAND_ECC_SW_HAMMING_SMC menuconfig MTD_RAW_NAND tristate "Raw/Parallel NAND Device Support" select MTD_NAND_CORE + select MTD_NAND_ECC select MTD_NAND_ECC_SW_HAMMING help This enables support for accessing all type of raw/parallel diff --git a/drivers/mtd/nand/raw/ams-delta.c b/drivers/mtd/nand/raw/ams-delta.c index fdba155416d2..d3c5cc513c8f 100644 --- a/drivers/mtd/nand/raw/ams-delta.c +++ b/drivers/mtd/nand/raw/ams-delta.c @@ -260,8 +260,8 @@ static int gpio_nand_probe(struct platform_device *pdev) return err; } - this->ecc.mode = NAND_ECC_SOFT; - this->ecc.algo = NAND_ECC_HAMMING; + this->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + this->ecc.algo = NAND_ECC_ALGO_HAMMING; platform_set_drvdata(pdev, priv); @@ -400,12 +400,14 @@ static int gpio_nand_remove(struct platform_device *pdev) return 0; } +#ifdef CONFIG_OF static const struct of_device_id gpio_nand_of_id_table[] = { { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, gpio_nand_of_id_table); +#endif static const struct platform_device_id gpio_nand_plat_id_table[] = { { diff --git a/drivers/mtd/nand/raw/arasan-nand-controller.c b/drivers/mtd/nand/raw/arasan-nand-controller.c index 12c643e97c85..fbb4ea751be8 100644 --- a/drivers/mtd/nand/raw/arasan-nand-controller.c +++ b/drivers/mtd/nand/raw/arasan-nand-controller.c @@ -980,10 +980,10 @@ static int anfc_init_hw_ecc_controller(struct arasan_nfc *nfc, return -EINVAL; } - mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops); + mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout()); ecc->steps = mtd->writesize / ecc->size; - ecc->algo = NAND_ECC_BCH; + ecc->algo = NAND_ECC_ALGO_BCH; anand->ecc_bits = bch_gf_mag * ecc->strength; ecc->bytes = DIV_ROUND_UP(anand->ecc_bits, 8); anand->ecc_total = DIV_ROUND_UP(anand->ecc_bits * ecc->steps, 8); @@ -1056,17 +1056,17 @@ static int anfc_attach_chip(struct nand_chip *chip) chip->ecc.read_page_raw = nand_monolithic_read_page_raw; chip->ecc.write_page_raw = nand_monolithic_write_page_raw; - switch (chip->ecc.mode) { - case NAND_ECC_NONE: - case NAND_ECC_SOFT: - case NAND_ECC_ON_DIE: + switch (chip->ecc.engine_type) { + case NAND_ECC_ENGINE_TYPE_NONE: + case NAND_ECC_ENGINE_TYPE_SOFT: + case NAND_ECC_ENGINE_TYPE_ON_DIE: break; - case NAND_ECC_HW: + case NAND_ECC_ENGINE_TYPE_ON_HOST: ret = anfc_init_hw_ecc_controller(nfc, chip); break; default: dev_err(nfc->dev, "Unsupported ECC mode: %d\n", - chip->ecc.mode); + chip->ecc.engine_type); return -EINVAL; } diff --git a/drivers/mtd/nand/raw/atmel/nand-controller.c b/drivers/mtd/nand/raw/atmel/nand-controller.c index c9818f548d07..e6ceec8f50dc 100644 --- a/drivers/mtd/nand/raw/atmel/nand-controller.c +++ b/drivers/mtd/nand/raw/atmel/nand-controller.c @@ -202,6 +202,8 @@ struct atmel_nand_controller_ops { int (*ecc_init)(struct nand_chip *chip); int (*setup_interface)(struct atmel_nand *nand, int csline, const struct nand_interface_config *conf); + int (*exec_op)(struct atmel_nand *nand, + const struct nand_operation *op, bool check_only); }; struct atmel_nand_controller_caps { @@ -259,6 +261,7 @@ struct atmel_hsmc_nand_controller { struct regmap *io; struct atmel_nfc_op op; struct completion complete; + u32 cfg; int irq; /* Only used when instantiating from legacy DT bindings. */ @@ -414,29 +417,62 @@ err: return -EIO; } -static u8 atmel_nand_read_byte(struct nand_chip *chip) +static int atmel_nfc_exec_op(struct atmel_hsmc_nand_controller *nc, bool poll) { - struct atmel_nand *nand = to_atmel_nand(chip); + u8 *addrs = nc->op.addrs; + unsigned int op = 0; + u32 addr, val; + int i, ret; - return ioread8(nand->activecs->io.virt); -} + nc->op.wait = ATMEL_HSMC_NFC_SR_CMDDONE; -static void atmel_nand_write_byte(struct nand_chip *chip, u8 byte) -{ - struct atmel_nand *nand = to_atmel_nand(chip); + for (i = 0; i < nc->op.ncmds; i++) + op |= ATMEL_NFC_CMD(i, nc->op.cmds[i]); - if (chip->options & NAND_BUSWIDTH_16) - iowrite16(byte | (byte << 8), nand->activecs->io.virt); - else - iowrite8(byte, nand->activecs->io.virt); + if (nc->op.naddrs == ATMEL_NFC_MAX_ADDR_CYCLES) + regmap_write(nc->base.smc, ATMEL_HSMC_NFC_ADDR, *addrs++); + + op |= ATMEL_NFC_CSID(nc->op.cs) | + ATMEL_NFC_ACYCLE(nc->op.naddrs); + + if (nc->op.ncmds > 1) + op |= ATMEL_NFC_VCMD2; + + addr = addrs[0] | (addrs[1] << 8) | (addrs[2] << 16) | + (addrs[3] << 24); + + if (nc->op.data != ATMEL_NFC_NO_DATA) { + op |= ATMEL_NFC_DATAEN; + nc->op.wait |= ATMEL_HSMC_NFC_SR_XFRDONE; + + if (nc->op.data == ATMEL_NFC_WRITE_DATA) + op |= ATMEL_NFC_NFCWR; + } + + /* Clear all flags. */ + regmap_read(nc->base.smc, ATMEL_HSMC_NFC_SR, &val); + + /* Send the command. */ + regmap_write(nc->io, op, addr); + + ret = atmel_nfc_wait(nc, poll, 0); + if (ret) + dev_err(nc->base.dev, + "Failed to send NAND command (err = %d)!", + ret); + + /* Reset the op state. */ + memset(&nc->op, 0, sizeof(nc->op)); + + return ret; } -static void atmel_nand_read_buf(struct nand_chip *chip, u8 *buf, int len) +static void atmel_nand_data_in(struct atmel_nand *nand, void *buf, + unsigned int len, bool force_8bit) { - struct atmel_nand *nand = to_atmel_nand(chip); struct atmel_nand_controller *nc; - nc = to_nand_controller(chip->controller); + nc = to_nand_controller(nand->base.controller); /* * If the controller supports DMA, the buffer address is DMA-able and @@ -444,23 +480,23 @@ static void atmel_nand_read_buf(struct nand_chip *chip, u8 *buf, int len) * a DMA transfer. If it fails, fallback to PIO mode. */ if (nc->dmac && virt_addr_valid(buf) && - len >= MIN_DMA_LEN && + len >= MIN_DMA_LEN && !force_8bit && !atmel_nand_dma_transfer(nc, buf, nand->activecs->io.dma, len, DMA_FROM_DEVICE)) return; - if (chip->options & NAND_BUSWIDTH_16) + if ((nand->base.options & NAND_BUSWIDTH_16) && !force_8bit) ioread16_rep(nand->activecs->io.virt, buf, len / 2); else ioread8_rep(nand->activecs->io.virt, buf, len); } -static void atmel_nand_write_buf(struct nand_chip *chip, const u8 *buf, int len) +static void atmel_nand_data_out(struct atmel_nand *nand, const void *buf, + unsigned int len, bool force_8bit) { - struct atmel_nand *nand = to_atmel_nand(chip); struct atmel_nand_controller *nc; - nc = to_nand_controller(chip->controller); + nc = to_nand_controller(nand->base.controller); /* * If the controller supports DMA, the buffer address is DMA-able and @@ -468,179 +504,213 @@ static void atmel_nand_write_buf(struct nand_chip *chip, const u8 *buf, int len) * a DMA transfer. If it fails, fallback to PIO mode. */ if (nc->dmac && virt_addr_valid(buf) && - len >= MIN_DMA_LEN && + len >= MIN_DMA_LEN && !force_8bit && !atmel_nand_dma_transfer(nc, (void *)buf, nand->activecs->io.dma, len, DMA_TO_DEVICE)) return; - if (chip->options & NAND_BUSWIDTH_16) + if ((nand->base.options & NAND_BUSWIDTH_16) && !force_8bit) iowrite16_rep(nand->activecs->io.virt, buf, len / 2); else iowrite8_rep(nand->activecs->io.virt, buf, len); } -static int atmel_nand_dev_ready(struct nand_chip *chip) +static int atmel_nand_waitrdy(struct atmel_nand *nand, unsigned int timeout_ms) { - struct atmel_nand *nand = to_atmel_nand(chip); + if (nand->activecs->rb.type == ATMEL_NAND_NO_RB) + return nand_soft_waitrdy(&nand->base, timeout_ms); - return gpiod_get_value(nand->activecs->rb.gpio); + return nand_gpio_waitrdy(&nand->base, nand->activecs->rb.gpio, + timeout_ms); } -static void atmel_nand_select_chip(struct nand_chip *chip, int cs) +static int atmel_hsmc_nand_waitrdy(struct atmel_nand *nand, + unsigned int timeout_ms) { - struct atmel_nand *nand = to_atmel_nand(chip); - - if (cs < 0 || cs >= nand->numcs) { - nand->activecs = NULL; - chip->legacy.dev_ready = NULL; - return; - } + struct atmel_hsmc_nand_controller *nc; + u32 status, mask; - nand->activecs = &nand->cs[cs]; + if (nand->activecs->rb.type != ATMEL_NAND_NATIVE_RB) + return atmel_nand_waitrdy(nand, timeout_ms); - if (nand->activecs->rb.type == ATMEL_NAND_GPIO_RB) - chip->legacy.dev_ready = atmel_nand_dev_ready; + nc = to_hsmc_nand_controller(nand->base.controller); + mask = ATMEL_HSMC_NFC_SR_RBEDGE(nand->activecs->rb.id); + return regmap_read_poll_timeout_atomic(nc->base.smc, ATMEL_HSMC_NFC_SR, + status, status & mask, + 10, timeout_ms * 1000); } -static int atmel_hsmc_nand_dev_ready(struct nand_chip *chip) +static void atmel_nand_select_target(struct atmel_nand *nand, + unsigned int cs) { - struct atmel_nand *nand = to_atmel_nand(chip); - struct atmel_hsmc_nand_controller *nc; - u32 status; - - nc = to_hsmc_nand_controller(chip->controller); - - regmap_read(nc->base.smc, ATMEL_HSMC_NFC_SR, &status); - - return status & ATMEL_HSMC_NFC_SR_RBEDGE(nand->activecs->rb.id); + nand->activecs = &nand->cs[cs]; } -static void atmel_hsmc_nand_select_chip(struct nand_chip *chip, int cs) +static void atmel_hsmc_nand_select_target(struct atmel_nand *nand, + unsigned int cs) { - struct mtd_info *mtd = nand_to_mtd(chip); - struct atmel_nand *nand = to_atmel_nand(chip); + struct mtd_info *mtd = nand_to_mtd(&nand->base); struct atmel_hsmc_nand_controller *nc; + u32 cfg = ATMEL_HSMC_NFC_CFG_PAGESIZE(mtd->writesize) | + ATMEL_HSMC_NFC_CFG_SPARESIZE(mtd->oobsize) | + ATMEL_HSMC_NFC_CFG_RSPARE; - nc = to_hsmc_nand_controller(chip->controller); - - atmel_nand_select_chip(chip, cs); - - if (!nand->activecs) { - regmap_write(nc->base.smc, ATMEL_HSMC_NFC_CTRL, - ATMEL_HSMC_NFC_CTRL_DIS); + nand->activecs = &nand->cs[cs]; + nc = to_hsmc_nand_controller(nand->base.controller); + if (nc->cfg == cfg) return; - } - - if (nand->activecs->rb.type == ATMEL_NAND_NATIVE_RB) - chip->legacy.dev_ready = atmel_hsmc_nand_dev_ready; regmap_update_bits(nc->base.smc, ATMEL_HSMC_NFC_CFG, ATMEL_HSMC_NFC_CFG_PAGESIZE_MASK | ATMEL_HSMC_NFC_CFG_SPARESIZE_MASK | ATMEL_HSMC_NFC_CFG_RSPARE | ATMEL_HSMC_NFC_CFG_WSPARE, - ATMEL_HSMC_NFC_CFG_PAGESIZE(mtd->writesize) | - ATMEL_HSMC_NFC_CFG_SPARESIZE(mtd->oobsize) | - ATMEL_HSMC_NFC_CFG_RSPARE); - regmap_write(nc->base.smc, ATMEL_HSMC_NFC_CTRL, - ATMEL_HSMC_NFC_CTRL_EN); + cfg); + nc->cfg = cfg; } -static int atmel_nfc_exec_op(struct atmel_hsmc_nand_controller *nc, bool poll) +static int atmel_smc_nand_exec_instr(struct atmel_nand *nand, + const struct nand_op_instr *instr) { - u8 *addrs = nc->op.addrs; - unsigned int op = 0; - u32 addr, val; - int i, ret; - - nc->op.wait = ATMEL_HSMC_NFC_SR_CMDDONE; - - for (i = 0; i < nc->op.ncmds; i++) - op |= ATMEL_NFC_CMD(i, nc->op.cmds[i]); - - if (nc->op.naddrs == ATMEL_NFC_MAX_ADDR_CYCLES) - regmap_write(nc->base.smc, ATMEL_HSMC_NFC_ADDR, *addrs++); - - op |= ATMEL_NFC_CSID(nc->op.cs) | - ATMEL_NFC_ACYCLE(nc->op.naddrs); - - if (nc->op.ncmds > 1) - op |= ATMEL_NFC_VCMD2; - - addr = addrs[0] | (addrs[1] << 8) | (addrs[2] << 16) | - (addrs[3] << 24); - - if (nc->op.data != ATMEL_NFC_NO_DATA) { - op |= ATMEL_NFC_DATAEN; - nc->op.wait |= ATMEL_HSMC_NFC_SR_XFRDONE; + struct atmel_nand_controller *nc; + unsigned int i; - if (nc->op.data == ATMEL_NFC_WRITE_DATA) - op |= ATMEL_NFC_NFCWR; + nc = to_nand_controller(nand->base.controller); + switch (instr->type) { + case NAND_OP_CMD_INSTR: + writeb(instr->ctx.cmd.opcode, + nand->activecs->io.virt + nc->caps->cle_offs); + return 0; + case NAND_OP_ADDR_INSTR: + for (i = 0; i < instr->ctx.addr.naddrs; i++) + writeb(instr->ctx.addr.addrs[i], + nand->activecs->io.virt + nc->caps->ale_offs); + return 0; + case NAND_OP_DATA_IN_INSTR: + atmel_nand_data_in(nand, instr->ctx.data.buf.in, + instr->ctx.data.len, + instr->ctx.data.force_8bit); + return 0; + case NAND_OP_DATA_OUT_INSTR: + atmel_nand_data_out(nand, instr->ctx.data.buf.out, + instr->ctx.data.len, + instr->ctx.data.force_8bit); + return 0; + case NAND_OP_WAITRDY_INSTR: + return atmel_nand_waitrdy(nand, + instr->ctx.waitrdy.timeout_ms); + default: + break; } - /* Clear all flags. */ - regmap_read(nc->base.smc, ATMEL_HSMC_NFC_SR, &val); + return -EINVAL; +} - /* Send the command. */ - regmap_write(nc->io, op, addr); +static int atmel_smc_nand_exec_op(struct atmel_nand *nand, + const struct nand_operation *op, + bool check_only) +{ + unsigned int i; + int ret = 0; - ret = atmel_nfc_wait(nc, poll, 0); - if (ret) - dev_err(nc->base.dev, - "Failed to send NAND command (err = %d)!", - ret); + if (check_only) + return 0; - /* Reset the op state. */ - memset(&nc->op, 0, sizeof(nc->op)); + atmel_nand_select_target(nand, op->cs); + gpiod_set_value(nand->activecs->csgpio, 0); + for (i = 0; i < op->ninstrs; i++) { + ret = atmel_smc_nand_exec_instr(nand, &op->instrs[i]); + if (ret) + break; + } + gpiod_set_value(nand->activecs->csgpio, 1); return ret; } -static void atmel_hsmc_nand_cmd_ctrl(struct nand_chip *chip, int dat, - unsigned int ctrl) +static int atmel_hsmc_exec_cmd_addr(struct nand_chip *chip, + const struct nand_subop *subop) { struct atmel_nand *nand = to_atmel_nand(chip); struct atmel_hsmc_nand_controller *nc; + unsigned int i, j; nc = to_hsmc_nand_controller(chip->controller); - if (ctrl & NAND_ALE) { - if (nc->op.naddrs == ATMEL_NFC_MAX_ADDR_CYCLES) - return; + nc->op.cs = nand->activecs->id; + for (i = 0; i < subop->ninstrs; i++) { + const struct nand_op_instr *instr = &subop->instrs[i]; - nc->op.addrs[nc->op.naddrs++] = dat; - } else if (ctrl & NAND_CLE) { - if (nc->op.ncmds > 1) - return; + if (instr->type == NAND_OP_CMD_INSTR) { + nc->op.cmds[nc->op.ncmds++] = instr->ctx.cmd.opcode; + continue; + } - nc->op.cmds[nc->op.ncmds++] = dat; + for (j = nand_subop_get_addr_start_off(subop, i); + j < nand_subop_get_num_addr_cyc(subop, i); j++) { + nc->op.addrs[nc->op.naddrs] = instr->ctx.addr.addrs[j]; + nc->op.naddrs++; + } } - if (dat == NAND_CMD_NONE) { - nc->op.cs = nand->activecs->id; - atmel_nfc_exec_op(nc, true); - } + return atmel_nfc_exec_op(nc, true); } -static void atmel_nand_cmd_ctrl(struct nand_chip *chip, int cmd, - unsigned int ctrl) +static int atmel_hsmc_exec_rw(struct nand_chip *chip, + const struct nand_subop *subop) { + const struct nand_op_instr *instr = subop->instrs; struct atmel_nand *nand = to_atmel_nand(chip); - struct atmel_nand_controller *nc; - nc = to_nand_controller(chip->controller); + if (instr->type == NAND_OP_DATA_IN_INSTR) + atmel_nand_data_in(nand, instr->ctx.data.buf.in, + instr->ctx.data.len, + instr->ctx.data.force_8bit); + else + atmel_nand_data_out(nand, instr->ctx.data.buf.out, + instr->ctx.data.len, + instr->ctx.data.force_8bit); - if ((ctrl & NAND_CTRL_CHANGE) && nand->activecs->csgpio) { - if (ctrl & NAND_NCE) - gpiod_set_value(nand->activecs->csgpio, 0); - else - gpiod_set_value(nand->activecs->csgpio, 1); - } + return 0; +} + +static int atmel_hsmc_exec_waitrdy(struct nand_chip *chip, + const struct nand_subop *subop) +{ + const struct nand_op_instr *instr = subop->instrs; + struct atmel_nand *nand = to_atmel_nand(chip); - if (ctrl & NAND_ALE) - writeb(cmd, nand->activecs->io.virt + nc->caps->ale_offs); - else if (ctrl & NAND_CLE) - writeb(cmd, nand->activecs->io.virt + nc->caps->cle_offs); + return atmel_hsmc_nand_waitrdy(nand, instr->ctx.waitrdy.timeout_ms); +} + +static const struct nand_op_parser atmel_hsmc_op_parser = NAND_OP_PARSER( + NAND_OP_PARSER_PATTERN(atmel_hsmc_exec_cmd_addr, + NAND_OP_PARSER_PAT_CMD_ELEM(true), + NAND_OP_PARSER_PAT_ADDR_ELEM(true, 5), + NAND_OP_PARSER_PAT_CMD_ELEM(true)), + NAND_OP_PARSER_PATTERN(atmel_hsmc_exec_rw, + NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 0)), + NAND_OP_PARSER_PATTERN(atmel_hsmc_exec_rw, + NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, 0)), + NAND_OP_PARSER_PATTERN(atmel_hsmc_exec_waitrdy, + NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)), +); + +static int atmel_hsmc_nand_exec_op(struct atmel_nand *nand, + const struct nand_operation *op, + bool check_only) +{ + int ret; + + if (check_only) + return nand_op_parser_exec_op(&nand->base, + &atmel_hsmc_op_parser, op, true); + + atmel_hsmc_nand_select_target(nand, op->cs); + ret = nand_op_parser_exec_op(&nand->base, &atmel_hsmc_op_parser, op, + false); + + return ret; } static void atmel_nfc_copy_to_sram(struct nand_chip *chip, const u8 *buf, @@ -838,7 +908,7 @@ static int atmel_nand_pmecc_write_pg(struct nand_chip *chip, const u8 *buf, if (ret) return ret; - atmel_nand_write_buf(chip, buf, mtd->writesize); + nand_write_data_op(chip, buf, mtd->writesize, false); ret = atmel_nand_pmecc_generate_eccbytes(chip, raw); if (ret) { @@ -848,7 +918,7 @@ static int atmel_nand_pmecc_write_pg(struct nand_chip *chip, const u8 *buf, atmel_nand_pmecc_disable(chip, raw); - atmel_nand_write_buf(chip, chip->oob_poi, mtd->oobsize); + nand_write_data_op(chip, chip->oob_poi, mtd->oobsize, false); return nand_prog_page_end_op(chip); } @@ -878,11 +948,17 @@ static int atmel_nand_pmecc_read_pg(struct nand_chip *chip, u8 *buf, if (ret) return ret; - atmel_nand_read_buf(chip, buf, mtd->writesize); - atmel_nand_read_buf(chip, chip->oob_poi, mtd->oobsize); + ret = nand_read_data_op(chip, buf, mtd->writesize, false, false); + if (ret) + goto out_disable; + + ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false, false); + if (ret) + goto out_disable; ret = atmel_nand_pmecc_correct_data(chip, buf, raw); +out_disable: atmel_nand_pmecc_disable(chip, raw); return ret; @@ -907,8 +983,9 @@ static int atmel_hsmc_nand_pmecc_write_pg(struct nand_chip *chip, struct mtd_info *mtd = nand_to_mtd(chip); struct atmel_nand *nand = to_atmel_nand(chip); struct atmel_hsmc_nand_controller *nc; - int ret, status; + int ret; + atmel_hsmc_nand_select_target(nand, chip->cur_cs); nc = to_hsmc_nand_controller(chip->controller); atmel_nfc_copy_to_sram(chip, buf, false); @@ -939,21 +1016,9 @@ static int atmel_hsmc_nand_pmecc_write_pg(struct nand_chip *chip, if (ret) return ret; - atmel_nand_write_buf(chip, chip->oob_poi, mtd->oobsize); - - nc->op.cmds[0] = NAND_CMD_PAGEPROG; - nc->op.ncmds = 1; - nc->op.cs = nand->activecs->id; - ret = atmel_nfc_exec_op(nc, false); - if (ret) - dev_err(nc->base.dev, "Failed to program NAND page (err = %d)\n", - ret); - - status = chip->legacy.waitfunc(chip); - if (status & NAND_STATUS_FAIL) - return -EIO; + nand_write_data_op(chip, chip->oob_poi, mtd->oobsize, false); - return ret; + return nand_prog_page_end_op(chip); } static int atmel_hsmc_nand_pmecc_write_page(struct nand_chip *chip, @@ -981,6 +1046,7 @@ static int atmel_hsmc_nand_pmecc_read_pg(struct nand_chip *chip, u8 *buf, struct atmel_hsmc_nand_controller *nc; int ret; + atmel_hsmc_nand_select_target(nand, chip->cur_cs); nc = to_hsmc_nand_controller(chip->controller); /* @@ -988,12 +1054,9 @@ static int atmel_hsmc_nand_pmecc_read_pg(struct nand_chip *chip, u8 *buf, * connected to a native SoC R/B pin. If that's not the case, fallback * to the non-optimized one. */ - if (nand->activecs->rb.type != ATMEL_NAND_NATIVE_RB) { - nand_read_page_op(chip, page, 0, NULL, 0); - + if (nand->activecs->rb.type != ATMEL_NAND_NATIVE_RB) return atmel_nand_pmecc_read_pg(chip, buf, oob_required, page, raw); - } nc->op.cmds[nc->op.ncmds++] = NAND_CMD_READ0; @@ -1043,7 +1106,10 @@ static int atmel_hsmc_nand_pmecc_read_page_raw(struct nand_chip *chip, static int atmel_nand_pmecc_init(struct nand_chip *chip) { + const struct nand_ecc_props *requirements = + nanddev_get_ecc_requirements(&chip->base); struct mtd_info *mtd = nand_to_mtd(chip); + struct nand_device *nanddev = mtd_to_nanddev(mtd); struct atmel_nand *nand = to_atmel_nand(chip); struct atmel_nand_controller *nc; struct atmel_pmecc_user_req req; @@ -1068,19 +1134,19 @@ static int atmel_nand_pmecc_init(struct nand_chip *chip) chip->ecc.size = val; } - if (chip->ecc.options & NAND_ECC_MAXIMIZE) + if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) req.ecc.strength = ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH; else if (chip->ecc.strength) req.ecc.strength = chip->ecc.strength; - else if (chip->base.eccreq.strength) - req.ecc.strength = chip->base.eccreq.strength; + else if (requirements->strength) + req.ecc.strength = requirements->strength; else req.ecc.strength = ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH; if (chip->ecc.size) req.ecc.sectorsize = chip->ecc.size; - else if (chip->base.eccreq.step_size) - req.ecc.sectorsize = chip->base.eccreq.step_size; + else if (requirements->step_size) + req.ecc.sectorsize = requirements->step_size; else req.ecc.sectorsize = ATMEL_PMECC_SECTOR_SIZE_AUTO; @@ -1099,14 +1165,14 @@ static int atmel_nand_pmecc_init(struct nand_chip *chip) if (IS_ERR(nand->pmecc)) return PTR_ERR(nand->pmecc); - chip->ecc.algo = NAND_ECC_BCH; + chip->ecc.algo = NAND_ECC_ALGO_BCH; chip->ecc.size = req.ecc.sectorsize; chip->ecc.bytes = req.ecc.bytes / req.ecc.nsectors; chip->ecc.strength = req.ecc.strength; chip->options |= NAND_NO_SUBPAGE_WRITE; - mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops); + mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout()); return 0; } @@ -1118,15 +1184,15 @@ static int atmel_nand_ecc_init(struct nand_chip *chip) nc = to_nand_controller(chip->controller); - switch (chip->ecc.mode) { - case NAND_ECC_NONE: - case NAND_ECC_SOFT: + switch (chip->ecc.engine_type) { + case NAND_ECC_ENGINE_TYPE_NONE: + case NAND_ECC_ENGINE_TYPE_SOFT: /* * Nothing to do, the core will initialize everything for us. */ break; - case NAND_ECC_HW: + case NAND_ECC_ENGINE_TYPE_ON_HOST: ret = atmel_nand_pmecc_init(chip); if (ret) return ret; @@ -1140,7 +1206,7 @@ static int atmel_nand_ecc_init(struct nand_chip *chip) default: /* Other modes are not supported. */ dev_err(nc->dev, "Unsupported ECC mode: %d\n", - chip->ecc.mode); + chip->ecc.engine_type); return -ENOTSUPP; } @@ -1155,7 +1221,7 @@ static int atmel_hsmc_nand_ecc_init(struct nand_chip *chip) if (ret) return ret; - if (chip->ecc.mode != NAND_ECC_HW) + if (chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST) return 0; /* Adjust the ECC operations for the HSMC IP. */ @@ -1467,6 +1533,18 @@ static int atmel_nand_setup_interface(struct nand_chip *chip, int csline, return nc->caps->ops->setup_interface(nand, csline, conf); } +static int atmel_nand_exec_op(struct nand_chip *chip, + const struct nand_operation *op, + bool check_only) +{ + struct atmel_nand *nand = to_atmel_nand(chip); + struct atmel_nand_controller *nc; + + nc = to_nand_controller(nand->base.controller); + + return nc->caps->ops->exec_op(nand, op, check_only); +} + static void atmel_nand_init(struct atmel_nand_controller *nc, struct atmel_nand *nand) { @@ -1476,19 +1554,9 @@ static void atmel_nand_init(struct atmel_nand_controller *nc, mtd->dev.parent = nc->dev; nand->base.controller = &nc->base; - chip->legacy.cmd_ctrl = atmel_nand_cmd_ctrl; - chip->legacy.read_byte = atmel_nand_read_byte; - chip->legacy.write_byte = atmel_nand_write_byte; - chip->legacy.read_buf = atmel_nand_read_buf; - chip->legacy.write_buf = atmel_nand_write_buf; - chip->legacy.select_chip = atmel_nand_select_chip; - if (!nc->mck || !nc->caps->ops->setup_interface) chip->options |= NAND_KEEP_TIMINGS; - /* Some NANDs require a longer delay than the default one (20us). */ - chip->legacy.chip_delay = 40; - /* * Use a bounce buffer when the buffer passed by the MTD user is not * suitable for DMA. @@ -1498,7 +1566,7 @@ static void atmel_nand_init(struct atmel_nand_controller *nc, /* Default to HW ECC if pmecc is available. */ if (nc->pmecc) - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; } static void atmel_smc_nand_init(struct atmel_nand_controller *nc, @@ -1527,18 +1595,6 @@ static void atmel_smc_nand_init(struct atmel_nand_controller *nc, smc_nc->ebi_csa->nfd0_on_d16); } -static void atmel_hsmc_nand_init(struct atmel_nand_controller *nc, - struct atmel_nand *nand) -{ - struct nand_chip *chip = &nand->base; - - atmel_nand_init(nc, nand); - - /* Overload some methods for the HSMC controller. */ - chip->legacy.cmd_ctrl = atmel_hsmc_nand_cmd_ctrl; - chip->legacy.select_chip = atmel_hsmc_nand_select_chip; -} - static int atmel_nand_controller_remove_nand(struct atmel_nand *nand) { struct nand_chip *chip = &nand->base; @@ -1957,6 +2013,7 @@ static int atmel_nand_attach_chip(struct nand_chip *chip) static const struct nand_controller_ops atmel_nand_controller_ops = { .attach_chip = atmel_nand_attach_chip, .setup_interface = atmel_nand_setup_interface, + .exec_op = atmel_nand_exec_op, }; static int atmel_nand_controller_init(struct atmel_nand_controller *nc, @@ -1976,13 +2033,9 @@ static int atmel_nand_controller_init(struct atmel_nand_controller *nc, platform_set_drvdata(pdev, nc); nc->pmecc = devm_atmel_pmecc_get(dev); - if (IS_ERR(nc->pmecc)) { - ret = PTR_ERR(nc->pmecc); - if (ret != -EPROBE_DEFER) - dev_err(dev, "Could not get PMECC object (err = %d)\n", - ret); - return ret; - } + if (IS_ERR(nc->pmecc)) + return dev_err_probe(dev, PTR_ERR(nc->pmecc), + "Could not get PMECC object\n"); if (nc->caps->has_dma && !atmel_nand_avoid_dma) { dma_cap_mask_t mask; @@ -2248,6 +2301,9 @@ atmel_hsmc_nand_controller_remove(struct atmel_nand_controller *nc) return ret; hsmc_nc = container_of(nc, struct atmel_hsmc_nand_controller, base); + regmap_write(hsmc_nc->base.smc, ATMEL_HSMC_NFC_CTRL, + ATMEL_HSMC_NFC_CTRL_DIS); + if (hsmc_nc->sram.pool) gen_pool_free(hsmc_nc->sram.pool, (unsigned long)hsmc_nc->sram.virt, @@ -2300,6 +2356,8 @@ static int atmel_hsmc_nand_controller_probe(struct platform_device *pdev, /* Initial NFC configuration. */ regmap_write(nc->base.smc, ATMEL_HSMC_NFC_CFG, ATMEL_HSMC_NFC_CFG_DTO_MAX); + regmap_write(nc->base.smc, ATMEL_HSMC_NFC_CTRL, + ATMEL_HSMC_NFC_CTRL_EN); ret = atmel_nand_controller_add_nands(&nc->base); if (ret) @@ -2317,8 +2375,9 @@ static const struct atmel_nand_controller_ops atmel_hsmc_nc_ops = { .probe = atmel_hsmc_nand_controller_probe, .remove = atmel_hsmc_nand_controller_remove, .ecc_init = atmel_hsmc_nand_ecc_init, - .nand_init = atmel_hsmc_nand_init, + .nand_init = atmel_nand_init, .setup_interface = atmel_hsmc_nand_setup_interface, + .exec_op = atmel_hsmc_nand_exec_op, }; static const struct atmel_nand_controller_caps atmel_sama5_nc_caps = { @@ -2385,6 +2444,7 @@ static const struct atmel_nand_controller_ops at91rm9200_nc_ops = { .remove = atmel_smc_nand_controller_remove, .ecc_init = atmel_nand_ecc_init, .nand_init = atmel_smc_nand_init, + .exec_op = atmel_smc_nand_exec_op, }; static const struct atmel_nand_controller_caps atmel_rm9200_nc_caps = { @@ -2400,6 +2460,7 @@ static const struct atmel_nand_controller_ops atmel_smc_nc_ops = { .ecc_init = atmel_nand_ecc_init, .nand_init = atmel_smc_nand_init, .setup_interface = atmel_smc_nand_setup_interface, + .exec_op = atmel_smc_nand_exec_op, }; static const struct atmel_nand_controller_caps atmel_sam9260_nc_caps = { diff --git a/drivers/mtd/nand/raw/au1550nd.c b/drivers/mtd/nand/raw/au1550nd.c index d865200ccd08..79b057400fe9 100644 --- a/drivers/mtd/nand/raw/au1550nd.c +++ b/drivers/mtd/nand/raw/au1550nd.c @@ -294,8 +294,8 @@ static int au1550nd_probe(struct platform_device *pdev) nand_controller_init(&ctx->controller); ctx->controller.ops = &au1550nd_ops; this->controller = &ctx->controller; - this->ecc.mode = NAND_ECC_SOFT; - this->ecc.algo = NAND_ECC_HAMMING; + this->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + this->ecc.algo = NAND_ECC_ALGO_HAMMING; if (pd->devwidth) this->options |= NAND_BUSWIDTH_16; diff --git a/drivers/mtd/nand/raw/bcm47xxnflash/ops_bcm4706.c b/drivers/mtd/nand/raw/bcm47xxnflash/ops_bcm4706.c index 591775173034..8bb17c5a66c3 100644 --- a/drivers/mtd/nand/raw/bcm47xxnflash/ops_bcm4706.c +++ b/drivers/mtd/nand/raw/bcm47xxnflash/ops_bcm4706.c @@ -391,7 +391,8 @@ int bcm47xxnflash_ops_bcm4706_init(struct bcm47xxnflash *b47n) nand_chip->legacy.chip_delay = 50; b47n->nand_chip.bbt_options = NAND_BBT_USE_FLASH; - b47n->nand_chip.ecc.mode = NAND_ECC_NONE; /* TODO: implement ECC */ + /* TODO: implement ECC */ + b47n->nand_chip.ecc.engine_type = NAND_ECC_ENGINE_TYPE_NONE; /* Enable NAND flash access */ bcma_cc_set32(b47n->cc, BCMA_CC_4706_FLASHSCFG, diff --git a/drivers/mtd/nand/raw/brcmnand/brcmnand.c b/drivers/mtd/nand/raw/brcmnand/brcmnand.c index a4033d32a710..2da39ab89286 100644 --- a/drivers/mtd/nand/raw/brcmnand/brcmnand.c +++ b/drivers/mtd/nand/raw/brcmnand/brcmnand.c @@ -2532,6 +2532,8 @@ static int brcmnand_setup_dev(struct brcmnand_host *host) { struct mtd_info *mtd = nand_to_mtd(&host->chip); struct nand_chip *chip = &host->chip; + const struct nand_ecc_props *requirements = + nanddev_get_ecc_requirements(&chip->base); struct brcmnand_controller *ctrl = host->ctrl; struct brcmnand_cfg *cfg = &host->hwcfg; char msg[128]; @@ -2565,34 +2567,34 @@ static int brcmnand_setup_dev(struct brcmnand_host *host) cfg->col_adr_bytes = 2; cfg->blk_adr_bytes = get_blk_adr_bytes(mtd->size, mtd->writesize); - if (chip->ecc.mode != NAND_ECC_HW) { + if (chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST) { dev_err(ctrl->dev, "only HW ECC supported; selected: %d\n", - chip->ecc.mode); + chip->ecc.engine_type); return -EINVAL; } - if (chip->ecc.algo == NAND_ECC_UNKNOWN) { + if (chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN) { if (chip->ecc.strength == 1 && chip->ecc.size == 512) /* Default to Hamming for 1-bit ECC, if unspecified */ - chip->ecc.algo = NAND_ECC_HAMMING; + chip->ecc.algo = NAND_ECC_ALGO_HAMMING; else /* Otherwise, BCH */ - chip->ecc.algo = NAND_ECC_BCH; + chip->ecc.algo = NAND_ECC_ALGO_BCH; } - if (chip->ecc.algo == NAND_ECC_HAMMING && (chip->ecc.strength != 1 || - chip->ecc.size != 512)) { + if (chip->ecc.algo == NAND_ECC_ALGO_HAMMING && + (chip->ecc.strength != 1 || chip->ecc.size != 512)) { dev_err(ctrl->dev, "invalid Hamming params: %d bits per %d bytes\n", chip->ecc.strength, chip->ecc.size); return -EINVAL; } - if (chip->ecc.mode != NAND_ECC_NONE && + if (chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_NONE && (!chip->ecc.size || !chip->ecc.strength)) { - if (chip->base.eccreq.step_size && chip->base.eccreq.strength) { + if (requirements->step_size && requirements->strength) { /* use detected ECC parameters */ - chip->ecc.size = chip->base.eccreq.step_size; - chip->ecc.strength = chip->base.eccreq.strength; + chip->ecc.size = requirements->step_size; + chip->ecc.strength = requirements->strength; dev_info(ctrl->dev, "Using ECC step-size %d, strength %d\n", chip->ecc.size, chip->ecc.strength); } @@ -2600,7 +2602,7 @@ static int brcmnand_setup_dev(struct brcmnand_host *host) switch (chip->ecc.size) { case 512: - if (chip->ecc.algo == NAND_ECC_HAMMING) + if (chip->ecc.algo == NAND_ECC_ALGO_HAMMING) cfg->ecc_level = 15; else cfg->ecc_level = chip->ecc.strength; @@ -2728,7 +2730,7 @@ static int brcmnand_init_cs(struct brcmnand_host *host, struct device_node *dn) chip->legacy.read_buf = brcmnand_read_buf; chip->legacy.write_buf = brcmnand_write_buf; - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; chip->ecc.read_page = brcmnand_read_page; chip->ecc.write_page = brcmnand_write_page; chip->ecc.read_page_raw = brcmnand_read_page_raw; diff --git a/drivers/mtd/nand/raw/cadence-nand-controller.c b/drivers/mtd/nand/raw/cadence-nand-controller.c index 71516af85f23..b46786cd53e0 100644 --- a/drivers/mtd/nand/raw/cadence-nand-controller.c +++ b/drivers/mtd/nand/raw/cadence-nand-controller.c @@ -2611,7 +2611,7 @@ static int cadence_nand_attach_chip(struct nand_chip *chip) chip->bbt_options |= NAND_BBT_USE_FLASH; chip->bbt_options |= NAND_BBT_NO_OOB; - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; chip->options |= NAND_NO_SUBPAGE_WRITE; @@ -2757,7 +2757,7 @@ static int cadence_nand_chip_init(struct cdns_nand_ctrl *cdns_ctrl, * Default to HW ECC engine mode. If the nand-ecc-mode property is given * in the DT node, this entry will be overwritten in nand_scan_ident(). */ - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; ret = nand_scan(chip, cdns_chip->nsels); if (ret) { @@ -2980,18 +2980,14 @@ static int cadence_nand_dt_probe(struct platform_device *ofdev) dev_info(cdns_ctrl->dev, "IRQ: nr %d\n", cdns_ctrl->irq); cdns_ctrl->reg = devm_platform_ioremap_resource(ofdev, 0); - if (IS_ERR(cdns_ctrl->reg)) { - dev_err(&ofdev->dev, "devm_ioremap_resource res 0 failed\n"); + if (IS_ERR(cdns_ctrl->reg)) return PTR_ERR(cdns_ctrl->reg); - } res = platform_get_resource(ofdev, IORESOURCE_MEM, 1); cdns_ctrl->io.dma = res->start; cdns_ctrl->io.virt = devm_ioremap_resource(&ofdev->dev, res); - if (IS_ERR(cdns_ctrl->io.virt)) { - dev_err(cdns_ctrl->dev, "devm_ioremap_resource res 1 failed\n"); + if (IS_ERR(cdns_ctrl->io.virt)) return PTR_ERR(cdns_ctrl->io.virt); - } dt->clk = devm_clk_get(cdns_ctrl->dev, "nf_clk"); if (IS_ERR(dt->clk)) diff --git a/drivers/mtd/nand/raw/cafe_nand.c b/drivers/mtd/nand/raw/cafe_nand.c index 92173790f20b..2b94f385a1a8 100644 --- a/drivers/mtd/nand/raw/cafe_nand.c +++ b/drivers/mtd/nand/raw/cafe_nand.c @@ -629,7 +629,8 @@ static int cafe_nand_attach_chip(struct nand_chip *chip) goto out_free_dma; } - cafe->nand.ecc.mode = NAND_ECC_HW_SYNDROME; + cafe->nand.ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; + cafe->nand.ecc.placement = NAND_ECC_PLACEMENT_INTERLEAVED; cafe->nand.ecc.size = mtd->writesize; cafe->nand.ecc.bytes = 14; cafe->nand.ecc.strength = 4; diff --git a/drivers/mtd/nand/raw/cs553x_nand.c b/drivers/mtd/nand/raw/cs553x_nand.c index 9472bf798ed5..b7f3f6347761 100644 --- a/drivers/mtd/nand/raw/cs553x_nand.c +++ b/drivers/mtd/nand/raw/cs553x_nand.c @@ -286,7 +286,7 @@ static int __init cs553x_init_one(int cs, int mmio, unsigned long adr) goto out_mtd; } - this->ecc.mode = NAND_ECC_HW; + this->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; this->ecc.size = 256; this->ecc.bytes = 3; this->ecc.hwctl = cs_enable_hwecc; diff --git a/drivers/mtd/nand/raw/davinci_nand.c b/drivers/mtd/nand/raw/davinci_nand.c index d975a62caaa5..427f320fb79b 100644 --- a/drivers/mtd/nand/raw/davinci_nand.c +++ b/drivers/mtd/nand/raw/davinci_nand.c @@ -168,7 +168,7 @@ static int nand_davinci_correct_1bit(struct nand_chip *chip, u_char *dat, /* * 4-bit hardware ECC ... context maintained over entire AEMIF * - * This is a syndrome engine, but we avoid NAND_ECC_HW_SYNDROME + * This is a syndrome engine, but we avoid NAND_ECC_PLACEMENT_INTERLEAVED * since that forces use of a problematic "infix OOB" layout. * Among other things, it trashes manufacturer bad block markers. * Also, and specific to this hardware, it ECC-protects the "prepad" @@ -530,11 +530,11 @@ static struct davinci_nand_pdata if (!of_property_read_string(pdev->dev.of_node, "ti,davinci-ecc-mode", &mode)) { if (!strncmp("none", mode, 4)) - pdata->ecc_mode = NAND_ECC_NONE; + pdata->engine_type = NAND_ECC_ENGINE_TYPE_NONE; if (!strncmp("soft", mode, 4)) - pdata->ecc_mode = NAND_ECC_SOFT; + pdata->engine_type = NAND_ECC_ENGINE_TYPE_SOFT; if (!strncmp("hw", mode, 2)) - pdata->ecc_mode = NAND_ECC_HW; + pdata->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; } if (!of_property_read_u32(pdev->dev.of_node, "ti,davinci-ecc-bits", &prop)) @@ -585,21 +585,21 @@ static int davinci_nand_attach_chip(struct nand_chip *chip) if (IS_ERR(pdata)) return PTR_ERR(pdata); - switch (info->chip.ecc.mode) { - case NAND_ECC_NONE: + switch (info->chip.ecc.engine_type) { + case NAND_ECC_ENGINE_TYPE_NONE: pdata->ecc_bits = 0; break; - case NAND_ECC_SOFT: + case NAND_ECC_ENGINE_TYPE_SOFT: pdata->ecc_bits = 0; /* - * This driver expects Hamming based ECC when ecc_mode is set - * to NAND_ECC_SOFT. Force ecc.algo to NAND_ECC_HAMMING to - * avoid adding an extra ->ecc_algo field to - * davinci_nand_pdata. + * This driver expects Hamming based ECC when engine_type is set + * to NAND_ECC_ENGINE_TYPE_SOFT. Force ecc.algo to + * NAND_ECC_ALGO_HAMMING to avoid adding an extra ->ecc_algo + * field to davinci_nand_pdata. */ - info->chip.ecc.algo = NAND_ECC_HAMMING; + info->chip.ecc.algo = NAND_ECC_ALGO_HAMMING; break; - case NAND_ECC_HW: + case NAND_ECC_ENGINE_TYPE_ON_HOST: if (pdata->ecc_bits == 4) { int chunks = mtd->writesize / 512; @@ -629,7 +629,7 @@ static int davinci_nand_attach_chip(struct nand_chip *chip) info->chip.ecc.hwctl = nand_davinci_hwctl_4bit; info->chip.ecc.bytes = 10; info->chip.ecc.options = NAND_ECC_GENERIC_ERASED_CHECK; - info->chip.ecc.algo = NAND_ECC_BCH; + info->chip.ecc.algo = NAND_ECC_ALGO_BCH; /* * Update ECC layout if needed ... for 1-bit HW ECC, the @@ -645,7 +645,8 @@ static int davinci_nand_attach_chip(struct nand_chip *chip) mtd_set_ooblayout(mtd, &hwecc4_small_ooblayout_ops); } else if (chunks == 4 || chunks == 8) { - mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops); + mtd_set_ooblayout(mtd, + nand_get_large_page_ooblayout()); info->chip.ecc.read_page = nand_davinci_read_page_hwecc_oob_first; } else { return -EIO; @@ -656,7 +657,7 @@ static int davinci_nand_attach_chip(struct nand_chip *chip) info->chip.ecc.correct = nand_davinci_correct_1bit; info->chip.ecc.hwctl = nand_davinci_hwctl_1bit; info->chip.ecc.bytes = 3; - info->chip.ecc.algo = NAND_ECC_HAMMING; + info->chip.ecc.algo = NAND_ECC_ALGO_HAMMING; } info->chip.ecc.size = 512; info->chip.ecc.strength = pdata->ecc_bits; @@ -850,7 +851,8 @@ static int nand_davinci_probe(struct platform_device *pdev) info->mask_cle = pdata->mask_cle ? : MASK_CLE; /* Use board-specific ECC config */ - info->chip.ecc.mode = pdata->ecc_mode; + info->chip.ecc.engine_type = pdata->engine_type; + info->chip.ecc.placement = pdata->ecc_placement; spin_lock_irq(&davinci_nand_lock); @@ -897,7 +899,7 @@ static int nand_davinci_remove(struct platform_device *pdev) int ret; spin_lock_irq(&davinci_nand_lock); - if (info->chip.ecc.mode == NAND_ECC_HW_SYNDROME) + if (info->chip.ecc.placement == NAND_ECC_PLACEMENT_INTERLEAVED) ecc4_busy = false; spin_unlock_irq(&davinci_nand_lock); diff --git a/drivers/mtd/nand/raw/denali.c b/drivers/mtd/nand/raw/denali.c index 9d99dade95ce..fa2439cb4daa 100644 --- a/drivers/mtd/nand/raw/denali.c +++ b/drivers/mtd/nand/raw/denali.c @@ -1237,7 +1237,8 @@ int denali_chip_init(struct denali_controller *denali, chip->bbt_options |= NAND_BBT_USE_FLASH; chip->bbt_options |= NAND_BBT_NO_OOB; chip->options |= NAND_NO_SUBPAGE_WRITE; - chip->ecc.mode = NAND_ECC_HW_SYNDROME; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; + chip->ecc.placement = NAND_ECC_PLACEMENT_INTERLEAVED; chip->ecc.read_page = denali_read_page; chip->ecc.write_page = denali_write_page; chip->ecc.read_page_raw = denali_read_page_raw; diff --git a/drivers/mtd/nand/raw/denali_pci.c b/drivers/mtd/nand/raw/denali_pci.c index 2f77ee55e1bf..20c085a30adc 100644 --- a/drivers/mtd/nand/raw/denali_pci.c +++ b/drivers/mtd/nand/raw/denali_pci.c @@ -100,7 +100,7 @@ static int denali_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) goto out_remove_denali; } - dchip->chip.ecc.options |= NAND_ECC_MAXIMIZE; + dchip->chip.base.ecc.user_conf.flags |= NAND_ECC_MAXIMIZE_STRENGTH; dchip->nsels = nsels; diff --git a/drivers/mtd/nand/raw/diskonchip.c b/drivers/mtd/nand/raw/diskonchip.c index 43721863a0d8..94432a453e5e 100644 --- a/drivers/mtd/nand/raw/diskonchip.c +++ b/drivers/mtd/nand/raw/diskonchip.c @@ -1456,7 +1456,8 @@ static int __init doc_probe(unsigned long physadr) nand->ecc.calculate = doc200x_calculate_ecc; nand->ecc.correct = doc200x_correct_data; - nand->ecc.mode = NAND_ECC_HW_SYNDROME; + nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; + nand->ecc.placement = NAND_ECC_PLACEMENT_INTERLEAVED; nand->ecc.size = 512; nand->ecc.bytes = 6; nand->ecc.strength = 2; diff --git a/drivers/mtd/nand/raw/fsl_elbc_nand.c b/drivers/mtd/nand/raw/fsl_elbc_nand.c index 088692b2e27a..b2af7f81fdf8 100644 --- a/drivers/mtd/nand/raw/fsl_elbc_nand.c +++ b/drivers/mtd/nand/raw/fsl_elbc_nand.c @@ -244,7 +244,7 @@ static int fsl_elbc_run_command(struct mtd_info *mtd) return -EIO; } - if (chip->ecc.mode != NAND_ECC_HW) + if (chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST) return 0; elbc_fcm_ctrl->max_bitflips = 0; @@ -727,12 +727,12 @@ static int fsl_elbc_attach_chip(struct nand_chip *chip) struct fsl_lbc_regs __iomem *lbc = ctrl->regs; unsigned int al; - switch (chip->ecc.mode) { + switch (chip->ecc.engine_type) { /* * if ECC was not chosen in DT, decide whether to use HW or SW ECC from * CS Base Register */ - case NAND_ECC_NONE: + case NAND_ECC_ENGINE_TYPE_NONE: /* If CS Base Register selects full hardware ECC then use it */ if ((in_be32(&lbc->bank[priv->bank].br) & BR_DECC) == BR_DECC_CHK_GEN) { @@ -740,23 +740,23 @@ static int fsl_elbc_attach_chip(struct nand_chip *chip) chip->ecc.write_page = fsl_elbc_write_page; chip->ecc.write_subpage = fsl_elbc_write_subpage; - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; mtd_set_ooblayout(mtd, &fsl_elbc_ooblayout_ops); chip->ecc.size = 512; chip->ecc.bytes = 3; chip->ecc.strength = 1; } else { /* otherwise fall back to default software ECC */ - chip->ecc.mode = NAND_ECC_SOFT; - chip->ecc.algo = NAND_ECC_HAMMING; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + chip->ecc.algo = NAND_ECC_ALGO_HAMMING; } break; /* if SW ECC was chosen in DT, we do not need to set anything here */ - case NAND_ECC_SOFT: + case NAND_ECC_ENGINE_TYPE_SOFT: break; - /* should we also implement NAND_ECC_HW to do as the code above? */ + /* should we also implement *_ECC_ENGINE_CONTROLLER to do as above? */ default: return -EINVAL; } @@ -786,8 +786,8 @@ static int fsl_elbc_attach_chip(struct nand_chip *chip) chip->page_shift); dev_dbg(priv->dev, "fsl_elbc_init: nand->phys_erase_shift = %d\n", chip->phys_erase_shift); - dev_dbg(priv->dev, "fsl_elbc_init: nand->ecc.mode = %d\n", - chip->ecc.mode); + dev_dbg(priv->dev, "fsl_elbc_init: nand->ecc.engine_type = %d\n", + chip->ecc.engine_type); dev_dbg(priv->dev, "fsl_elbc_init: nand->ecc.steps = %d\n", chip->ecc.steps); dev_dbg(priv->dev, "fsl_elbc_init: nand->ecc.bytes = %d\n", diff --git a/drivers/mtd/nand/raw/fsl_ifc_nand.c b/drivers/mtd/nand/raw/fsl_ifc_nand.c index 00ae7a910b03..0e7a9b64301e 100644 --- a/drivers/mtd/nand/raw/fsl_ifc_nand.c +++ b/drivers/mtd/nand/raw/fsl_ifc_nand.c @@ -309,7 +309,7 @@ static void fsl_ifc_cmdfunc(struct nand_chip *chip, unsigned int command, ifc_nand_ctrl->read_bytes = mtd->writesize + mtd->oobsize; ifc_nand_ctrl->index += column; - if (chip->ecc.mode == NAND_ECC_HW) + if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) ifc_nand_ctrl->eccread = 1; fsl_ifc_do_read(chip, 0, mtd); @@ -724,8 +724,8 @@ static int fsl_ifc_attach_chip(struct nand_chip *chip) chip->page_shift); dev_dbg(priv->dev, "%s: nand->phys_erase_shift = %d\n", __func__, chip->phys_erase_shift); - dev_dbg(priv->dev, "%s: nand->ecc.mode = %d\n", __func__, - chip->ecc.mode); + dev_dbg(priv->dev, "%s: nand->ecc.engine_type = %d\n", __func__, + chip->ecc.engine_type); dev_dbg(priv->dev, "%s: nand->ecc.steps = %d\n", __func__, chip->ecc.steps); dev_dbg(priv->dev, "%s: nand->ecc.bytes = %d\n", __func__, @@ -912,7 +912,7 @@ static int fsl_ifc_chip_init(struct fsl_ifc_mtd *priv) /* Must also set CSOR_NAND_ECC_ENC_EN if DEC_EN set */ if (csor & CSOR_NAND_ECC_DEC_EN) { - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; mtd_set_ooblayout(mtd, &fsl_ifc_ooblayout_ops); /* Hardware generates ECC per 512 Bytes */ @@ -925,8 +925,8 @@ static int fsl_ifc_chip_init(struct fsl_ifc_mtd *priv) chip->ecc.strength = 8; } } else { - chip->ecc.mode = NAND_ECC_SOFT; - chip->ecc.algo = NAND_ECC_HAMMING; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + chip->ecc.algo = NAND_ECC_ALGO_HAMMING; } ret = fsl_ifc_sram_init(priv); diff --git a/drivers/mtd/nand/raw/fsl_upm.c b/drivers/mtd/nand/raw/fsl_upm.c index 197850aeb261..d5813b9abc8e 100644 --- a/drivers/mtd/nand/raw/fsl_upm.c +++ b/drivers/mtd/nand/raw/fsl_upm.c @@ -47,8 +47,8 @@ static int fun_chip_init(struct fsl_upm_nand *fun, int ret; struct device_node *flash_np; - fun->chip.ecc.mode = NAND_ECC_SOFT; - fun->chip.ecc.algo = NAND_ECC_HAMMING; + fun->chip.ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + fun->chip.ecc.algo = NAND_ECC_ALGO_HAMMING; fun->chip.controller = &fun->base; mtd->dev.parent = fun->dev; diff --git a/drivers/mtd/nand/raw/fsmc_nand.c b/drivers/mtd/nand/raw/fsmc_nand.c index 92ddc41d0ff0..4191831df182 100644 --- a/drivers/mtd/nand/raw/fsmc_nand.c +++ b/drivers/mtd/nand/raw/fsmc_nand.c @@ -900,8 +900,8 @@ static int fsmc_nand_attach_chip(struct nand_chip *nand) return 0; } - switch (nand->ecc.mode) { - case NAND_ECC_HW: + switch (nand->ecc.engine_type) { + case NAND_ECC_ENGINE_TYPE_ON_HOST: dev_info(host->dev, "Using 1-bit HW ECC scheme\n"); nand->ecc.calculate = fsmc_read_hwecc_ecc1; nand->ecc.correct = nand_correct_data; @@ -910,14 +910,14 @@ static int fsmc_nand_attach_chip(struct nand_chip *nand) nand->ecc.options |= NAND_ECC_SOFT_HAMMING_SM_ORDER; break; - case NAND_ECC_SOFT: - if (nand->ecc.algo == NAND_ECC_BCH) { + case NAND_ECC_ENGINE_TYPE_SOFT: + if (nand->ecc.algo == NAND_ECC_ALGO_BCH) { dev_info(host->dev, "Using 4-bit SW BCH ECC scheme\n"); break; } - case NAND_ECC_ON_DIE: + case NAND_ECC_ENGINE_TYPE_ON_DIE: break; default: @@ -929,7 +929,7 @@ static int fsmc_nand_attach_chip(struct nand_chip *nand) * Don't set layout for BCH4 SW ECC. This will be * generated later in nand_bch_init() later. */ - if (nand->ecc.mode == NAND_ECC_HW) { + if (nand->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) { switch (mtd->oobsize) { case 16: case 64: @@ -1059,7 +1059,7 @@ static int __init fsmc_nand_probe(struct platform_device *pdev) * Setup default ECC mode. nand_dt_init() called from nand_scan_ident() * can overwrite this value if the DT provides a different value. */ - nand->ecc.mode = NAND_ECC_HW; + nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; nand->ecc.hwctl = fsmc_enable_hwecc; nand->ecc.size = 512; nand->badblockbits = 7; diff --git a/drivers/mtd/nand/raw/gpio.c b/drivers/mtd/nand/raw/gpio.c index 3bd847ccc3f3..4ec0a1e10867 100644 --- a/drivers/mtd/nand/raw/gpio.c +++ b/drivers/mtd/nand/raw/gpio.c @@ -342,8 +342,8 @@ static int gpio_nand_probe(struct platform_device *pdev) gpiomtd->base.ops = &gpio_nand_ops; nand_set_flash_node(chip, pdev->dev.of_node); - chip->ecc.mode = NAND_ECC_SOFT; - chip->ecc.algo = NAND_ECC_HAMMING; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + chip->ecc.algo = NAND_ECC_ALGO_HAMMING; chip->options = gpiomtd->plat.options; chip->controller = &gpiomtd->base; diff --git a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c index 5d4aee46cc55..dc8104e67506 100644 --- a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c +++ b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c @@ -272,8 +272,8 @@ static int set_geometry_by_ecc_info(struct gpmi_nand_data *this, default: dev_err(this->dev, "unsupported nand chip. ecc bits : %d, ecc size : %d\n", - chip->base.eccreq.strength, - chip->base.eccreq.step_size); + nanddev_get_ecc_requirements(&chip->base)->strength, + nanddev_get_ecc_requirements(&chip->base)->step_size); return -EINVAL; } geo->ecc_chunk_size = ecc_step; @@ -510,6 +510,8 @@ static int legacy_set_geometry(struct gpmi_nand_data *this) static int common_nfc_set_geometry(struct gpmi_nand_data *this) { struct nand_chip *chip = &this->nand; + const struct nand_ecc_props *requirements = + nanddev_get_ecc_requirements(&chip->base); if (chip->ecc.strength > 0 && chip->ecc.size > 0) return set_geometry_by_ecc_info(this, chip->ecc.strength, @@ -517,13 +519,12 @@ static int common_nfc_set_geometry(struct gpmi_nand_data *this) if ((of_property_read_bool(this->dev->of_node, "fsl,use-minimum-ecc")) || legacy_set_geometry(this)) { - if (!(chip->base.eccreq.strength > 0 && - chip->base.eccreq.step_size > 0)) + if (!(requirements->strength > 0 && requirements->step_size > 0)) return -EINVAL; return set_geometry_by_ecc_info(this, - chip->base.eccreq.strength, - chip->base.eccreq.step_size); + requirements->strength, + requirements->step_size); } return 0; @@ -1003,10 +1004,8 @@ static int acquire_dma_channels(struct gpmi_nand_data *this) /* request dma channel */ dma_chan = dma_request_chan(&pdev->dev, "rx-tx"); if (IS_ERR(dma_chan)) { - ret = PTR_ERR(dma_chan); - if (ret != -EPROBE_DEFER) - dev_err(this->dev, "DMA channel request failed: %d\n", - ret); + ret = dev_err_probe(this->dev, PTR_ERR(dma_chan), + "DMA channel request failed\n"); release_dma_channels(this); } else { this->dma_chans[0] = dma_chan; @@ -2032,7 +2031,7 @@ static int gpmi_init_last(struct gpmi_nand_data *this) ecc->write_page_raw = gpmi_ecc_write_page_raw; ecc->read_oob_raw = gpmi_ecc_read_oob_raw; ecc->write_oob_raw = gpmi_ecc_write_oob_raw; - ecc->mode = NAND_ECC_HW; + ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; ecc->size = bch_geo->ecc_chunk_size; ecc->strength = bch_geo->ecc_strength; mtd_set_ooblayout(mtd, &gpmi_ooblayout_ops); diff --git a/drivers/mtd/nand/raw/hisi504_nand.c b/drivers/mtd/nand/raw/hisi504_nand.c index b84238e2268a..8b2122ce6ec3 100644 --- a/drivers/mtd/nand/raw/hisi504_nand.c +++ b/drivers/mtd/nand/raw/hisi504_nand.c @@ -186,7 +186,7 @@ static void hisi_nfc_dma_transfer(struct hinfc_host *host, int todev) hinfc_write(host, host->dma_buffer, HINFC504_DMA_ADDR_DATA); hinfc_write(host, host->dma_oob, HINFC504_DMA_ADDR_OOB); - if (chip->ecc.mode == NAND_ECC_NONE) { + if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_NONE) { hinfc_write(host, ((mtd->oobsize & HINFC504_DMA_LEN_OOB_MASK) << HINFC504_DMA_LEN_OOB_SHIFT), HINFC504_DMA_LEN); @@ -468,7 +468,7 @@ static void hisi_nfc_cmdfunc(struct nand_chip *chip, unsigned command, case NAND_CMD_STATUS: flag = hinfc_read(host, HINFC504_CON); - if (chip->ecc.mode == NAND_ECC_HW) + if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) hinfc_write(host, flag & ~(HINFC504_CON_ECCTYPE_MASK << HINFC504_CON_ECCTYPE_SHIFT), HINFC504_CON); @@ -721,7 +721,7 @@ static int hisi_nfc_attach_chip(struct nand_chip *chip) } hinfc_write(host, flag, HINFC504_CON); - if (chip->ecc.mode == NAND_ECC_HW) + if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) hisi_nfc_ecc_probe(host); return 0; diff --git a/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c b/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c index 69423bb29adb..0e9d426fe4f2 100644 --- a/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c +++ b/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c @@ -194,8 +194,8 @@ static int ingenic_nand_attach_chip(struct nand_chip *chip) (chip->ecc.strength / 8); } - switch (chip->ecc.mode) { - case NAND_ECC_HW: + switch (chip->ecc.engine_type) { + case NAND_ECC_ENGINE_TYPE_ON_HOST: if (!nfc->ecc) { dev_err(nfc->dev, "HW ECC selected, but ECC controller not found\n"); return -ENODEV; @@ -205,22 +205,22 @@ static int ingenic_nand_attach_chip(struct nand_chip *chip) chip->ecc.calculate = ingenic_nand_ecc_calculate; chip->ecc.correct = ingenic_nand_ecc_correct; fallthrough; - case NAND_ECC_SOFT: + case NAND_ECC_ENGINE_TYPE_SOFT: dev_info(nfc->dev, "using %s (strength %d, size %d, bytes %d)\n", (nfc->ecc) ? "hardware ECC" : "software ECC", chip->ecc.strength, chip->ecc.size, chip->ecc.bytes); break; - case NAND_ECC_NONE: + case NAND_ECC_ENGINE_TYPE_NONE: dev_info(nfc->dev, "not using ECC\n"); break; default: dev_err(nfc->dev, "ECC mode %d not supported\n", - chip->ecc.mode); + chip->ecc.engine_type); return -EINVAL; } /* The NAND core will generate the ECC layout for SW ECC */ - if (chip->ecc.mode != NAND_ECC_HW) + if (chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST) return 0; /* Generate ECC layout. ECC codes are right aligned in the OOB area. */ @@ -243,8 +243,10 @@ static int ingenic_nand_attach_chip(struct nand_chip *chip) /* For legacy reasons we use a different layout on the qi,lb60 board. */ if (of_machine_is_compatible("qi,lb60")) mtd_set_ooblayout(mtd, &qi_lb60_ooblayout_ops); - else + else if (nfc->soc_info->oob_layout) mtd_set_ooblayout(mtd, nfc->soc_info->oob_layout); + else + mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout()); return 0; } @@ -404,7 +406,7 @@ static int ingenic_nand_init_chip(struct platform_device *pdev, mtd->dev.parent = dev; chip->options = NAND_NO_SUBPAGE_WRITE; - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; chip->controller = &nfc->controller; nand_set_flash_node(chip, np); @@ -532,7 +534,6 @@ static const struct jz_soc_info jz4740_soc_info = { .data_offset = 0x00000000, .cmd_offset = 0x00008000, .addr_offset = 0x00010000, - .oob_layout = &nand_ooblayout_lp_ops, }; static const struct jz_soc_info jz4725b_soc_info = { @@ -546,7 +547,6 @@ static const struct jz_soc_info jz4780_soc_info = { .data_offset = 0x00000000, .cmd_offset = 0x00400000, .addr_offset = 0x00800000, - .oob_layout = &nand_ooblayout_lp_ops, }; static const struct of_device_id ingenic_nand_dt_match[] = { diff --git a/drivers/mtd/nand/raw/lpc32xx_mlc.c b/drivers/mtd/nand/raw/lpc32xx_mlc.c index 7521038af2ef..4940bb2e3c07 100644 --- a/drivers/mtd/nand/raw/lpc32xx_mlc.c +++ b/drivers/mtd/nand/raw/lpc32xx_mlc.c @@ -656,7 +656,7 @@ static int lpc32xx_nand_attach_chip(struct nand_chip *chip) if (!host->dummy_buf) return -ENOMEM; - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; chip->ecc.size = 512; mtd_set_ooblayout(mtd, &lpc32xx_ooblayout_ops); host->mlcsubpages = mtd->writesize / 512; diff --git a/drivers/mtd/nand/raw/lpc32xx_slc.c b/drivers/mtd/nand/raw/lpc32xx_slc.c index b151fd000815..6db9d2ed6881 100644 --- a/drivers/mtd/nand/raw/lpc32xx_slc.c +++ b/drivers/mtd/nand/raw/lpc32xx_slc.c @@ -881,7 +881,8 @@ static int lpc32xx_nand_probe(struct platform_device *pdev) platform_set_drvdata(pdev, host); /* NAND callbacks for LPC32xx SLC hardware */ - chip->ecc.mode = NAND_ECC_HW_SYNDROME; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; + chip->ecc.placement = NAND_ECC_PLACEMENT_INTERLEAVED; chip->legacy.read_byte = lpc32xx_nand_read_byte; chip->legacy.read_buf = lpc32xx_nand_read_buf; chip->legacy.write_buf = lpc32xx_nand_write_buf; diff --git a/drivers/mtd/nand/raw/marvell_nand.c b/drivers/mtd/nand/raw/marvell_nand.c index 8482d3bd8b1f..f5ca2002d08e 100644 --- a/drivers/mtd/nand/raw/marvell_nand.c +++ b/drivers/mtd/nand/raw/marvell_nand.c @@ -227,6 +227,8 @@ #define XTYPE_MASK 7 /** + * struct marvell_hw_ecc_layout - layout of Marvell ECC + * * Marvell ECC engine works differently than the others, in order to limit the * size of the IP, hardware engineers chose to set a fixed strength at 16 bits * per subpage, and depending on a the desired strength needed by the NAND chip, @@ -292,6 +294,8 @@ static const struct marvell_hw_ecc_layout marvell_nfc_layouts[] = { }; /** + * struct marvell_nand_chip_sel - CS line description + * * The Nand Flash Controller has up to 4 CE and 2 RB pins. The CE selection * is made by a field in NDCB0 register, and in another field in NDCB2 register. * The datasheet describes the logic with an error: ADDR5 field is once @@ -312,14 +316,15 @@ struct marvell_nand_chip_sel { }; /** - * NAND chip structure: stores NAND chip device related information + * struct marvell_nand_chip - stores NAND chip device related information * * @chip: Base NAND chip structure * @node: Used to store NAND chips into a list - * @layout NAND layout when using hardware ECC + * @layout: NAND layout when using hardware ECC * @ndcr: Controller register value for this NAND chip * @ndtr0: Timing registers 0 value for this NAND chip * @ndtr1: Timing registers 1 value for this NAND chip + * @addr_cyc: Amount of cycles needed to pass column address * @selected_die: Current active CS * @nsels: Number of CS lines required by the NAND chip * @sels: Array of CS lines descriptions @@ -349,7 +354,8 @@ static inline struct marvell_nand_chip_sel *to_nand_sel(struct marvell_nand_chip } /** - * NAND controller capabilities for distinction between compatible strings + * struct marvell_nfc_caps - NAND controller capabilities for distinction + * between compatible strings * * @max_cs_nb: Number of Chip Select lines available * @max_rb_nb: Number of Ready/Busy lines available @@ -372,7 +378,7 @@ struct marvell_nfc_caps { }; /** - * NAND controller structure: stores Marvell NAND controller information + * struct marvell_nfc - stores Marvell NAND controller information * * @controller: Base controller structure * @dev: Parent device (used to print error messages) @@ -383,7 +389,9 @@ struct marvell_nfc_caps { * @assigned_cs: Bitmask describing already assigned CS lines * @chips: List containing all the NAND chips attached to * this NAND controller + * @selected_chip: Currently selected target chip * @caps: NAND controller capabilities for each compatible string + * @use_dma: Whetner DMA is used * @dma_chan: DMA channel (NFCv1 only) * @dma_buf: 32-bit aligned buffer for DMA transfers (NFCv1 only) */ @@ -411,7 +419,8 @@ static inline struct marvell_nfc *to_marvell_nfc(struct nand_controller *ctrl) } /** - * NAND controller timings expressed in NAND Controller clock cycles + * struct marvell_nfc_timings - NAND controller timings expressed in NAND + * Controller clock cycles * * @tRP: ND_nRE pulse width * @tRH: ND_nRE high duration @@ -455,8 +464,8 @@ struct marvell_nfc_timings { period_ns)) /** - * NAND driver structure filled during the parsing of the ->exec_op() subop - * subset of instructions. + * struct marvell_nfc_op - filled during the parsing of the ->exec_op() + * subop subset of instructions. * * @ndcb: Array of values written to NDCBx registers * @cle_ale_delay_ns: Optional delay after the last CMD or ADDR cycle @@ -685,9 +694,31 @@ static int marvell_nfc_wait_cmdd(struct nand_chip *chip) return marvell_nfc_end_cmd(chip, cs_flag, "CMDD"); } +static int marvell_nfc_poll_status(struct marvell_nfc *nfc, u32 mask, + u32 expected_val, unsigned long timeout_ms) +{ + unsigned long limit; + u32 st; + + limit = jiffies + msecs_to_jiffies(timeout_ms); + do { + st = readl_relaxed(nfc->regs + NDSR); + if (st & NDSR_RDY(1)) + st |= NDSR_RDY(0); + + if ((st & mask) == expected_val) + return 0; + + cpu_relax(); + } while (time_after(limit, jiffies)); + + return -ETIMEDOUT; +} + static int marvell_nfc_wait_op(struct nand_chip *chip, unsigned int timeout_ms) { struct marvell_nfc *nfc = to_marvell_nfc(chip->controller); + struct mtd_info *mtd = nand_to_mtd(chip); u32 pending; int ret; @@ -695,12 +726,18 @@ static int marvell_nfc_wait_op(struct nand_chip *chip, unsigned int timeout_ms) if (!timeout_ms) timeout_ms = IRQ_TIMEOUT; - init_completion(&nfc->complete); + if (mtd->oops_panic_write) { + ret = marvell_nfc_poll_status(nfc, NDSR_RDY(0), + NDSR_RDY(0), + timeout_ms); + } else { + init_completion(&nfc->complete); - marvell_nfc_enable_int(nfc, NDCR_RDYM); - ret = wait_for_completion_timeout(&nfc->complete, - msecs_to_jiffies(timeout_ms)); - marvell_nfc_disable_int(nfc, NDCR_RDYM); + marvell_nfc_enable_int(nfc, NDCR_RDYM); + ret = wait_for_completion_timeout(&nfc->complete, + msecs_to_jiffies(timeout_ms)); + marvell_nfc_disable_int(nfc, NDCR_RDYM); + } pending = marvell_nfc_clear_int(nfc, NDSR_RDY(0) | NDSR_RDY(1)); /* @@ -780,7 +817,7 @@ static void marvell_nfc_enable_hw_ecc(struct nand_chip *chip) * When enabling BCH, set threshold to 0 to always know the * number of corrected bitflips. */ - if (chip->ecc.algo == NAND_ECC_BCH) + if (chip->ecc.algo == NAND_ECC_ALGO_BCH) writel_relaxed(NDECCCTRL_BCH_EN, nfc->regs + NDECCCTRL); } } @@ -792,7 +829,7 @@ static void marvell_nfc_disable_hw_ecc(struct nand_chip *chip) if (ndcr & NDCR_ECC_EN) { writel_relaxed(ndcr & ~NDCR_ECC_EN, nfc->regs + NDCR); - if (chip->ecc.algo == NAND_ECC_BCH) + if (chip->ecc.algo == NAND_ECC_ALGO_BCH) writel_relaxed(0, nfc->regs + NDECCCTRL); } } @@ -966,7 +1003,7 @@ static int marvell_nfc_hw_ecc_check_bitflips(struct nand_chip *chip, if (ndsr & NDSR_CORERR) { writel_relaxed(ndsr, nfc->regs + NDSR); - if (chip->ecc.algo == NAND_ECC_BCH) + if (chip->ecc.algo == NAND_ECC_ALGO_BCH) bf = NDSR_ERRCNT(ndsr); else bf = 1; @@ -2218,7 +2255,7 @@ static int marvell_nand_hw_ecc_controller_init(struct mtd_info *mtd, ecc->size = l->data_bytes; if (ecc->strength == 1) { - chip->ecc.algo = NAND_ECC_HAMMING; + chip->ecc.algo = NAND_ECC_ALGO_HAMMING; ecc->read_page_raw = marvell_nfc_hw_ecc_hmg_read_page_raw; ecc->read_page = marvell_nfc_hw_ecc_hmg_read_page; ecc->read_oob_raw = marvell_nfc_hw_ecc_hmg_read_oob_raw; @@ -2228,7 +2265,7 @@ static int marvell_nand_hw_ecc_controller_init(struct mtd_info *mtd, ecc->write_oob_raw = marvell_nfc_hw_ecc_hmg_write_oob_raw; ecc->write_oob = ecc->write_oob_raw; } else { - chip->ecc.algo = NAND_ECC_BCH; + chip->ecc.algo = NAND_ECC_ALGO_BCH; ecc->strength = 16; ecc->read_page_raw = marvell_nfc_hw_ecc_bch_read_page_raw; ecc->read_page = marvell_nfc_hw_ecc_bch_read_page; @@ -2247,13 +2284,16 @@ static int marvell_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc) { struct nand_chip *chip = mtd_to_nand(mtd); + const struct nand_ecc_props *requirements = + nanddev_get_ecc_requirements(&chip->base); struct marvell_nfc *nfc = to_marvell_nfc(chip->controller); int ret; - if (ecc->mode != NAND_ECC_NONE && (!ecc->size || !ecc->strength)) { - if (chip->base.eccreq.step_size && chip->base.eccreq.strength) { - ecc->size = chip->base.eccreq.step_size; - ecc->strength = chip->base.eccreq.strength; + if (ecc->engine_type != NAND_ECC_ENGINE_TYPE_NONE && + (!ecc->size || !ecc->strength)) { + if (requirements->step_size && requirements->strength) { + ecc->size = requirements->step_size; + ecc->strength = requirements->strength; } else { dev_info(nfc->dev, "No minimum ECC strength, using 1b/512B\n"); @@ -2262,15 +2302,15 @@ static int marvell_nand_ecc_init(struct mtd_info *mtd, } } - switch (ecc->mode) { - case NAND_ECC_HW: + switch (ecc->engine_type) { + case NAND_ECC_ENGINE_TYPE_ON_HOST: ret = marvell_nand_hw_ecc_controller_init(mtd, ecc); if (ret) return ret; break; - case NAND_ECC_NONE: - case NAND_ECC_SOFT: - case NAND_ECC_ON_DIE: + case NAND_ECC_ENGINE_TYPE_NONE: + case NAND_ECC_ENGINE_TYPE_SOFT: + case NAND_ECC_ENGINE_TYPE_ON_DIE: if (!nfc->caps->is_nfcv2 && mtd->writesize != SZ_512 && mtd->writesize != SZ_2K) { dev_err(nfc->dev, "NFCv1 cannot write %d bytes pages\n", @@ -2467,7 +2507,7 @@ static int marvell_nand_attach_chip(struct nand_chip *chip) return ret; } - if (chip->ecc.mode == NAND_ECC_HW) { + if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) { /* * Subpage write not available with hardware ECC, prohibit also * subpage read as in userspace subpage access would still be @@ -2642,7 +2682,7 @@ static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc, * Default to HW ECC engine mode. If the nand-ecc-mode property is given * in the DT node, this entry will be overwritten in nand_scan_ident(). */ - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; /* * Save a reference value for timing registers before @@ -2759,10 +2799,7 @@ static int marvell_nfc_init_dma(struct marvell_nfc *nfc) if (IS_ERR(nfc->dma_chan)) { ret = PTR_ERR(nfc->dma_chan); nfc->dma_chan = NULL; - if (ret != -EPROBE_DEFER) - dev_err(nfc->dev, "DMA channel request failed: %d\n", - ret); - return ret; + return dev_err_probe(nfc->dev, ret, "DMA channel request failed\n"); } r = platform_get_resource(pdev, IORESOURCE_MEM, 0); diff --git a/drivers/mtd/nand/raw/meson_nand.c b/drivers/mtd/nand/raw/meson_nand.c index 0e5829a2b54f..48e6dac96be6 100644 --- a/drivers/mtd/nand/raw/meson_nand.c +++ b/drivers/mtd/nand/raw/meson_nand.c @@ -1197,7 +1197,7 @@ static int meson_nand_attach_chip(struct nand_chip *nand) if (ret) return -EINVAL; - nand->ecc.mode = NAND_ECC_HW; + nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; nand->ecc.write_page_raw = meson_nfc_write_page_raw; nand->ecc.write_page = meson_nfc_write_page_hwecc; nand->ecc.write_oob_raw = nand_write_oob_std; diff --git a/drivers/mtd/nand/raw/mpc5121_nfc.c b/drivers/mtd/nand/raw/mpc5121_nfc.c index 18ecb096a32d..dfd0d3ed5ed0 100644 --- a/drivers/mtd/nand/raw/mpc5121_nfc.c +++ b/drivers/mtd/nand/raw/mpc5121_nfc.c @@ -688,8 +688,8 @@ static int mpc5121_nfc_probe(struct platform_device *op) chip->legacy.set_features = nand_get_set_features_notsupp; chip->legacy.get_features = nand_get_set_features_notsupp; chip->bbt_options = NAND_BBT_USE_FLASH; - chip->ecc.mode = NAND_ECC_SOFT; - chip->ecc.algo = NAND_ECC_HAMMING; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + chip->ecc.algo = NAND_ECC_ALGO_HAMMING; /* Support external chip-select logic on ADS5121 board */ if (of_machine_is_compatible("fsl,mpc5121ads")) { diff --git a/drivers/mtd/nand/raw/mtk_nand.c b/drivers/mtd/nand/raw/mtk_nand.c index ad1b55dab211..57f1f1708994 100644 --- a/drivers/mtd/nand/raw/mtk_nand.c +++ b/drivers/mtd/nand/raw/mtk_nand.c @@ -1253,21 +1253,23 @@ static int mtk_nfc_set_spare_per_sector(u32 *sps, struct mtd_info *mtd) static int mtk_nfc_ecc_init(struct device *dev, struct mtd_info *mtd) { struct nand_chip *nand = mtd_to_nand(mtd); + const struct nand_ecc_props *requirements = + nanddev_get_ecc_requirements(&nand->base); struct mtk_nfc *nfc = nand_get_controller_data(nand); u32 spare; int free, ret; /* support only ecc hw mode */ - if (nand->ecc.mode != NAND_ECC_HW) { - dev_err(dev, "ecc.mode not supported\n"); + if (nand->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST) { + dev_err(dev, "ecc.engine_type not supported\n"); return -EINVAL; } /* if optional dt settings not present */ if (!nand->ecc.size || !nand->ecc.strength) { /* use datasheet requirements */ - nand->ecc.strength = nand->base.eccreq.strength; - nand->ecc.size = nand->base.eccreq.step_size; + nand->ecc.strength = requirements->strength; + nand->ecc.size = requirements->step_size; /* * align eccstrength and eccsize @@ -1416,7 +1418,7 @@ static int mtk_nfc_nand_chip_init(struct device *dev, struct mtk_nfc *nfc, nand->options |= NAND_USES_DMA | NAND_SUBPAGE_READ; /* set default mode in case dt entry is missing */ - nand->ecc.mode = NAND_ECC_HW; + nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; nand->ecc.write_subpage = mtk_nfc_write_subpage_hwecc; nand->ecc.write_page_raw = mtk_nfc_write_page_raw; diff --git a/drivers/mtd/nand/raw/mxc_nand.c b/drivers/mtd/nand/raw/mxc_nand.c index a043d76b48cb..d4200eb2ad32 100644 --- a/drivers/mtd/nand/raw/mxc_nand.c +++ b/drivers/mtd/nand/raw/mxc_nand.c @@ -669,7 +669,7 @@ static void mxc_nand_enable_hwecc_v1_v2(struct nand_chip *chip, bool enable) struct mxc_nand_host *host = nand_get_controller_data(chip); uint16_t config1; - if (chip->ecc.mode != NAND_ECC_HW) + if (chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST) return; config1 = readw(NFC_V1_V2_CONFIG1); @@ -687,7 +687,7 @@ static void mxc_nand_enable_hwecc_v3(struct nand_chip *chip, bool enable) struct mxc_nand_host *host = nand_get_controller_data(chip); uint32_t config2; - if (chip->ecc.mode != NAND_ECC_HW) + if (chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST) return; config2 = readl(NFC_V3_CONFIG2); @@ -1117,7 +1117,8 @@ static void preset_v1(struct mtd_info *mtd) struct mxc_nand_host *host = nand_get_controller_data(nand_chip); uint16_t config1 = 0; - if (nand_chip->ecc.mode == NAND_ECC_HW && mtd->writesize) + if (nand_chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST && + mtd->writesize) config1 |= NFC_V1_V2_CONFIG1_ECC_EN; if (!host->devtype_data->irqpending_quirk) @@ -1227,7 +1228,7 @@ static void preset_v2(struct mtd_info *mtd) if (mtd->writesize) { uint16_t pages_per_block = mtd->erasesize / mtd->writesize; - if (nand_chip->ecc.mode == NAND_ECC_HW) + if (nand_chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) config1 |= NFC_V1_V2_CONFIG1_ECC_EN; host->eccsize = get_eccsize(mtd); @@ -1303,7 +1304,7 @@ static void preset_v3(struct mtd_info *mtd) } if (mtd->writesize) { - if (chip->ecc.mode == NAND_ECC_HW) + if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) config2 |= NFC_V3_CONFIG2_ECC_EN; config2 |= NFC_V3_CONFIG2_PPB( @@ -1680,8 +1681,8 @@ static int mxcnd_attach_chip(struct nand_chip *chip) struct mxc_nand_host *host = nand_get_controller_data(chip); struct device *dev = mtd->dev.parent; - switch (chip->ecc.mode) { - case NAND_ECC_HW: + switch (chip->ecc.engine_type) { + case NAND_ECC_ENGINE_TYPE_ON_HOST: chip->ecc.read_page = mxc_nand_read_page; chip->ecc.read_page_raw = mxc_nand_read_page_raw; chip->ecc.read_oob = mxc_nand_read_oob; @@ -1690,7 +1691,7 @@ static int mxcnd_attach_chip(struct nand_chip *chip) chip->ecc.write_oob = mxc_nand_write_oob; break; - case NAND_ECC_SOFT: + case NAND_ECC_ENGINE_TYPE_SOFT: break; default: @@ -1728,7 +1729,7 @@ static int mxcnd_attach_chip(struct nand_chip *chip) */ host->used_oobsize = min(mtd->oobsize, 218U); - if (chip->ecc.mode == NAND_ECC_HW) { + if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) { if (is_imx21_nfc(host) || is_imx27_nfc(host)) chip->ecc.strength = 1; else @@ -1843,10 +1844,10 @@ static int mxcnd_probe(struct platform_device *pdev) mtd_set_ooblayout(mtd, host->devtype_data->ooblayout); if (host->pdata.hw_ecc) { - this->ecc.mode = NAND_ECC_HW; + this->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; } else { - this->ecc.mode = NAND_ECC_SOFT; - this->ecc.algo = NAND_ECC_HAMMING; + this->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + this->ecc.algo = NAND_ECC_ALGO_HAMMING; } /* NAND bus width determines access functions used by upper layer */ diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c index 0c768cb88f96..1f0d542d5923 100644 --- a/drivers/mtd/nand/raw/nand_base.c +++ b/drivers/mtd/nand/raw/nand_base.c @@ -34,6 +34,7 @@ #include <linux/mm.h> #include <linux/types.h> #include <linux/mtd/mtd.h> +#include <linux/mtd/nand.h> #include <linux/mtd/nand_ecc.h> #include <linux/mtd/nand_bch.h> #include <linux/interrupt.h> @@ -45,166 +46,6 @@ #include "internals.h" -/* Define default oob placement schemes for large and small page devices */ -static int nand_ooblayout_ecc_sp(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - struct nand_ecc_ctrl *ecc = &chip->ecc; - - if (section > 1) - return -ERANGE; - - if (!section) { - oobregion->offset = 0; - if (mtd->oobsize == 16) - oobregion->length = 4; - else - oobregion->length = 3; - } else { - if (mtd->oobsize == 8) - return -ERANGE; - - oobregion->offset = 6; - oobregion->length = ecc->total - 4; - } - - return 0; -} - -static int nand_ooblayout_free_sp(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) -{ - if (section > 1) - return -ERANGE; - - if (mtd->oobsize == 16) { - if (section) - return -ERANGE; - - oobregion->length = 8; - oobregion->offset = 8; - } else { - oobregion->length = 2; - if (!section) - oobregion->offset = 3; - else - oobregion->offset = 6; - } - - return 0; -} - -const struct mtd_ooblayout_ops nand_ooblayout_sp_ops = { - .ecc = nand_ooblayout_ecc_sp, - .free = nand_ooblayout_free_sp, -}; -EXPORT_SYMBOL_GPL(nand_ooblayout_sp_ops); - -static int nand_ooblayout_ecc_lp(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - struct nand_ecc_ctrl *ecc = &chip->ecc; - - if (section || !ecc->total) - return -ERANGE; - - oobregion->length = ecc->total; - oobregion->offset = mtd->oobsize - oobregion->length; - - return 0; -} - -static int nand_ooblayout_free_lp(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - struct nand_ecc_ctrl *ecc = &chip->ecc; - - if (section) - return -ERANGE; - - oobregion->length = mtd->oobsize - ecc->total - 2; - oobregion->offset = 2; - - return 0; -} - -const struct mtd_ooblayout_ops nand_ooblayout_lp_ops = { - .ecc = nand_ooblayout_ecc_lp, - .free = nand_ooblayout_free_lp, -}; -EXPORT_SYMBOL_GPL(nand_ooblayout_lp_ops); - -/* - * Support the old "large page" layout used for 1-bit Hamming ECC where ECC - * are placed at a fixed offset. - */ -static int nand_ooblayout_ecc_lp_hamming(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - struct nand_ecc_ctrl *ecc = &chip->ecc; - - if (section) - return -ERANGE; - - switch (mtd->oobsize) { - case 64: - oobregion->offset = 40; - break; - case 128: - oobregion->offset = 80; - break; - default: - return -EINVAL; - } - - oobregion->length = ecc->total; - if (oobregion->offset + oobregion->length > mtd->oobsize) - return -ERANGE; - - return 0; -} - -static int nand_ooblayout_free_lp_hamming(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - struct nand_ecc_ctrl *ecc = &chip->ecc; - int ecc_offset = 0; - - if (section < 0 || section > 1) - return -ERANGE; - - switch (mtd->oobsize) { - case 64: - ecc_offset = 40; - break; - case 128: - ecc_offset = 80; - break; - default: - return -EINVAL; - } - - if (section == 0) { - oobregion->offset = 2; - oobregion->length = ecc_offset - 2; - } else { - oobregion->offset = ecc_offset + ecc->total; - oobregion->length = mtd->oobsize - oobregion->offset; - } - - return 0; -} - -static const struct mtd_ooblayout_ops nand_ooblayout_lp_hamming_ops = { - .ecc = nand_ooblayout_ecc_lp_hamming, - .free = nand_ooblayout_free_lp_hamming, -}; - static int nand_pairing_dist3_get_info(struct mtd_info *mtd, int page, struct mtd_pairing_info *info) { @@ -4750,6 +4591,8 @@ static inline bool is_full_id_nand(struct nand_flash_dev *type) static bool find_full_id_nand(struct nand_chip *chip, struct nand_flash_dev *type) { + struct nand_device *base = &chip->base; + struct nand_ecc_props requirements; struct mtd_info *mtd = nand_to_mtd(chip); struct nand_memory_organization *memorg; u8 *id_data = chip->id.data; @@ -4771,8 +4614,9 @@ static bool find_full_id_nand(struct nand_chip *chip, memorg->pagesize * memorg->pages_per_eraseblock); chip->options |= type->options; - chip->base.eccreq.strength = NAND_ECC_STRENGTH(type); - chip->base.eccreq.step_size = NAND_ECC_STEP(type); + requirements.strength = NAND_ECC_STRENGTH(type); + requirements.step_size = NAND_ECC_STEP(type); + nanddev_set_ecc_requirements(base, &requirements); chip->parameters.model = kstrdup(type->name, GFP_KERNEL); if (!chip->parameters.model) @@ -5033,91 +4877,101 @@ free_detect_allocation: return ret; } -static const char * const nand_ecc_modes[] = { - [NAND_ECC_NONE] = "none", - [NAND_ECC_SOFT] = "soft", - [NAND_ECC_HW] = "hw", - [NAND_ECC_HW_SYNDROME] = "hw_syndrome", - [NAND_ECC_ON_DIE] = "on-die", -}; - -static int of_get_nand_ecc_mode(struct device_node *np) +static enum nand_ecc_engine_type +of_get_rawnand_ecc_engine_type_legacy(struct device_node *np) { + enum nand_ecc_legacy_mode { + NAND_ECC_INVALID, + NAND_ECC_NONE, + NAND_ECC_SOFT, + NAND_ECC_SOFT_BCH, + NAND_ECC_HW, + NAND_ECC_HW_SYNDROME, + NAND_ECC_ON_DIE, + }; + const char * const nand_ecc_legacy_modes[] = { + [NAND_ECC_NONE] = "none", + [NAND_ECC_SOFT] = "soft", + [NAND_ECC_SOFT_BCH] = "soft_bch", + [NAND_ECC_HW] = "hw", + [NAND_ECC_HW_SYNDROME] = "hw_syndrome", + [NAND_ECC_ON_DIE] = "on-die", + }; + enum nand_ecc_legacy_mode eng_type; const char *pm; - int err, i; + int err; err = of_property_read_string(np, "nand-ecc-mode", &pm); - if (err < 0) - return err; - - for (i = NAND_ECC_NONE; i < ARRAY_SIZE(nand_ecc_modes); i++) - if (!strcasecmp(pm, nand_ecc_modes[i])) - return i; - - /* - * For backward compatibility we support few obsoleted values that don't - * have their mappings into the nand_ecc_mode enum anymore (they were - * merged with other enums). - */ - if (!strcasecmp(pm, "soft_bch")) - return NAND_ECC_SOFT; + if (err) + return NAND_ECC_ENGINE_TYPE_INVALID; + + for (eng_type = NAND_ECC_NONE; + eng_type < ARRAY_SIZE(nand_ecc_legacy_modes); eng_type++) { + if (!strcasecmp(pm, nand_ecc_legacy_modes[eng_type])) { + switch (eng_type) { + case NAND_ECC_NONE: + return NAND_ECC_ENGINE_TYPE_NONE; + case NAND_ECC_SOFT: + case NAND_ECC_SOFT_BCH: + return NAND_ECC_ENGINE_TYPE_SOFT; + case NAND_ECC_HW: + case NAND_ECC_HW_SYNDROME: + return NAND_ECC_ENGINE_TYPE_ON_HOST; + case NAND_ECC_ON_DIE: + return NAND_ECC_ENGINE_TYPE_ON_DIE; + default: + break; + } + } + } - return -ENODEV; + return NAND_ECC_ENGINE_TYPE_INVALID; } -static const char * const nand_ecc_algos[] = { - [NAND_ECC_HAMMING] = "hamming", - [NAND_ECC_BCH] = "bch", - [NAND_ECC_RS] = "rs", -}; - -static enum nand_ecc_algo of_get_nand_ecc_algo(struct device_node *np) +static enum nand_ecc_placement +of_get_rawnand_ecc_placement_legacy(struct device_node *np) { - enum nand_ecc_algo ecc_algo; const char *pm; int err; - err = of_property_read_string(np, "nand-ecc-algo", &pm); + err = of_property_read_string(np, "nand-ecc-mode", &pm); if (!err) { - for (ecc_algo = NAND_ECC_HAMMING; - ecc_algo < ARRAY_SIZE(nand_ecc_algos); - ecc_algo++) { - if (!strcasecmp(pm, nand_ecc_algos[ecc_algo])) - return ecc_algo; - } + if (!strcasecmp(pm, "hw_syndrome")) + return NAND_ECC_PLACEMENT_INTERLEAVED; } - /* - * For backward compatibility we also read "nand-ecc-mode" checking - * for some obsoleted values that were specifying ECC algorithm. - */ + return NAND_ECC_PLACEMENT_UNKNOWN; +} + +static enum nand_ecc_algo of_get_rawnand_ecc_algo_legacy(struct device_node *np) +{ + const char *pm; + int err; + err = of_property_read_string(np, "nand-ecc-mode", &pm); if (!err) { if (!strcasecmp(pm, "soft")) - return NAND_ECC_HAMMING; + return NAND_ECC_ALGO_HAMMING; else if (!strcasecmp(pm, "soft_bch")) - return NAND_ECC_BCH; + return NAND_ECC_ALGO_BCH; } - return NAND_ECC_UNKNOWN; + return NAND_ECC_ALGO_UNKNOWN; } -static int of_get_nand_ecc_step_size(struct device_node *np) +static void of_get_nand_ecc_legacy_user_config(struct nand_chip *chip) { - int ret; - u32 val; + struct device_node *dn = nand_get_flash_node(chip); + struct nand_ecc_props *user_conf = &chip->base.ecc.user_conf; - ret = of_property_read_u32(np, "nand-ecc-step-size", &val); - return ret ? ret : val; -} + if (user_conf->engine_type == NAND_ECC_ENGINE_TYPE_INVALID) + user_conf->engine_type = of_get_rawnand_ecc_engine_type_legacy(dn); -static int of_get_nand_ecc_strength(struct device_node *np) -{ - int ret; - u32 val; + if (user_conf->algo == NAND_ECC_ALGO_UNKNOWN) + user_conf->algo = of_get_rawnand_ecc_algo_legacy(dn); - ret = of_property_read_u32(np, "nand-ecc-strength", &val); - return ret ? ret : val; + if (user_conf->placement == NAND_ECC_PLACEMENT_UNKNOWN) + user_conf->placement = of_get_rawnand_ecc_placement_legacy(dn); } static int of_get_nand_bus_width(struct device_node *np) @@ -5141,11 +4995,10 @@ static bool of_get_nand_on_flash_bbt(struct device_node *np) return of_property_read_bool(np, "nand-on-flash-bbt"); } -static int nand_dt_init(struct nand_chip *chip) +static int rawnand_dt_init(struct nand_chip *chip) { + struct nand_device *nand = mtd_to_nanddev(nand_to_mtd(chip)); struct device_node *dn = nand_get_flash_node(chip); - enum nand_ecc_algo ecc_algo; - int ecc_mode, ecc_strength, ecc_step; if (!dn) return 0; @@ -5159,25 +5012,29 @@ static int nand_dt_init(struct nand_chip *chip) if (of_get_nand_on_flash_bbt(dn)) chip->bbt_options |= NAND_BBT_USE_FLASH; - ecc_mode = of_get_nand_ecc_mode(dn); - ecc_algo = of_get_nand_ecc_algo(dn); - ecc_strength = of_get_nand_ecc_strength(dn); - ecc_step = of_get_nand_ecc_step_size(dn); - - if (ecc_mode >= 0) - chip->ecc.mode = ecc_mode; + of_get_nand_ecc_user_config(nand); + of_get_nand_ecc_legacy_user_config(chip); - if (ecc_algo != NAND_ECC_UNKNOWN) - chip->ecc.algo = ecc_algo; - - if (ecc_strength >= 0) - chip->ecc.strength = ecc_strength; + /* + * If neither the user nor the NAND controller have requested a specific + * ECC engine type, we will default to NAND_ECC_ENGINE_TYPE_ON_HOST. + */ + nand->ecc.defaults.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; - if (ecc_step > 0) - chip->ecc.size = ecc_step; + /* + * Use the user requested engine type, unless there is none, in this + * case default to the NAND controller choice, otherwise fallback to + * the raw NAND default one. + */ + if (nand->ecc.user_conf.engine_type != NAND_ECC_ENGINE_TYPE_INVALID) + chip->ecc.engine_type = nand->ecc.user_conf.engine_type; + if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_INVALID) + chip->ecc.engine_type = nand->ecc.defaults.engine_type; - if (of_property_read_bool(dn, "nand-ecc-maximize")) - chip->ecc.options |= NAND_ECC_MAXIMIZE; + chip->ecc.placement = nand->ecc.user_conf.placement; + chip->ecc.algo = nand->ecc.user_conf.algo; + chip->ecc.strength = nand->ecc.user_conf.strength; + chip->ecc.size = nand->ecc.user_conf.step_size; return 0; } @@ -5215,7 +5072,7 @@ static int nand_scan_ident(struct nand_chip *chip, unsigned int maxchips, /* Enforce the right timings for reset/detection */ chip->current_interface_config = nand_get_reset_interface_config(); - ret = nand_dt_init(chip); + ret = rawnand_dt_init(chip); if (ret) return ret; @@ -5282,16 +5139,76 @@ static void nand_scan_ident_cleanup(struct nand_chip *chip) kfree(chip->parameters.onfi); } +static int nand_set_ecc_on_host_ops(struct nand_chip *chip) +{ + struct nand_ecc_ctrl *ecc = &chip->ecc; + + switch (ecc->placement) { + case NAND_ECC_PLACEMENT_UNKNOWN: + case NAND_ECC_PLACEMENT_OOB: + /* Use standard hwecc read page function? */ + if (!ecc->read_page) + ecc->read_page = nand_read_page_hwecc; + if (!ecc->write_page) + ecc->write_page = nand_write_page_hwecc; + if (!ecc->read_page_raw) + ecc->read_page_raw = nand_read_page_raw; + if (!ecc->write_page_raw) + ecc->write_page_raw = nand_write_page_raw; + if (!ecc->read_oob) + ecc->read_oob = nand_read_oob_std; + if (!ecc->write_oob) + ecc->write_oob = nand_write_oob_std; + if (!ecc->read_subpage) + ecc->read_subpage = nand_read_subpage; + if (!ecc->write_subpage && ecc->hwctl && ecc->calculate) + ecc->write_subpage = nand_write_subpage_hwecc; + fallthrough; + + case NAND_ECC_PLACEMENT_INTERLEAVED: + if ((!ecc->calculate || !ecc->correct || !ecc->hwctl) && + (!ecc->read_page || + ecc->read_page == nand_read_page_hwecc || + !ecc->write_page || + ecc->write_page == nand_write_page_hwecc)) { + WARN(1, "No ECC functions supplied; hardware ECC not possible\n"); + return -EINVAL; + } + /* Use standard syndrome read/write page function? */ + if (!ecc->read_page) + ecc->read_page = nand_read_page_syndrome; + if (!ecc->write_page) + ecc->write_page = nand_write_page_syndrome; + if (!ecc->read_page_raw) + ecc->read_page_raw = nand_read_page_raw_syndrome; + if (!ecc->write_page_raw) + ecc->write_page_raw = nand_write_page_raw_syndrome; + if (!ecc->read_oob) + ecc->read_oob = nand_read_oob_syndrome; + if (!ecc->write_oob) + ecc->write_oob = nand_write_oob_syndrome; + break; + + default: + pr_warn("Invalid NAND_ECC_PLACEMENT %d\n", + ecc->placement); + return -EINVAL; + } + + return 0; +} + static int nand_set_ecc_soft_ops(struct nand_chip *chip) { struct mtd_info *mtd = nand_to_mtd(chip); + struct nand_device *nanddev = mtd_to_nanddev(mtd); struct nand_ecc_ctrl *ecc = &chip->ecc; - if (WARN_ON(ecc->mode != NAND_ECC_SOFT)) + if (WARN_ON(ecc->engine_type != NAND_ECC_ENGINE_TYPE_SOFT)) return -EINVAL; switch (ecc->algo) { - case NAND_ECC_HAMMING: + case NAND_ECC_ALGO_HAMMING: ecc->calculate = nand_calculate_ecc; ecc->correct = nand_correct_data; ecc->read_page = nand_read_page_swecc; @@ -5312,7 +5229,7 @@ static int nand_set_ecc_soft_ops(struct nand_chip *chip) ecc->options |= NAND_ECC_SOFT_HAMMING_SM_ORDER; return 0; - case NAND_ECC_BCH: + case NAND_ECC_ALGO_BCH: if (!mtd_nand_has_bch()) { WARN(1, "CONFIG_MTD_NAND_ECC_SW_BCH not enabled\n"); return -EINVAL; @@ -5350,7 +5267,7 @@ static int nand_set_ecc_soft_ops(struct nand_chip *chip) return -EINVAL; } - mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops); + mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout()); } @@ -5359,8 +5276,8 @@ static int nand_set_ecc_soft_ops(struct nand_chip *chip) * used, otherwise we don't know how many bytes can really be * used. */ - if (mtd->ooblayout == &nand_ooblayout_lp_ops && - ecc->options & NAND_ECC_MAXIMIZE) { + if (mtd->ooblayout == nand_get_large_page_ooblayout() && + nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) { int steps, bytes; /* Always prefer 1k blocks over 512bytes ones */ @@ -5454,10 +5371,12 @@ static int nand_match_ecc_req(struct nand_chip *chip, const struct nand_ecc_caps *caps, int oobavail) { + const struct nand_ecc_props *requirements = + nanddev_get_ecc_requirements(&chip->base); struct mtd_info *mtd = nand_to_mtd(chip); const struct nand_ecc_step_info *stepinfo; - int req_step = chip->base.eccreq.step_size; - int req_strength = chip->base.eccreq.strength; + int req_step = requirements->step_size; + int req_strength = requirements->strength; int req_corr, step_size, strength, nsteps, ecc_bytes, ecc_bytes_total; int best_step, best_strength, best_ecc_bytes; int best_ecc_bytes_total = INT_MAX; @@ -5598,11 +5517,12 @@ nand_maximize_ecc(struct nand_chip *chip, * @caps: ECC engine caps info structure * @oobavail: OOB size that the ECC engine can use * - * Choose the ECC configuration according to following logic + * Choose the ECC configuration according to following logic. * * 1. If both ECC step size and ECC strength are already set (usually by DT) * then check if it is supported by this controller. - * 2. If NAND_ECC_MAXIMIZE is set, then select maximum ECC strength. + * 2. If the user provided the nand-ecc-maximize property, then select maximum + * ECC strength. * 3. Otherwise, try to match the ECC step size and ECC strength closest * to the chip's requirement. If available OOB size can't fit the chip * requirement then fallback to the maximum ECC step size and ECC strength. @@ -5613,6 +5533,7 @@ int nand_ecc_choose_conf(struct nand_chip *chip, const struct nand_ecc_caps *caps, int oobavail) { struct mtd_info *mtd = nand_to_mtd(chip); + struct nand_device *nanddev = mtd_to_nanddev(mtd); if (WARN_ON(oobavail < 0 || oobavail > mtd->oobsize)) return -EINVAL; @@ -5620,7 +5541,7 @@ int nand_ecc_choose_conf(struct nand_chip *chip, if (chip->ecc.size && chip->ecc.strength) return nand_check_ecc_caps(chip, caps, oobavail); - if (chip->ecc.options & NAND_ECC_MAXIMIZE) + if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) return nand_maximize_ecc(chip, caps, oobavail); if (!nand_match_ecc_req(chip, caps, oobavail)) @@ -5630,41 +5551,6 @@ int nand_ecc_choose_conf(struct nand_chip *chip, } EXPORT_SYMBOL_GPL(nand_ecc_choose_conf); -/* - * Check if the chip configuration meet the datasheet requirements. - - * If our configuration corrects A bits per B bytes and the minimum - * required correction level is X bits per Y bytes, then we must ensure - * both of the following are true: - * - * (1) A / B >= X / Y - * (2) A >= X - * - * Requirement (1) ensures we can correct for the required bitflip density. - * Requirement (2) ensures we can correct even when all bitflips are clumped - * in the same sector. - */ -static bool nand_ecc_strength_good(struct nand_chip *chip) -{ - struct mtd_info *mtd = nand_to_mtd(chip); - struct nand_ecc_ctrl *ecc = &chip->ecc; - int corr, ds_corr; - - if (ecc->size == 0 || chip->base.eccreq.step_size == 0) - /* Not enough information */ - return true; - - /* - * We get the number of corrected bits per page to compare - * the correction density. - */ - corr = (mtd->writesize * ecc->strength) / ecc->size; - ds_corr = (mtd->writesize * chip->base.eccreq.strength) / - chip->base.eccreq.step_size; - - return corr >= ds_corr && ecc->strength >= chip->base.eccreq.strength; -} - static int rawnand_erase(struct nand_device *nand, const struct nand_pos *pos) { struct nand_chip *chip = container_of(nand, struct nand_chip, @@ -5752,15 +5638,17 @@ static int nand_scan_tail(struct nand_chip *chip) * If no default placement scheme is given, select an appropriate one. */ if (!mtd->ooblayout && - !(ecc->mode == NAND_ECC_SOFT && ecc->algo == NAND_ECC_BCH)) { + !(ecc->engine_type == NAND_ECC_ENGINE_TYPE_SOFT && + ecc->algo == NAND_ECC_ALGO_BCH)) { switch (mtd->oobsize) { case 8: case 16: - mtd_set_ooblayout(mtd, &nand_ooblayout_sp_ops); + mtd_set_ooblayout(mtd, nand_get_small_page_ooblayout()); break; case 64: case 128: - mtd_set_ooblayout(mtd, &nand_ooblayout_lp_hamming_ops); + mtd_set_ooblayout(mtd, + nand_get_large_page_hamming_ooblayout()); break; default: /* @@ -5770,9 +5658,9 @@ static int nand_scan_tail(struct nand_chip *chip) * page with ECC layout when ->oobsize <= 128 for * compatibility reasons. */ - if (ecc->mode == NAND_ECC_NONE) { + if (ecc->engine_type == NAND_ECC_ENGINE_TYPE_NONE) { mtd_set_ooblayout(mtd, - &nand_ooblayout_lp_ops); + nand_get_large_page_ooblayout()); break; } @@ -5788,49 +5676,11 @@ static int nand_scan_tail(struct nand_chip *chip) * selected and we have 256 byte pagesize fallback to software ECC */ - switch (ecc->mode) { - case NAND_ECC_HW: - /* Use standard hwecc read page function? */ - if (!ecc->read_page) - ecc->read_page = nand_read_page_hwecc; - if (!ecc->write_page) - ecc->write_page = nand_write_page_hwecc; - if (!ecc->read_page_raw) - ecc->read_page_raw = nand_read_page_raw; - if (!ecc->write_page_raw) - ecc->write_page_raw = nand_write_page_raw; - if (!ecc->read_oob) - ecc->read_oob = nand_read_oob_std; - if (!ecc->write_oob) - ecc->write_oob = nand_write_oob_std; - if (!ecc->read_subpage) - ecc->read_subpage = nand_read_subpage; - if (!ecc->write_subpage && ecc->hwctl && ecc->calculate) - ecc->write_subpage = nand_write_subpage_hwecc; - fallthrough; - case NAND_ECC_HW_SYNDROME: - if ((!ecc->calculate || !ecc->correct || !ecc->hwctl) && - (!ecc->read_page || - ecc->read_page == nand_read_page_hwecc || - !ecc->write_page || - ecc->write_page == nand_write_page_hwecc)) { - WARN(1, "No ECC functions supplied; hardware ECC not possible\n"); - ret = -EINVAL; + switch (ecc->engine_type) { + case NAND_ECC_ENGINE_TYPE_ON_HOST: + ret = nand_set_ecc_on_host_ops(chip); + if (ret) goto err_nand_manuf_cleanup; - } - /* Use standard syndrome read/write page function? */ - if (!ecc->read_page) - ecc->read_page = nand_read_page_syndrome; - if (!ecc->write_page) - ecc->write_page = nand_write_page_syndrome; - if (!ecc->read_page_raw) - ecc->read_page_raw = nand_read_page_raw_syndrome; - if (!ecc->write_page_raw) - ecc->write_page_raw = nand_write_page_raw_syndrome; - if (!ecc->read_oob) - ecc->read_oob = nand_read_oob_syndrome; - if (!ecc->write_oob) - ecc->write_oob = nand_write_oob_syndrome; if (mtd->writesize >= ecc->size) { if (!ecc->strength) { @@ -5842,18 +5692,17 @@ static int nand_scan_tail(struct nand_chip *chip) } pr_warn("%d byte HW ECC not possible on %d byte page size, fallback to SW ECC\n", ecc->size, mtd->writesize); - ecc->mode = NAND_ECC_SOFT; - ecc->algo = NAND_ECC_HAMMING; + ecc->engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + ecc->algo = NAND_ECC_ALGO_HAMMING; fallthrough; - case NAND_ECC_SOFT: + + case NAND_ECC_ENGINE_TYPE_SOFT: ret = nand_set_ecc_soft_ops(chip); - if (ret) { - ret = -EINVAL; + if (ret) goto err_nand_manuf_cleanup; - } break; - case NAND_ECC_ON_DIE: + case NAND_ECC_ENGINE_TYPE_ON_DIE: if (!ecc->read_page || !ecc->write_page) { WARN(1, "No ECC functions supplied; on-die ECC not possible\n"); ret = -EINVAL; @@ -5865,8 +5714,8 @@ static int nand_scan_tail(struct nand_chip *chip) ecc->write_oob = nand_write_oob_std; break; - case NAND_ECC_NONE: - pr_warn("NAND_ECC_NONE selected by board driver. This is not recommended!\n"); + case NAND_ECC_ENGINE_TYPE_NONE: + pr_warn("NAND_ECC_ENGINE_TYPE_NONE selected by board driver. This is not recommended!\n"); ecc->read_page = nand_read_page_raw; ecc->write_page = nand_write_page_raw; ecc->read_oob = nand_read_oob_std; @@ -5879,7 +5728,7 @@ static int nand_scan_tail(struct nand_chip *chip) break; default: - WARN(1, "Invalid NAND_ECC_MODE %d\n", ecc->mode); + WARN(1, "Invalid NAND_ECC_MODE %d\n", ecc->engine_type); ret = -EINVAL; goto err_nand_manuf_cleanup; } @@ -5913,7 +5762,10 @@ static int nand_scan_tail(struct nand_chip *chip) ret = -EINVAL; goto err_nand_manuf_cleanup; } + ecc->total = ecc->steps * ecc->bytes; + chip->base.ecc.ctx.total = ecc->total; + if (ecc->total > mtd->oobsize) { WARN(1, "Total number of ECC bytes exceeded oobsize\n"); ret = -EINVAL; @@ -5931,11 +5783,11 @@ static int nand_scan_tail(struct nand_chip *chip) mtd->oobavail = ret; /* ECC sanity check: warn if it's too weak */ - if (!nand_ecc_strength_good(chip)) + if (!nand_ecc_is_strong_enough(&chip->base)) pr_warn("WARNING: %s: the ECC used on your system (%db/%dB) is too weak compared to the one required by the NAND chip (%db/%dB)\n", mtd->name, chip->ecc.strength, chip->ecc.size, - chip->base.eccreq.strength, - chip->base.eccreq.step_size); + nanddev_get_ecc_requirements(&chip->base)->strength, + nanddev_get_ecc_requirements(&chip->base)->step_size); /* Allow subpage writes up to ecc.steps. Not possible for MLC flash */ if (!(chip->options & NAND_NO_SUBPAGE_WRITE) && nand_is_slc(chip)) { @@ -5956,8 +5808,8 @@ static int nand_scan_tail(struct nand_chip *chip) chip->pagecache.page = -1; /* Large page NAND with SOFT_ECC should support subpage reads */ - switch (ecc->mode) { - case NAND_ECC_SOFT: + switch (ecc->engine_type) { + case NAND_ECC_ENGINE_TYPE_SOFT: if (chip->page_shift > 9) chip->options |= NAND_SUBPAGE_READ; break; @@ -6101,8 +5953,8 @@ EXPORT_SYMBOL(nand_scan_with_ids); */ void nand_cleanup(struct nand_chip *chip) { - if (chip->ecc.mode == NAND_ECC_SOFT && - chip->ecc.algo == NAND_ECC_BCH) + if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT && + chip->ecc.algo == NAND_ECC_ALGO_BCH) nand_bch_free((struct nand_bch_control *)chip->ecc.priv); nanddev_cleanup(&chip->base); diff --git a/drivers/mtd/nand/raw/nand_bch.c b/drivers/mtd/nand/raw/nand_bch.c index d5af8c5fd02f..9d19ac14c196 100644 --- a/drivers/mtd/nand/raw/nand_bch.c +++ b/drivers/mtd/nand/raw/nand_bch.c @@ -165,6 +165,7 @@ struct nand_bch_control *nand_bch_init(struct mtd_info *mtd) */ nand->ecc.steps = eccsteps; nand->ecc.total = eccsteps * eccbytes; + nand->base.ecc.ctx.total = nand->ecc.total; if (mtd_ooblayout_count_eccbytes(mtd) != (eccsteps*eccbytes)) { pr_warn("invalid ecc layout\n"); goto fail; diff --git a/drivers/mtd/nand/raw/nand_esmt.c b/drivers/mtd/nand/raw/nand_esmt.c index 3338c68aaaf1..4412c407aef3 100644 --- a/drivers/mtd/nand/raw/nand_esmt.c +++ b/drivers/mtd/nand/raw/nand_esmt.c @@ -10,27 +10,32 @@ static void esmt_nand_decode_id(struct nand_chip *chip) { + struct nand_device *base = &chip->base; + struct nand_ecc_props requirements = {}; + nand_decode_ext_id(chip); /* Extract ECC requirements from 5th id byte. */ if (chip->id.len >= 5 && nand_is_slc(chip)) { - chip->base.eccreq.step_size = 512; + requirements.step_size = 512; switch (chip->id.data[4] & 0x3) { case 0x0: - chip->base.eccreq.strength = 4; + requirements.strength = 4; break; case 0x1: - chip->base.eccreq.strength = 2; + requirements.strength = 2; break; case 0x2: - chip->base.eccreq.strength = 1; + requirements.strength = 1; break; default: WARN(1, "Could not get ECC info"); - chip->base.eccreq.step_size = 0; + requirements.step_size = 0; break; } } + + nanddev_set_ecc_requirements(base, &requirements); } static int esmt_nand_init(struct nand_chip *chip) diff --git a/drivers/mtd/nand/raw/nand_hynix.c b/drivers/mtd/nand/raw/nand_hynix.c index 6d08eb834456..a9f50c9af109 100644 --- a/drivers/mtd/nand/raw/nand_hynix.c +++ b/drivers/mtd/nand/raw/nand_hynix.c @@ -495,34 +495,36 @@ static void hynix_nand_extract_oobsize(struct nand_chip *chip, static void hynix_nand_extract_ecc_requirements(struct nand_chip *chip, bool valid_jedecid) { + struct nand_device *base = &chip->base; + struct nand_ecc_props requirements = {}; u8 ecc_level = (chip->id.data[4] >> 4) & 0x7; if (valid_jedecid) { /* Reference: H27UCG8T2E datasheet */ - chip->base.eccreq.step_size = 1024; + requirements.step_size = 1024; switch (ecc_level) { case 0: - chip->base.eccreq.step_size = 0; - chip->base.eccreq.strength = 0; + requirements.step_size = 0; + requirements.strength = 0; break; case 1: - chip->base.eccreq.strength = 4; + requirements.strength = 4; break; case 2: - chip->base.eccreq.strength = 24; + requirements.strength = 24; break; case 3: - chip->base.eccreq.strength = 32; + requirements.strength = 32; break; case 4: - chip->base.eccreq.strength = 40; + requirements.strength = 40; break; case 5: - chip->base.eccreq.strength = 50; + requirements.strength = 50; break; case 6: - chip->base.eccreq.strength = 60; + requirements.strength = 60; break; default: /* @@ -543,14 +545,14 @@ static void hynix_nand_extract_ecc_requirements(struct nand_chip *chip, if (nand_tech < 3) { /* > 26nm, reference: H27UBG8T2A datasheet */ if (ecc_level < 5) { - chip->base.eccreq.step_size = 512; - chip->base.eccreq.strength = 1 << ecc_level; + requirements.step_size = 512; + requirements.strength = 1 << ecc_level; } else if (ecc_level < 7) { if (ecc_level == 5) - chip->base.eccreq.step_size = 2048; + requirements.step_size = 2048; else - chip->base.eccreq.step_size = 1024; - chip->base.eccreq.strength = 24; + requirements.step_size = 1024; + requirements.strength = 24; } else { /* * We should never reach this case, but if that @@ -563,18 +565,20 @@ static void hynix_nand_extract_ecc_requirements(struct nand_chip *chip, } else { /* <= 26nm, reference: H27UBG8T2B datasheet */ if (!ecc_level) { - chip->base.eccreq.step_size = 0; - chip->base.eccreq.strength = 0; + requirements.step_size = 0; + requirements.strength = 0; } else if (ecc_level < 5) { - chip->base.eccreq.step_size = 512; - chip->base.eccreq.strength = 1 << (ecc_level - 1); + requirements.step_size = 512; + requirements.strength = 1 << (ecc_level - 1); } else { - chip->base.eccreq.step_size = 1024; - chip->base.eccreq.strength = 24 + + requirements.step_size = 1024; + requirements.strength = 24 + (8 * (ecc_level - 5)); } } } + + nanddev_set_ecc_requirements(base, &requirements); } static void hynix_nand_extract_scrambling_requirements(struct nand_chip *chip, diff --git a/drivers/mtd/nand/raw/nand_jedec.c b/drivers/mtd/nand/raw/nand_jedec.c index b15c42f48755..85b6d9372d80 100644 --- a/drivers/mtd/nand/raw/nand_jedec.c +++ b/drivers/mtd/nand/raw/nand_jedec.c @@ -23,6 +23,7 @@ */ int nand_jedec_detect(struct nand_chip *chip) { + struct nand_device *base = &chip->base; struct mtd_info *mtd = nand_to_mtd(chip); struct nand_memory_organization *memorg; struct nand_jedec_params *p; @@ -120,8 +121,12 @@ int nand_jedec_detect(struct nand_chip *chip) ecc = &p->ecc_info[0]; if (ecc->codeword_size >= 9) { - chip->base.eccreq.strength = ecc->ecc_bits; - chip->base.eccreq.step_size = 1 << ecc->codeword_size; + struct nand_ecc_props requirements = { + .strength = ecc->ecc_bits, + .step_size = 1 << ecc->codeword_size, + }; + + nanddev_set_ecc_requirements(base, &requirements); } else { pr_warn("Invalid codeword size\n"); } diff --git a/drivers/mtd/nand/raw/nand_micron.c b/drivers/mtd/nand/raw/nand_micron.c index 4385092a9325..c0192881906b 100644 --- a/drivers/mtd/nand/raw/nand_micron.c +++ b/drivers/mtd/nand/raw/nand_micron.c @@ -413,6 +413,8 @@ enum { */ static int micron_supports_on_die_ecc(struct nand_chip *chip) { + const struct nand_ecc_props *requirements = + nanddev_get_ecc_requirements(&chip->base); u8 id[5]; int ret; @@ -425,7 +427,7 @@ static int micron_supports_on_die_ecc(struct nand_chip *chip) /* * We only support on-die ECC of 4/512 or 8/512 */ - if (chip->base.eccreq.strength != 4 && chip->base.eccreq.strength != 8) + if (requirements->strength != 4 && requirements->strength != 8) return MICRON_ON_DIE_UNSUPPORTED; /* 0x2 means on-die ECC is available. */ @@ -466,7 +468,7 @@ static int micron_supports_on_die_ecc(struct nand_chip *chip) /* * We only support on-die ECC of 4/512 or 8/512 */ - if (chip->base.eccreq.strength != 4 && chip->base.eccreq.strength != 8) + if (requirements->strength != 4 && requirements->strength != 8) return MICRON_ON_DIE_UNSUPPORTED; return MICRON_ON_DIE_SUPPORTED; @@ -474,6 +476,9 @@ static int micron_supports_on_die_ecc(struct nand_chip *chip) static int micron_nand_init(struct nand_chip *chip) { + struct nand_device *base = &chip->base; + const struct nand_ecc_props *requirements = + nanddev_get_ecc_requirements(base); struct mtd_info *mtd = nand_to_mtd(chip); struct micron_nand *micron; int ondie; @@ -497,13 +502,13 @@ static int micron_nand_init(struct nand_chip *chip) ondie = micron_supports_on_die_ecc(chip); if (ondie == MICRON_ON_DIE_MANDATORY && - chip->ecc.mode != NAND_ECC_ON_DIE) { + chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_DIE) { pr_err("On-die ECC forcefully enabled, not supported\n"); ret = -EINVAL; goto err_free_manuf_data; } - if (chip->ecc.mode == NAND_ECC_ON_DIE) { + if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE) { if (ondie == MICRON_ON_DIE_UNSUPPORTED) { pr_err("On-die ECC selected but not supported\n"); ret = -EINVAL; @@ -523,7 +528,7 @@ static int micron_nand_init(struct nand_chip *chip) * That's not needed for 8-bit ECC, because the status expose * a better approximation of the number of bitflips in a page. */ - if (chip->base.eccreq.strength == 4) { + if (requirements->strength == 4) { micron->ecc.rawbuf = kmalloc(mtd->writesize + mtd->oobsize, GFP_KERNEL); @@ -533,17 +538,17 @@ static int micron_nand_init(struct nand_chip *chip) } } - if (chip->base.eccreq.strength == 4) + if (requirements->strength == 4) mtd_set_ooblayout(mtd, µn_nand_on_die_4_ooblayout_ops); else mtd_set_ooblayout(mtd, µn_nand_on_die_8_ooblayout_ops); - chip->ecc.bytes = chip->base.eccreq.strength * 2; + chip->ecc.bytes = requirements->strength * 2; chip->ecc.size = 512; - chip->ecc.strength = chip->base.eccreq.strength; - chip->ecc.algo = NAND_ECC_BCH; + chip->ecc.strength = requirements->strength; + chip->ecc.algo = NAND_ECC_ALGO_BCH; chip->ecc.read_page = micron_nand_read_page_on_die_ecc; chip->ecc.write_page = micron_nand_write_page_on_die_ecc; diff --git a/drivers/mtd/nand/raw/nand_onfi.c b/drivers/mtd/nand/raw/nand_onfi.c index be3456627288..45649e03797d 100644 --- a/drivers/mtd/nand/raw/nand_onfi.c +++ b/drivers/mtd/nand/raw/nand_onfi.c @@ -34,6 +34,8 @@ u16 onfi_crc16(u16 crc, u8 const *p, size_t len) static int nand_flash_detect_ext_param_page(struct nand_chip *chip, struct nand_onfi_params *p) { + struct nand_device *base = &chip->base; + struct nand_ecc_props requirements; struct onfi_ext_param_page *ep; struct onfi_ext_section *s; struct onfi_ext_ecc_info *ecc; @@ -94,8 +96,10 @@ static int nand_flash_detect_ext_param_page(struct nand_chip *chip, goto ext_out; } - chip->base.eccreq.strength = ecc->ecc_bits; - chip->base.eccreq.step_size = 1 << ecc->codeword_size; + requirements.strength = ecc->ecc_bits; + requirements.step_size = 1 << ecc->codeword_size; + nanddev_set_ecc_requirements(base, &requirements); + ret = 0; ext_out: @@ -139,6 +143,7 @@ static void nand_bit_wise_majority(const void **srcbufs, */ int nand_onfi_detect(struct nand_chip *chip) { + struct nand_device *base = &chip->base; struct mtd_info *mtd = nand_to_mtd(chip); struct nand_memory_organization *memorg; struct nand_onfi_params *p = NULL, *pbuf; @@ -265,8 +270,12 @@ int nand_onfi_detect(struct nand_chip *chip) chip->options |= NAND_BUSWIDTH_16; if (p->ecc_bits != 0xff) { - chip->base.eccreq.strength = p->ecc_bits; - chip->base.eccreq.step_size = 512; + struct nand_ecc_props requirements = { + .strength = p->ecc_bits, + .step_size = 512, + }; + + nanddev_set_ecc_requirements(base, &requirements); } else if (onfi_version >= 21 && (le16_to_cpu(p->features) & ONFI_FEATURE_EXT_PARAM_PAGE)) { diff --git a/drivers/mtd/nand/raw/nand_samsung.c b/drivers/mtd/nand/raw/nand_samsung.c index 3a4a19e808f6..0be6b7563805 100644 --- a/drivers/mtd/nand/raw/nand_samsung.c +++ b/drivers/mtd/nand/raw/nand_samsung.c @@ -10,6 +10,8 @@ static void samsung_nand_decode_id(struct nand_chip *chip) { + struct nand_device *base = &chip->base; + struct nand_ecc_props requirements = {}; struct mtd_info *mtd = nand_to_mtd(chip); struct nand_memory_organization *memorg; @@ -71,23 +73,23 @@ static void samsung_nand_decode_id(struct nand_chip *chip) /* Extract ECC requirements from 5th id byte*/ extid = (chip->id.data[4] >> 4) & 0x07; if (extid < 5) { - chip->base.eccreq.step_size = 512; - chip->base.eccreq.strength = 1 << extid; + requirements.step_size = 512; + requirements.strength = 1 << extid; } else { - chip->base.eccreq.step_size = 1024; + requirements.step_size = 1024; switch (extid) { case 5: - chip->base.eccreq.strength = 24; + requirements.strength = 24; break; case 6: - chip->base.eccreq.strength = 40; + requirements.strength = 40; break; case 7: - chip->base.eccreq.strength = 60; + requirements.strength = 60; break; default: WARN(1, "Could not decode ECC info"); - chip->base.eccreq.step_size = 0; + requirements.step_size = 0; } } } else { @@ -97,8 +99,8 @@ static void samsung_nand_decode_id(struct nand_chip *chip) switch (chip->id.data[1]) { /* K9F4G08U0D-S[I|C]B0(T00) */ case 0xDC: - chip->base.eccreq.step_size = 512; - chip->base.eccreq.strength = 1; + requirements.step_size = 512; + requirements.strength = 1; break; /* K9F1G08U0E 21nm chips do not support subpage write */ @@ -112,6 +114,8 @@ static void samsung_nand_decode_id(struct nand_chip *chip) } } } + + nanddev_set_ecc_requirements(base, &requirements); } static int samsung_nand_init(struct nand_chip *chip) diff --git a/drivers/mtd/nand/raw/nand_toshiba.c b/drivers/mtd/nand/raw/nand_toshiba.c index f746c19f3b2c..cf4f37959421 100644 --- a/drivers/mtd/nand/raw/nand_toshiba.c +++ b/drivers/mtd/nand/raw/nand_toshiba.c @@ -140,11 +140,13 @@ static void toshiba_nand_benand_init(struct nand_chip *chip) chip->options |= NAND_SUBPAGE_READ; - mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops); + mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout()); } static void toshiba_nand_decode_id(struct nand_chip *chip) { + struct nand_device *base = &chip->base; + struct nand_ecc_props requirements = {}; struct mtd_info *mtd = nand_to_mtd(chip); struct nand_memory_organization *memorg; @@ -175,23 +177,25 @@ static void toshiba_nand_decode_id(struct nand_chip *chip) * - 24nm: 8 bit ECC for each 512Byte is required. */ if (chip->id.len >= 6 && nand_is_slc(chip)) { - chip->base.eccreq.step_size = 512; + requirements.step_size = 512; switch (chip->id.data[5] & 0x7) { case 0x4: - chip->base.eccreq.strength = 1; + requirements.strength = 1; break; case 0x5: - chip->base.eccreq.strength = 4; + requirements.strength = 4; break; case 0x6: - chip->base.eccreq.strength = 8; + requirements.strength = 8; break; default: WARN(1, "Could not get ECC info"); - chip->base.eccreq.step_size = 0; + requirements.step_size = 0; break; } } + + nanddev_set_ecc_requirements(base, &requirements); } static int @@ -273,7 +277,8 @@ static int toshiba_nand_init(struct nand_chip *chip) chip->options |= NAND_BBM_FIRSTPAGE | NAND_BBM_SECONDPAGE; /* Check that chip is BENAND and ECC mode is on-die */ - if (nand_is_slc(chip) && chip->ecc.mode == NAND_ECC_ON_DIE && + if (nand_is_slc(chip) && + chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE && chip->id.data[4] & TOSHIBA_NAND_ID4_IS_BENAND) toshiba_nand_benand_init(chip); diff --git a/drivers/mtd/nand/raw/nandsim.c b/drivers/mtd/nand/raw/nandsim.c index f5a53aac3c5f..a8048cb8d220 100644 --- a/drivers/mtd/nand/raw/nandsim.c +++ b/drivers/mtd/nand/raw/nandsim.c @@ -2234,8 +2234,8 @@ static int ns_attach_chip(struct nand_chip *chip) return -EINVAL; } - chip->ecc.mode = NAND_ECC_SOFT; - chip->ecc.algo = NAND_ECC_BCH; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + chip->ecc.algo = NAND_ECC_ALGO_BCH; chip->ecc.size = 512; chip->ecc.strength = bch; chip->ecc.bytes = eccbytes; @@ -2274,8 +2274,8 @@ static int __init ns_init_module(void) nsmtd = nand_to_mtd(chip); nand_set_controller_data(chip, (void *)ns); - chip->ecc.mode = NAND_ECC_SOFT; - chip->ecc.algo = NAND_ECC_HAMMING; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + chip->ecc.algo = NAND_ECC_ALGO_HAMMING; /* The NAND_SKIP_BBTSCAN option is necessary for 'overridesize' */ /* and 'badblocks' parameters to work */ chip->options |= NAND_SKIP_BBTSCAN; diff --git a/drivers/mtd/nand/raw/ndfc.c b/drivers/mtd/nand/raw/ndfc.c index ed38338c1383..0fb4ba93c41e 100644 --- a/drivers/mtd/nand/raw/ndfc.c +++ b/drivers/mtd/nand/raw/ndfc.c @@ -149,7 +149,7 @@ static int ndfc_chip_init(struct ndfc_controller *ndfc, chip->ecc.correct = nand_correct_data; chip->ecc.hwctl = ndfc_enable_hwecc; chip->ecc.calculate = ndfc_calculate_ecc; - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; chip->ecc.size = 256; chip->ecc.bytes = 3; chip->ecc.strength = 1; diff --git a/drivers/mtd/nand/raw/omap2.c b/drivers/mtd/nand/raw/omap2.c index eb7fcfd9276b..512f60780a50 100644 --- a/drivers/mtd/nand/raw/omap2.c +++ b/drivers/mtd/nand/raw/omap2.c @@ -884,8 +884,8 @@ static int omap_correct_data(struct nand_chip *chip, u_char *dat, int stat = 0; /* Ex NAND_ECC_HW12_2048 */ - if ((info->nand.ecc.mode == NAND_ECC_HW) && - (info->nand.ecc.size == 2048)) + if (info->nand.ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST && + info->nand.ecc.size == 2048) blockCnt = 4; else blockCnt = 1; @@ -2006,12 +2006,12 @@ static int omap_nand_attach_chip(struct nand_chip *chip) return -EINVAL; /* - * Bail out earlier to let NAND_ECC_SOFT code create its own + * Bail out earlier to let NAND_ECC_ENGINE_TYPE_SOFT code create its own * ooblayout instead of using ours. */ if (info->ecc_opt == OMAP_ECC_HAM1_CODE_SW) { - chip->ecc.mode = NAND_ECC_SOFT; - chip->ecc.algo = NAND_ECC_HAMMING; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + chip->ecc.algo = NAND_ECC_ALGO_HAMMING; return 0; } @@ -2019,7 +2019,7 @@ static int omap_nand_attach_chip(struct nand_chip *chip) switch (info->ecc_opt) { case OMAP_ECC_HAM1_CODE_HW: dev_info(dev, "nand: using OMAP_ECC_HAM1_CODE_HW\n"); - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; chip->ecc.bytes = 3; chip->ecc.size = 512; chip->ecc.strength = 1; @@ -2036,7 +2036,7 @@ static int omap_nand_attach_chip(struct nand_chip *chip) case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: pr_info("nand: using OMAP_ECC_BCH4_CODE_HW_DETECTION_SW\n"); - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; chip->ecc.size = 512; chip->ecc.bytes = 7; chip->ecc.strength = 4; @@ -2056,7 +2056,7 @@ static int omap_nand_attach_chip(struct nand_chip *chip) case OMAP_ECC_BCH4_CODE_HW: pr_info("nand: using OMAP_ECC_BCH4_CODE_HW ECC scheme\n"); - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; chip->ecc.size = 512; /* 14th bit is kept reserved for ROM-code compatibility */ chip->ecc.bytes = 7 + 1; @@ -2078,7 +2078,7 @@ static int omap_nand_attach_chip(struct nand_chip *chip) case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: pr_info("nand: using OMAP_ECC_BCH8_CODE_HW_DETECTION_SW\n"); - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; chip->ecc.size = 512; chip->ecc.bytes = 13; chip->ecc.strength = 8; @@ -2098,7 +2098,7 @@ static int omap_nand_attach_chip(struct nand_chip *chip) case OMAP_ECC_BCH8_CODE_HW: pr_info("nand: using OMAP_ECC_BCH8_CODE_HW ECC scheme\n"); - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; chip->ecc.size = 512; /* 14th bit is kept reserved for ROM-code compatibility */ chip->ecc.bytes = 13 + 1; @@ -2121,7 +2121,7 @@ static int omap_nand_attach_chip(struct nand_chip *chip) case OMAP_ECC_BCH16_CODE_HW: pr_info("Using OMAP_ECC_BCH16_CODE_HW ECC scheme\n"); - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; chip->ecc.size = 512; chip->ecc.bytes = 26; chip->ecc.strength = 16; diff --git a/drivers/mtd/nand/raw/orion_nand.c b/drivers/mtd/nand/raw/orion_nand.c index 880b54ca1b41..df9c0f8e4b4e 100644 --- a/drivers/mtd/nand/raw/orion_nand.c +++ b/drivers/mtd/nand/raw/orion_nand.c @@ -139,8 +139,8 @@ static int __init orion_nand_probe(struct platform_device *pdev) nc->legacy.IO_ADDR_R = nc->legacy.IO_ADDR_W = io_base; nc->legacy.cmd_ctrl = orion_nand_cmd_ctrl; nc->legacy.read_buf = orion_nand_read_buf; - nc->ecc.mode = NAND_ECC_SOFT; - nc->ecc.algo = NAND_ECC_HAMMING; + nc->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + nc->ecc.algo = NAND_ECC_ALGO_HAMMING; if (board->chip_delay) nc->legacy.chip_delay = board->chip_delay; diff --git a/drivers/mtd/nand/raw/pasemi_nand.c b/drivers/mtd/nand/raw/pasemi_nand.c index d8eca8c3fdcd..2b8f155cc0c5 100644 --- a/drivers/mtd/nand/raw/pasemi_nand.c +++ b/drivers/mtd/nand/raw/pasemi_nand.c @@ -68,7 +68,7 @@ static void pasemi_hwcontrol(struct nand_chip *chip, int cmd, inl(lpcctl); } -int pasemi_device_ready(struct nand_chip *chip) +static int pasemi_device_ready(struct nand_chip *chip) { return !!(inl(lpcctl) & LBICTRL_LPCCTL_NR); } @@ -132,8 +132,8 @@ static int pasemi_nand_probe(struct platform_device *ofdev) chip->legacy.read_buf = pasemi_read_buf; chip->legacy.write_buf = pasemi_write_buf; chip->legacy.chip_delay = 0; - chip->ecc.mode = NAND_ECC_SOFT; - chip->ecc.algo = NAND_ECC_HAMMING; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + chip->ecc.algo = NAND_ECC_ALGO_HAMMING; /* Enable the following for a flash based bad block table */ chip->bbt_options = NAND_BBT_USE_FLASH; diff --git a/drivers/mtd/nand/raw/plat_nand.c b/drivers/mtd/nand/raw/plat_nand.c index 556182f26057..b98c0d5c413f 100644 --- a/drivers/mtd/nand/raw/plat_nand.c +++ b/drivers/mtd/nand/raw/plat_nand.c @@ -66,8 +66,8 @@ static int plat_nand_probe(struct platform_device *pdev) data->chip.options |= pdata->chip.options; data->chip.bbt_options |= pdata->chip.bbt_options; - data->chip.ecc.mode = NAND_ECC_SOFT; - data->chip.ecc.algo = NAND_ECC_HAMMING; + data->chip.ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + data->chip.ecc.algo = NAND_ECC_ALGO_HAMMING; platform_set_drvdata(pdev, data); diff --git a/drivers/mtd/nand/raw/qcom_nandc.c b/drivers/mtd/nand/raw/qcom_nandc.c index bd7a7251429b..777fb0de0680 100644 --- a/drivers/mtd/nand/raw/qcom_nandc.c +++ b/drivers/mtd/nand/raw/qcom_nandc.c @@ -2550,7 +2550,7 @@ static int qcom_nand_attach_chip(struct nand_chip *chip) ecc->write_page_raw = qcom_nandc_write_page_raw; ecc->write_oob = qcom_nandc_write_oob; - ecc->mode = NAND_ECC_HW; + ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops); @@ -2702,10 +2702,8 @@ static int qcom_nandc_alloc(struct qcom_nand_controller *nandc) if (IS_ERR(nandc->tx_chan)) { ret = PTR_ERR(nandc->tx_chan); nandc->tx_chan = NULL; - if (ret != -EPROBE_DEFER) - dev_err(nandc->dev, - "tx DMA channel request failed: %d\n", - ret); + dev_err_probe(nandc->dev, ret, + "tx DMA channel request failed\n"); goto unalloc; } @@ -2713,10 +2711,8 @@ static int qcom_nandc_alloc(struct qcom_nand_controller *nandc) if (IS_ERR(nandc->rx_chan)) { ret = PTR_ERR(nandc->rx_chan); nandc->rx_chan = NULL; - if (ret != -EPROBE_DEFER) - dev_err(nandc->dev, - "rx DMA channel request failed: %d\n", - ret); + dev_err_probe(nandc->dev, ret, + "rx DMA channel request failed\n"); goto unalloc; } @@ -2724,10 +2720,8 @@ static int qcom_nandc_alloc(struct qcom_nand_controller *nandc) if (IS_ERR(nandc->cmd_chan)) { ret = PTR_ERR(nandc->cmd_chan); nandc->cmd_chan = NULL; - if (ret != -EPROBE_DEFER) - dev_err(nandc->dev, - "cmd DMA channel request failed: %d\n", - ret); + dev_err_probe(nandc->dev, ret, + "cmd DMA channel request failed\n"); goto unalloc; } @@ -2750,10 +2744,8 @@ static int qcom_nandc_alloc(struct qcom_nand_controller *nandc) if (IS_ERR(nandc->chan)) { ret = PTR_ERR(nandc->chan); nandc->chan = NULL; - if (ret != -EPROBE_DEFER) - dev_err(nandc->dev, - "rxtx DMA channel request failed: %d\n", - ret); + dev_err_probe(nandc->dev, ret, + "rxtx DMA channel request failed\n"); return ret; } } diff --git a/drivers/mtd/nand/raw/r852.c b/drivers/mtd/nand/raw/r852.c index f865e3a47b01..6b7addd2c420 100644 --- a/drivers/mtd/nand/raw/r852.c +++ b/drivers/mtd/nand/raw/r852.c @@ -859,7 +859,8 @@ static int r852_probe(struct pci_dev *pci_dev, const struct pci_device_id *id) chip->legacy.write_buf = r852_write_buf; /* ecc */ - chip->ecc.mode = NAND_ECC_HW_SYNDROME; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; + chip->ecc.placement = NAND_ECC_PLACEMENT_INTERLEAVED; chip->ecc.size = R852_DMA_LEN; chip->ecc.bytes = SM_OOB_SIZE; chip->ecc.strength = 2; diff --git a/drivers/mtd/nand/raw/s3c2410.c b/drivers/mtd/nand/raw/s3c2410.c index 105522205979..fbd0fa48e063 100644 --- a/drivers/mtd/nand/raw/s3c2410.c +++ b/drivers/mtd/nand/raw/s3c2410.c @@ -904,7 +904,7 @@ static void s3c2410_nand_init_chip(struct s3c2410_nand_info *info, nmtd->info = info; nmtd->set = set; - chip->ecc.mode = info->platform->ecc_mode; + chip->ecc.engine_type = info->platform->engine_type; /* * If you use u-boot BBT creation code, specifying this flag will @@ -929,24 +929,24 @@ static int s3c2410_nand_attach_chip(struct nand_chip *chip) struct mtd_info *mtd = nand_to_mtd(chip); struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); - switch (chip->ecc.mode) { + switch (chip->ecc.engine_type) { - case NAND_ECC_NONE: + case NAND_ECC_ENGINE_TYPE_NONE: dev_info(info->device, "ECC disabled\n"); break; - case NAND_ECC_SOFT: + case NAND_ECC_ENGINE_TYPE_SOFT: /* - * This driver expects Hamming based ECC when ecc_mode is set - * to NAND_ECC_SOFT. Force ecc.algo to NAND_ECC_HAMMING to - * avoid adding an extra ecc_algo field to - * s3c2410_platform_nand. + * This driver expects Hamming based ECC when engine_type is set + * to NAND_ECC_ENGINE_TYPE_SOFT. Force ecc.algo to + * NAND_ECC_ALGO_HAMMING to avoid adding an extra ecc_algo field + * to s3c2410_platform_nand. */ - chip->ecc.algo = NAND_ECC_HAMMING; + chip->ecc.algo = NAND_ECC_ALGO_HAMMING; dev_info(info->device, "soft ECC\n"); break; - case NAND_ECC_HW: + case NAND_ECC_ENGINE_TYPE_ON_HOST: chip->ecc.calculate = s3c2410_nand_calculate_ecc; chip->ecc.correct = s3c2410_nand_correct_data; chip->ecc.strength = 1; diff --git a/drivers/mtd/nand/raw/sh_flctl.c b/drivers/mtd/nand/raw/sh_flctl.c index a661b8bb2dd5..13df4bdf792a 100644 --- a/drivers/mtd/nand/raw/sh_flctl.c +++ b/drivers/mtd/nand/raw/sh_flctl.c @@ -1039,13 +1039,13 @@ static int flctl_chip_attach_chip(struct nand_chip *chip) chip->ecc.strength = 4; chip->ecc.read_page = flctl_read_page_hwecc; chip->ecc.write_page = flctl_write_page_hwecc; - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; /* 4 symbols ECC enabled */ flctl->flcmncr_base |= _4ECCEN; } else { - chip->ecc.mode = NAND_ECC_SOFT; - chip->ecc.algo = NAND_ECC_HAMMING; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + chip->ecc.algo = NAND_ECC_ALGO_HAMMING; } return 0; diff --git a/drivers/mtd/nand/raw/sharpsl.c b/drivers/mtd/nand/raw/sharpsl.c index 51286f7acf54..1327bfb3d5d3 100644 --- a/drivers/mtd/nand/raw/sharpsl.c +++ b/drivers/mtd/nand/raw/sharpsl.c @@ -157,7 +157,7 @@ static int sharpsl_nand_probe(struct platform_device *pdev) /* 15 us command delay time */ this->legacy.chip_delay = 15; /* set eccmode using hardware ECC */ - this->ecc.mode = NAND_ECC_HW; + this->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; this->ecc.size = 256; this->ecc.bytes = 3; this->ecc.strength = 1; diff --git a/drivers/mtd/nand/raw/socrates_nand.c b/drivers/mtd/nand/raw/socrates_nand.c index 243b34cfbc1b..0f63ff6f7fe7 100644 --- a/drivers/mtd/nand/raw/socrates_nand.c +++ b/drivers/mtd/nand/raw/socrates_nand.c @@ -153,8 +153,9 @@ static int socrates_nand_probe(struct platform_device *ofdev) nand_chip->legacy.read_buf = socrates_nand_read_buf; nand_chip->legacy.dev_ready = socrates_nand_device_ready; - nand_chip->ecc.mode = NAND_ECC_SOFT; /* enable ECC */ - nand_chip->ecc.algo = NAND_ECC_HAMMING; + /* enable ECC */ + nand_chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + nand_chip->ecc.algo = NAND_ECC_ALGO_HAMMING; /* TODO: I have no idea what real delay is. */ nand_chip->legacy.chip_delay = 20; /* 20us command delay time */ diff --git a/drivers/mtd/nand/raw/stm32_fmc2_nand.c b/drivers/mtd/nand/raw/stm32_fmc2_nand.c index 7f4546ae9130..b31a5818234d 100644 --- a/drivers/mtd/nand/raw/stm32_fmc2_nand.c +++ b/drivers/mtd/nand/raw/stm32_fmc2_nand.c @@ -1696,14 +1696,15 @@ static int stm32_fmc2_nfc_attach_chip(struct nand_chip *chip) int ret; /* - * Only NAND_ECC_HW mode is actually supported + * Only NAND_ECC_ENGINE_TYPE_ON_HOST mode is actually supported * Hamming => ecc.strength = 1 * BCH4 => ecc.strength = 4 * BCH8 => ecc.strength = 8 * ECC sector size = 512 */ - if (chip->ecc.mode != NAND_ECC_HW) { - dev_err(nfc->dev, "nand_ecc_mode is not well defined in the DT\n"); + if (chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST) { + dev_err(nfc->dev, + "nand_ecc_engine_type is not well defined in the DT\n"); return -EINVAL; } @@ -1762,7 +1763,7 @@ static int stm32_fmc2_nfc_parse_child(struct stm32_fmc2_nfc *nfc, return ret; } - if (cs > FMC2_MAX_CE) { + if (cs >= FMC2_MAX_CE) { dev_err(nfc->dev, "invalid reg value: %d\n", cs); return -EINVAL; } @@ -1952,7 +1953,7 @@ static int stm32_fmc2_nfc_probe(struct platform_device *pdev) NAND_USES_DMA; /* Default ECC settings */ - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; chip->ecc.size = FMC2_ECC_STEP_SIZE; chip->ecc.strength = FMC2_ECC_BCH8; diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c index 9c50c2b965e1..2a7ca3072f35 100644 --- a/drivers/mtd/nand/raw/sunxi_nand.c +++ b/drivers/mtd/nand/raw/sunxi_nand.c @@ -1575,7 +1575,7 @@ static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section, * only have 2 bytes available in the first user data * section. */ - if (!section && ecc->mode == NAND_ECC_HW) { + if (!section && ecc->engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) { oobregion->offset = 2; oobregion->length = 2; @@ -1609,12 +1609,13 @@ static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand, static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 }; struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); struct mtd_info *mtd = nand_to_mtd(nand); + struct nand_device *nanddev = mtd_to_nanddev(mtd); struct sunxi_nand_hw_ecc *data; int nsectors; int ret; int i; - if (ecc->options & NAND_ECC_MAXIMIZE) { + if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) { int bytes; ecc->size = 1024; @@ -1720,11 +1721,11 @@ err: static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc) { - switch (ecc->mode) { - case NAND_ECC_HW: + switch (ecc->engine_type) { + case NAND_ECC_ENGINE_TYPE_ON_HOST: sunxi_nand_hw_ecc_ctrl_cleanup(ecc); break; - case NAND_ECC_NONE: + case NAND_ECC_ENGINE_TYPE_NONE: default: break; } @@ -1732,6 +1733,8 @@ static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc) static int sunxi_nand_attach_chip(struct nand_chip *nand) { + const struct nand_ecc_props *requirements = + nanddev_get_ecc_requirements(&nand->base); struct nand_ecc_ctrl *ecc = &nand->ecc; struct device_node *np = nand_get_flash_node(nand); int ret; @@ -1745,21 +1748,21 @@ static int sunxi_nand_attach_chip(struct nand_chip *nand) nand->options |= NAND_SUBPAGE_READ; if (!ecc->size) { - ecc->size = nand->base.eccreq.step_size; - ecc->strength = nand->base.eccreq.strength; + ecc->size = requirements->step_size; + ecc->strength = requirements->strength; } if (!ecc->size || !ecc->strength) return -EINVAL; - switch (ecc->mode) { - case NAND_ECC_HW: + switch (ecc->engine_type) { + case NAND_ECC_ENGINE_TYPE_ON_HOST: ret = sunxi_nand_hw_ecc_ctrl_init(nand, ecc, np); if (ret) return ret; break; - case NAND_ECC_NONE: - case NAND_ECC_SOFT: + case NAND_ECC_ENGINE_TYPE_NONE: + case NAND_ECC_ENGINE_TYPE_SOFT: break; default: return -EINVAL; @@ -1991,7 +1994,7 @@ static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc, * Set the ECC mode to the default value in case nothing is specified * in the DT. */ - nand->ecc.mode = NAND_ECC_HW; + nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; nand_set_flash_node(nand, np); mtd = nand_to_mtd(nand); diff --git a/drivers/mtd/nand/raw/tango_nand.c b/drivers/mtd/nand/raw/tango_nand.c index bdb965ae7a4a..359187b5a4be 100644 --- a/drivers/mtd/nand/raw/tango_nand.c +++ b/drivers/mtd/nand/raw/tango_nand.c @@ -549,8 +549,8 @@ static int tango_attach_chip(struct nand_chip *chip) { struct nand_ecc_ctrl *ecc = &chip->ecc; - ecc->mode = NAND_ECC_HW; - ecc->algo = NAND_ECC_BCH; + ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; + ecc->algo = NAND_ECC_ALGO_BCH; ecc->bytes = DIV_ROUND_UP(ecc->strength * FIELD_ORDER, BITS_PER_BYTE); ecc->read_page_raw = tango_read_page_raw; diff --git a/drivers/mtd/nand/raw/tegra_nand.c b/drivers/mtd/nand/raw/tegra_nand.c index 6b6212ffa01c..fbf67722a049 100644 --- a/drivers/mtd/nand/raw/tegra_nand.c +++ b/drivers/mtd/nand/raw/tegra_nand.c @@ -479,7 +479,7 @@ static void tegra_nand_hw_ecc(struct tegra_nand_controller *ctrl, { struct tegra_nand_chip *nand = to_tegra_chip(chip); - if (chip->ecc.algo == NAND_ECC_BCH && enable) + if (chip->ecc.algo == NAND_ECC_ALGO_BCH && enable) writel_relaxed(nand->bch_config, ctrl->regs + BCH_CONFIG); else writel_relaxed(0, ctrl->regs + BCH_CONFIG); @@ -840,7 +840,10 @@ static int tegra_nand_get_strength(struct nand_chip *chip, const int *strength, int strength_len, int bits_per_step, int oobsize) { - bool maximize = chip->ecc.options & NAND_ECC_MAXIMIZE; + struct nand_device *base = mtd_to_nanddev(nand_to_mtd(chip)); + const struct nand_ecc_props *requirements = + nanddev_get_ecc_requirements(base); + bool maximize = base->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH; int i; /* @@ -855,7 +858,7 @@ static int tegra_nand_get_strength(struct nand_chip *chip, const int *strength, } else { strength_sel = strength[i]; - if (strength_sel < chip->base.eccreq.strength) + if (strength_sel < requirements->strength) continue; } @@ -877,7 +880,7 @@ static int tegra_nand_select_strength(struct nand_chip *chip, int oobsize) int strength_len, bits_per_step; switch (chip->ecc.algo) { - case NAND_ECC_RS: + case NAND_ECC_ALGO_RS: bits_per_step = BITS_PER_STEP_RS; if (chip->options & NAND_IS_BOOT_MEDIUM) { strength = rs_strength_bootable; @@ -887,7 +890,7 @@ static int tegra_nand_select_strength(struct nand_chip *chip, int oobsize) strength_len = ARRAY_SIZE(rs_strength); } break; - case NAND_ECC_BCH: + case NAND_ECC_ALGO_BCH: bits_per_step = BITS_PER_STEP_BCH; if (chip->options & NAND_IS_BOOT_MEDIUM) { strength = bch_strength_bootable; @@ -908,6 +911,8 @@ static int tegra_nand_select_strength(struct nand_chip *chip, int oobsize) static int tegra_nand_attach_chip(struct nand_chip *chip) { struct tegra_nand_controller *ctrl = to_tegra_ctrl(chip->controller); + const struct nand_ecc_props *requirements = + nanddev_get_ecc_requirements(&chip->base); struct tegra_nand_chip *nand = to_tegra_chip(chip); struct mtd_info *mtd = nand_to_mtd(chip); int bits_per_step; @@ -916,12 +921,12 @@ static int tegra_nand_attach_chip(struct nand_chip *chip) if (chip->bbt_options & NAND_BBT_USE_FLASH) chip->bbt_options |= NAND_BBT_NO_OOB; - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; chip->ecc.size = 512; chip->ecc.steps = mtd->writesize / chip->ecc.size; - if (chip->base.eccreq.step_size != 512) { + if (requirements->step_size != 512) { dev_err(ctrl->dev, "Unsupported step size %d\n", - chip->base.eccreq.step_size); + requirements->step_size); return -EINVAL; } @@ -935,14 +940,14 @@ static int tegra_nand_attach_chip(struct nand_chip *chip) if (chip->options & NAND_BUSWIDTH_16) nand->config |= CONFIG_BUS_WIDTH_16; - if (chip->ecc.algo == NAND_ECC_UNKNOWN) { + if (chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN) { if (mtd->writesize < 2048) - chip->ecc.algo = NAND_ECC_RS; + chip->ecc.algo = NAND_ECC_ALGO_RS; else - chip->ecc.algo = NAND_ECC_BCH; + chip->ecc.algo = NAND_ECC_ALGO_BCH; } - if (chip->ecc.algo == NAND_ECC_BCH && mtd->writesize < 2048) { + if (chip->ecc.algo == NAND_ECC_ALGO_BCH && mtd->writesize < 2048) { dev_err(ctrl->dev, "BCH supports 2K or 4K page size only\n"); return -EINVAL; } @@ -952,7 +957,7 @@ static int tegra_nand_attach_chip(struct nand_chip *chip) if (ret < 0) { dev_err(ctrl->dev, "No valid strength found, minimum %d\n", - chip->base.eccreq.strength); + requirements->strength); return ret; } @@ -963,7 +968,7 @@ static int tegra_nand_attach_chip(struct nand_chip *chip) CONFIG_SKIP_SPARE_SIZE_4; switch (chip->ecc.algo) { - case NAND_ECC_RS: + case NAND_ECC_ALGO_RS: bits_per_step = BITS_PER_STEP_RS * chip->ecc.strength; mtd_set_ooblayout(mtd, &tegra_nand_oob_rs_ops); nand->config_ecc |= CONFIG_HW_ECC | CONFIG_ECC_SEL | @@ -984,7 +989,7 @@ static int tegra_nand_attach_chip(struct nand_chip *chip) return -EINVAL; } break; - case NAND_ECC_BCH: + case NAND_ECC_ALGO_BCH: bits_per_step = BITS_PER_STEP_BCH * chip->ecc.strength; mtd_set_ooblayout(mtd, &tegra_nand_oob_bch_ops); nand->bch_config = BCH_ENABLE; @@ -1013,7 +1018,7 @@ static int tegra_nand_attach_chip(struct nand_chip *chip) } dev_info(ctrl->dev, "Using %s with strength %d per 512 byte step\n", - chip->ecc.algo == NAND_ECC_BCH ? "BCH" : "RS", + chip->ecc.algo == NAND_ECC_ALGO_BCH ? "BCH" : "RS", chip->ecc.strength); chip->ecc.bytes = DIV_ROUND_UP(bits_per_step, BITS_PER_BYTE); diff --git a/drivers/mtd/nand/raw/tmio_nand.c b/drivers/mtd/nand/raw/tmio_nand.c index 843a8683b737..235a2f7b1bad 100644 --- a/drivers/mtd/nand/raw/tmio_nand.c +++ b/drivers/mtd/nand/raw/tmio_nand.c @@ -410,7 +410,7 @@ static int tmio_probe(struct platform_device *dev) nand_chip->legacy.read_buf = tmio_nand_read_buf; /* set eccmode using hardware ECC */ - nand_chip->ecc.mode = NAND_ECC_HW; + nand_chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; nand_chip->ecc.size = 512; nand_chip->ecc.bytes = 6; nand_chip->ecc.strength = 2; diff --git a/drivers/mtd/nand/raw/txx9ndfmc.c b/drivers/mtd/nand/raw/txx9ndfmc.c index 47d966871445..ef81dce6b5c4 100644 --- a/drivers/mtd/nand/raw/txx9ndfmc.c +++ b/drivers/mtd/nand/raw/txx9ndfmc.c @@ -329,7 +329,7 @@ static int __init txx9ndfmc_probe(struct platform_device *dev) chip->ecc.calculate = txx9ndfmc_calculate_ecc; chip->ecc.correct = txx9ndfmc_correct_data; chip->ecc.hwctl = txx9ndfmc_enable_hwecc; - chip->ecc.mode = NAND_ECC_HW; + chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; chip->ecc.strength = 1; chip->legacy.chip_delay = 100; chip->controller = &drvdata->controller; diff --git a/drivers/mtd/nand/raw/vf610_nfc.c b/drivers/mtd/nand/raw/vf610_nfc.c index 7248c5901183..40d70f991d89 100644 --- a/drivers/mtd/nand/raw/vf610_nfc.c +++ b/drivers/mtd/nand/raw/vf610_nfc.c @@ -323,11 +323,6 @@ static inline void vf610_nfc_ecc_mode(struct vf610_nfc *nfc, int ecc_mode) CONFIG_ECC_MODE_SHIFT, ecc_mode); } -static inline void vf610_nfc_transfer_size(struct vf610_nfc *nfc, int size) -{ - vf610_nfc_write(nfc, NFC_SECTOR_SIZE, size); -} - static inline void vf610_nfc_run(struct vf610_nfc *nfc, u32 col, u32 row, u32 cmd1, u32 cmd2, u32 trfr_sz) { @@ -732,7 +727,7 @@ static void vf610_nfc_init_controller(struct vf610_nfc *nfc) else vf610_nfc_clear(nfc, NFC_FLASH_CONFIG, CONFIG_16BIT); - if (nfc->chip.ecc.mode == NAND_ECC_HW) { + if (nfc->chip.ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) { /* Set ECC status offset in SRAM */ vf610_nfc_set_field(nfc, NFC_FLASH_CONFIG, CONFIG_ECC_SRAM_ADDR_MASK, @@ -761,7 +756,7 @@ static int vf610_nfc_attach_chip(struct nand_chip *chip) return -ENXIO; } - if (chip->ecc.mode != NAND_ECC_HW) + if (chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST) return 0; if (mtd->writesize != PAGE_2K && mtd->oobsize < 64) { @@ -779,7 +774,7 @@ static int vf610_nfc_attach_chip(struct nand_chip *chip) mtd->oobsize = 64; /* Use default large page ECC layout defined in NAND core */ - mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops); + mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout()); if (chip->ecc.strength == 32) { nfc->ecc_mode = ECC_60_BYTE; chip->ecc.bytes = 60; @@ -852,8 +847,10 @@ static int vf610_nfc_probe(struct platform_device *pdev) } of_id = of_match_device(vf610_nfc_dt_ids, &pdev->dev); - if (!of_id) - return -ENODEV; + if (!of_id) { + err = -ENODEV; + goto err_disable_clk; + } nfc->variant = (enum vf610_nfc_variant)of_id->data; diff --git a/drivers/mtd/nand/raw/xway_nand.c b/drivers/mtd/nand/raw/xway_nand.c index 29255476afdb..f2dbd63a5c1f 100644 --- a/drivers/mtd/nand/raw/xway_nand.c +++ b/drivers/mtd/nand/raw/xway_nand.c @@ -180,8 +180,8 @@ static int xway_nand_probe(struct platform_device *pdev) data->chip.legacy.read_byte = xway_read_byte; data->chip.legacy.chip_delay = 30; - data->chip.ecc.mode = NAND_ECC_SOFT; - data->chip.ecc.algo = NAND_ECC_HAMMING; + data->chip.ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + data->chip.ecc.algo = NAND_ECC_ALGO_HAMMING; platform_set_drvdata(pdev, data); nand_set_controller_data(&data->chip, data); diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index e2c382ffc5b6..c35221794645 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -419,7 +419,7 @@ static int spinand_check_ecc_status(struct spinand_device *spinand, u8 status) * fixed, so let's return the maximum possible value so that * wear-leveling layers move the data immediately. */ - return nand->eccreq.strength; + return nanddev_get_ecc_conf(nand)->strength; case STATUS_ECC_UNCOR_ERROR: return -EBADMSG; @@ -497,7 +497,7 @@ static int spinand_mtd_read(struct mtd_info *mtd, loff_t from, mutex_lock(&spinand->lock); - nanddev_io_for_each_page(nand, from, ops, &iter) { + nanddev_io_for_each_page(nand, NAND_PAGE_READ, from, ops, &iter) { ret = spinand_select_target(spinand, iter.req.pos.target); if (ret) break; @@ -545,7 +545,7 @@ static int spinand_mtd_write(struct mtd_info *mtd, loff_t to, mutex_lock(&spinand->lock); - nanddev_io_for_each_page(nand, to, ops, &iter) { + nanddev_io_for_each_page(nand, NAND_PAGE_WRITE, to, ops, &iter) { ret = spinand_select_target(spinand, iter.req.pos.target); if (ret) break; @@ -902,7 +902,7 @@ int spinand_match_and_init(struct spinand_device *spinand, continue; nand->memorg = table[i].memorg; - nand->eccreq = table[i].eccreq; + nanddev_set_ecc_requirements(nand, &table[i].eccreq); spinand->eccinfo = table[i].eccinfo; spinand->flags = table[i].flags; spinand->id.len = 1 + table[i].devid.len; @@ -1090,8 +1090,8 @@ static int spinand_init(struct spinand_device *spinand) mtd->oobavail = ret; /* Propagate ECC information to mtd_info */ - mtd->ecc_strength = nand->eccreq.strength; - mtd->ecc_step_size = nand->eccreq.step_size; + mtd->ecc_strength = nanddev_get_ecc_conf(nand)->strength; + mtd->ecc_step_size = nanddev_get_ecc_conf(nand)->step_size; return 0; diff --git a/drivers/mtd/nand/spi/gigadevice.c b/drivers/mtd/nand/spi/gigadevice.c index d219c970042a..33c67403c4aa 100644 --- a/drivers/mtd/nand/spi/gigadevice.c +++ b/drivers/mtd/nand/spi/gigadevice.c @@ -21,7 +21,7 @@ #define GD5FXGQ4UXFXXG_STATUS_ECC_UNCOR_ERROR (7 << 4) static SPINAND_OP_VARIANTS(read_cache_variants, - SPINAND_PAGE_READ_FROM_CACHE_QUADIO_OP(0, 2, NULL, 0), + SPINAND_PAGE_READ_FROM_CACHE_QUADIO_OP(0, 1, NULL, 0), SPINAND_PAGE_READ_FROM_CACHE_X4_OP(0, 1, NULL, 0), SPINAND_PAGE_READ_FROM_CACHE_DUALIO_OP(0, 1, NULL, 0), SPINAND_PAGE_READ_FROM_CACHE_X2_OP(0, 1, NULL, 0), @@ -29,7 +29,7 @@ static SPINAND_OP_VARIANTS(read_cache_variants, SPINAND_PAGE_READ_FROM_CACHE_OP(false, 0, 1, NULL, 0)); static SPINAND_OP_VARIANTS(read_cache_variants_f, - SPINAND_PAGE_READ_FROM_CACHE_QUADIO_OP(0, 2, NULL, 0), + SPINAND_PAGE_READ_FROM_CACHE_QUADIO_OP(0, 1, NULL, 0), SPINAND_PAGE_READ_FROM_CACHE_X4_OP_3A(0, 1, NULL, 0), SPINAND_PAGE_READ_FROM_CACHE_DUALIO_OP(0, 1, NULL, 0), SPINAND_PAGE_READ_FROM_CACHE_X2_OP_3A(0, 1, NULL, 0), @@ -132,6 +132,35 @@ static const struct mtd_ooblayout_ops gd5fxgq4_variant2_ooblayout = { .free = gd5fxgq4_variant2_ooblayout_free, }; +static int gd5fxgq4xc_ooblayout_256_ecc(struct mtd_info *mtd, int section, + struct mtd_oob_region *oobregion) +{ + if (section) + return -ERANGE; + + oobregion->offset = 128; + oobregion->length = 128; + + return 0; +} + +static int gd5fxgq4xc_ooblayout_256_free(struct mtd_info *mtd, int section, + struct mtd_oob_region *oobregion) +{ + if (section) + return -ERANGE; + + oobregion->offset = 1; + oobregion->length = 127; + + return 0; +} + +static const struct mtd_ooblayout_ops gd5fxgq4xc_oob_256_ops = { + .ecc = gd5fxgq4xc_ooblayout_256_ecc, + .free = gd5fxgq4xc_ooblayout_256_free, +}; + static int gd5fxgq4uexxg_ecc_get_status(struct spinand_device *spinand, u8 status) { @@ -202,7 +231,7 @@ static const struct spinand_info gigadevice_spinand_table[] = { SPINAND_INFO_OP_VARIANTS(&read_cache_variants, &write_cache_variants, &update_cache_variants), - 0, + SPINAND_HAS_QE_BIT, SPINAND_ECCINFO(&gd5fxgq4xa_ooblayout, gd5fxgq4xa_ecc_get_status)), SPINAND_INFO("GD5F2GQ4xA", @@ -212,7 +241,7 @@ static const struct spinand_info gigadevice_spinand_table[] = { SPINAND_INFO_OP_VARIANTS(&read_cache_variants, &write_cache_variants, &update_cache_variants), - 0, + SPINAND_HAS_QE_BIT, SPINAND_ECCINFO(&gd5fxgq4xa_ooblayout, gd5fxgq4xa_ecc_get_status)), SPINAND_INFO("GD5F4GQ4xA", @@ -222,9 +251,29 @@ static const struct spinand_info gigadevice_spinand_table[] = { SPINAND_INFO_OP_VARIANTS(&read_cache_variants, &write_cache_variants, &update_cache_variants), - 0, + SPINAND_HAS_QE_BIT, SPINAND_ECCINFO(&gd5fxgq4xa_ooblayout, gd5fxgq4xa_ecc_get_status)), + SPINAND_INFO("GD5F4GQ4RC", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE, 0xa4, 0x68), + NAND_MEMORG(1, 4096, 256, 64, 2048, 40, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants_f, + &write_cache_variants, + &update_cache_variants), + SPINAND_HAS_QE_BIT, + SPINAND_ECCINFO(&gd5fxgq4xc_oob_256_ops, + gd5fxgq4ufxxg_ecc_get_status)), + SPINAND_INFO("GD5F4GQ4UC", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE, 0xb4, 0x68), + NAND_MEMORG(1, 4096, 256, 64, 2048, 40, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants_f, + &write_cache_variants, + &update_cache_variants), + SPINAND_HAS_QE_BIT, + SPINAND_ECCINFO(&gd5fxgq4xc_oob_256_ops, + gd5fxgq4ufxxg_ecc_get_status)), SPINAND_INFO("GD5F1GQ4UExxG", SPINAND_ID(SPINAND_READID_METHOD_OPCODE_ADDR, 0xd1), NAND_MEMORG(1, 2048, 128, 64, 1024, 20, 1, 1, 1), @@ -232,7 +281,7 @@ static const struct spinand_info gigadevice_spinand_table[] = { SPINAND_INFO_OP_VARIANTS(&read_cache_variants, &write_cache_variants, &update_cache_variants), - 0, + SPINAND_HAS_QE_BIT, SPINAND_ECCINFO(&gd5fxgq4_variant2_ooblayout, gd5fxgq4uexxg_ecc_get_status)), SPINAND_INFO("GD5F1GQ4UFxxG", @@ -242,7 +291,7 @@ static const struct spinand_info gigadevice_spinand_table[] = { SPINAND_INFO_OP_VARIANTS(&read_cache_variants_f, &write_cache_variants, &update_cache_variants), - 0, + SPINAND_HAS_QE_BIT, SPINAND_ECCINFO(&gd5fxgq4_variant2_ooblayout, gd5fxgq4ufxxg_ecc_get_status)), }; diff --git a/drivers/mtd/nand/spi/macronix.c b/drivers/mtd/nand/spi/macronix.c index 0f900f3aa21a..8e801e4c3a00 100644 --- a/drivers/mtd/nand/spi/macronix.c +++ b/drivers/mtd/nand/spi/macronix.c @@ -84,10 +84,11 @@ static int mx35lf1ge4ab_ecc_get_status(struct spinand_device *spinand, * data around if it's not necessary. */ if (mx35lf1ge4ab_get_eccsr(spinand, &eccsr)) - return nand->eccreq.strength; + return nanddev_get_ecc_conf(nand)->strength; - if (WARN_ON(eccsr > nand->eccreq.strength || !eccsr)) - return nand->eccreq.strength; + if (WARN_ON(eccsr > nanddev_get_ecc_conf(nand)->strength || + !eccsr)) + return nanddev_get_ecc_conf(nand)->strength; return eccsr; @@ -118,6 +119,26 @@ static const struct spinand_info macronix_spinand_table[] = { &update_cache_variants), SPINAND_HAS_QE_BIT, SPINAND_ECCINFO(&mx35lfxge4ab_ooblayout, NULL)), + SPINAND_INFO("MX31LF1GE4BC", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x1e), + NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + 0 /*SPINAND_HAS_QE_BIT*/, + SPINAND_ECCINFO(&mx35lfxge4ab_ooblayout, + mx35lf1ge4ab_ecc_get_status)), + SPINAND_INFO("MX31UF1GE4BC", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x9e), + NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + 0 /*SPINAND_HAS_QE_BIT*/, + SPINAND_ECCINFO(&mx35lfxge4ab_ooblayout, + mx35lf1ge4ab_ecc_get_status)), }; static const struct spinand_manufacturer_ops macronix_spinand_manuf_ops = { diff --git a/drivers/mtd/nand/spi/toshiba.c b/drivers/mtd/nand/spi/toshiba.c index bc801d83343e..21fde2875674 100644 --- a/drivers/mtd/nand/spi/toshiba.c +++ b/drivers/mtd/nand/spi/toshiba.c @@ -90,12 +90,12 @@ static int tx58cxgxsxraix_ecc_get_status(struct spinand_device *spinand, * data around if it's not necessary. */ if (spi_mem_exec_op(spinand->spimem, &op)) - return nand->eccreq.strength; + return nanddev_get_ecc_conf(nand)->strength; mbf >>= 4; - if (WARN_ON(mbf > nand->eccreq.strength || !mbf)) - return nand->eccreq.strength; + if (WARN_ON(mbf > nanddev_get_ecc_conf(nand)->strength || !mbf)) + return nanddev_get_ecc_conf(nand)->strength; return mbf; diff --git a/drivers/mtd/parsers/Kconfig b/drivers/mtd/parsers/Kconfig index f98363c9b363..e72354322f62 100644 --- a/drivers/mtd/parsers/Kconfig +++ b/drivers/mtd/parsers/Kconfig @@ -12,7 +12,7 @@ config MTD_BCM47XX_PARTS boards. config MTD_BCM63XX_PARTS - tristate "BCM63XX CFE partitioning parser" + bool "BCM63XX CFE partitioning parser" depends on BCM63XX || BMIPS_GENERIC || COMPILE_TEST select CRC32 select MTD_PARSER_IMAGETAG diff --git a/drivers/mtd/spi-nor/controllers/intel-spi-pci.c b/drivers/mtd/spi-nor/controllers/intel-spi-pci.c index c72aa1ab71ad..555fe55d14ae 100644 --- a/drivers/mtd/spi-nor/controllers/intel-spi-pci.c +++ b/drivers/mtd/spi-nor/controllers/intel-spi-pci.c @@ -73,6 +73,7 @@ static const struct pci_device_id intel_spi_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x43a4), (unsigned long)&cnl_info }, { PCI_VDEVICE(INTEL, 0x4b24), (unsigned long)&bxt_info }, { PCI_VDEVICE(INTEL, 0x4da4), (unsigned long)&bxt_info }, + { PCI_VDEVICE(INTEL, 0x7aa4), (unsigned long)&cnl_info }, { PCI_VDEVICE(INTEL, 0xa0a4), (unsigned long)&bxt_info }, { PCI_VDEVICE(INTEL, 0xa1a4), (unsigned long)&bxt_info }, { PCI_VDEVICE(INTEL, 0xa224), (unsigned long)&bxt_info }, diff --git a/drivers/mtd/spi-nor/macronix.c b/drivers/mtd/spi-nor/macronix.c index f97f3d127575..9203abaac229 100644 --- a/drivers/mtd/spi-nor/macronix.c +++ b/drivers/mtd/spi-nor/macronix.c @@ -50,7 +50,7 @@ static const struct flash_info macronix_parts[] = { { "mx25u4035", INFO(0xc22533, 0, 64 * 1024, 8, SECT_4K) }, { "mx25u8035", INFO(0xc22534, 0, 64 * 1024, 16, SECT_4K) }, { "mx25u6435f", INFO(0xc22537, 0, 64 * 1024, 128, SECT_4K) }, - { "mx25l12805d", INFO(0xc22018, 0, 64 * 1024, 256, 0) }, + { "mx25l12805d", INFO(0xc22018, 0, 64 * 1024, 256, SECT_4K) }, { "mx25l12855e", INFO(0xc22618, 0, 64 * 1024, 256, 0) }, { "mx25r1635f", INFO(0xc22815, 0, 64 * 1024, 32, SECT_4K | SPI_NOR_DUAL_READ | diff --git a/drivers/mtd/spi-nor/winbond.c b/drivers/mtd/spi-nor/winbond.c index 6dcde15fb1aa..e5dfa786f190 100644 --- a/drivers/mtd/spi-nor/winbond.c +++ b/drivers/mtd/spi-nor/winbond.c @@ -63,6 +63,15 @@ static const struct flash_info winbond_parts[] = { { "w25q32jwm", INFO(0xef8016, 0, 64 * 1024, 64, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "w25q64jwm", INFO(0xef8017, 0, 64 * 1024, 128, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "w25q128jwm", INFO(0xef8018, 0, 64 * 1024, 256, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "w25q256jwm", INFO(0xef8019, 0, 64 * 1024, 512, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, { "w25x64", INFO(0xef3017, 0, 64 * 1024, 128, SECT_4K) }, { "w25q64", INFO(0xef4017, 0, 64 * 1024, 128, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, diff --git a/drivers/mtd/ubi/wl.c b/drivers/mtd/ubi/wl.c index 42cac572f82d..7847de75a74c 100644 --- a/drivers/mtd/ubi/wl.c +++ b/drivers/mtd/ubi/wl.c @@ -1639,6 +1639,19 @@ int ubi_thread(void *u) !ubi->thread_enabled || ubi_dbg_is_bgt_disabled(ubi)) { set_current_state(TASK_INTERRUPTIBLE); spin_unlock(&ubi->wl_lock); + + /* + * Check kthread_should_stop() after we set the task + * state to guarantee that we either see the stop bit + * and exit or the task state is reset to runnable such + * that it's not scheduled out indefinitely and detects + * the stop bit at kthread_should_stop(). + */ + if (kthread_should_stop()) { + set_current_state(TASK_RUNNING); + break; + } + schedule(); continue; } diff --git a/drivers/net/appletalk/Kconfig b/drivers/net/appletalk/Kconfig index d4f22a2e5be4..43918398f0d3 100644 --- a/drivers/net/appletalk/Kconfig +++ b/drivers/net/appletalk/Kconfig @@ -48,7 +48,7 @@ config LTPC If you are in doubt, this card is the one with the 65C02 chip on it. You also need version 1.3.3 or later of the netatalk package. This driver is experimental, which means that it may not work. - See the file <file:Documentation/networking/ltpc.rst>. + See the file <file:Documentation/networking/device_drivers/appletalk/ltpc.rst>. config COPS tristate "COPS LocalTalk PC support" diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c index 0b5b2b33b3b6..1e9a0adda2d6 100644 --- a/drivers/net/dsa/bcm_sf2.c +++ b/drivers/net/dsa/bcm_sf2.c @@ -54,7 +54,7 @@ static void bcm_sf2_recalc_clock(struct dsa_switch *ds) unsigned long new_rate; unsigned int ports_active; /* Frequenty in Mhz */ - const unsigned long rate_table[] = { + static const unsigned long rate_table[] = { 59220000, 60820000, 62500000, diff --git a/drivers/net/dsa/ocelot/seville_vsc9953.c b/drivers/net/dsa/ocelot/seville_vsc9953.c index 76576cf0ba8a..1d420c4a2f0f 100644 --- a/drivers/net/dsa/ocelot/seville_vsc9953.c +++ b/drivers/net/dsa/ocelot/seville_vsc9953.c @@ -1181,7 +1181,7 @@ static const struct felix_info seville_info_vsc9953 = { .stats_layout = vsc9953_stats_layout, .num_stats = ARRAY_SIZE(vsc9953_stats_layout), .vcap = vsc9953_vcap_props, - .shared_queue_sz = 2048 * 1024, + .shared_queue_sz = 256 * 1024, .num_mact_rows = 2048, .num_ports = 10, .mdio_bus_alloc = vsc9953_mdio_bus_alloc, diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c index 0f865daeb36d..bf5e0e9bd0e2 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c @@ -1163,7 +1163,6 @@ int aq_nic_set_link_ksettings(struct aq_nic_s *self, default: err = -1; goto err_exit; - break; } if (!(self->aq_nic_cfg.aq_hw_caps->link_speed_msk & rate)) { err = -1; diff --git a/drivers/net/ethernet/chelsio/inline_crypto/Kconfig b/drivers/net/ethernet/chelsio/inline_crypto/Kconfig index 7dfa57348d54..bc06e83fd3c6 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/Kconfig +++ b/drivers/net/ethernet/chelsio/inline_crypto/Kconfig @@ -16,6 +16,7 @@ if CHELSIO_INLINE_CRYPTO config CRYPTO_DEV_CHELSIO_TLS tristate "Chelsio Crypto Inline TLS Driver" depends on CHELSIO_T4 + depends on TLS depends on TLS_TOE help Support Chelsio Inline TLS with Chelsio crypto accelerator. diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c index 05520dccd906..ec4f79049a06 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c @@ -92,11 +92,13 @@ static void chtls_sock_release(struct kref *ref) static struct net_device *chtls_find_netdev(struct chtls_dev *cdev, struct sock *sk) { + struct adapter *adap = pci_get_drvdata(cdev->pdev); struct net_device *ndev = cdev->ports[0]; #if IS_ENABLED(CONFIG_IPV6) struct net_device *temp; int addr_type; #endif + int i; switch (sk->sk_family) { case PF_INET: @@ -127,8 +129,12 @@ static struct net_device *chtls_find_netdev(struct chtls_dev *cdev, return NULL; if (is_vlan_dev(ndev)) - return vlan_dev_real_dev(ndev); - return ndev; + ndev = vlan_dev_real_dev(ndev); + + for_each_port(adap, i) + if (cdev->ports[i] == ndev) + return ndev; + return NULL; } static void assign_rxopt(struct sock *sk, unsigned int opt) @@ -477,7 +483,6 @@ void chtls_destroy_sock(struct sock *sk) chtls_purge_write_queue(sk); free_tls_keyid(sk); kref_put(&csk->kref, chtls_sock_release); - csk->cdev = NULL; if (sk->sk_family == AF_INET) sk->sk_prot = &tcp_prot; #if IS_ENABLED(CONFIG_IPV6) @@ -736,14 +741,13 @@ void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk) #if IS_ENABLED(CONFIG_IPV6) if (sk->sk_family == PF_INET6) { - struct chtls_sock *csk; + struct net_device *ndev = chtls_find_netdev(cdev, sk); int addr_type = 0; - csk = rcu_dereference_sk_user_data(sk); addr_type = ipv6_addr_type((const struct in6_addr *) &sk->sk_v6_rcv_saddr); if (addr_type != IPV6_ADDR_ANY) - cxgb4_clip_release(csk->egress_dev, (const u32 *) + cxgb4_clip_release(ndev, (const u32 *) &sk->sk_v6_rcv_saddr, 1); } #endif @@ -1157,6 +1161,9 @@ static struct sock *chtls_recv_sock(struct sock *lsk, ndev = n->dev; if (!ndev) goto free_dst; + if (is_vlan_dev(ndev)) + ndev = vlan_dev_real_dev(ndev); + port_id = cxgb4_port_idx(ndev); csk = chtls_sock_create(cdev); diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c index 2e9acae1cba3..9fb5ca6682ea 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c @@ -902,9 +902,9 @@ static int chtls_skb_copy_to_page_nocache(struct sock *sk, return 0; } -static int csk_mem_free(struct chtls_dev *cdev, struct sock *sk) +static bool csk_mem_free(struct chtls_dev *cdev, struct sock *sk) { - return (cdev->max_host_sndbuf - sk->sk_wmem_queued); + return (cdev->max_host_sndbuf - sk->sk_wmem_queued > 0); } static int csk_wait_memory(struct chtls_dev *cdev, @@ -1240,6 +1240,7 @@ int chtls_sendpage(struct sock *sk, struct page *page, copied = 0; csk = rcu_dereference_sk_user_data(sk); cdev = csk->cdev; + lock_sock(sk); timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); err = sk_stream_wait_connect(sk, &timeo); diff --git a/drivers/net/ethernet/cisco/enic/enic_ethtool.c b/drivers/net/ethernet/cisco/enic/enic_ethtool.c index a4dd52bba2c3..1a9803f2073e 100644 --- a/drivers/net/ethernet/cisco/enic/enic_ethtool.c +++ b/drivers/net/ethernet/cisco/enic/enic_ethtool.c @@ -434,7 +434,6 @@ static int enic_grxclsrule(struct enic *enic, struct ethtool_rxnfc *cmd) break; default: return -EINVAL; - break; } fsp->h_u.tcp_ip4_spec.ip4src = flow_get_u32_src(&n->keys); diff --git a/drivers/net/ethernet/faraday/ftgmac100.c b/drivers/net/ethernet/faraday/ftgmac100.c index 87236206366f..00024dd41147 100644 --- a/drivers/net/ethernet/faraday/ftgmac100.c +++ b/drivers/net/ethernet/faraday/ftgmac100.c @@ -1817,6 +1817,11 @@ static int ftgmac100_probe(struct platform_device *pdev) priv->rxdes0_edorr_mask = BIT(30); priv->txdes0_edotr_mask = BIT(30); priv->is_aspeed = true; + /* Disable ast2600 problematic HW arbitration */ + if (of_device_is_compatible(np, "aspeed,ast2600-mac")) { + iowrite32(FTGMAC100_TM_DEFAULT, + priv->base + FTGMAC100_OFFSET_TM); + } } else { priv->rxdes0_edorr_mask = BIT(15); priv->txdes0_edotr_mask = BIT(15); diff --git a/drivers/net/ethernet/faraday/ftgmac100.h b/drivers/net/ethernet/faraday/ftgmac100.h index e5876a3fda91..63b3e02fab16 100644 --- a/drivers/net/ethernet/faraday/ftgmac100.h +++ b/drivers/net/ethernet/faraday/ftgmac100.h @@ -170,6 +170,14 @@ #define FTGMAC100_MACCR_SW_RST (1 << 31) /* + * test mode control register + */ +#define FTGMAC100_TM_RQ_TX_VALID_DIS (1 << 28) +#define FTGMAC100_TM_RQ_RR_IDLE_PREV (1 << 27) +#define FTGMAC100_TM_DEFAULT \ + (FTGMAC100_TM_RQ_TX_VALID_DIS | FTGMAC100_TM_RQ_RR_IDLE_PREV) + +/* * PHY control register */ #define FTGMAC100_PHYCR_MDC_CYCTHR_MASK 0x3f diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 1f7fe6b3dd5a..8148f796a807 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -4235,8 +4235,13 @@ static int handle_change_mac_rsp(union ibmvnic_crq *crq, dev_err(dev, "Error %ld in CHANGE_MAC_ADDR_RSP\n", rc); goto out; } + /* crq->change_mac_addr.mac_addr is the requested one + * crq->change_mac_addr_rsp.mac_addr is the returned valid one. + */ ether_addr_copy(netdev->dev_addr, &crq->change_mac_addr_rsp.mac_addr[0]); + ether_addr_copy(adapter->mac_addr, + &crq->change_mac_addr_rsp.mac_addr[0]); out: complete(&adapter->fw_done); return rc; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c index f77fa3e4fdd1..fc389eecdd2b 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c @@ -901,15 +901,13 @@ static bool ixgbe_x550em_a_has_mii(struct ixgbe_hw *hw) **/ s32 ixgbe_mii_bus_init(struct ixgbe_hw *hw) { + s32 (*write)(struct mii_bus *bus, int addr, int regnum, u16 val); + s32 (*read)(struct mii_bus *bus, int addr, int regnum); struct ixgbe_adapter *adapter = hw->back; struct pci_dev *pdev = adapter->pdev; struct device *dev = &adapter->netdev->dev; struct mii_bus *bus; - bus = devm_mdiobus_alloc(dev); - if (!bus) - return -ENOMEM; - switch (hw->device_id) { /* C3000 SoCs */ case IXGBE_DEV_ID_X550EM_A_KR: @@ -922,16 +920,23 @@ s32 ixgbe_mii_bus_init(struct ixgbe_hw *hw) case IXGBE_DEV_ID_X550EM_A_1G_T: case IXGBE_DEV_ID_X550EM_A_1G_T_L: if (!ixgbe_x550em_a_has_mii(hw)) - return -ENODEV; - bus->read = &ixgbe_x550em_a_mii_bus_read; - bus->write = &ixgbe_x550em_a_mii_bus_write; + return 0; + read = &ixgbe_x550em_a_mii_bus_read; + write = &ixgbe_x550em_a_mii_bus_write; break; default: - bus->read = &ixgbe_mii_bus_read; - bus->write = &ixgbe_mii_bus_write; + read = &ixgbe_mii_bus_read; + write = &ixgbe_mii_bus_write; break; } + bus = devm_mdiobus_alloc(dev); + if (!bus) + return -ENOMEM; + + bus->read = read; + bus->write = write; + /* Use the position of the device in the PCI hierarchy as the id */ snprintf(bus->id, MII_BUS_ID_SIZE, "%s-mdio-%s", ixgbe_driver_name, pci_name(pdev)); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.c index de563cfd294d..4b93ba149ec5 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.c @@ -350,7 +350,6 @@ static s32 ixgbe_calc_eeprom_checksum_X540(struct ixgbe_hw *hw) if (ixgbe_read_eerd_generic(hw, pointer, &length)) { hw_dbg(hw, "EEPROM read failed\n"); return IXGBE_ERR_EEPROM; - break; } /* Skip pointer section if length is invalid. */ diff --git a/drivers/net/ethernet/korina.c b/drivers/net/ethernet/korina.c index af441d699a57..bf48f0ded9c7 100644 --- a/drivers/net/ethernet/korina.c +++ b/drivers/net/ethernet/korina.c @@ -1113,7 +1113,7 @@ out: return rc; probe_err_register: - kfree(KSEG0ADDR(lp->td_ring)); + kfree((struct dma_desc *)KSEG0ADDR(lp->td_ring)); probe_err_td_ring: iounmap(lp->tx_dma_regs); probe_err_dma_tx: @@ -1133,7 +1133,7 @@ static int korina_remove(struct platform_device *pdev) iounmap(lp->eth_regs); iounmap(lp->rx_dma_regs); iounmap(lp->tx_dma_regs); - kfree(KSEG0ADDR(lp->td_ring)); + kfree((struct dma_desc *)KSEG0ADDR(lp->td_ring)); unregister_netdev(bif->dev); free_netdev(bif->dev); diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c index 344864275ed5..25981a7a43b5 100644 --- a/drivers/net/ethernet/marvell/sky2.c +++ b/drivers/net/ethernet/marvell/sky2.c @@ -5105,7 +5105,7 @@ static int sky2_probe(struct pci_dev *pdev, const struct pci_device_id *ent) INIT_WORK(&hw->restart_work, sky2_restart); pci_set_drvdata(pdev, hw); - pdev->d3_delay = 300; + pdev->d3hot_delay = 300; return 0; diff --git a/drivers/net/ethernet/mediatek/Kconfig b/drivers/net/ethernet/mediatek/Kconfig index 62a820b1eb16..3362b148de23 100644 --- a/drivers/net/ethernet/mediatek/Kconfig +++ b/drivers/net/ethernet/mediatek/Kconfig @@ -17,6 +17,7 @@ config NET_MEDIATEK_SOC config NET_MEDIATEK_STAR_EMAC tristate "MediaTek STAR Ethernet MAC support" select PHYLIB + select REGMAP_MMIO help This driver supports the ethernet MAC IP first used on MediaTek MT85** SoCs. diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c index 1eef66ee849e..cac8f085b16d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c @@ -130,14 +130,6 @@ static int mlx5i_flash_device(struct net_device *netdev, return mlx5e_ethtool_flash_device(priv, flash); } -enum mlx5_ptys_width { - MLX5_PTYS_WIDTH_1X = 1 << 0, - MLX5_PTYS_WIDTH_2X = 1 << 1, - MLX5_PTYS_WIDTH_4X = 1 << 2, - MLX5_PTYS_WIDTH_8X = 1 << 3, - MLX5_PTYS_WIDTH_12X = 1 << 4, -}; - static inline int mlx5_ptys_width_enum_to_int(enum mlx5_ptys_width width) { switch (width) { @@ -174,24 +166,6 @@ static inline int mlx5_ptys_rate_enum_to_int(enum mlx5_ptys_rate rate) } } -static int mlx5i_get_port_settings(struct net_device *netdev, - u16 *ib_link_width_oper, u16 *ib_proto_oper) -{ - struct mlx5e_priv *priv = mlx5i_epriv(netdev); - struct mlx5_core_dev *mdev = priv->mdev; - u32 out[MLX5_ST_SZ_DW(ptys_reg)] = {0}; - int ret; - - ret = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_IB, 1); - if (ret) - return ret; - - *ib_link_width_oper = MLX5_GET(ptys_reg, out, ib_link_width_oper); - *ib_proto_oper = MLX5_GET(ptys_reg, out, ib_proto_oper); - - return 0; -} - static int mlx5i_get_speed_settings(u16 ib_link_width_oper, u16 ib_proto_oper) { int rate, width; @@ -209,11 +183,14 @@ static int mlx5i_get_speed_settings(u16 ib_link_width_oper, u16 ib_proto_oper) static int mlx5i_get_link_ksettings(struct net_device *netdev, struct ethtool_link_ksettings *link_ksettings) { + struct mlx5e_priv *priv = mlx5i_epriv(netdev); + struct mlx5_core_dev *mdev = priv->mdev; u16 ib_link_width_oper; u16 ib_proto_oper; int speed, ret; - ret = mlx5i_get_port_settings(netdev, &ib_link_width_oper, &ib_proto_oper); + ret = mlx5_query_ib_port_oper(mdev, &ib_link_width_oper, &ib_proto_oper, + 1); if (ret) return ret; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c index e4186e84b3ff..4bb219565c58 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c @@ -154,24 +154,8 @@ int mlx5_set_port_beacon(struct mlx5_core_dev *dev, u16 beacon_duration) sizeof(out), MLX5_REG_MLCR, 0, 1); } -int mlx5_query_port_link_width_oper(struct mlx5_core_dev *dev, - u8 *link_width_oper, u8 local_port) -{ - u32 out[MLX5_ST_SZ_DW(ptys_reg)]; - int err; - - err = mlx5_query_port_ptys(dev, out, sizeof(out), MLX5_PTYS_IB, local_port); - if (err) - return err; - - *link_width_oper = MLX5_GET(ptys_reg, out, ib_link_width_oper); - - return 0; -} -EXPORT_SYMBOL_GPL(mlx5_query_port_link_width_oper); - -int mlx5_query_port_ib_proto_oper(struct mlx5_core_dev *dev, - u8 *proto_oper, u8 local_port) +int mlx5_query_ib_port_oper(struct mlx5_core_dev *dev, u16 *link_width_oper, + u16 *proto_oper, u8 local_port) { u32 out[MLX5_ST_SZ_DW(ptys_reg)]; int err; @@ -181,11 +165,12 @@ int mlx5_query_port_ib_proto_oper(struct mlx5_core_dev *dev, if (err) return err; + *link_width_oper = MLX5_GET(ptys_reg, out, ib_link_width_oper); *proto_oper = MLX5_GET(ptys_reg, out, ib_proto_oper); return 0; } -EXPORT_SYMBOL(mlx5_query_port_ib_proto_oper); +EXPORT_SYMBOL(mlx5_query_ib_port_oper); /* This function should be used after setting a port register only */ void mlx5_toggle_port_link(struct mlx5_core_dev *dev) diff --git a/drivers/net/ethernet/qlogic/qed/qed_rdma.c b/drivers/net/ethernet/qlogic/qed/qed_rdma.c index d3136556a1e9..da864d12916b 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_rdma.c +++ b/drivers/net/ethernet/qlogic/qed/qed_rdma.c @@ -504,7 +504,8 @@ static void qed_rdma_init_devinfo(struct qed_hwfn *p_hwfn, dev->max_mw = 0; dev->max_mr_mw_fmr_pbl = (PAGE_SIZE / 8) * (PAGE_SIZE / 8); dev->max_mr_mw_fmr_size = dev->max_mr_mw_fmr_pbl * PAGE_SIZE; - dev->max_pkey = QED_RDMA_MAX_P_KEY; + if (QED_IS_ROCE_PERSONALITY(p_hwfn)) + dev->max_pkey = QED_RDMA_MAX_P_KEY; dev->max_srq = p_hwfn->p_rdma_info->num_srqs; dev->max_srq_wr = QED_RDMA_MAX_SRQ_WQE_ELEM; @@ -1519,7 +1520,7 @@ qed_rdma_register_tid(void *rdma_cxt, params->pbl_two_level); SET_FIELD(flags, RDMA_REGISTER_TID_RAMROD_DATA_ZERO_BASED, - params->zbva); + false); SET_FIELD(flags, RDMA_REGISTER_TID_RAMROD_DATA_PHY_MR, params->phy_mr); @@ -1581,15 +1582,7 @@ qed_rdma_register_tid(void *rdma_cxt, p_ramrod->pd = cpu_to_le16(params->pd); p_ramrod->length_hi = (u8)(params->length >> 32); p_ramrod->length_lo = DMA_LO_LE(params->length); - if (params->zbva) { - /* Lower 32 bits of the registered MR address. - * In case of zero based MR, will hold FBO - */ - p_ramrod->va.hi = 0; - p_ramrod->va.lo = cpu_to_le32(params->fbo); - } else { - DMA_REGPAIR_LE(p_ramrod->va, params->vaddr); - } + DMA_REGPAIR_LE(p_ramrod->va, params->vaddr); DMA_REGPAIR_LE(p_ramrod->pbl_base, params->pbl_ptr); /* DIF */ diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c index b9aa6384563b..bedbb85a179a 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c +++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c @@ -1026,7 +1026,9 @@ int qede_change_mtu(struct net_device *ndev, int new_mtu) args.u.mtu = new_mtu; args.func = &qede_update_mtu; qede_reload(edev, &args, false); - +#if IS_ENABLED(CONFIG_QED_RDMA) + qede_rdma_event_change_mtu(edev); +#endif edev->ops->common->update_mtu(edev->cdev, new_mtu); return 0; diff --git a/drivers/net/ethernet/qlogic/qede/qede_rdma.c b/drivers/net/ethernet/qlogic/qede/qede_rdma.c index 769ec2f4d0b7..2f6598086d9b 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_rdma.c +++ b/drivers/net/ethernet/qlogic/qede/qede_rdma.c @@ -234,6 +234,15 @@ static void qede_rdma_changeaddr(struct qede_dev *edev) qedr_drv->notify(edev->rdma_info.qedr_dev, QEDE_CHANGE_ADDR); } +static void qede_rdma_change_mtu(struct qede_dev *edev) +{ + if (qede_rdma_supported(edev)) { + if (qedr_drv && edev->rdma_info.qedr_dev && qedr_drv->notify) + qedr_drv->notify(edev->rdma_info.qedr_dev, + QEDE_CHANGE_MTU); + } +} + static struct qede_rdma_event_work * qede_rdma_get_free_event_node(struct qede_dev *edev) { @@ -287,6 +296,9 @@ static void qede_rdma_handle_event(struct work_struct *work) case QEDE_CHANGE_ADDR: qede_rdma_changeaddr(edev); break; + case QEDE_CHANGE_MTU: + qede_rdma_change_mtu(edev); + break; default: DP_NOTICE(edev, "Invalid rdma event %d", event); } @@ -338,3 +350,8 @@ void qede_rdma_event_changeaddr(struct qede_dev *edev) { qede_rdma_add_event(edev, QEDE_CHANGE_ADDR); } + +void qede_rdma_event_change_mtu(struct qede_dev *edev) +{ + qede_rdma_add_event(edev, QEDE_CHANGE_MTU); +} diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index 7d366b0362cb..3b6ddc706e92 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -4694,7 +4694,7 @@ static int rtl8169_close(struct net_device *dev) phy_disconnect(tp->phydev); - pci_free_irq(pdev, 0, tp); + free_irq(pci_irq_vector(pdev, 0), tp); dma_free_coherent(&pdev->dev, R8169_RX_RING_BYTES, tp->RxDescArray, tp->RxPhyAddr); @@ -4745,8 +4745,8 @@ static int rtl_open(struct net_device *dev) rtl_request_firmware(tp); - retval = pci_request_irq(pdev, 0, rtl8169_interrupt, NULL, tp, - dev->name); + retval = request_irq(pci_irq_vector(pdev, 0), rtl8169_interrupt, + IRQF_NO_THREAD | IRQF_SHARED, dev->name, tp); if (retval < 0) goto err_release_fw_2; @@ -4763,7 +4763,7 @@ out: return retval; err_free_irq: - pci_free_irq(pdev, 0, tp); + free_irq(pci_irq_vector(pdev, 0), tp); err_release_fw_2: rtl_release_firmware(tp); rtl8169_rx_clear(tp); diff --git a/drivers/net/ethernet/sfc/efx_common.c b/drivers/net/ethernet/sfc/efx_common.c index 72a3f0e09f52..de797e1ac5a9 100644 --- a/drivers/net/ethernet/sfc/efx_common.c +++ b/drivers/net/ethernet/sfc/efx_common.c @@ -1014,6 +1014,7 @@ int efx_init_struct(struct efx_nic *efx, efx->num_mac_stats = MC_CMD_MAC_NSTATS; BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END); mutex_init(&efx->mac_lock); + init_rwsem(&efx->filter_sem); #ifdef CONFIG_RFS_ACCEL mutex_init(&efx->rps_mutex); spin_lock_init(&efx->rps_hash_lock); diff --git a/drivers/net/ethernet/sfc/rx_common.c b/drivers/net/ethernet/sfc/rx_common.c index 5e29284c89c9..19cf7cac1e6e 100644 --- a/drivers/net/ethernet/sfc/rx_common.c +++ b/drivers/net/ethernet/sfc/rx_common.c @@ -797,7 +797,6 @@ int efx_probe_filters(struct efx_nic *efx) { int rc; - init_rwsem(&efx->filter_sem); mutex_lock(&efx->mac_lock); down_write(&efx->filter_sem); rc = efx->type->filter_table_probe(efx); diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c index 806eb651cea3..1503cc9ec6e2 100644 --- a/drivers/net/ethernet/socionext/netsec.c +++ b/drivers/net/ethernet/socionext/netsec.c @@ -6,6 +6,7 @@ #include <linux/pm_runtime.h> #include <linux/acpi.h> #include <linux/of_mdio.h> +#include <linux/of_net.h> #include <linux/etherdevice.h> #include <linux/interrupt.h> #include <linux/io.h> @@ -1833,6 +1834,14 @@ static const struct net_device_ops netsec_netdev_ops = { static int netsec_of_probe(struct platform_device *pdev, struct netsec_priv *priv, u32 *phy_addr) { + int err; + + err = of_get_phy_mode(pdev->dev.of_node, &priv->phy_interface); + if (err) { + dev_err(&pdev->dev, "missing required property 'phy-mode'\n"); + return err; + } + priv->phy_np = of_parse_phandle(pdev->dev.of_node, "phy-handle", 0); if (!priv->phy_np) { dev_err(&pdev->dev, "missing required property 'phy-handle'\n"); @@ -1859,6 +1868,14 @@ static int netsec_acpi_probe(struct platform_device *pdev, if (!IS_ENABLED(CONFIG_ACPI)) return -ENODEV; + /* ACPI systems are assumed to configure the PHY in firmware, so + * there is really no need to discover the PHY mode from the DSDT. + * Since firmware is known to exist in the field that configures the + * PHY correctly but passes the wrong mode string in the phy-mode + * device property, we have no choice but to ignore it. + */ + priv->phy_interface = PHY_INTERFACE_MODE_NA; + ret = device_property_read_u32(&pdev->dev, "phy-channel", phy_addr); if (ret) { dev_err(&pdev->dev, @@ -1995,13 +2012,6 @@ static int netsec_probe(struct platform_device *pdev) priv->msg_enable = NETIF_MSG_TX_ERR | NETIF_MSG_HW | NETIF_MSG_DRV | NETIF_MSG_LINK | NETIF_MSG_PROBE; - priv->phy_interface = device_get_phy_mode(&pdev->dev); - if ((int)priv->phy_interface < 0) { - dev_err(&pdev->dev, "missing required property 'phy-mode'\n"); - ret = -ENODEV; - goto free_ndev; - } - priv->ioaddr = devm_ioremap(&pdev->dev, mmio_res->start, resource_size(mmio_res)); if (!priv->ioaddr) { diff --git a/drivers/net/hamradio/scc.c b/drivers/net/hamradio/scc.c index 1e915871baa7..36eeb80406f2 100644 --- a/drivers/net/hamradio/scc.c +++ b/drivers/net/hamradio/scc.c @@ -7,7 +7,7 @@ * ------------------ * * You can find a subset of the documentation in - * Documentation/networking/device_drivers/wan/z8530drv.rst. + * Documentation/networking/device_drivers/hamradio/z8530drv.rst. */ /* diff --git a/drivers/net/pcs/Kconfig b/drivers/net/pcs/Kconfig index 074fb3f5db18..22ba7b0b476d 100644 --- a/drivers/net/pcs/Kconfig +++ b/drivers/net/pcs/Kconfig @@ -7,8 +7,7 @@ menu "PCS device drivers" config PCS_XPCS tristate "Synopsys DesignWare XPCS controller" - select MDIO_BUS - depends on MDIO_DEVICE + depends on MDIO_DEVICE && MDIO_BUS help This module provides helper functions for Synopsys DesignWare XPCS controllers. diff --git a/drivers/net/usb/rtl8150.c b/drivers/net/usb/rtl8150.c index f020401adf04..bf8a60533f3e 100644 --- a/drivers/net/usb/rtl8150.c +++ b/drivers/net/usb/rtl8150.c @@ -261,7 +261,7 @@ static void set_ethernet_addr(rtl8150_t *dev) ret = get_registers(dev, IDR, sizeof(node_id), node_id); - if (ret == sizeof(node_id)) { + if (!ret) { ether_addr_copy(dev->netdev->dev_addr, node_id); } else { eth_hw_addr_random(dev->netdev); diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index d2d2c4a53cf2..21b71148c532 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -68,8 +68,6 @@ static const unsigned long guest_offloads[] = { (1ULL << VIRTIO_NET_F_GUEST_ECN) | \ (1ULL << VIRTIO_NET_F_GUEST_UFO)) -#define GUEST_OFFLOAD_CSUM_MASK (1ULL << VIRTIO_NET_F_GUEST_CSUM) - struct virtnet_stat_desc { char desc[ETH_GSTRING_LEN]; size_t offset; @@ -2524,48 +2522,29 @@ static int virtnet_get_phys_port_name(struct net_device *dev, char *buf, return 0; } -static netdev_features_t virtnet_fix_features(struct net_device *netdev, - netdev_features_t features) -{ - /* If Rx checksum is disabled, LRO should also be disabled. */ - if (!(features & NETIF_F_RXCSUM)) - features &= ~NETIF_F_LRO; - - return features; -} - static int virtnet_set_features(struct net_device *dev, netdev_features_t features) { struct virtnet_info *vi = netdev_priv(dev); - u64 offloads = vi->guest_offloads; + u64 offloads; int err; - /* Don't allow configuration while XDP is active. */ - if (vi->xdp_queue_pairs) - return -EBUSY; - if ((dev->features ^ features) & NETIF_F_LRO) { + if (vi->xdp_queue_pairs) + return -EBUSY; + if (features & NETIF_F_LRO) - offloads |= GUEST_OFFLOAD_LRO_MASK & - vi->guest_offloads_capable; + offloads = vi->guest_offloads_capable; else - offloads &= ~GUEST_OFFLOAD_LRO_MASK; - } + offloads = vi->guest_offloads_capable & + ~GUEST_OFFLOAD_LRO_MASK; - if ((dev->features ^ features) & NETIF_F_RXCSUM) { - if (features & NETIF_F_RXCSUM) - offloads |= GUEST_OFFLOAD_CSUM_MASK & - vi->guest_offloads_capable; - else - offloads &= ~GUEST_OFFLOAD_CSUM_MASK; + err = virtnet_set_guest_offloads(vi, offloads); + if (err) + return err; + vi->guest_offloads = offloads; } - err = virtnet_set_guest_offloads(vi, offloads); - if (err) - return err; - - vi->guest_offloads = offloads; return 0; } @@ -2584,7 +2563,6 @@ static const struct net_device_ops virtnet_netdev = { .ndo_features_check = passthru_features_check, .ndo_get_phys_port_name = virtnet_get_phys_port_name, .ndo_set_features = virtnet_set_features, - .ndo_fix_features = virtnet_fix_features, }; static void virtnet_config_changed_work(struct work_struct *work) @@ -3035,10 +3013,8 @@ static int virtnet_probe(struct virtio_device *vdev) if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) || virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6)) dev->features |= NETIF_F_LRO; - if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) { - dev->hw_features |= NETIF_F_RXCSUM; + if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) dev->hw_features |= NETIF_F_LRO; - } dev->vlan_features = dev->features; diff --git a/drivers/net/wan/hdlc.c b/drivers/net/wan/hdlc.c index 9b00708676cf..1bdd3df0867a 100644 --- a/drivers/net/wan/hdlc.c +++ b/drivers/net/wan/hdlc.c @@ -46,7 +46,15 @@ static struct hdlc_proto *first_proto; static int hdlc_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *p, struct net_device *orig_dev) { - struct hdlc_device *hdlc = dev_to_hdlc(dev); + struct hdlc_device *hdlc; + + /* First make sure "dev" is an HDLC device */ + if (!(dev->priv_flags & IFF_WAN_HDLC)) { + kfree_skb(skb); + return NET_RX_SUCCESS; + } + + hdlc = dev_to_hdlc(dev); if (!net_eq(dev_net(dev), &init_net)) { kfree_skb(skb); diff --git a/drivers/net/wan/hdlc_raw_eth.c b/drivers/net/wan/hdlc_raw_eth.c index 08e0a46501de..c70a518b8b47 100644 --- a/drivers/net/wan/hdlc_raw_eth.c +++ b/drivers/net/wan/hdlc_raw_eth.c @@ -99,6 +99,7 @@ static int raw_eth_ioctl(struct net_device *dev, struct ifreq *ifr) old_qlen = dev->tx_queue_len; ether_setup(dev); dev->tx_queue_len = old_qlen; + dev->priv_flags &= ~IFF_TX_SKB_SHARING; eth_hw_addr_random(dev); call_netdevice_notifiers(NETDEV_POST_TYPE_CHANGE, dev); netif_dormant_off(dev); diff --git a/drivers/net/wan/lmc/lmc_proto.c b/drivers/net/wan/lmc/lmc_proto.c index e8b0b902b424..4e9cc83b615a 100644 --- a/drivers/net/wan/lmc/lmc_proto.c +++ b/drivers/net/wan/lmc/lmc_proto.c @@ -89,17 +89,13 @@ __be16 lmc_proto_type(lmc_softc_t *sc, struct sk_buff *skb) /*FOLD00*/ switch(sc->if_type){ case LMC_PPP: return hdlc_type_trans(skb, sc->lmc_device); - break; case LMC_NET: return htons(ETH_P_802_2); - break; case LMC_RAW: /* Packet type for skbuff kind of useless */ return htons(ETH_P_802_2); - break; default: printk(KERN_WARNING "%s: No protocol set for this interface, assuming 802.2 (which is wrong!!)\n", sc->name); return htons(ETH_P_802_2); - break; } } diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h index ae477f7756af..8ee24e351bdc 100644 --- a/drivers/net/xen-netback/common.h +++ b/drivers/net/xen-netback/common.h @@ -140,6 +140,20 @@ struct xenvif_queue { /* Per-queue data for xenvif */ char name[QUEUE_NAME_SIZE]; /* DEVNAME-qN */ struct xenvif *vif; /* Parent VIF */ + /* + * TX/RX common EOI handling. + * When feature-split-event-channels = 0, interrupt handler sets + * NETBK_COMMON_EOI, otherwise NETBK_RX_EOI and NETBK_TX_EOI are set + * by the RX and TX interrupt handlers. + * RX and TX handler threads will issue an EOI when either + * NETBK_COMMON_EOI or their specific bits (NETBK_RX_EOI or + * NETBK_TX_EOI) are set and they will reset those bits. + */ + atomic_t eoi_pending; +#define NETBK_RX_EOI 0x01 +#define NETBK_TX_EOI 0x02 +#define NETBK_COMMON_EOI 0x04 + /* Use NAPI for guest TX */ struct napi_struct napi; /* When feature-split-event-channels = 0, tx_irq = rx_irq. */ @@ -378,6 +392,7 @@ int xenvif_dealloc_kthread(void *data); irqreturn_t xenvif_ctrl_irq_fn(int irq, void *data); +bool xenvif_have_rx_work(struct xenvif_queue *queue, bool test_kthread); void xenvif_rx_action(struct xenvif_queue *queue); void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb); diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c index 8af497285691..acb786d8b1d8 100644 --- a/drivers/net/xen-netback/interface.c +++ b/drivers/net/xen-netback/interface.c @@ -77,12 +77,28 @@ int xenvif_schedulable(struct xenvif *vif) !vif->disabled; } +static bool xenvif_handle_tx_interrupt(struct xenvif_queue *queue) +{ + bool rc; + + rc = RING_HAS_UNCONSUMED_REQUESTS(&queue->tx); + if (rc) + napi_schedule(&queue->napi); + return rc; +} + static irqreturn_t xenvif_tx_interrupt(int irq, void *dev_id) { struct xenvif_queue *queue = dev_id; + int old; - if (RING_HAS_UNCONSUMED_REQUESTS(&queue->tx)) - napi_schedule(&queue->napi); + old = atomic_fetch_or(NETBK_TX_EOI, &queue->eoi_pending); + WARN(old & NETBK_TX_EOI, "Interrupt while EOI pending\n"); + + if (!xenvif_handle_tx_interrupt(queue)) { + atomic_andnot(NETBK_TX_EOI, &queue->eoi_pending); + xen_irq_lateeoi(irq, XEN_EOI_FLAG_SPURIOUS); + } return IRQ_HANDLED; } @@ -116,19 +132,46 @@ static int xenvif_poll(struct napi_struct *napi, int budget) return work_done; } +static bool xenvif_handle_rx_interrupt(struct xenvif_queue *queue) +{ + bool rc; + + rc = xenvif_have_rx_work(queue, false); + if (rc) + xenvif_kick_thread(queue); + return rc; +} + static irqreturn_t xenvif_rx_interrupt(int irq, void *dev_id) { struct xenvif_queue *queue = dev_id; + int old; - xenvif_kick_thread(queue); + old = atomic_fetch_or(NETBK_RX_EOI, &queue->eoi_pending); + WARN(old & NETBK_RX_EOI, "Interrupt while EOI pending\n"); + + if (!xenvif_handle_rx_interrupt(queue)) { + atomic_andnot(NETBK_RX_EOI, &queue->eoi_pending); + xen_irq_lateeoi(irq, XEN_EOI_FLAG_SPURIOUS); + } return IRQ_HANDLED; } irqreturn_t xenvif_interrupt(int irq, void *dev_id) { - xenvif_tx_interrupt(irq, dev_id); - xenvif_rx_interrupt(irq, dev_id); + struct xenvif_queue *queue = dev_id; + int old; + + old = atomic_fetch_or(NETBK_COMMON_EOI, &queue->eoi_pending); + WARN(old, "Interrupt while EOI pending\n"); + + /* Use bitwise or as we need to call both functions. */ + if ((!xenvif_handle_tx_interrupt(queue) | + !xenvif_handle_rx_interrupt(queue))) { + atomic_andnot(NETBK_COMMON_EOI, &queue->eoi_pending); + xen_irq_lateeoi(irq, XEN_EOI_FLAG_SPURIOUS); + } return IRQ_HANDLED; } @@ -605,7 +648,7 @@ int xenvif_connect_ctrl(struct xenvif *vif, grant_ref_t ring_ref, if (req_prod - rsp_prod > RING_SIZE(&vif->ctrl)) goto err_unmap; - err = bind_interdomain_evtchn_to_irq(vif->domid, evtchn); + err = bind_interdomain_evtchn_to_irq_lateeoi(vif->domid, evtchn); if (err < 0) goto err_unmap; @@ -709,7 +752,7 @@ int xenvif_connect_data(struct xenvif_queue *queue, if (tx_evtchn == rx_evtchn) { /* feature-split-event-channels == 0 */ - err = bind_interdomain_evtchn_to_irqhandler( + err = bind_interdomain_evtchn_to_irqhandler_lateeoi( queue->vif->domid, tx_evtchn, xenvif_interrupt, 0, queue->name, queue); if (err < 0) @@ -720,7 +763,7 @@ int xenvif_connect_data(struct xenvif_queue *queue, /* feature-split-event-channels == 1 */ snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name), "%s-tx", queue->name); - err = bind_interdomain_evtchn_to_irqhandler( + err = bind_interdomain_evtchn_to_irqhandler_lateeoi( queue->vif->domid, tx_evtchn, xenvif_tx_interrupt, 0, queue->tx_irq_name, queue); if (err < 0) @@ -730,7 +773,7 @@ int xenvif_connect_data(struct xenvif_queue *queue, snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name), "%s-rx", queue->name); - err = bind_interdomain_evtchn_to_irqhandler( + err = bind_interdomain_evtchn_to_irqhandler_lateeoi( queue->vif->domid, rx_evtchn, xenvif_rx_interrupt, 0, queue->rx_irq_name, queue); if (err < 0) diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c index 6dfca7265644..bc3421d14576 100644 --- a/drivers/net/xen-netback/netback.c +++ b/drivers/net/xen-netback/netback.c @@ -169,6 +169,10 @@ void xenvif_napi_schedule_or_enable_events(struct xenvif_queue *queue) if (more_to_do) napi_schedule(&queue->napi); + else if (atomic_fetch_andnot(NETBK_TX_EOI | NETBK_COMMON_EOI, + &queue->eoi_pending) & + (NETBK_TX_EOI | NETBK_COMMON_EOI)) + xen_irq_lateeoi(queue->tx_irq, 0); } static void tx_add_credit(struct xenvif_queue *queue) @@ -1643,9 +1647,14 @@ static bool xenvif_ctrl_work_todo(struct xenvif *vif) irqreturn_t xenvif_ctrl_irq_fn(int irq, void *data) { struct xenvif *vif = data; + unsigned int eoi_flag = XEN_EOI_FLAG_SPURIOUS; - while (xenvif_ctrl_work_todo(vif)) + while (xenvif_ctrl_work_todo(vif)) { xenvif_ctrl_action(vif); + eoi_flag = 0; + } + + xen_irq_lateeoi(irq, eoi_flag); return IRQ_HANDLED; } diff --git a/drivers/net/xen-netback/rx.c b/drivers/net/xen-netback/rx.c index ac034f69a170..b8febe1d1bfd 100644 --- a/drivers/net/xen-netback/rx.c +++ b/drivers/net/xen-netback/rx.c @@ -503,13 +503,13 @@ static bool xenvif_rx_queue_ready(struct xenvif_queue *queue) return queue->stalled && prod - cons >= 1; } -static bool xenvif_have_rx_work(struct xenvif_queue *queue) +bool xenvif_have_rx_work(struct xenvif_queue *queue, bool test_kthread) { return xenvif_rx_ring_slots_available(queue) || (queue->vif->stall_timeout && (xenvif_rx_queue_stalled(queue) || xenvif_rx_queue_ready(queue))) || - kthread_should_stop() || + (test_kthread && kthread_should_stop()) || queue->vif->disabled; } @@ -540,15 +540,20 @@ static void xenvif_wait_for_rx_work(struct xenvif_queue *queue) { DEFINE_WAIT(wait); - if (xenvif_have_rx_work(queue)) + if (xenvif_have_rx_work(queue, true)) return; for (;;) { long ret; prepare_to_wait(&queue->wq, &wait, TASK_INTERRUPTIBLE); - if (xenvif_have_rx_work(queue)) + if (xenvif_have_rx_work(queue, true)) break; + if (atomic_fetch_andnot(NETBK_RX_EOI | NETBK_COMMON_EOI, + &queue->eoi_pending) & + (NETBK_RX_EOI | NETBK_COMMON_EOI)) + xen_irq_lateeoi(queue->rx_irq, 0); + ret = schedule_timeout(xenvif_rx_queue_timeout(queue)); if (!ret) break; diff --git a/drivers/nfc/st21nfca/core.c b/drivers/nfc/st21nfca/core.c index 2ce17932a073..6ca0d2f56b18 100644 --- a/drivers/nfc/st21nfca/core.c +++ b/drivers/nfc/st21nfca/core.c @@ -794,7 +794,6 @@ static int st21nfca_hci_im_transceive(struct nfc_hci_dev *hdev, skb->len, st21nfca_hci_data_exchange_cb, info); - break; default: return 1; } diff --git a/drivers/nfc/trf7970a.c b/drivers/nfc/trf7970a.c index 3bd97c73f983..c70f62fe321e 100644 --- a/drivers/nfc/trf7970a.c +++ b/drivers/nfc/trf7970a.c @@ -1382,7 +1382,6 @@ static int trf7970a_is_iso15693_write_or_lock(u8 cmd) case ISO15693_CMD_WRITE_DSFID: case ISO15693_CMD_LOCK_DSFID: return 1; - break; default: return 0; } diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig index 438a792d2cf7..0c473d75e625 100644 --- a/drivers/pci/Kconfig +++ b/drivers/pci/Kconfig @@ -190,6 +190,68 @@ config PCI_HYPERV The PCI device frontend driver allows the kernel to import arbitrary PCI devices from a PCI backend to support PCI driver domains. +choice + prompt "PCI Express hierarchy optimization setting" + default PCIE_BUS_DEFAULT + depends on PCI && EXPERT + help + MPS (Max Payload Size) and MRRS (Max Read Request Size) are PCIe + device parameters that affect performance and the ability to + support hotplug and peer-to-peer DMA. + + The following choices set the MPS and MRRS optimization strategy + at compile-time. The choices are the same as those offered for + the kernel command-line parameter 'pci', i.e., + 'pci=pcie_bus_tune_off', 'pci=pcie_bus_safe', + 'pci=pcie_bus_perf', and 'pci=pcie_bus_peer2peer'. + + This is a compile-time setting and can be overridden by the above + command-line parameters. If unsure, choose PCIE_BUS_DEFAULT. + +config PCIE_BUS_TUNE_OFF + bool "Tune Off" + depends on PCI + help + Use the BIOS defaults; don't touch MPS at all. This is the same + as booting with 'pci=pcie_bus_tune_off'. + +config PCIE_BUS_DEFAULT + bool "Default" + depends on PCI + help + Default choice; ensure that the MPS matches upstream bridge. + +config PCIE_BUS_SAFE + bool "Safe" + depends on PCI + help + Use largest MPS that boot-time devices support. If you have a + closed system with no possibility of adding new devices, this + will use the largest MPS that's supported by all devices. This + is the same as booting with 'pci=pcie_bus_safe'. + +config PCIE_BUS_PERFORMANCE + bool "Performance" + depends on PCI + help + Use MPS and MRRS for best performance. Ensure that a given + device's MPS is no larger than its parent MPS, which allows us to + keep all switches/bridges to the max MPS supported by their + parent. This is the same as booting with 'pci=pcie_bus_perf'. + +config PCIE_BUS_PEER2PEER + bool "Peer2peer" + depends on PCI + help + Set MPS = 128 for all devices. MPS configuration effected by the + other options could cause the MPS on one root port to be + different than that of the MPS on another, which may cause + hot-added devices or peer-to-peer DMA to fail. Set MPS to the + smallest possible value (128B) system-wide to avoid these issues. + This is the same as booting with 'pci=pcie_bus_peer2peer'. + +endchoice + source "drivers/pci/hotplug/Kconfig" source "drivers/pci/controller/Kconfig" source "drivers/pci/endpoint/Kconfig" diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 4a7afbe189f8..64e2f5e379aa 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -12,7 +12,7 @@ config PCI_MVEBU select PCI_BRIDGE_EMUL config PCI_AARDVARK - bool "Aardvark PCIe controller" + tristate "Aardvark PCIe controller" depends on (ARCH_MVEBU && ARM64) || COMPILE_TEST depends on OF depends on PCI_MSI_IRQ_DOMAIN @@ -273,9 +273,10 @@ config VMD config PCIE_BRCMSTB tristate "Broadcom Brcmstb PCIe host controller" - depends on ARCH_BCM2835 || COMPILE_TEST + depends on ARCH_BRCMSTB || ARCH_BCM2835 || COMPILE_TEST depends on OF depends on PCI_MSI_IRQ_DOMAIN + default ARCH_BRCMSTB help Say Y here to enable PCIe host controller support for Broadcom STB based SoCs, like the Raspberry Pi 4. @@ -297,6 +298,13 @@ config PCI_LOONGSON Say Y here if you want to enable PCI controller support on Loongson systems. +config PCIE_HISI_ERR + depends on ACPI_APEI_GHES && (ARM64 || COMPILE_TEST) + bool "HiSilicon HIP PCIe controller error handling driver" + help + Say Y here if you want error handling support + for the PCIe controller's errors on HiSilicon HIP SoCs + source "drivers/pci/controller/dwc/Kconfig" source "drivers/pci/controller/mobiveil/Kconfig" source "drivers/pci/controller/cadence/Kconfig" diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index bcdbf49ab1e4..04c6edc285c5 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -31,6 +31,7 @@ obj-$(CONFIG_PCIE_TANGO_SMP8759) += pcie-tango.o obj-$(CONFIG_VMD) += vmd.o obj-$(CONFIG_PCIE_BRCMSTB) += pcie-brcmstb.o obj-$(CONFIG_PCI_LOONGSON) += pci-loongson.o +obj-$(CONFIG_PCIE_HISI_ERR) += pcie-hisi-error.o # pcie-hisi.o quirks are needed even without CONFIG_PCIE_DW obj-y += dwc/ obj-y += mobiveil/ diff --git a/drivers/pci/controller/cadence/pcie-cadence-ep.c b/drivers/pci/controller/cadence/pcie-cadence-ep.c index 254a3e1eff50..84cc58dc8512 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-ep.c +++ b/drivers/pci/controller/cadence/pcie-cadence-ep.c @@ -328,7 +328,6 @@ static int cdns_pcie_ep_send_legacy_irq(struct cdns_pcie_ep *ep, u8 fn, u8 intx) cdns_pcie_ep_assert_intx(ep, fn, intx, true); /* * The mdelay() value was taken from dra7xx_pcie_raise_legacy_irq() - * from drivers/pci/dwc/pci-dra7xx.c */ mdelay(1); cdns_pcie_ep_assert_intx(ep, fn, intx, false); diff --git a/drivers/pci/controller/cadence/pcie-cadence-host.c b/drivers/pci/controller/cadence/pcie-cadence-host.c index 4550e0d469ca..811c1cb2e8de 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-host.c +++ b/drivers/pci/controller/cadence/pcie-cadence-host.c @@ -337,7 +337,7 @@ static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc) struct resource_entry *entry; u64 cpu_addr = cfg_res->start; u32 addr0, addr1, desc1; - int r, err, busnr = 0; + int r, busnr = 0; entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS); if (entry) @@ -383,11 +383,7 @@ static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc) r++; } - err = cdns_pcie_host_map_dma_ranges(rc); - if (err) - return err; - - return 0; + return cdns_pcie_host_map_dma_ranges(rc); } static int cdns_pcie_host_init(struct device *dev, diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig index 044a3761c44f..bc049865f8e0 100644 --- a/drivers/pci/controller/dwc/Kconfig +++ b/drivers/pci/controller/dwc/Kconfig @@ -237,8 +237,9 @@ config PCIE_HISI_STB Say Y here if you want PCIe controller support on HiSilicon STB SoCs config PCI_MESON - bool "MESON PCIe controller" + tristate "MESON PCIe controller" depends on PCI_MSI_IRQ_DOMAIN + default m if ARCH_MESON select PCIE_DW_HOST help Say Y here if you want to enable PCI controller support on Amlogic diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c index dc387724cf08..6d012d2b1e90 100644 --- a/drivers/pci/controller/dwc/pci-dra7xx.c +++ b/drivers/pci/controller/dwc/pci-dra7xx.c @@ -73,8 +73,6 @@ #define LINK_UP BIT(16) #define DRA7XX_CPU_TO_BUS_ADDR 0x0FFFFFFF -#define EXP_CAP_ID_OFFSET 0x70 - #define PCIECTRL_TI_CONF_INTX_ASSERT 0x0124 #define PCIECTRL_TI_CONF_INTX_DEASSERT 0x0128 @@ -91,7 +89,6 @@ struct dra7xx_pcie { void __iomem *base; /* DT ti_conf */ int phy_count; /* DT phy-names count */ struct phy **phy; - int link_gen; struct irq_domain *irq_domain; enum dw_pcie_device_mode mode; }; @@ -142,33 +139,12 @@ static int dra7xx_pcie_establish_link(struct dw_pcie *pci) struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci); struct device *dev = pci->dev; u32 reg; - u32 exp_cap_off = EXP_CAP_ID_OFFSET; if (dw_pcie_link_up(pci)) { dev_err(dev, "link is already up\n"); return 0; } - if (dra7xx->link_gen == 1) { - dw_pcie_read(pci->dbi_base + exp_cap_off + PCI_EXP_LNKCAP, - 4, ®); - if ((reg & PCI_EXP_LNKCAP_SLS) != PCI_EXP_LNKCAP_SLS_2_5GB) { - reg &= ~((u32)PCI_EXP_LNKCAP_SLS); - reg |= PCI_EXP_LNKCAP_SLS_2_5GB; - dw_pcie_write(pci->dbi_base + exp_cap_off + - PCI_EXP_LNKCAP, 4, reg); - } - - dw_pcie_read(pci->dbi_base + exp_cap_off + PCI_EXP_LNKCTL2, - 2, ®); - if ((reg & PCI_EXP_LNKCAP_SLS) != PCI_EXP_LNKCAP_SLS_2_5GB) { - reg &= ~((u32)PCI_EXP_LNKCAP_SLS); - reg |= PCI_EXP_LNKCAP_SLS_2_5GB; - dw_pcie_write(pci->dbi_base + exp_cap_off + - PCI_EXP_LNKCTL2, 2, reg); - } - } - reg = dra7xx_pcie_readl(dra7xx, PCIECTRL_DRA7XX_CONF_DEVICE_CMD); reg |= LTSSM_EN; dra7xx_pcie_writel(dra7xx, PCIECTRL_DRA7XX_CONF_DEVICE_CMD, reg); @@ -490,7 +466,9 @@ static struct irq_chip dra7xx_pci_msi_bottom_irq_chip = { static int dra7xx_pcie_msi_host_init(struct pcie_port *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct device *dev = pci->dev; u32 ctrl, num_ctrls; + int ret; pp->msi_irq_chip = &dra7xx_pci_msi_bottom_irq_chip; @@ -506,7 +484,21 @@ static int dra7xx_pcie_msi_host_init(struct pcie_port *pp) ~0); } - return dw_pcie_allocate_domains(pp); + ret = dw_pcie_allocate_domains(pp); + if (ret) + return ret; + + pp->msi_data = dma_map_single_attrs(dev, &pp->msi_msg, + sizeof(pp->msi_msg), + DMA_FROM_DEVICE, + DMA_ATTR_SKIP_CPU_SYNC); + ret = dma_mapping_error(dev, pp->msi_data); + if (ret) { + dev_err(dev, "Failed to map MSI data\n"); + pp->msi_data = 0; + dw_pcie_free_msi(pp); + } + return ret; } static const struct dw_pcie_host_ops dra7xx_pcie_host_ops = { @@ -937,10 +929,6 @@ static int __init dra7xx_pcie_probe(struct platform_device *pdev) reg &= ~LTSSM_EN; dra7xx_pcie_writel(dra7xx, PCIECTRL_DRA7XX_CONF_DEVICE_CMD, reg); - dra7xx->link_gen = of_pci_get_max_link_speed(np); - if (dra7xx->link_gen < 0 || dra7xx->link_gen > 2) - dra7xx->link_gen = 2; - switch (mode) { case DW_PCIE_RC_TYPE: if (!IS_ENABLED(CONFIG_PCI_DRA7XX_HOST)) { diff --git a/drivers/pci/controller/dwc/pci-exynos.c b/drivers/pci/controller/dwc/pci-exynos.c index 8d82c43ae299..242683cde04a 100644 --- a/drivers/pci/controller/dwc/pci-exynos.c +++ b/drivers/pci/controller/dwc/pci-exynos.c @@ -336,32 +336,37 @@ static void exynos_pcie_write_dbi(struct dw_pcie *pci, void __iomem *base, exynos_pcie_sideband_dbi_w_mode(ep, false); } -static int exynos_pcie_rd_own_conf(struct pcie_port *pp, int where, int size, - u32 *val) +static int exynos_pcie_rd_own_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) { - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct exynos_pcie *ep = to_exynos_pcie(pci); - int ret; + struct dw_pcie *pci = to_dw_pcie_from_pp(bus->sysdata); - exynos_pcie_sideband_dbi_r_mode(ep, true); - ret = dw_pcie_read(pci->dbi_base + where, size, val); - exynos_pcie_sideband_dbi_r_mode(ep, false); - return ret; + if (PCI_SLOT(devfn)) { + *val = ~0; + return PCIBIOS_DEVICE_NOT_FOUND; + } + + *val = dw_pcie_read_dbi(pci, where, size); + return PCIBIOS_SUCCESSFUL; } -static int exynos_pcie_wr_own_conf(struct pcie_port *pp, int where, int size, - u32 val) +static int exynos_pcie_wr_own_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) { - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct exynos_pcie *ep = to_exynos_pcie(pci); - int ret; + struct dw_pcie *pci = to_dw_pcie_from_pp(bus->sysdata); - exynos_pcie_sideband_dbi_w_mode(ep, true); - ret = dw_pcie_write(pci->dbi_base + where, size, val); - exynos_pcie_sideband_dbi_w_mode(ep, false); - return ret; + if (PCI_SLOT(devfn)) + return PCIBIOS_DEVICE_NOT_FOUND; + + dw_pcie_write_dbi(pci, where, size, val); + return PCIBIOS_SUCCESSFUL; } +static struct pci_ops exynos_pci_ops = { + .read = exynos_pcie_rd_own_conf, + .write = exynos_pcie_wr_own_conf, +}; + static int exynos_pcie_link_up(struct dw_pcie *pci) { struct exynos_pcie *ep = to_exynos_pcie(pci); @@ -379,6 +384,8 @@ static int exynos_pcie_host_init(struct pcie_port *pp) struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct exynos_pcie *ep = to_exynos_pcie(pci); + pp->bridge->ops = &exynos_pci_ops; + exynos_pcie_establish_link(ep); exynos_pcie_enable_interrupts(ep); @@ -386,8 +393,6 @@ static int exynos_pcie_host_init(struct pcie_port *pp) } static const struct dw_pcie_host_ops exynos_pcie_host_ops = { - .rd_own_conf = exynos_pcie_rd_own_conf, - .wr_own_conf = exynos_pcie_wr_own_conf, .host_init = exynos_pcie_host_init, }; diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 5fef2613b223..5cf1ef12fb9b 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -79,7 +79,6 @@ struct imx6_pcie { u32 tx_deemph_gen2_6db; u32 tx_swing_full; u32 tx_swing_low; - int link_gen; struct regulator *vpcie; void __iomem *phy_base; @@ -94,15 +93,6 @@ struct imx6_pcie { #define PHY_PLL_LOCK_WAIT_USLEEP_MAX 200 #define PHY_PLL_LOCK_WAIT_TIMEOUT (2000 * PHY_PLL_LOCK_WAIT_USLEEP_MAX) -/* PCIe Root Complex registers (memory-mapped) */ -#define PCIE_RC_IMX6_MSI_CAP 0x50 -#define PCIE_RC_LCR 0x7c -#define PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN1 0x1 -#define PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN2 0x2 -#define PCIE_RC_LCR_MAX_LINK_SPEEDS_MASK 0xf - -#define PCIE_RC_LCSR 0x80 - /* PCIe Port Logic registers (memory-mapped) */ #define PL_OFFSET 0x700 @@ -116,8 +106,6 @@ struct imx6_pcie { #define PCIE_PHY_STAT (PL_OFFSET + 0x110) #define PCIE_PHY_STAT_ACK BIT(16) -#define PCIE_LINK_WIDTH_SPEED_CONTROL 0x80C - /* PHY registers (not memory-mapped) */ #define PCIE_PHY_ATEOVRD 0x10 #define PCIE_PHY_ATEOVRD_EN BIT(2) @@ -761,6 +749,7 @@ static int imx6_pcie_establish_link(struct imx6_pcie *imx6_pcie) { struct dw_pcie *pci = imx6_pcie->pci; struct device *dev = pci->dev; + u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); u32 tmp; int ret; @@ -769,10 +758,10 @@ static int imx6_pcie_establish_link(struct imx6_pcie *imx6_pcie) * started in Gen2 mode, there is a possibility the devices on the * bus will not be detected at all. This happens with PCIe switches. */ - tmp = dw_pcie_readl_dbi(pci, PCIE_RC_LCR); - tmp &= ~PCIE_RC_LCR_MAX_LINK_SPEEDS_MASK; - tmp |= PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN1; - dw_pcie_writel_dbi(pci, PCIE_RC_LCR, tmp); + tmp = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); + tmp &= ~PCI_EXP_LNKCAP_SLS; + tmp |= PCI_EXP_LNKCAP_SLS_2_5GB; + dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, tmp); /* Start LTSSM. */ imx6_pcie_ltssm_enable(dev); @@ -781,12 +770,12 @@ static int imx6_pcie_establish_link(struct imx6_pcie *imx6_pcie) if (ret) goto err_reset_phy; - if (imx6_pcie->link_gen == 2) { + if (pci->link_gen == 2) { /* Allow Gen2 mode after the link is up. */ - tmp = dw_pcie_readl_dbi(pci, PCIE_RC_LCR); - tmp &= ~PCIE_RC_LCR_MAX_LINK_SPEEDS_MASK; - tmp |= PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN2; - dw_pcie_writel_dbi(pci, PCIE_RC_LCR, tmp); + tmp = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); + tmp &= ~PCI_EXP_LNKCAP_SLS; + tmp |= PCI_EXP_LNKCAP_SLS_5_0GB; + dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, tmp); /* * Start Directed Speed Change so the best possible @@ -824,8 +813,8 @@ static int imx6_pcie_establish_link(struct imx6_pcie *imx6_pcie) dev_info(dev, "Link: Gen2 disabled\n"); } - tmp = dw_pcie_readl_dbi(pci, PCIE_RC_LCSR); - dev_info(dev, "Link up, Gen%i\n", (tmp >> 16) & 0xf); + tmp = dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKSTA); + dev_info(dev, "Link up, Gen%i\n", tmp & PCI_EXP_LNKSTA_CLS); return 0; err_reset_phy: @@ -847,9 +836,7 @@ static int imx6_pcie_host_init(struct pcie_port *pp) imx6_setup_phy_mpll(imx6_pcie); dw_pcie_setup_rc(pp); imx6_pcie_establish_link(imx6_pcie); - - if (IS_ENABLED(CONFIG_PCI_MSI)) - dw_pcie_msi_init(pp); + dw_pcie_msi_init(pp); return 0; } @@ -1073,38 +1060,33 @@ static int imx6_pcie_probe(struct platform_device *pdev) /* Fetch clocks */ imx6_pcie->pcie_phy = devm_clk_get(dev, "pcie_phy"); - if (IS_ERR(imx6_pcie->pcie_phy)) { - dev_err(dev, "pcie_phy clock source missing or invalid\n"); - return PTR_ERR(imx6_pcie->pcie_phy); - } + if (IS_ERR(imx6_pcie->pcie_phy)) + return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_phy), + "pcie_phy clock source missing or invalid\n"); imx6_pcie->pcie_bus = devm_clk_get(dev, "pcie_bus"); - if (IS_ERR(imx6_pcie->pcie_bus)) { - dev_err(dev, "pcie_bus clock source missing or invalid\n"); - return PTR_ERR(imx6_pcie->pcie_bus); - } + if (IS_ERR(imx6_pcie->pcie_bus)) + return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_bus), + "pcie_bus clock source missing or invalid\n"); imx6_pcie->pcie = devm_clk_get(dev, "pcie"); - if (IS_ERR(imx6_pcie->pcie)) { - dev_err(dev, "pcie clock source missing or invalid\n"); - return PTR_ERR(imx6_pcie->pcie); - } + if (IS_ERR(imx6_pcie->pcie)) + return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie), + "pcie clock source missing or invalid\n"); switch (imx6_pcie->drvdata->variant) { case IMX6SX: imx6_pcie->pcie_inbound_axi = devm_clk_get(dev, "pcie_inbound_axi"); - if (IS_ERR(imx6_pcie->pcie_inbound_axi)) { - dev_err(dev, "pcie_inbound_axi clock missing or invalid\n"); - return PTR_ERR(imx6_pcie->pcie_inbound_axi); - } + if (IS_ERR(imx6_pcie->pcie_inbound_axi)) + return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_inbound_axi), + "pcie_inbound_axi clock missing or invalid\n"); break; case IMX8MQ: imx6_pcie->pcie_aux = devm_clk_get(dev, "pcie_aux"); - if (IS_ERR(imx6_pcie->pcie_aux)) { - dev_err(dev, "pcie_aux clock source missing or invalid\n"); - return PTR_ERR(imx6_pcie->pcie_aux); - } + if (IS_ERR(imx6_pcie->pcie_aux)) + return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_aux), + "pcie_aux clock source missing or invalid\n"); fallthrough; case IMX7D: if (dbi_base->start == IMX8MQ_PCIE2_BASE_ADDR) @@ -1165,10 +1147,8 @@ static int imx6_pcie_probe(struct platform_device *pdev) imx6_pcie->tx_swing_low = 127; /* Limit link speed */ - ret = of_property_read_u32(node, "fsl,max-link-speed", - &imx6_pcie->link_gen); - if (ret) - imx6_pcie->link_gen = 1; + pci->link_gen = 1; + ret = of_property_read_u32(node, "fsl,max-link-speed", &pci->link_gen); imx6_pcie->vpcie = devm_regulator_get_optional(&pdev->dev, "vpcie"); if (IS_ERR(imx6_pcie->vpcie)) { @@ -1188,11 +1168,10 @@ static int imx6_pcie_probe(struct platform_device *pdev) return ret; if (pci_msi_enabled()) { - val = dw_pcie_readw_dbi(pci, PCIE_RC_IMX6_MSI_CAP + - PCI_MSI_FLAGS); + u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_MSI); + val = dw_pcie_readw_dbi(pci, offset + PCI_MSI_FLAGS); val |= PCI_MSI_FLAGS_ENABLE; - dw_pcie_writew_dbi(pci, PCIE_RC_IMX6_MSI_CAP + PCI_MSI_FLAGS, - val); + dw_pcie_writew_dbi(pci, offset + PCI_MSI_FLAGS, val); } return 0; diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index c8c9d6a75f17..a222728238ca 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -96,8 +96,6 @@ #define LEG_EP 0x1 #define RC 0x2 -#define EXP_CAP_ID_OFFSET 0x70 - #define KS_PCIE_SYSCLOCKOUTEN BIT(0) #define AM654_PCIE_DEV_TYPE_MASK 0x3 @@ -123,7 +121,6 @@ struct keystone_pcie { int msi_host_irq; int num_lanes; - u32 num_viewport; struct phy **phy; struct device_link **link; struct device_node *msi_intc_np; @@ -397,13 +394,17 @@ static void ks_pcie_clear_dbi_mode(struct keystone_pcie *ks_pcie) static void ks_pcie_setup_rc_app_regs(struct keystone_pcie *ks_pcie) { u32 val; - u32 num_viewport = ks_pcie->num_viewport; struct dw_pcie *pci = ks_pcie->pci; struct pcie_port *pp = &pci->pp; - u64 start = pp->mem->start; - u64 end = pp->mem->end; + u32 num_viewport = pci->num_viewport; + u64 start, end; + struct resource *mem; int i; + mem = resource_list_first_type(&pp->bridge->windows, IORESOURCE_MEM)->res; + start = mem->start; + end = mem->end; + /* Disable BARs for inbound access */ ks_pcie_set_dbi_mode(ks_pcie); dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); @@ -430,10 +431,10 @@ static void ks_pcie_setup_rc_app_regs(struct keystone_pcie *ks_pcie) ks_pcie_app_writel(ks_pcie, CMD_STATUS, val); } -static int ks_pcie_rd_other_conf(struct pcie_port *pp, struct pci_bus *bus, - unsigned int devfn, int where, int size, - u32 *val) +static void __iomem *ks_pcie_other_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) { + struct pcie_port *pp = bus->sysdata; struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); u32 reg; @@ -444,36 +445,29 @@ static int ks_pcie_rd_other_conf(struct pcie_port *pp, struct pci_bus *bus, reg |= CFG_TYPE1; ks_pcie_app_writel(ks_pcie, CFG_SETUP, reg); - return dw_pcie_read(pp->va_cfg0_base + where, size, val); + return pp->va_cfg0_base + where; } -static int ks_pcie_wr_other_conf(struct pcie_port *pp, struct pci_bus *bus, - unsigned int devfn, int where, int size, - u32 val) -{ - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); - u32 reg; - - reg = CFG_BUS(bus->number) | CFG_DEVICE(PCI_SLOT(devfn)) | - CFG_FUNC(PCI_FUNC(devfn)); - if (!pci_is_root_bus(bus->parent)) - reg |= CFG_TYPE1; - ks_pcie_app_writel(ks_pcie, CFG_SETUP, reg); - - return dw_pcie_write(pp->va_cfg0_base + where, size, val); -} +static struct pci_ops ks_child_pcie_ops = { + .map_bus = ks_pcie_other_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; /** - * ks_pcie_v3_65_scan_bus() - keystone scan_bus post initialization + * ks_pcie_v3_65_add_bus() - keystone add_bus post initialization * * This sets BAR0 to enable inbound access for MSI_IRQ register */ -static void ks_pcie_v3_65_scan_bus(struct pcie_port *pp) +static int ks_pcie_v3_65_add_bus(struct pci_bus *bus) { + struct pcie_port *pp = bus->sysdata; struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); + if (!pci_is_root_bus(bus)) + return 0; + /* Configure and set up BAR0 */ ks_pcie_set_dbi_mode(ks_pcie); @@ -488,8 +482,17 @@ static void ks_pcie_v3_65_scan_bus(struct pcie_port *pp) * be sufficient. Use physical address to avoid any conflicts. */ dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, ks_pcie->app.start); + + return 0; } +static struct pci_ops ks_pcie_ops = { + .map_bus = dw_pcie_own_conf_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + .add_bus = ks_pcie_v3_65_add_bus, +}; + /** * ks_pcie_link_up() - Check if link up */ @@ -807,6 +810,9 @@ static int __init ks_pcie_host_init(struct pcie_port *pp) struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); int ret; + pp->bridge->ops = &ks_pcie_ops; + pp->bridge->child_ops = &ks_child_pcie_ops; + ret = ks_pcie_config_legacy_irq(ks_pcie); if (ret) return ret; @@ -842,11 +848,8 @@ static int __init ks_pcie_host_init(struct pcie_port *pp) } static const struct dw_pcie_host_ops ks_pcie_host_ops = { - .rd_other_conf = ks_pcie_rd_other_conf, - .wr_other_conf = ks_pcie_wr_other_conf, .host_init = ks_pcie_host_init, .msi_host_init = ks_pcie_msi_host_init, - .scan_bus = ks_pcie_v3_65_scan_bus, }; static const struct dw_pcie_host_ops ks_pcie_am654_host_ops = { @@ -867,16 +870,8 @@ static int __init ks_pcie_add_pcie_port(struct keystone_pcie *ks_pcie, struct dw_pcie *pci = ks_pcie->pci; struct pcie_port *pp = &pci->pp; struct device *dev = &pdev->dev; - struct resource *res; int ret; - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config"); - pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res); - if (IS_ERR(pp->va_cfg0_base)) - return PTR_ERR(pp->va_cfg0_base); - - pp->va_cfg1_base = pp->va_cfg0_base; - ret = dw_pcie_host_init(pp); if (ret) { dev_err(dev, "failed to initialize host\n"); @@ -886,18 +881,6 @@ static int __init ks_pcie_add_pcie_port(struct keystone_pcie *ks_pcie, return 0; } -static u32 ks_pcie_am654_read_dbi2(struct dw_pcie *pci, void __iomem *base, - u32 reg, size_t size) -{ - struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); - u32 val; - - ks_pcie_set_dbi_mode(ks_pcie); - dw_pcie_read(base + reg, size, &val); - ks_pcie_clear_dbi_mode(ks_pcie); - return val; -} - static void ks_pcie_am654_write_dbi2(struct dw_pcie *pci, void __iomem *base, u32 reg, size_t size, u32 val) { @@ -912,7 +895,6 @@ static const struct dw_pcie_ops ks_pcie_dw_pcie_ops = { .start_link = ks_pcie_start_link, .stop_link = ks_pcie_stop_link, .link_up = ks_pcie_link_up, - .read_dbi2 = ks_pcie_am654_read_dbi2, .write_dbi2 = ks_pcie_am654_write_dbi2, }; @@ -1125,31 +1107,6 @@ static int ks_pcie_am654_set_mode(struct device *dev, return 0; } -static void ks_pcie_set_link_speed(struct dw_pcie *pci, int link_speed) -{ - u32 val; - - dw_pcie_dbi_ro_wr_en(pci); - - val = dw_pcie_readl_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCAP); - if ((val & PCI_EXP_LNKCAP_SLS) != link_speed) { - val &= ~((u32)PCI_EXP_LNKCAP_SLS); - val |= link_speed; - dw_pcie_writel_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCAP, - val); - } - - val = dw_pcie_readl_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCTL2); - if ((val & PCI_EXP_LNKCAP_SLS) != link_speed) { - val &= ~((u32)PCI_EXP_LNKCAP_SLS); - val |= link_speed; - dw_pcie_writel_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCTL2, - val); - } - - dw_pcie_dbi_ro_wr_dis(pci); -} - static const struct ks_pcie_of_data ks_pcie_rc_of_data = { .host_ops = &ks_pcie_host_ops, .version = 0x365A, @@ -1197,13 +1154,10 @@ static int __init ks_pcie_probe(struct platform_device *pdev) struct keystone_pcie *ks_pcie; struct device_link **link; struct gpio_desc *gpiod; - void __iomem *atu_base; struct resource *res; unsigned int version; void __iomem *base; - u32 num_viewport; struct phy **phy; - int link_speed; u32 num_lanes; char name[10]; int ret; @@ -1320,29 +1274,12 @@ static int __init ks_pcie_probe(struct platform_device *pdev) goto err_get_sync; } - if (pci->version >= 0x480A) { - atu_base = devm_platform_ioremap_resource_byname(pdev, "atu"); - if (IS_ERR(atu_base)) { - ret = PTR_ERR(atu_base); - goto err_get_sync; - } - - pci->atu_base = atu_base; - + if (pci->version >= 0x480A) ret = ks_pcie_am654_set_mode(dev, mode); - if (ret < 0) - goto err_get_sync; - } else { + else ret = ks_pcie_set_mode(dev); - if (ret < 0) - goto err_get_sync; - } - - link_speed = of_pci_get_max_link_speed(np); - if (link_speed < 0) - link_speed = 2; - - ks_pcie_set_link_speed(pci, link_speed); + if (ret < 0) + goto err_get_sync; switch (mode) { case DW_PCIE_RC_TYPE: @@ -1351,12 +1288,6 @@ static int __init ks_pcie_probe(struct platform_device *pdev) goto err_get_sync; } - ret = of_property_read_u32(np, "num-viewport", &num_viewport); - if (ret < 0) { - dev_err(dev, "unable to read *num-viewport* property\n"); - goto err_get_sync; - } - /* * "Power Sequencing and Reset Signal Timings" table in * PCI EXPRESS CARD ELECTROMECHANICAL SPECIFICATION, REV. 2.0 @@ -1370,7 +1301,6 @@ static int __init ks_pcie_probe(struct platform_device *pdev) gpiod_set_value_cansleep(gpiod, 1); } - ks_pcie->num_viewport = num_viewport; pci->pp.ops = host_ops; ret = ks_pcie_add_pcie_port(ks_pcie, pdev); if (ret < 0) diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c index 0d151cead1b7..84206f265e54 100644 --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c @@ -20,50 +20,58 @@ #define PCIE_DBI2_OFFSET 0x1000 /* DBI2 base address*/ -struct ls_pcie_ep { - struct dw_pcie *pci; +#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) + +struct ls_pcie_ep_drvdata { + u32 func_offset; + const struct dw_pcie_ep_ops *ops; + const struct dw_pcie_ops *dw_pcie_ops; }; -#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) +struct ls_pcie_ep { + struct dw_pcie *pci; + struct pci_epc_features *ls_epc; + const struct ls_pcie_ep_drvdata *drvdata; +}; static int ls_pcie_establish_link(struct dw_pcie *pci) { return 0; } -static const struct dw_pcie_ops ls_pcie_ep_ops = { +static const struct dw_pcie_ops dw_ls_pcie_ep_ops = { .start_link = ls_pcie_establish_link, }; -static const struct of_device_id ls_pcie_ep_of_match[] = { - { .compatible = "fsl,ls-pcie-ep",}, - { }, -}; - -static const struct pci_epc_features ls_pcie_epc_features = { - .linkup_notifier = false, - .msi_capable = true, - .msix_capable = false, - .bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4), -}; - static const struct pci_epc_features* ls_pcie_ep_get_features(struct dw_pcie_ep *ep) { - return &ls_pcie_epc_features; + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); + + return pcie->ls_epc; } static void ls_pcie_ep_init(struct dw_pcie_ep *ep) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); + struct dw_pcie_ep_func *ep_func; enum pci_barno bar; + ep_func = dw_pcie_ep_get_func_from_ep(ep, 0); + if (!ep_func) + return; + for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) dw_pcie_ep_reset_bar(pci, bar); + + pcie->ls_epc->msi_capable = ep_func->msi_cap ? true : false; + pcie->ls_epc->msix_capable = ep_func->msix_cap ? true : false; } static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, - enum pci_epc_irq_type type, u16 interrupt_num) + enum pci_epc_irq_type type, u16 interrupt_num) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); @@ -73,21 +81,51 @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, case PCI_EPC_IRQ_MSI: return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); case PCI_EPC_IRQ_MSIX: - return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num); + return dw_pcie_ep_raise_msix_irq_doorbell(ep, func_no, + interrupt_num); default: dev_err(pci->dev, "UNKNOWN IRQ type\n"); return -EINVAL; } } -static const struct dw_pcie_ep_ops pcie_ep_ops = { +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, + u8 func_no) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); + + WARN_ON(func_no && !pcie->drvdata->func_offset); + return pcie->drvdata->func_offset * func_no; +} + +static const struct dw_pcie_ep_ops ls_pcie_ep_ops = { .ep_init = ls_pcie_ep_init, .raise_irq = ls_pcie_ep_raise_irq, .get_features = ls_pcie_ep_get_features, + .func_conf_select = ls_pcie_ep_func_conf_select, +}; + +static const struct ls_pcie_ep_drvdata ls1_ep_drvdata = { + .ops = &ls_pcie_ep_ops, + .dw_pcie_ops = &dw_ls_pcie_ep_ops, +}; + +static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { + .func_offset = 0x20000, + .ops = &ls_pcie_ep_ops, + .dw_pcie_ops = &dw_ls_pcie_ep_ops, +}; + +static const struct of_device_id ls_pcie_ep_of_match[] = { + { .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata }, + { .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata }, + { .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata }, + { }, }; static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, - struct platform_device *pdev) + struct platform_device *pdev) { struct dw_pcie *pci = pcie->pci; struct device *dev = pci->dev; @@ -96,7 +134,7 @@ static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, int ret; ep = &pci->ep; - ep->ops = &pcie_ep_ops; + ep->ops = pcie->drvdata->ops; res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space"); if (!res) @@ -119,6 +157,7 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct dw_pcie *pci; struct ls_pcie_ep *pcie; + struct pci_epc_features *ls_epc; struct resource *dbi_base; int ret; @@ -130,15 +169,26 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev) if (!pci) return -ENOMEM; + ls_epc = devm_kzalloc(dev, sizeof(*ls_epc), GFP_KERNEL); + if (!ls_epc) + return -ENOMEM; + + pcie->drvdata = of_device_get_match_data(dev); + + pci->dev = dev; + pci->ops = pcie->drvdata->dw_pcie_ops; + + ls_epc->bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4), + + pcie->pci = pci; + pcie->ls_epc = ls_epc; + dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); if (IS_ERR(pci->dbi_base)) return PTR_ERR(pci->dbi_base); pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; - pci->dev = dev; - pci->ops = &ls_pcie_ep_ops; - pcie->pci = pci; platform_set_drvdata(pdev, pcie); diff --git a/drivers/pci/controller/dwc/pci-meson.c b/drivers/pci/controller/dwc/pci-meson.c index 4f183b96afbb..1913dc2c8fa0 100644 --- a/drivers/pci/controller/dwc/pci-meson.c +++ b/drivers/pci/controller/dwc/pci-meson.c @@ -17,37 +17,13 @@ #include <linux/resource.h> #include <linux/types.h> #include <linux/phy/phy.h> +#include <linux/module.h> #include "pcie-designware.h" #define to_meson_pcie(x) dev_get_drvdata((x)->dev) -/* External local bus interface registers */ -#define PLR_OFFSET 0x700 -#define PCIE_PORT_LINK_CTRL_OFF (PLR_OFFSET + 0x10) -#define FAST_LINK_MODE BIT(7) -#define LINK_CAPABLE_MASK GENMASK(21, 16) -#define LINK_CAPABLE_X1 BIT(16) - -#define PCIE_GEN2_CTRL_OFF (PLR_OFFSET + 0x10c) -#define NUM_OF_LANES_MASK GENMASK(12, 8) -#define NUM_OF_LANES_X1 BIT(8) -#define DIRECT_SPEED_CHANGE BIT(17) - -#define TYPE1_HDR_OFFSET 0x0 -#define PCIE_STATUS_COMMAND (TYPE1_HDR_OFFSET + 0x04) -#define PCI_IO_EN BIT(0) -#define PCI_MEM_SPACE_EN BIT(1) -#define PCI_BUS_MASTER_EN BIT(2) - -#define PCIE_BASE_ADDR0 (TYPE1_HDR_OFFSET + 0x10) -#define PCIE_BASE_ADDR1 (TYPE1_HDR_OFFSET + 0x14) - -#define PCIE_CAP_OFFSET 0x70 -#define PCIE_DEV_CTRL_DEV_STUS (PCIE_CAP_OFFSET + 0x08) -#define PCIE_CAP_MAX_PAYLOAD_MASK GENMASK(7, 5) #define PCIE_CAP_MAX_PAYLOAD_SIZE(x) ((x) << 5) -#define PCIE_CAP_MAX_READ_REQ_MASK GENMASK(14, 12) #define PCIE_CAP_MAX_READ_REQ_SIZE(x) ((x) << 12) /* PCIe specific config registers */ @@ -77,11 +53,6 @@ enum pcie_data_rate { PCIE_GEN4 }; -struct meson_pcie_mem_res { - void __iomem *elbi_base; - void __iomem *cfg_base; -}; - struct meson_pcie_clk_res { struct clk *clk; struct clk *port_clk; @@ -95,7 +66,7 @@ struct meson_pcie_rc_reset { struct meson_pcie { struct dw_pcie pci; - struct meson_pcie_mem_res mem_res; + void __iomem *cfg_base; struct meson_pcie_clk_res clk_res; struct meson_pcie_rc_reset mrst; struct gpio_desc *reset_gpio; @@ -134,28 +105,18 @@ static int meson_pcie_get_resets(struct meson_pcie *mp) return 0; } -static void __iomem *meson_pcie_get_mem(struct platform_device *pdev, - struct meson_pcie *mp, - const char *id) -{ - struct device *dev = mp->pci.dev; - struct resource *res; - - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, id); - - return devm_ioremap_resource(dev, res); -} - static int meson_pcie_get_mems(struct platform_device *pdev, struct meson_pcie *mp) { - mp->mem_res.elbi_base = meson_pcie_get_mem(pdev, mp, "elbi"); - if (IS_ERR(mp->mem_res.elbi_base)) - return PTR_ERR(mp->mem_res.elbi_base); + struct dw_pcie *pci = &mp->pci; + + pci->dbi_base = devm_platform_ioremap_resource_byname(pdev, "elbi"); + if (IS_ERR(pci->dbi_base)) + return PTR_ERR(pci->dbi_base); - mp->mem_res.cfg_base = meson_pcie_get_mem(pdev, mp, "cfg"); - if (IS_ERR(mp->mem_res.cfg_base)) - return PTR_ERR(mp->mem_res.cfg_base); + mp->cfg_base = devm_platform_ioremap_resource_byname(pdev, "cfg"); + if (IS_ERR(mp->cfg_base)) + return PTR_ERR(mp->cfg_base); return 0; } @@ -253,24 +214,14 @@ static int meson_pcie_probe_clocks(struct meson_pcie *mp) return 0; } -static inline void meson_elb_writel(struct meson_pcie *mp, u32 val, u32 reg) -{ - writel(val, mp->mem_res.elbi_base + reg); -} - -static inline u32 meson_elb_readl(struct meson_pcie *mp, u32 reg) -{ - return readl(mp->mem_res.elbi_base + reg); -} - static inline u32 meson_cfg_readl(struct meson_pcie *mp, u32 reg) { - return readl(mp->mem_res.cfg_base + reg); + return readl(mp->cfg_base + reg); } static inline void meson_cfg_writel(struct meson_pcie *mp, u32 val, u32 reg) { - writel(val, mp->mem_res.cfg_base + reg); + writel(val, mp->cfg_base + reg); } static void meson_pcie_assert_reset(struct meson_pcie *mp) @@ -287,25 +238,6 @@ static void meson_pcie_init_dw(struct meson_pcie *mp) val = meson_cfg_readl(mp, PCIE_CFG0); val |= APP_LTSSM_ENABLE; meson_cfg_writel(mp, val, PCIE_CFG0); - - val = meson_elb_readl(mp, PCIE_PORT_LINK_CTRL_OFF); - val &= ~(LINK_CAPABLE_MASK | FAST_LINK_MODE); - meson_elb_writel(mp, val, PCIE_PORT_LINK_CTRL_OFF); - - val = meson_elb_readl(mp, PCIE_PORT_LINK_CTRL_OFF); - val |= LINK_CAPABLE_X1; - meson_elb_writel(mp, val, PCIE_PORT_LINK_CTRL_OFF); - - val = meson_elb_readl(mp, PCIE_GEN2_CTRL_OFF); - val &= ~NUM_OF_LANES_MASK; - meson_elb_writel(mp, val, PCIE_GEN2_CTRL_OFF); - - val = meson_elb_readl(mp, PCIE_GEN2_CTRL_OFF); - val |= NUM_OF_LANES_X1 | DIRECT_SPEED_CHANGE; - meson_elb_writel(mp, val, PCIE_GEN2_CTRL_OFF); - - meson_elb_writel(mp, 0x0, PCIE_BASE_ADDR0); - meson_elb_writel(mp, 0x0, PCIE_BASE_ADDR1); } static int meson_size_to_payload(struct meson_pcie *mp, int size) @@ -327,37 +259,34 @@ static int meson_size_to_payload(struct meson_pcie *mp, int size) static void meson_set_max_payload(struct meson_pcie *mp, int size) { + struct dw_pcie *pci = &mp->pci; u32 val; + u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); int max_payload_size = meson_size_to_payload(mp, size); - val = meson_elb_readl(mp, PCIE_DEV_CTRL_DEV_STUS); - val &= ~PCIE_CAP_MAX_PAYLOAD_MASK; - meson_elb_writel(mp, val, PCIE_DEV_CTRL_DEV_STUS); + val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_DEVCTL); + val &= ~PCI_EXP_DEVCTL_PAYLOAD; + dw_pcie_writel_dbi(pci, offset + PCI_EXP_DEVCTL, val); - val = meson_elb_readl(mp, PCIE_DEV_CTRL_DEV_STUS); + val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_DEVCTL); val |= PCIE_CAP_MAX_PAYLOAD_SIZE(max_payload_size); - meson_elb_writel(mp, val, PCIE_DEV_CTRL_DEV_STUS); + dw_pcie_writel_dbi(pci, offset + PCI_EXP_DEVCTL, val); } static void meson_set_max_rd_req_size(struct meson_pcie *mp, int size) { + struct dw_pcie *pci = &mp->pci; u32 val; + u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); int max_rd_req_size = meson_size_to_payload(mp, size); - val = meson_elb_readl(mp, PCIE_DEV_CTRL_DEV_STUS); - val &= ~PCIE_CAP_MAX_READ_REQ_MASK; - meson_elb_writel(mp, val, PCIE_DEV_CTRL_DEV_STUS); + val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_DEVCTL); + val &= ~PCI_EXP_DEVCTL_READRQ; + dw_pcie_writel_dbi(pci, offset + PCI_EXP_DEVCTL, val); - val = meson_elb_readl(mp, PCIE_DEV_CTRL_DEV_STUS); + val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_DEVCTL); val |= PCIE_CAP_MAX_READ_REQ_SIZE(max_rd_req_size); - meson_elb_writel(mp, val, PCIE_DEV_CTRL_DEV_STUS); -} - -static inline void meson_enable_memory_space(struct meson_pcie *mp) -{ - /* Set the RC Bus Master, Memory Space and I/O Space enables */ - meson_elb_writel(mp, PCI_IO_EN | PCI_MEM_SPACE_EN | PCI_BUS_MASTER_EN, - PCIE_STATUS_COMMAND); + dw_pcie_writel_dbi(pci, offset + PCI_EXP_DEVCTL, val); } static int meson_pcie_establish_link(struct meson_pcie *mp) @@ -370,26 +299,18 @@ static int meson_pcie_establish_link(struct meson_pcie *mp) meson_set_max_rd_req_size(mp, MAX_READ_REQ_SIZE); dw_pcie_setup_rc(pp); - meson_enable_memory_space(mp); meson_pcie_assert_reset(mp); return dw_pcie_wait_for_link(pci); } -static void meson_pcie_enable_interrupts(struct meson_pcie *mp) +static int meson_pcie_rd_own_conf(struct pci_bus *bus, u32 devfn, + int where, int size, u32 *val) { - if (IS_ENABLED(CONFIG_PCI_MSI)) - dw_pcie_msi_init(&mp->pci.pp); -} - -static int meson_pcie_rd_own_conf(struct pcie_port *pp, int where, int size, - u32 *val) -{ - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); int ret; - ret = dw_pcie_read(pci->dbi_base + where, size, val); + ret = pci_generic_config_read(bus, devfn, where, size, val); if (ret != PCIBIOS_SUCCESSFUL) return ret; @@ -410,13 +331,11 @@ static int meson_pcie_rd_own_conf(struct pcie_port *pp, int where, int size, return PCIBIOS_SUCCESSFUL; } -static int meson_pcie_wr_own_conf(struct pcie_port *pp, int where, - int size, u32 val) -{ - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - - return dw_pcie_write(pci->dbi_base + where, size, val); -} +static struct pci_ops meson_pci_ops = { + .map_bus = dw_pcie_own_conf_map_bus, + .read = meson_pcie_rd_own_conf, + .write = pci_generic_config_write, +}; static int meson_pcie_link_up(struct dw_pcie *pci) { @@ -463,18 +382,18 @@ static int meson_pcie_host_init(struct pcie_port *pp) struct meson_pcie *mp = to_meson_pcie(pci); int ret; + pp->bridge->ops = &meson_pci_ops; + ret = meson_pcie_establish_link(mp); if (ret) return ret; - meson_pcie_enable_interrupts(mp); + dw_pcie_msi_init(pp); return 0; } static const struct dw_pcie_host_ops meson_pcie_host_ops = { - .rd_own_conf = meson_pcie_rd_own_conf, - .wr_own_conf = meson_pcie_wr_own_conf, .host_init = meson_pcie_host_init, }; @@ -493,7 +412,6 @@ static int meson_add_pcie_port(struct meson_pcie *mp, } pp->ops = &meson_pcie_host_ops; - pci->dbi_base = mp->mem_res.elbi_base; ret = dw_pcie_host_init(pp); if (ret) { @@ -522,6 +440,7 @@ static int meson_pcie_probe(struct platform_device *pdev) pci = &mp->pci; pci->dev = dev; pci->ops = &dw_pcie_ops; + pci->num_lanes = 1; mp->phy = devm_phy_get(dev, "pcie"); if (IS_ERR(mp->phy)) { @@ -589,6 +508,7 @@ static const struct of_device_id meson_pcie_of_match[] = { }, {}, }; +MODULE_DEVICE_TABLE(of, meson_pcie_of_match); static struct platform_driver meson_pcie_driver = { .probe = meson_pcie_probe, @@ -598,4 +518,8 @@ static struct platform_driver meson_pcie_driver = { }, }; -builtin_platform_driver(meson_pcie_driver); +module_platform_driver(meson_pcie_driver); + +MODULE_AUTHOR("Yue Wang <yue.wang@amlogic.com>"); +MODULE_DESCRIPTION("Amlogic PCIe Controller driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/dwc/pcie-al.c b/drivers/pci/controller/dwc/pcie-al.c index d57d4ee15848..f973fbca90cf 100644 --- a/drivers/pci/controller/dwc/pcie-al.c +++ b/drivers/pci/controller/dwc/pcie-al.c @@ -217,14 +217,15 @@ static inline void al_pcie_target_bus_set(struct al_pcie *pcie, reg); } -static void __iomem *al_pcie_conf_addr_map(struct al_pcie *pcie, - unsigned int busnr, - unsigned int devfn) +static void __iomem *al_pcie_conf_addr_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) { + struct pcie_port *pp = bus->sysdata; + struct al_pcie *pcie = to_al_pcie(to_dw_pcie_from_pp(pp)); + unsigned int busnr = bus->number; struct al_pcie_target_bus_cfg *target_bus_cfg = &pcie->target_bus_cfg; unsigned int busnr_ecam = busnr & target_bus_cfg->ecam_mask; unsigned int busnr_reg = busnr & target_bus_cfg->reg_mask; - struct pcie_port *pp = &pcie->pci->pp; void __iomem *pci_base_addr; pci_base_addr = (void __iomem *)((uintptr_t)pp->va_cfg0_base + @@ -240,52 +241,14 @@ static void __iomem *al_pcie_conf_addr_map(struct al_pcie *pcie, target_bus_cfg->reg_mask); } - return pci_base_addr; -} - -static int al_pcie_rd_other_conf(struct pcie_port *pp, struct pci_bus *bus, - unsigned int devfn, int where, int size, - u32 *val) -{ - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct al_pcie *pcie = to_al_pcie(pci); - unsigned int busnr = bus->number; - void __iomem *pci_addr; - int rc; - - pci_addr = al_pcie_conf_addr_map(pcie, busnr, devfn); - - rc = dw_pcie_read(pci_addr + where, size, val); - - dev_dbg(pci->dev, "%d-byte config read from %04x:%02x:%02x.%d offset 0x%x (pci_addr: 0x%px) - val:0x%x\n", - size, pci_domain_nr(bus), bus->number, - PCI_SLOT(devfn), PCI_FUNC(devfn), where, - (pci_addr + where), *val); - - return rc; + return pci_base_addr + where; } -static int al_pcie_wr_other_conf(struct pcie_port *pp, struct pci_bus *bus, - unsigned int devfn, int where, int size, - u32 val) -{ - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct al_pcie *pcie = to_al_pcie(pci); - unsigned int busnr = bus->number; - void __iomem *pci_addr; - int rc; - - pci_addr = al_pcie_conf_addr_map(pcie, busnr, devfn); - - rc = dw_pcie_write(pci_addr + where, size, val); - - dev_dbg(pci->dev, "%d-byte config write to %04x:%02x:%02x.%d offset 0x%x (pci_addr: 0x%px) - val:0x%x\n", - size, pci_domain_nr(bus), bus->number, - PCI_SLOT(devfn), PCI_FUNC(devfn), where, - (pci_addr + where), val); - - return rc; -} +static struct pci_ops al_child_pci_ops = { + .map_bus = al_pcie_conf_addr_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; static void al_pcie_config_prepare(struct al_pcie *pcie) { @@ -297,6 +260,7 @@ static void al_pcie_config_prepare(struct al_pcie *pcie) u8 secondary_bus; u32 cfg_control; u32 reg; + struct resource *bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS)->res; target_bus_cfg = &pcie->target_bus_cfg; @@ -310,13 +274,13 @@ static void al_pcie_config_prepare(struct al_pcie *pcie) target_bus_cfg->ecam_mask = ecam_bus_mask; /* This portion is taken from the cfg_target_bus reg */ target_bus_cfg->reg_mask = ~target_bus_cfg->ecam_mask; - target_bus_cfg->reg_val = pp->busn->start & target_bus_cfg->reg_mask; + target_bus_cfg->reg_val = bus->start & target_bus_cfg->reg_mask; al_pcie_target_bus_set(pcie, target_bus_cfg->reg_val, target_bus_cfg->reg_mask); - secondary_bus = pp->busn->start + 1; - subordinate_bus = pp->busn->end; + secondary_bus = bus->start + 1; + subordinate_bus = bus->end; /* Set the valid values of secondary and subordinate buses */ cfg_control_offset = AXI_BASE_OFFSET + pcie->reg_offsets.ob_ctrl + @@ -339,6 +303,8 @@ static int al_pcie_host_init(struct pcie_port *pp) struct al_pcie *pcie = to_al_pcie(pci); int rc; + pp->bridge->child_ops = &al_child_pci_ops; + rc = al_pcie_rev_id_get(pcie, &pcie->controller_rev_id); if (rc) return rc; @@ -353,8 +319,6 @@ static int al_pcie_host_init(struct pcie_port *pp) } static const struct dw_pcie_host_ops al_pcie_host_ops = { - .rd_other_conf = al_pcie_rd_other_conf, - .wr_other_conf = al_pcie_wr_other_conf, .host_init = al_pcie_host_init, }; diff --git a/drivers/pci/controller/dwc/pcie-artpec6.c b/drivers/pci/controller/dwc/pcie-artpec6.c index 97d50bb50f06..929448e9e0bc 100644 --- a/drivers/pci/controller/dwc/pcie-artpec6.c +++ b/drivers/pci/controller/dwc/pcie-artpec6.c @@ -44,13 +44,6 @@ struct artpec_pcie_of_data { static const struct of_device_id artpec6_pcie_of_match[]; -/* PCIe Port Logic registers (memory-mapped) */ -#define PL_OFFSET 0x700 - -#define ACK_F_ASPM_CTRL_OFF (PL_OFFSET + 0xc) -#define ACK_N_FTS_MASK GENMASK(15, 8) -#define ACK_N_FTS(x) (((x) << 8) & ACK_N_FTS_MASK) - /* ARTPEC-6 specific registers */ #define PCIECFG 0x18 #define PCIECFG_DBG_OEN BIT(24) @@ -289,30 +282,6 @@ static void artpec6_pcie_init_phy(struct artpec6_pcie *artpec6_pcie) } } -static void artpec6_pcie_set_nfts(struct artpec6_pcie *artpec6_pcie) -{ - struct dw_pcie *pci = artpec6_pcie->pci; - u32 val; - - if (artpec6_pcie->variant != ARTPEC7) - return; - - /* - * Increase the N_FTS (Number of Fast Training Sequences) - * to be transmitted when transitioning from L0s to L0. - */ - val = dw_pcie_readl_dbi(pci, ACK_F_ASPM_CTRL_OFF); - val &= ~ACK_N_FTS_MASK; - val |= ACK_N_FTS(180); - dw_pcie_writel_dbi(pci, ACK_F_ASPM_CTRL_OFF, val); - - /* - * Set the Number of Fast Training Sequences that the core - * advertises as its N_FTS during Gen2 or Gen3 link training. - */ - dw_pcie_link_set_n_fts(pci, 180); -} - static void artpec6_pcie_assert_core_reset(struct artpec6_pcie *artpec6_pcie) { u32 val; @@ -346,29 +315,23 @@ static void artpec6_pcie_deassert_core_reset(struct artpec6_pcie *artpec6_pcie) usleep_range(100, 200); } -static void artpec6_pcie_enable_interrupts(struct artpec6_pcie *artpec6_pcie) -{ - struct dw_pcie *pci = artpec6_pcie->pci; - struct pcie_port *pp = &pci->pp; - - if (IS_ENABLED(CONFIG_PCI_MSI)) - dw_pcie_msi_init(pp); -} - static int artpec6_pcie_host_init(struct pcie_port *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct artpec6_pcie *artpec6_pcie = to_artpec6_pcie(pci); + if (artpec6_pcie->variant == ARTPEC7) { + pci->n_fts[0] = 180; + pci->n_fts[1] = 180; + } artpec6_pcie_assert_core_reset(artpec6_pcie); artpec6_pcie_init_phy(artpec6_pcie); artpec6_pcie_deassert_core_reset(artpec6_pcie); artpec6_pcie_wait_for_phy(artpec6_pcie); - artpec6_pcie_set_nfts(artpec6_pcie); dw_pcie_setup_rc(pp); artpec6_pcie_establish_link(pci); dw_pcie_wait_for_link(pci); - artpec6_pcie_enable_interrupts(artpec6_pcie); + dw_pcie_msi_init(pp); return 0; } @@ -412,7 +375,6 @@ static void artpec6_pcie_ep_init(struct dw_pcie_ep *ep) artpec6_pcie_init_phy(artpec6_pcie); artpec6_pcie_deassert_core_reset(artpec6_pcie); artpec6_pcie_wait_for_phy(artpec6_pcie); - artpec6_pcie_set_nfts(artpec6_pcie); for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) dw_pcie_ep_reset_bar(pci, bar); diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 305bfec2424d..ad7da4ea43a5 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -12,6 +12,8 @@ #include <linux/pci-epc.h> #include <linux/pci-epf.h> +#include "../../pci.h" + void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) { struct pci_epc *epc = ep->epc; @@ -28,12 +30,39 @@ void dw_pcie_ep_init_notify(struct dw_pcie_ep *ep) } EXPORT_SYMBOL_GPL(dw_pcie_ep_init_notify); -static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar, - int flags) +struct dw_pcie_ep_func * +dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no) +{ + struct dw_pcie_ep_func *ep_func; + + list_for_each_entry(ep_func, &ep->func_list, list) { + if (ep_func->func_no == func_no) + return ep_func; + } + + return NULL; +} + +static unsigned int dw_pcie_ep_func_select(struct dw_pcie_ep *ep, u8 func_no) +{ + unsigned int func_offset = 0; + + if (ep->ops->func_conf_select) + func_offset = ep->ops->func_conf_select(ep, func_no); + + return func_offset; +} + +static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no, + enum pci_barno bar, int flags) { u32 reg; + unsigned int func_offset = 0; + struct dw_pcie_ep *ep = &pci->ep; - reg = PCI_BASE_ADDRESS_0 + (4 * bar); + func_offset = dw_pcie_ep_func_select(ep, func_no); + + reg = func_offset + PCI_BASE_ADDRESS_0 + (4 * bar); dw_pcie_dbi_ro_wr_en(pci); dw_pcie_writel_dbi2(pci, reg, 0x0); dw_pcie_writel_dbi(pci, reg, 0x0); @@ -46,7 +75,53 @@ static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar, void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) { - __dw_pcie_ep_reset_bar(pci, bar, 0); + u8 func_no, funcs; + + funcs = pci->ep.epc->max_functions; + + for (func_no = 0; func_no < funcs; func_no++) + __dw_pcie_ep_reset_bar(pci, func_no, bar, 0); +} + +static u8 __dw_pcie_ep_find_next_cap(struct dw_pcie_ep *ep, u8 func_no, + u8 cap_ptr, u8 cap) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + unsigned int func_offset = 0; + u8 cap_id, next_cap_ptr; + u16 reg; + + if (!cap_ptr) + return 0; + + func_offset = dw_pcie_ep_func_select(ep, func_no); + + reg = dw_pcie_readw_dbi(pci, func_offset + cap_ptr); + cap_id = (reg & 0x00ff); + + if (cap_id > PCI_CAP_ID_MAX) + return 0; + + if (cap_id == cap) + return cap_ptr; + + next_cap_ptr = (reg & 0xff00) >> 8; + return __dw_pcie_ep_find_next_cap(ep, func_no, next_cap_ptr, cap); +} + +static u8 dw_pcie_ep_find_capability(struct dw_pcie_ep *ep, u8 func_no, u8 cap) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + unsigned int func_offset = 0; + u8 next_cap_ptr; + u16 reg; + + func_offset = dw_pcie_ep_func_select(ep, func_no); + + reg = dw_pcie_readw_dbi(pci, func_offset + PCI_CAPABILITY_LIST); + next_cap_ptr = (reg & 0x00ff); + + return __dw_pcie_ep_find_next_cap(ep, func_no, next_cap_ptr, cap); } static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, @@ -54,28 +129,31 @@ static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, { struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + unsigned int func_offset = 0; + + func_offset = dw_pcie_ep_func_select(ep, func_no); dw_pcie_dbi_ro_wr_en(pci); - dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, hdr->vendorid); - dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, hdr->deviceid); - dw_pcie_writeb_dbi(pci, PCI_REVISION_ID, hdr->revid); - dw_pcie_writeb_dbi(pci, PCI_CLASS_PROG, hdr->progif_code); - dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE, + dw_pcie_writew_dbi(pci, func_offset + PCI_VENDOR_ID, hdr->vendorid); + dw_pcie_writew_dbi(pci, func_offset + PCI_DEVICE_ID, hdr->deviceid); + dw_pcie_writeb_dbi(pci, func_offset + PCI_REVISION_ID, hdr->revid); + dw_pcie_writeb_dbi(pci, func_offset + PCI_CLASS_PROG, hdr->progif_code); + dw_pcie_writew_dbi(pci, func_offset + PCI_CLASS_DEVICE, hdr->subclass_code | hdr->baseclass_code << 8); - dw_pcie_writeb_dbi(pci, PCI_CACHE_LINE_SIZE, + dw_pcie_writeb_dbi(pci, func_offset + PCI_CACHE_LINE_SIZE, hdr->cache_line_size); - dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_VENDOR_ID, + dw_pcie_writew_dbi(pci, func_offset + PCI_SUBSYSTEM_VENDOR_ID, hdr->subsys_vendor_id); - dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_ID, hdr->subsys_id); - dw_pcie_writeb_dbi(pci, PCI_INTERRUPT_PIN, + dw_pcie_writew_dbi(pci, func_offset + PCI_SUBSYSTEM_ID, hdr->subsys_id); + dw_pcie_writeb_dbi(pci, func_offset + PCI_INTERRUPT_PIN, hdr->interrupt_pin); dw_pcie_dbi_ro_wr_dis(pci); return 0; } -static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum pci_barno bar, - dma_addr_t cpu_addr, +static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, u8 func_no, + enum pci_barno bar, dma_addr_t cpu_addr, enum dw_pcie_as_type as_type) { int ret; @@ -88,7 +166,7 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum pci_barno bar, return -EINVAL; } - ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr, + ret = dw_pcie_prog_inbound_atu(pci, func_no, free_win, bar, cpu_addr, as_type); if (ret < 0) { dev_err(pci->dev, "Failed to program IB window\n"); @@ -101,7 +179,8 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum pci_barno bar, return 0; } -static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, phys_addr_t phys_addr, +static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, u8 func_no, + phys_addr_t phys_addr, u64 pci_addr, size_t size) { u32 free_win; @@ -113,8 +192,8 @@ static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, phys_addr_t phys_addr, return -EINVAL; } - dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM, - phys_addr, pci_addr, size); + dw_pcie_prog_ep_outbound_atu(pci, func_no, free_win, PCIE_ATU_TYPE_MEM, + phys_addr, pci_addr, size); set_bit(free_win, ep->ob_window_map); ep->outbound_addr[free_win] = phys_addr; @@ -130,7 +209,7 @@ static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, enum pci_barno bar = epf_bar->barno; u32 atu_index = ep->bar_to_atu[bar]; - __dw_pcie_ep_reset_bar(pci, bar, epf_bar->flags); + __dw_pcie_ep_reset_bar(pci, func_no, bar, epf_bar->flags); dw_pcie_disable_atu(pci, atu_index, DW_PCIE_REGION_INBOUND); clear_bit(atu_index, ep->ib_window_map); @@ -147,14 +226,20 @@ static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, size_t size = epf_bar->size; int flags = epf_bar->flags; enum dw_pcie_as_type as_type; - u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar); + u32 reg; + unsigned int func_offset = 0; + + func_offset = dw_pcie_ep_func_select(ep, func_no); + + reg = PCI_BASE_ADDRESS_0 + (4 * bar) + func_offset; if (!(flags & PCI_BASE_ADDRESS_SPACE)) as_type = DW_PCIE_AS_MEM; else as_type = DW_PCIE_AS_IO; - ret = dw_pcie_ep_inbound_atu(ep, bar, epf_bar->phys_addr, as_type); + ret = dw_pcie_ep_inbound_atu(ep, func_no, bar, + epf_bar->phys_addr, as_type); if (ret) return ret; @@ -213,7 +298,7 @@ static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no, struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); - ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size); + ret = dw_pcie_ep_outbound_atu(ep, func_no, addr, pci_addr, size); if (ret) { dev_err(pci->dev, "Failed to enable address\n"); return ret; @@ -227,11 +312,16 @@ static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no) struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); u32 val, reg; + unsigned int func_offset = 0; + struct dw_pcie_ep_func *ep_func; - if (!ep->msi_cap) + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (!ep_func || !ep_func->msi_cap) return -EINVAL; - reg = ep->msi_cap + PCI_MSI_FLAGS; + func_offset = dw_pcie_ep_func_select(ep, func_no); + + reg = ep_func->msi_cap + func_offset + PCI_MSI_FLAGS; val = dw_pcie_readw_dbi(pci, reg); if (!(val & PCI_MSI_FLAGS_ENABLE)) return -EINVAL; @@ -246,11 +336,16 @@ static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 interrupts) struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); u32 val, reg; + unsigned int func_offset = 0; + struct dw_pcie_ep_func *ep_func; - if (!ep->msi_cap) + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (!ep_func || !ep_func->msi_cap) return -EINVAL; - reg = ep->msi_cap + PCI_MSI_FLAGS; + func_offset = dw_pcie_ep_func_select(ep, func_no); + + reg = ep_func->msi_cap + func_offset + PCI_MSI_FLAGS; val = dw_pcie_readw_dbi(pci, reg); val &= ~PCI_MSI_FLAGS_QMASK; val |= (interrupts << 1) & PCI_MSI_FLAGS_QMASK; @@ -266,11 +361,16 @@ static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no) struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); u32 val, reg; + unsigned int func_offset = 0; + struct dw_pcie_ep_func *ep_func; - if (!ep->msix_cap) + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (!ep_func || !ep_func->msix_cap) return -EINVAL; - reg = ep->msix_cap + PCI_MSIX_FLAGS; + func_offset = dw_pcie_ep_func_select(ep, func_no); + + reg = ep_func->msix_cap + func_offset + PCI_MSIX_FLAGS; val = dw_pcie_readw_dbi(pci, reg); if (!(val & PCI_MSIX_FLAGS_ENABLE)) return -EINVAL; @@ -286,23 +386,28 @@ static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u16 interrupts, struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); u32 val, reg; + unsigned int func_offset = 0; + struct dw_pcie_ep_func *ep_func; - if (!ep->msix_cap) + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (!ep_func || !ep_func->msix_cap) return -EINVAL; dw_pcie_dbi_ro_wr_en(pci); - reg = ep->msix_cap + PCI_MSIX_FLAGS; + func_offset = dw_pcie_ep_func_select(ep, func_no); + + reg = ep_func->msix_cap + func_offset + PCI_MSIX_FLAGS; val = dw_pcie_readw_dbi(pci, reg); val &= ~PCI_MSIX_FLAGS_QSIZE; val |= interrupts; dw_pcie_writew_dbi(pci, reg, val); - reg = ep->msix_cap + PCI_MSIX_TABLE; + reg = ep_func->msix_cap + func_offset + PCI_MSIX_TABLE; val = offset | bir; dw_pcie_writel_dbi(pci, reg, val); - reg = ep->msix_cap + PCI_MSIX_PBA; + reg = ep_func->msix_cap + func_offset + PCI_MSIX_PBA; val = (offset + (interrupts * PCI_MSIX_ENTRY_SIZE)) | bir; dw_pcie_writel_dbi(pci, reg, val); @@ -385,31 +490,36 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, u8 interrupt_num) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_pcie_ep_func *ep_func; struct pci_epc *epc = ep->epc; unsigned int aligned_offset; + unsigned int func_offset = 0; u16 msg_ctrl, msg_data; u32 msg_addr_lower, msg_addr_upper, reg; u64 msg_addr; bool has_upper; int ret; - if (!ep->msi_cap) + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (!ep_func || !ep_func->msi_cap) return -EINVAL; + func_offset = dw_pcie_ep_func_select(ep, func_no); + /* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. */ - reg = ep->msi_cap + PCI_MSI_FLAGS; + reg = ep_func->msi_cap + func_offset + PCI_MSI_FLAGS; msg_ctrl = dw_pcie_readw_dbi(pci, reg); has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT); - reg = ep->msi_cap + PCI_MSI_ADDRESS_LO; + reg = ep_func->msi_cap + func_offset + PCI_MSI_ADDRESS_LO; msg_addr_lower = dw_pcie_readl_dbi(pci, reg); if (has_upper) { - reg = ep->msi_cap + PCI_MSI_ADDRESS_HI; + reg = ep_func->msi_cap + func_offset + PCI_MSI_ADDRESS_HI; msg_addr_upper = dw_pcie_readl_dbi(pci, reg); - reg = ep->msi_cap + PCI_MSI_DATA_64; + reg = ep_func->msi_cap + func_offset + PCI_MSI_DATA_64; msg_data = dw_pcie_readw_dbi(pci, reg); } else { msg_addr_upper = 0; - reg = ep->msi_cap + PCI_MSI_DATA_32; + reg = ep_func->msi_cap + func_offset + PCI_MSI_DATA_32; msg_data = dw_pcie_readw_dbi(pci, reg); } aligned_offset = msg_addr_lower & (epc->mem->window.page_size - 1); @@ -427,12 +537,33 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, return 0; } +int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, + u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_pcie_ep_func *ep_func; + u32 msg_data; + + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (!ep_func || !ep_func->msix_cap) + return -EINVAL; + + msg_data = (func_no << PCIE_MSIX_DOORBELL_PF_SHIFT) | + (interrupt_num - 1); + + dw_pcie_writel_dbi(pci, PCIE_MSIX_DOORBELL, msg_data); + + return 0; +} + int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, - u16 interrupt_num) + u16 interrupt_num) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_pcie_ep_func *ep_func; struct pci_epf_msix_tbl *msix_tbl; struct pci_epc *epc = ep->epc; + unsigned int func_offset = 0; u32 reg, msg_data, vec_ctrl; unsigned int aligned_offset; u32 tbl_offset; @@ -440,7 +571,13 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, int ret; u8 bir; - reg = ep->msix_cap + PCI_MSIX_TABLE; + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (!ep_func || !ep_func->msix_cap) + return -EINVAL; + + func_offset = dw_pcie_ep_func_select(ep, func_no); + + reg = ep_func->msix_cap + func_offset + PCI_MSIX_TABLE; tbl_offset = dw_pcie_readl_dbi(pci, reg); bir = (tbl_offset & PCI_MSIX_TABLE_BIR); tbl_offset &= PCI_MSIX_TABLE_OFFSET; @@ -505,7 +642,8 @@ int dw_pcie_ep_init_complete(struct dw_pcie_ep *ep) u32 reg; int i; - hdr_type = dw_pcie_readb_dbi(pci, PCI_HEADER_TYPE); + hdr_type = dw_pcie_readb_dbi(pci, PCI_HEADER_TYPE) & + PCI_HEADER_TYPE_MASK; if (hdr_type != PCI_HEADER_TYPE_NORMAL) { dev_err(pci->dev, "PCIe controller is not set to EP mode (hdr_type:0x%x)!\n", @@ -513,23 +651,21 @@ int dw_pcie_ep_init_complete(struct dw_pcie_ep *ep) return -EIO; } - ep->msi_cap = dw_pcie_find_capability(pci, PCI_CAP_ID_MSI); + offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); - ep->msix_cap = dw_pcie_find_capability(pci, PCI_CAP_ID_MSIX); + dw_pcie_dbi_ro_wr_en(pci); - offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); if (offset) { reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); nbars = (reg & PCI_REBAR_CTRL_NBAR_MASK) >> PCI_REBAR_CTRL_NBAR_SHIFT; - dw_pcie_dbi_ro_wr_en(pci); for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) dw_pcie_writel_dbi(pci, offset + PCI_REBAR_CAP, 0x0); - dw_pcie_dbi_ro_wr_dis(pci); } dw_pcie_setup(pci); + dw_pcie_dbi_ro_wr_dis(pci); return 0; } @@ -539,11 +675,15 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) { int ret; void *addr; + u8 func_no; struct pci_epc *epc; struct dw_pcie *pci = to_dw_pcie_from_ep(ep); struct device *dev = pci->dev; struct device_node *np = dev->of_node; const struct pci_epc_features *epc_features; + struct dw_pcie_ep_func *ep_func; + + INIT_LIST_HEAD(&ep->func_list); if (!pci->dbi_base || !pci->dbi_base2) { dev_err(dev, "dbi_base/dbi_base2 is not populated\n"); @@ -590,6 +730,9 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) return -ENOMEM; ep->outbound_addr = addr; + if (pci->link_gen < 1) + pci->link_gen = of_pci_get_max_link_speed(np); + epc = devm_pci_epc_create(dev, &epc_ops); if (IS_ERR(epc)) { dev_err(dev, "Failed to create epc device\n"); @@ -599,13 +742,27 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) ep->epc = epc; epc_set_drvdata(epc, ep); - if (ep->ops->ep_init) - ep->ops->ep_init(ep); - ret = of_property_read_u8(np, "max-functions", &epc->max_functions); if (ret < 0) epc->max_functions = 1; + for (func_no = 0; func_no < epc->max_functions; func_no++) { + ep_func = devm_kzalloc(dev, sizeof(*ep_func), GFP_KERNEL); + if (!ep_func) + return -ENOMEM; + + ep_func->func_no = func_no; + ep_func->msi_cap = dw_pcie_ep_find_capability(ep, func_no, + PCI_CAP_ID_MSI); + ep_func->msix_cap = dw_pcie_ep_find_capability(ep, func_no, + PCI_CAP_ID_MSIX); + + list_add_tail(&ep_func->list, &ep->func_list); + } + + if (ep->ops->ep_init) + ep->ops->ep_init(ep); + ret = pci_epc_mem_init(epc, ep->phys_base, ep->addr_size, ep->page_size); if (ret < 0) { diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c index 9dafecba347f..674f32db85ca 100644 --- a/drivers/pci/controller/dwc/pcie-designware-host.c +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -20,30 +20,7 @@ #include "pcie-designware.h" static struct pci_ops dw_pcie_ops; - -static int dw_pcie_rd_own_conf(struct pcie_port *pp, int where, int size, - u32 *val) -{ - struct dw_pcie *pci; - - if (pp->ops->rd_own_conf) - return pp->ops->rd_own_conf(pp, where, size, val); - - pci = to_dw_pcie_from_pp(pp); - return dw_pcie_read(pci->dbi_base + where, size, val); -} - -static int dw_pcie_wr_own_conf(struct pcie_port *pp, int where, int size, - u32 val) -{ - struct dw_pcie *pci; - - if (pp->ops->wr_own_conf) - return pp->ops->wr_own_conf(pp, where, size, val); - - pci = to_dw_pcie_from_pp(pp); - return dw_pcie_write(pci->dbi_base + where, size, val); -} +static struct pci_ops dw_child_pcie_ops; static void dw_msi_ack_irq(struct irq_data *d) { @@ -82,13 +59,13 @@ irqreturn_t dw_handle_msi_irq(struct pcie_port *pp) unsigned long val; u32 status, num_ctrls; irqreturn_t ret = IRQ_NONE; + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; for (i = 0; i < num_ctrls; i++) { - dw_pcie_rd_own_conf(pp, PCIE_MSI_INTR0_STATUS + - (i * MSI_REG_CTRL_BLOCK_SIZE), - 4, &status); + status = dw_pcie_readl_dbi(pci, PCIE_MSI_INTR0_STATUS + + (i * MSI_REG_CTRL_BLOCK_SIZE)); if (!status) continue; @@ -148,6 +125,7 @@ static int dw_pci_msi_set_affinity(struct irq_data *d, static void dw_pci_bottom_mask(struct irq_data *d) { struct pcie_port *pp = irq_data_get_irq_chip_data(d); + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); unsigned int res, bit, ctrl; unsigned long flags; @@ -158,8 +136,7 @@ static void dw_pci_bottom_mask(struct irq_data *d) bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; pp->irq_mask[ctrl] |= BIT(bit); - dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + res, 4, - pp->irq_mask[ctrl]); + dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]); raw_spin_unlock_irqrestore(&pp->lock, flags); } @@ -167,6 +144,7 @@ static void dw_pci_bottom_mask(struct irq_data *d) static void dw_pci_bottom_unmask(struct irq_data *d) { struct pcie_port *pp = irq_data_get_irq_chip_data(d); + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); unsigned int res, bit, ctrl; unsigned long flags; @@ -177,8 +155,7 @@ static void dw_pci_bottom_unmask(struct irq_data *d) bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; pp->irq_mask[ctrl] &= ~BIT(bit); - dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + res, 4, - pp->irq_mask[ctrl]); + dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]); raw_spin_unlock_irqrestore(&pp->lock, flags); } @@ -186,13 +163,14 @@ static void dw_pci_bottom_unmask(struct irq_data *d) static void dw_pci_bottom_ack(struct irq_data *d) { struct pcie_port *pp = irq_data_get_irq_chip_data(d); + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); unsigned int res, bit, ctrl; ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; - dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_STATUS + res, 4, BIT(bit)); + dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_STATUS + res, BIT(bit)); } static struct irq_chip dw_pci_msi_bottom_irq_chip = { @@ -288,32 +266,26 @@ void dw_pcie_free_msi(struct pcie_port *pp) irq_domain_remove(pp->msi_domain); irq_domain_remove(pp->irq_domain); - if (pp->msi_page) - __free_page(pp->msi_page); + if (pp->msi_data) { + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct device *dev = pci->dev; + + dma_unmap_single_attrs(dev, pp->msi_data, sizeof(pp->msi_msg), + DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); + } } void dw_pcie_msi_init(struct pcie_port *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct device *dev = pci->dev; - u64 msi_target; + u64 msi_target = (u64)pp->msi_data; - pp->msi_page = alloc_page(GFP_KERNEL); - pp->msi_data = dma_map_page(dev, pp->msi_page, 0, PAGE_SIZE, - DMA_FROM_DEVICE); - if (dma_mapping_error(dev, pp->msi_data)) { - dev_err(dev, "Failed to map MSI data\n"); - __free_page(pp->msi_page); - pp->msi_page = NULL; + if (!IS_ENABLED(CONFIG_PCI_MSI)) return; - } - msi_target = (u64)pp->msi_data; /* Program the msi_data */ - dw_pcie_wr_own_conf(pp, PCIE_MSI_ADDR_LO, 4, - lower_32_bits(msi_target)); - dw_pcie_wr_own_conf(pp, PCIE_MSI_ADDR_HI, 4, - upper_32_bits(msi_target)); + dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_LO, lower_32_bits(msi_target)); + dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_HI, upper_32_bits(msi_target)); } EXPORT_SYMBOL_GPL(dw_pcie_msi_init); @@ -324,20 +296,16 @@ int dw_pcie_host_init(struct pcie_port *pp) struct device_node *np = dev->of_node; struct platform_device *pdev = to_platform_device(dev); struct resource_entry *win; - struct pci_bus *child; struct pci_host_bridge *bridge; struct resource *cfg_res; - u32 hdr_type; int ret; raw_spin_lock_init(&pci->pp.lock); cfg_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config"); if (cfg_res) { - pp->cfg0_size = resource_size(cfg_res) >> 1; - pp->cfg1_size = resource_size(cfg_res) >> 1; + pp->cfg0_size = resource_size(cfg_res); pp->cfg0_base = cfg_res->start; - pp->cfg1_base = cfg_res->start + pp->cfg0_size; } else if (!pp->va_cfg0_base) { dev_err(dev, "Missing *config* reg space\n"); } @@ -346,47 +314,33 @@ int dw_pcie_host_init(struct pcie_port *pp) if (!bridge) return -ENOMEM; + pp->bridge = bridge; + /* Get the I/O and memory ranges from DT */ resource_list_for_each_entry(win, &bridge->windows) { switch (resource_type(win->res)) { case IORESOURCE_IO: - pp->io = win->res; - pp->io->name = "I/O"; - pp->io_size = resource_size(pp->io); - pp->io_bus_addr = pp->io->start - win->offset; - pp->io_base = pci_pio_to_address(pp->io->start); - break; - case IORESOURCE_MEM: - pp->mem = win->res; - pp->mem->name = "MEM"; - pp->mem_size = resource_size(pp->mem); - pp->mem_bus_addr = pp->mem->start - win->offset; + pp->io_size = resource_size(win->res); + pp->io_bus_addr = win->res->start - win->offset; + pp->io_base = pci_pio_to_address(win->res->start); break; case 0: - pp->cfg = win->res; - pp->cfg0_size = resource_size(pp->cfg) >> 1; - pp->cfg1_size = resource_size(pp->cfg) >> 1; - pp->cfg0_base = pp->cfg->start; - pp->cfg1_base = pp->cfg->start + pp->cfg0_size; - break; - case IORESOURCE_BUS: - pp->busn = win->res; + dev_err(dev, "Missing *config* reg space\n"); + pp->cfg0_size = resource_size(win->res); + pp->cfg0_base = win->res->start; + if (!pci->dbi_base) { + pci->dbi_base = devm_pci_remap_cfgspace(dev, + pp->cfg0_base, + pp->cfg0_size); + if (!pci->dbi_base) { + dev_err(dev, "Error with ioremap\n"); + return -ENOMEM; + } + } break; } } - if (!pci->dbi_base) { - pci->dbi_base = devm_pci_remap_cfgspace(dev, - pp->cfg->start, - resource_size(pp->cfg)); - if (!pci->dbi_base) { - dev_err(dev, "Error with ioremap\n"); - return -ENOMEM; - } - } - - pp->mem_base = pp->mem->start; - if (!pp->va_cfg0_base) { pp->va_cfg0_base = devm_pci_remap_cfgspace(dev, pp->cfg0_base, pp->cfg0_size); @@ -396,20 +350,13 @@ int dw_pcie_host_init(struct pcie_port *pp) } } - if (!pp->va_cfg1_base) { - pp->va_cfg1_base = devm_pci_remap_cfgspace(dev, - pp->cfg1_base, - pp->cfg1_size); - if (!pp->va_cfg1_base) { - dev_err(dev, "Error with ioremap\n"); - return -ENOMEM; - } - } - ret = of_property_read_u32(np, "num-viewport", &pci->num_viewport); if (ret) pci->num_viewport = 2; + if (pci->link_gen < 1) + pci->link_gen = of_pci_get_max_link_speed(np); + if (pci_msi_enabled()) { /* * If a specific SoC driver needs to change the @@ -440,6 +387,16 @@ int dw_pcie_host_init(struct pcie_port *pp) irq_set_chained_handler_and_data(pp->msi_irq, dw_chained_msi_isr, pp); + + pp->msi_data = dma_map_single_attrs(pci->dev, &pp->msi_msg, + sizeof(pp->msi_msg), + DMA_FROM_DEVICE, + DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(pci->dev, pp->msi_data)) { + dev_err(pci->dev, "Failed to map MSI data\n"); + pp->msi_data = 0; + goto err_free_msi; + } } else { ret = pp->ops->msi_host_init(pp); if (ret < 0) @@ -447,47 +404,21 @@ int dw_pcie_host_init(struct pcie_port *pp) } } + /* Set default bus ops */ + bridge->ops = &dw_pcie_ops; + bridge->child_ops = &dw_child_pcie_ops; + if (pp->ops->host_init) { ret = pp->ops->host_init(pp); if (ret) goto err_free_msi; } - ret = dw_pcie_rd_own_conf(pp, PCI_HEADER_TYPE, 1, &hdr_type); - if (ret != PCIBIOS_SUCCESSFUL) { - dev_err(pci->dev, "Failed reading PCI_HEADER_TYPE cfg space reg (ret: 0x%x)\n", - ret); - ret = pcibios_err_to_errno(ret); - goto err_free_msi; - } - if (hdr_type != PCI_HEADER_TYPE_BRIDGE) { - dev_err(pci->dev, - "PCIe controller is not set to bridge type (hdr_type: 0x%x)!\n", - hdr_type); - ret = -EIO; - goto err_free_msi; - } - bridge->sysdata = pp; - bridge->ops = &dw_pcie_ops; - - ret = pci_scan_root_bus_bridge(bridge); - if (ret) - goto err_free_msi; - - pp->root_bus = bridge->bus; - - if (pp->ops->scan_bus) - pp->ops->scan_bus(pp); - pci_bus_size_bridges(pp->root_bus); - pci_bus_assign_resources(pp->root_bus); - - list_for_each_entry(child, &pp->root_bus->children, node) - pcie_bus_configure_settings(child); - - pci_bus_add_devices(pp->root_bus); - return 0; + ret = pci_host_probe(bridge); + if (!ret) + return 0; err_free_msi: if (pci_msi_enabled() && !pp->ops->msi_host_init) @@ -498,125 +429,104 @@ EXPORT_SYMBOL_GPL(dw_pcie_host_init); void dw_pcie_host_deinit(struct pcie_port *pp) { - pci_stop_root_bus(pp->root_bus); - pci_remove_root_bus(pp->root_bus); + pci_stop_root_bus(pp->bridge->bus); + pci_remove_root_bus(pp->bridge->bus); if (pci_msi_enabled() && !pp->ops->msi_host_init) dw_pcie_free_msi(pp); } EXPORT_SYMBOL_GPL(dw_pcie_host_deinit); -static int dw_pcie_access_other_conf(struct pcie_port *pp, struct pci_bus *bus, - u32 devfn, int where, int size, u32 *val, - bool write) +static void __iomem *dw_pcie_other_conf_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) { - int ret, type; - u32 busdev, cfg_size; - u64 cpu_addr; - void __iomem *va_cfg_base; + int type; + u32 busdev; + struct pcie_port *pp = bus->sysdata; struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + /* + * Checking whether the link is up here is a last line of defense + * against platforms that forward errors on the system bus as + * SError upon PCI configuration transactions issued when the link + * is down. This check is racy by definition and does not stop + * the system from triggering an SError if the link goes down + * after this check is performed. + */ + if (!dw_pcie_link_up(pci)) + return NULL; + busdev = PCIE_ATU_BUS(bus->number) | PCIE_ATU_DEV(PCI_SLOT(devfn)) | PCIE_ATU_FUNC(PCI_FUNC(devfn)); - if (pci_is_root_bus(bus->parent)) { + if (pci_is_root_bus(bus->parent)) type = PCIE_ATU_TYPE_CFG0; - cpu_addr = pp->cfg0_base; - cfg_size = pp->cfg0_size; - va_cfg_base = pp->va_cfg0_base; - } else { - type = PCIE_ATU_TYPE_CFG1; - cpu_addr = pp->cfg1_base; - cfg_size = pp->cfg1_size; - va_cfg_base = pp->va_cfg1_base; - } - - dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX1, - type, cpu_addr, - busdev, cfg_size); - if (write) - ret = dw_pcie_write(va_cfg_base + where, size, *val); else - ret = dw_pcie_read(va_cfg_base + where, size, val); - - if (pci->num_viewport <= 2) - dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX1, - PCIE_ATU_TYPE_IO, pp->io_base, - pp->io_bus_addr, pp->io_size); - - return ret; -} - -static int dw_pcie_rd_other_conf(struct pcie_port *pp, struct pci_bus *bus, - u32 devfn, int where, int size, u32 *val) -{ - if (pp->ops->rd_other_conf) - return pp->ops->rd_other_conf(pp, bus, devfn, where, - size, val); + type = PCIE_ATU_TYPE_CFG1; - return dw_pcie_access_other_conf(pp, bus, devfn, where, size, val, - false); -} -static int dw_pcie_wr_other_conf(struct pcie_port *pp, struct pci_bus *bus, - u32 devfn, int where, int size, u32 val) -{ - if (pp->ops->wr_other_conf) - return pp->ops->wr_other_conf(pp, bus, devfn, where, - size, val); + dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX1, + type, pp->cfg0_base, + busdev, pp->cfg0_size); - return dw_pcie_access_other_conf(pp, bus, devfn, where, size, &val, - true); + return pp->va_cfg0_base + where; } -static int dw_pcie_valid_device(struct pcie_port *pp, struct pci_bus *bus, - int dev) +static int dw_pcie_rd_other_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) { + int ret; + struct pcie_port *pp = bus->sysdata; struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - /* If there is no link, then there is no device */ - if (!pci_is_root_bus(bus)) { - if (!dw_pcie_link_up(pci)) - return 0; - } else if (dev > 0) - /* Access only one slot on each root port */ - return 0; + ret = pci_generic_config_read(bus, devfn, where, size, val); - return 1; + if (!ret && pci->num_viewport <= 2) + dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX1, + PCIE_ATU_TYPE_IO, pp->io_base, + pp->io_bus_addr, pp->io_size); + + return ret; } -static int dw_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where, - int size, u32 *val) +static int dw_pcie_wr_other_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) { + int ret; struct pcie_port *pp = bus->sysdata; + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - if (!dw_pcie_valid_device(pp, bus, PCI_SLOT(devfn))) { - *val = 0xffffffff; - return PCIBIOS_DEVICE_NOT_FOUND; - } + ret = pci_generic_config_write(bus, devfn, where, size, val); - if (pci_is_root_bus(bus)) - return dw_pcie_rd_own_conf(pp, where, size, val); + if (!ret && pci->num_viewport <= 2) + dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX1, + PCIE_ATU_TYPE_IO, pp->io_base, + pp->io_bus_addr, pp->io_size); - return dw_pcie_rd_other_conf(pp, bus, devfn, where, size, val); + return ret; } -static int dw_pcie_wr_conf(struct pci_bus *bus, u32 devfn, - int where, int size, u32 val) +static struct pci_ops dw_child_pcie_ops = { + .map_bus = dw_pcie_other_conf_map_bus, + .read = dw_pcie_rd_other_conf, + .write = dw_pcie_wr_other_conf, +}; + +void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, unsigned int devfn, int where) { struct pcie_port *pp = bus->sysdata; + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - if (!dw_pcie_valid_device(pp, bus, PCI_SLOT(devfn))) - return PCIBIOS_DEVICE_NOT_FOUND; - - if (pci_is_root_bus(bus)) - return dw_pcie_wr_own_conf(pp, where, size, val); + if (PCI_SLOT(devfn) > 0) + return NULL; - return dw_pcie_wr_other_conf(pp, bus, devfn, where, size, val); + return pci->dbi_base + where; } +EXPORT_SYMBOL_GPL(dw_pcie_own_conf_map_bus); static struct pci_ops dw_pcie_ops = { - .read = dw_pcie_rd_conf, - .write = dw_pcie_wr_conf, + .map_bus = dw_pcie_own_conf_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, }; void dw_pcie_setup_rc(struct pcie_port *pp) @@ -632,18 +542,18 @@ void dw_pcie_setup_rc(struct pcie_port *pp) dw_pcie_setup(pci); - if (!pp->ops->msi_host_init) { + if (pci_msi_enabled() && !pp->ops->msi_host_init) { num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; /* Initialize IRQ Status array */ for (ctrl = 0; ctrl < num_ctrls; ctrl++) { pp->irq_mask[ctrl] = ~0; - dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + + dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + (ctrl * MSI_REG_CTRL_BLOCK_SIZE), - 4, pp->irq_mask[ctrl]); - dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_ENABLE + + pp->irq_mask[ctrl]); + dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_ENABLE + (ctrl * MSI_REG_CTRL_BLOCK_SIZE), - 4, ~0); + ~0); } } @@ -671,28 +581,32 @@ void dw_pcie_setup_rc(struct pcie_port *pp) dw_pcie_writel_dbi(pci, PCI_COMMAND, val); /* - * If the platform provides ->rd_other_conf, it means the platform - * uses its own address translation component rather than ATU, so - * we should not program the ATU here. + * If the platform provides its own child bus config accesses, it means + * the platform uses its own address translation component rather than + * ATU, so we should not program the ATU here. */ - if (!pp->ops->rd_other_conf) { + if (pp->bridge->child_ops == &dw_child_pcie_ops) { + struct resource_entry *entry = + resource_list_first_type(&pp->bridge->windows, IORESOURCE_MEM); + dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX0, - PCIE_ATU_TYPE_MEM, pp->mem_base, - pp->mem_bus_addr, pp->mem_size); + PCIE_ATU_TYPE_MEM, entry->res->start, + entry->res->start - entry->offset, + resource_size(entry->res)); if (pci->num_viewport > 2) dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX2, PCIE_ATU_TYPE_IO, pp->io_base, pp->io_bus_addr, pp->io_size); } - dw_pcie_wr_own_conf(pp, PCI_BASE_ADDRESS_0, 4, 0); + dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); /* Program correct class for RC */ - dw_pcie_wr_own_conf(pp, PCI_CLASS_DEVICE, 2, PCI_CLASS_BRIDGE_PCI); + dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI); - dw_pcie_rd_own_conf(pp, PCIE_LINK_WIDTH_SPEED_CONTROL, 4, &val); + val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); val |= PORT_LOGIC_SPEED_CHANGE; - dw_pcie_wr_own_conf(pp, PCIE_LINK_WIDTH_SPEED_CONTROL, 4, val); + dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); dw_pcie_dbi_ro_wr_dis(pci); } diff --git a/drivers/pci/controller/dwc/pcie-designware-plat.c b/drivers/pci/controller/dwc/pcie-designware-plat.c index 712456f6ce36..e3e300669ed5 100644 --- a/drivers/pci/controller/dwc/pcie-designware-plat.c +++ b/drivers/pci/controller/dwc/pcie-designware-plat.c @@ -39,9 +39,7 @@ static int dw_plat_pcie_host_init(struct pcie_port *pp) dw_pcie_setup_rc(pp); dw_pcie_wait_for_link(pci); - - if (IS_ENABLED(CONFIG_PCI_MSI)) - dw_pcie_msi_init(pp); + dw_pcie_msi_init(pp); return 0; } diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index b723e0cc41fb..c2dea8fc97c8 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -10,6 +10,7 @@ #include <linux/delay.h> #include <linux/of.h> +#include <linux/of_platform.h> #include <linux/types.h> #include "../../pci.h" @@ -166,21 +167,6 @@ void dw_pcie_write_dbi(struct dw_pcie *pci, u32 reg, size_t size, u32 val) } EXPORT_SYMBOL_GPL(dw_pcie_write_dbi); -u32 dw_pcie_read_dbi2(struct dw_pcie *pci, u32 reg, size_t size) -{ - int ret; - u32 val; - - if (pci->ops->read_dbi2) - return pci->ops->read_dbi2(pci, pci->dbi_base2, reg, size); - - ret = dw_pcie_read(pci->dbi_base2 + reg, size, &val); - if (ret) - dev_err(pci->dev, "read DBI address failed\n"); - - return val; -} - void dw_pcie_write_dbi2(struct dw_pcie *pci, u32 reg, size_t size, u32 val) { int ret; @@ -195,31 +181,31 @@ void dw_pcie_write_dbi2(struct dw_pcie *pci, u32 reg, size_t size, u32 val) dev_err(pci->dev, "write DBI address failed\n"); } -u32 dw_pcie_read_atu(struct dw_pcie *pci, u32 reg, size_t size) +static u32 dw_pcie_readl_atu(struct dw_pcie *pci, u32 reg) { int ret; u32 val; if (pci->ops->read_dbi) - return pci->ops->read_dbi(pci, pci->atu_base, reg, size); + return pci->ops->read_dbi(pci, pci->atu_base, reg, 4); - ret = dw_pcie_read(pci->atu_base + reg, size, &val); + ret = dw_pcie_read(pci->atu_base + reg, 4, &val); if (ret) dev_err(pci->dev, "Read ATU address failed\n"); return val; } -void dw_pcie_write_atu(struct dw_pcie *pci, u32 reg, size_t size, u32 val) +static void dw_pcie_writel_atu(struct dw_pcie *pci, u32 reg, u32 val) { int ret; if (pci->ops->write_dbi) { - pci->ops->write_dbi(pci, pci->atu_base, reg, size, val); + pci->ops->write_dbi(pci, pci->atu_base, reg, 4, val); return; } - ret = dw_pcie_write(pci->atu_base + reg, size, val); + ret = dw_pcie_write(pci->atu_base + reg, 4, val); if (ret) dev_err(pci->dev, "Write ATU address failed\n"); } @@ -239,9 +225,10 @@ static void dw_pcie_writel_ob_unroll(struct dw_pcie *pci, u32 index, u32 reg, dw_pcie_writel_atu(pci, offset + reg, val); } -static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, - int type, u64 cpu_addr, - u64 pci_addr, u32 size) +static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, u8 func_no, + int index, int type, + u64 cpu_addr, u64 pci_addr, + u32 size) { u32 retries, val; u64 limit_addr = cpu_addr + size - 1; @@ -259,7 +246,7 @@ static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_TARGET, upper_32_bits(pci_addr)); dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, - type); + type | PCIE_ATU_FUNC_NUM(func_no)); dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2, PCIE_ATU_ENABLE); @@ -278,8 +265,9 @@ static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, dev_err(pci->dev, "Outbound iATU is not being enabled\n"); } -void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, - u64 cpu_addr, u64 pci_addr, u32 size) +static void __dw_pcie_prog_outbound_atu(struct dw_pcie *pci, u8 func_no, + int index, int type, u64 cpu_addr, + u64 pci_addr, u32 size) { u32 retries, val; @@ -287,8 +275,8 @@ void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr); if (pci->iatu_unroll_enabled) { - dw_pcie_prog_outbound_atu_unroll(pci, index, type, cpu_addr, - pci_addr, size); + dw_pcie_prog_outbound_atu_unroll(pci, func_no, index, type, + cpu_addr, pci_addr, size); return; } @@ -304,7 +292,8 @@ void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, lower_32_bits(pci_addr)); dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_TARGET, upper_32_bits(pci_addr)); - dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type); + dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type | + PCIE_ATU_FUNC_NUM(func_no)); dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE); /* @@ -321,6 +310,21 @@ void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, dev_err(pci->dev, "Outbound iATU is not being enabled\n"); } +void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, + u64 cpu_addr, u64 pci_addr, u32 size) +{ + __dw_pcie_prog_outbound_atu(pci, 0, index, type, + cpu_addr, pci_addr, size); +} + +void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index, + int type, u64 cpu_addr, u64 pci_addr, + u32 size) +{ + __dw_pcie_prog_outbound_atu(pci, func_no, index, type, + cpu_addr, pci_addr, size); +} + static u32 dw_pcie_readl_ib_unroll(struct dw_pcie *pci, u32 index, u32 reg) { u32 offset = PCIE_GET_ATU_INB_UNR_REG_OFFSET(index); @@ -336,8 +340,8 @@ static void dw_pcie_writel_ib_unroll(struct dw_pcie *pci, u32 index, u32 reg, dw_pcie_writel_atu(pci, offset + reg, val); } -static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index, - int bar, u64 cpu_addr, +static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, u8 func_no, + int index, int bar, u64 cpu_addr, enum dw_pcie_as_type as_type) { int type; @@ -359,8 +363,10 @@ static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index, return -EINVAL; } - dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, type); + dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, type | + PCIE_ATU_FUNC_NUM(func_no)); dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2, + PCIE_ATU_FUNC_NUM_MATCH_EN | PCIE_ATU_ENABLE | PCIE_ATU_BAR_MODE_ENABLE | (bar << 8)); @@ -381,14 +387,15 @@ static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index, return -EBUSY; } -int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar, - u64 cpu_addr, enum dw_pcie_as_type as_type) +int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index, + int bar, u64 cpu_addr, + enum dw_pcie_as_type as_type) { int type; u32 retries, val; if (pci->iatu_unroll_enabled) - return dw_pcie_prog_inbound_atu_unroll(pci, index, bar, + return dw_pcie_prog_inbound_atu_unroll(pci, func_no, index, bar, cpu_addr, as_type); dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT, PCIE_ATU_REGION_INBOUND | @@ -407,9 +414,11 @@ int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar, return -EINVAL; } - dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type); - dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE - | PCIE_ATU_BAR_MODE_ENABLE | (bar << 8)); + dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type | + PCIE_ATU_FUNC_NUM(func_no)); + dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE | + PCIE_ATU_FUNC_NUM_MATCH_EN | + PCIE_ATU_BAR_MODE_ENABLE | (bar << 8)); /* * Make sure ATU enable takes effect before any subsequent config @@ -444,7 +453,7 @@ void dw_pcie_disable_atu(struct dw_pcie *pci, int index, } dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT, region | index); - dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, (u32)~PCIE_ATU_ENABLE); + dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, ~(u32)PCIE_ATU_ENABLE); } int dw_pcie_wait_for_link(struct dw_pcie *pci) @@ -488,50 +497,41 @@ void dw_pcie_upconfig_setup(struct dw_pcie *pci) } EXPORT_SYMBOL_GPL(dw_pcie_upconfig_setup); -void dw_pcie_link_set_max_speed(struct dw_pcie *pci, u32 link_gen) +static void dw_pcie_link_set_max_speed(struct dw_pcie *pci, u32 link_gen) { - u32 reg, val; + u32 cap, ctrl2, link_speed; u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); - reg = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCTL2); - reg &= ~PCI_EXP_LNKCTL2_TLS; + cap = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); + ctrl2 = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCTL2); + ctrl2 &= ~PCI_EXP_LNKCTL2_TLS; switch (pcie_link_speed[link_gen]) { case PCIE_SPEED_2_5GT: - reg |= PCI_EXP_LNKCTL2_TLS_2_5GT; + link_speed = PCI_EXP_LNKCTL2_TLS_2_5GT; break; case PCIE_SPEED_5_0GT: - reg |= PCI_EXP_LNKCTL2_TLS_5_0GT; + link_speed = PCI_EXP_LNKCTL2_TLS_5_0GT; break; case PCIE_SPEED_8_0GT: - reg |= PCI_EXP_LNKCTL2_TLS_8_0GT; + link_speed = PCI_EXP_LNKCTL2_TLS_8_0GT; break; case PCIE_SPEED_16_0GT: - reg |= PCI_EXP_LNKCTL2_TLS_16_0GT; + link_speed = PCI_EXP_LNKCTL2_TLS_16_0GT; break; default: /* Use hardware capability */ - val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); - val = FIELD_GET(PCI_EXP_LNKCAP_SLS, val); - reg &= ~PCI_EXP_LNKCTL2_HASD; - reg |= FIELD_PREP(PCI_EXP_LNKCTL2_TLS, val); + link_speed = FIELD_GET(PCI_EXP_LNKCAP_SLS, cap); + ctrl2 &= ~PCI_EXP_LNKCTL2_HASD; break; } - dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCTL2, reg); -} -EXPORT_SYMBOL_GPL(dw_pcie_link_set_max_speed); + dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCTL2, ctrl2 | link_speed); -void dw_pcie_link_set_n_fts(struct dw_pcie *pci, u32 n_fts) -{ - u32 val; + cap &= ~((u32)PCI_EXP_LNKCAP_SLS); + dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, cap | link_speed); - val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); - val &= ~PORT_LOGIC_N_FTS_MASK; - val |= n_fts & PORT_LOGIC_N_FTS_MASK; - dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); } -EXPORT_SYMBOL_GPL(dw_pcie_link_set_n_fts); static u8 dw_pcie_iatu_unroll_enabled(struct dw_pcie *pci) { @@ -546,32 +546,58 @@ static u8 dw_pcie_iatu_unroll_enabled(struct dw_pcie *pci) void dw_pcie_setup(struct dw_pcie *pci) { - int ret; u32 val; - u32 lanes; struct device *dev = pci->dev; struct device_node *np = dev->of_node; + struct platform_device *pdev = to_platform_device(dev); if (pci->version >= 0x480A || (!pci->version && dw_pcie_iatu_unroll_enabled(pci))) { pci->iatu_unroll_enabled = true; if (!pci->atu_base) + pci->atu_base = + devm_platform_ioremap_resource_byname(pdev, "atu"); + if (IS_ERR(pci->atu_base)) pci->atu_base = pci->dbi_base + DEFAULT_DBI_ATU_OFFSET; } dev_dbg(pci->dev, "iATU unroll: %s\n", pci->iatu_unroll_enabled ? "enabled" : "disabled"); + if (pci->link_gen > 0) + dw_pcie_link_set_max_speed(pci, pci->link_gen); - ret = of_property_read_u32(np, "num-lanes", &lanes); - if (ret) { - dev_dbg(pci->dev, "property num-lanes isn't found\n"); + /* Configure Gen1 N_FTS */ + if (pci->n_fts[0]) { + val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR); + val &= ~(PORT_AFR_N_FTS_MASK | PORT_AFR_CC_N_FTS_MASK); + val |= PORT_AFR_N_FTS(pci->n_fts[0]); + val |= PORT_AFR_CC_N_FTS(pci->n_fts[0]); + dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val); + } + + /* Configure Gen2+ N_FTS */ + if (pci->n_fts[1]) { + val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); + val &= ~PORT_LOGIC_N_FTS_MASK; + val |= pci->n_fts[pci->link_gen - 1]; + dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); + } + + val = dw_pcie_readl_dbi(pci, PCIE_PORT_LINK_CONTROL); + val &= ~PORT_LINK_FAST_LINK_MODE; + val |= PORT_LINK_DLL_LINK_EN; + dw_pcie_writel_dbi(pci, PCIE_PORT_LINK_CONTROL, val); + + of_property_read_u32(np, "num-lanes", &pci->num_lanes); + if (!pci->num_lanes) { + dev_dbg(pci->dev, "Using h/w default number of lanes\n"); return; } /* Set the number of lanes */ - val = dw_pcie_readl_dbi(pci, PCIE_PORT_LINK_CONTROL); + val &= ~PORT_LINK_FAST_LINK_MODE; val &= ~PORT_LINK_MODE_MASK; - switch (lanes) { + switch (pci->num_lanes) { case 1: val |= PORT_LINK_MODE_1_LANES; break; @@ -585,7 +611,7 @@ void dw_pcie_setup(struct dw_pcie *pci) val |= PORT_LINK_MODE_8_LANES; break; default: - dev_err(pci->dev, "num-lanes %u: invalid value\n", lanes); + dev_err(pci->dev, "num-lanes %u: invalid value\n", pci->num_lanes); return; } dw_pcie_writel_dbi(pci, PCIE_PORT_LINK_CONTROL, val); @@ -593,7 +619,7 @@ void dw_pcie_setup(struct dw_pcie *pci) /* Set link width speed control register */ val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); val &= ~PORT_LOGIC_LINK_WIDTH_MASK; - switch (lanes) { + switch (pci->num_lanes) { case 1: val |= PORT_LOGIC_LINK_WIDTH_1_LANES; break; diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index f911760dcc69..9d2f511f13fa 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -32,10 +32,18 @@ /* Synopsys-specific PCIe configuration registers */ #define PCIE_PORT_AFR 0x70C #define PORT_AFR_N_FTS_MASK GENMASK(15, 8) +#define PORT_AFR_N_FTS(n) FIELD_PREP(PORT_AFR_N_FTS_MASK, n) #define PORT_AFR_CC_N_FTS_MASK GENMASK(23, 16) +#define PORT_AFR_CC_N_FTS(n) FIELD_PREP(PORT_AFR_CC_N_FTS_MASK, n) +#define PORT_AFR_ENTER_ASPM BIT(30) +#define PORT_AFR_L0S_ENTRANCE_LAT_SHIFT 24 +#define PORT_AFR_L0S_ENTRANCE_LAT_MASK GENMASK(26, 24) +#define PORT_AFR_L1_ENTRANCE_LAT_SHIFT 27 +#define PORT_AFR_L1_ENTRANCE_LAT_MASK GENMASK(29, 27) #define PCIE_PORT_LINK_CONTROL 0x710 #define PORT_LINK_DLL_LINK_EN BIT(5) +#define PORT_LINK_FAST_LINK_MODE BIT(7) #define PORT_LINK_MODE_MASK GENMASK(21, 16) #define PORT_LINK_MODE(n) FIELD_PREP(PORT_LINK_MODE_MASK, n) #define PORT_LINK_MODE_1_LANES PORT_LINK_MODE(0x1) @@ -80,9 +88,11 @@ #define PCIE_ATU_TYPE_IO 0x2 #define PCIE_ATU_TYPE_CFG0 0x4 #define PCIE_ATU_TYPE_CFG1 0x5 +#define PCIE_ATU_FUNC_NUM(pf) ((pf) << 20) #define PCIE_ATU_CR2 0x908 #define PCIE_ATU_ENABLE BIT(31) #define PCIE_ATU_BAR_MODE_ENABLE BIT(30) +#define PCIE_ATU_FUNC_NUM_MATCH_EN BIT(19) #define PCIE_ATU_LOWER_BASE 0x90C #define PCIE_ATU_UPPER_BASE 0x910 #define PCIE_ATU_LIMIT 0x914 @@ -95,6 +105,9 @@ #define PCIE_MISC_CONTROL_1_OFF 0x8BC #define PCIE_DBI_RO_WR_EN BIT(0) +#define PCIE_MSIX_DOORBELL 0x948 +#define PCIE_MSIX_DOORBELL_PF_SHIFT 24 + #define PCIE_PL_CHK_REG_CONTROL_STATUS 0xB20 #define PCIE_PL_CHK_REG_CHK_REG_START BIT(0) #define PCIE_PL_CHK_REG_CHK_REG_CONTINUOUS BIT(1) @@ -160,14 +173,7 @@ enum dw_pcie_device_mode { }; struct dw_pcie_host_ops { - int (*rd_own_conf)(struct pcie_port *pp, int where, int size, u32 *val); - int (*wr_own_conf)(struct pcie_port *pp, int where, int size, u32 val); - int (*rd_other_conf)(struct pcie_port *pp, struct pci_bus *bus, - unsigned int devfn, int where, int size, u32 *val); - int (*wr_other_conf)(struct pcie_port *pp, struct pci_bus *bus, - unsigned int devfn, int where, int size, u32 val); int (*host_init)(struct pcie_port *pp); - void (*scan_bus)(struct pcie_port *pp); void (*set_num_vectors)(struct pcie_port *pp); int (*msi_host_init)(struct pcie_port *pp); }; @@ -176,30 +182,20 @@ struct pcie_port { u64 cfg0_base; void __iomem *va_cfg0_base; u32 cfg0_size; - u64 cfg1_base; - void __iomem *va_cfg1_base; - u32 cfg1_size; resource_size_t io_base; phys_addr_t io_bus_addr; u32 io_size; - u64 mem_base; - phys_addr_t mem_bus_addr; - u32 mem_size; - struct resource *cfg; - struct resource *io; - struct resource *mem; - struct resource *busn; int irq; const struct dw_pcie_host_ops *ops; int msi_irq; struct irq_domain *irq_domain; struct irq_domain *msi_domain; + u16 msi_msg; dma_addr_t msi_data; - struct page *msi_page; struct irq_chip *msi_irq_chip; u32 num_vectors; u32 irq_mask[MAX_MSI_CTRLS]; - struct pci_bus *root_bus; + struct pci_host_bridge *bridge; raw_spinlock_t lock; DECLARE_BITMAP(msi_irq_in_use, MAX_MSI_IRQS); }; @@ -215,10 +211,26 @@ struct dw_pcie_ep_ops { int (*raise_irq)(struct dw_pcie_ep *ep, u8 func_no, enum pci_epc_irq_type type, u16 interrupt_num); const struct pci_epc_features* (*get_features)(struct dw_pcie_ep *ep); + /* + * Provide a method to implement the different func config space + * access for different platform, if different func have different + * offset, return the offset of func. if use write a register way + * return a 0, and implement code in callback function of platform + * driver. + */ + unsigned int (*func_conf_select)(struct dw_pcie_ep *ep, u8 func_no); +}; + +struct dw_pcie_ep_func { + struct list_head list; + u8 func_no; + u8 msi_cap; /* MSI capability offset */ + u8 msix_cap; /* MSI-X capability offset */ }; struct dw_pcie_ep { struct pci_epc *epc; + struct list_head func_list; const struct dw_pcie_ep_ops *ops; phys_addr_t phys_base; size_t addr_size; @@ -231,8 +243,6 @@ struct dw_pcie_ep { u32 num_ob_windows; void __iomem *msi_mem; phys_addr_t msi_mem_phys; - u8 msi_cap; /* MSI capability offset */ - u8 msix_cap; /* MSI-X capability offset */ struct pci_epf_bar *epf_bar[PCI_STD_NUM_BARS]; }; @@ -242,8 +252,6 @@ struct dw_pcie_ops { size_t size); void (*write_dbi)(struct dw_pcie *pcie, void __iomem *base, u32 reg, size_t size, u32 val); - u32 (*read_dbi2)(struct dw_pcie *pcie, void __iomem *base, u32 reg, - size_t size); void (*write_dbi2)(struct dw_pcie *pcie, void __iomem *base, u32 reg, size_t size, u32 val); int (*link_up)(struct dw_pcie *pcie); @@ -263,6 +271,9 @@ struct dw_pcie { struct dw_pcie_ep ep; const struct dw_pcie_ops *ops; unsigned int version; + int num_lanes; + int link_gen; + u8 n_fts[2]; }; #define to_dw_pcie_from_pp(port) container_of((port), struct dw_pcie, pp) @@ -278,20 +289,19 @@ int dw_pcie_write(void __iomem *addr, int size, u32 val); u32 dw_pcie_read_dbi(struct dw_pcie *pci, u32 reg, size_t size); void dw_pcie_write_dbi(struct dw_pcie *pci, u32 reg, size_t size, u32 val); -u32 dw_pcie_read_dbi2(struct dw_pcie *pci, u32 reg, size_t size); void dw_pcie_write_dbi2(struct dw_pcie *pci, u32 reg, size_t size, u32 val); -u32 dw_pcie_read_atu(struct dw_pcie *pci, u32 reg, size_t size); -void dw_pcie_write_atu(struct dw_pcie *pci, u32 reg, size_t size, u32 val); int dw_pcie_link_up(struct dw_pcie *pci); void dw_pcie_upconfig_setup(struct dw_pcie *pci); -void dw_pcie_link_set_max_speed(struct dw_pcie *pci, u32 link_gen); -void dw_pcie_link_set_n_fts(struct dw_pcie *pci, u32 n_fts); int dw_pcie_wait_for_link(struct dw_pcie *pci); void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, u64 cpu_addr, u64 pci_addr, u32 size); -int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar, - u64 cpu_addr, enum dw_pcie_as_type as_type); +void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index, + int type, u64 cpu_addr, u64 pci_addr, + u32 size); +int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index, + int bar, u64 cpu_addr, + enum dw_pcie_as_type as_type); void dw_pcie_disable_atu(struct dw_pcie *pci, int index, enum dw_pcie_region_type type); void dw_pcie_setup(struct dw_pcie *pci); @@ -331,21 +341,6 @@ static inline void dw_pcie_writel_dbi2(struct dw_pcie *pci, u32 reg, u32 val) dw_pcie_write_dbi2(pci, reg, 0x4, val); } -static inline u32 dw_pcie_readl_dbi2(struct dw_pcie *pci, u32 reg) -{ - return dw_pcie_read_dbi2(pci, reg, 0x4); -} - -static inline void dw_pcie_writel_atu(struct dw_pcie *pci, u32 reg, u32 val) -{ - dw_pcie_write_atu(pci, reg, 0x4, val); -} - -static inline u32 dw_pcie_readl_atu(struct dw_pcie *pci, u32 reg) -{ - return dw_pcie_read_atu(pci, reg, 0x4); -} - static inline void dw_pcie_dbi_ro_wr_en(struct dw_pcie *pci) { u32 reg; @@ -376,6 +371,8 @@ void dw_pcie_setup_rc(struct pcie_port *pp); int dw_pcie_host_init(struct pcie_port *pp); void dw_pcie_host_deinit(struct pcie_port *pp); int dw_pcie_allocate_domains(struct pcie_port *pp); +void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, unsigned int devfn, + int where); #else static inline irqreturn_t dw_handle_msi_irq(struct pcie_port *pp) { @@ -407,6 +404,12 @@ static inline int dw_pcie_allocate_domains(struct pcie_port *pp) { return 0; } +static inline void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, + unsigned int devfn, + int where) +{ + return NULL; +} #endif #ifdef CONFIG_PCIE_DW_EP @@ -420,7 +423,11 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, u8 interrupt_num); int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, u16 interrupt_num); +int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, + u16 interrupt_num); void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar); +struct dw_pcie_ep_func * +dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no); #else static inline void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) { @@ -461,8 +468,21 @@ static inline int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, return 0; } +static inline int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, + u8 func_no, + u16 interrupt_num) +{ + return 0; +} + static inline void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) { } + +static inline struct dw_pcie_ep_func * +dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no) +{ + return NULL; +} #endif #endif /* _PCIE_DESIGNWARE_H */ diff --git a/drivers/pci/controller/dwc/pcie-histb.c b/drivers/pci/controller/dwc/pcie-histb.c index 2a2835746077..afc1abbe49aa 100644 --- a/drivers/pci/controller/dwc/pcie-histb.c +++ b/drivers/pci/controller/dwc/pcie-histb.c @@ -122,32 +122,37 @@ static void histb_pcie_write_dbi(struct dw_pcie *pci, void __iomem *base, histb_pcie_dbi_w_mode(&pci->pp, false); } -static int histb_pcie_rd_own_conf(struct pcie_port *pp, int where, - int size, u32 *val) +static int histb_pcie_rd_own_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) { - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - int ret; + struct dw_pcie *pci = to_dw_pcie_from_pp(bus->sysdata); - histb_pcie_dbi_r_mode(pp, true); - ret = dw_pcie_read(pci->dbi_base + where, size, val); - histb_pcie_dbi_r_mode(pp, false); + if (PCI_SLOT(devfn)) { + *val = ~0; + return PCIBIOS_DEVICE_NOT_FOUND; + } - return ret; + *val = dw_pcie_read_dbi(pci, where, size); + return PCIBIOS_SUCCESSFUL; } -static int histb_pcie_wr_own_conf(struct pcie_port *pp, int where, - int size, u32 val) +static int histb_pcie_wr_own_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) { - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - int ret; + struct dw_pcie *pci = to_dw_pcie_from_pp(bus->sysdata); - histb_pcie_dbi_w_mode(pp, true); - ret = dw_pcie_write(pci->dbi_base + where, size, val); - histb_pcie_dbi_w_mode(pp, false); + if (PCI_SLOT(devfn)) + return PCIBIOS_DEVICE_NOT_FOUND; - return ret; + dw_pcie_write_dbi(pci, where, size, val); + return PCIBIOS_SUCCESSFUL; } +static struct pci_ops histb_pci_ops = { + .read = histb_pcie_rd_own_conf, + .write = histb_pcie_wr_own_conf, +}; + static int histb_pcie_link_up(struct dw_pcie *pci) { struct histb_pcie *hipcie = to_histb_pcie(pci); @@ -194,17 +199,15 @@ static int histb_pcie_establish_link(struct pcie_port *pp) static int histb_pcie_host_init(struct pcie_port *pp) { - histb_pcie_establish_link(pp); + pp->bridge->ops = &histb_pci_ops; - if (IS_ENABLED(CONFIG_PCI_MSI)) - dw_pcie_msi_init(pp); + histb_pcie_establish_link(pp); + dw_pcie_msi_init(pp); return 0; } static const struct dw_pcie_host_ops histb_pcie_host_ops = { - .rd_own_conf = histb_pcie_rd_own_conf, - .wr_own_conf = histb_pcie_wr_own_conf, .host_init = histb_pcie_host_init, }; diff --git a/drivers/pci/controller/dwc/pcie-intel-gw.c b/drivers/pci/controller/dwc/pcie-intel-gw.c index c3b3a1d162b5..5650cb78acba 100644 --- a/drivers/pci/controller/dwc/pcie-intel-gw.c +++ b/drivers/pci/controller/dwc/pcie-intel-gw.c @@ -67,14 +67,9 @@ struct intel_pcie_port { void __iomem *app_base; struct gpio_desc *reset_gpio; u32 rst_intrvl; - u32 max_speed; - u32 link_gen; - u32 max_width; - u32 n_fts; struct clk *core_clk; struct reset_control *core_rst; struct phy *phy; - u8 pcie_cap_ofst; }; static void pcie_update_bits(void __iomem *base, u32 ofs, u32 mask, u32 val) @@ -134,11 +129,7 @@ static void intel_pcie_ltssm_disable(struct intel_pcie_port *lpp) static void intel_pcie_link_setup(struct intel_pcie_port *lpp) { u32 val; - u8 offset = lpp->pcie_cap_ofst; - - val = pcie_rc_cfg_rd(lpp, offset + PCI_EXP_LNKCAP); - lpp->max_speed = FIELD_GET(PCI_EXP_LNKCAP_SLS, val); - lpp->max_width = FIELD_GET(PCI_EXP_LNKCAP_MLW, val); + u8 offset = dw_pcie_find_capability(&lpp->pci, PCI_CAP_ID_EXP); val = pcie_rc_cfg_rd(lpp, offset + PCI_EXP_LNKCTL); @@ -146,41 +137,29 @@ static void intel_pcie_link_setup(struct intel_pcie_port *lpp) pcie_rc_cfg_wr(lpp, offset + PCI_EXP_LNKCTL, val); } -static void intel_pcie_port_logic_setup(struct intel_pcie_port *lpp) +static void intel_pcie_init_n_fts(struct dw_pcie *pci) { - u32 val, mask; - - switch (pcie_link_speed[lpp->max_speed]) { - case PCIE_SPEED_8_0GT: - lpp->n_fts = PORT_AFR_N_FTS_GEN3; + switch (pci->link_gen) { + case 3: + pci->n_fts[1] = PORT_AFR_N_FTS_GEN3; break; - case PCIE_SPEED_16_0GT: - lpp->n_fts = PORT_AFR_N_FTS_GEN4; + case 4: + pci->n_fts[1] = PORT_AFR_N_FTS_GEN4; break; default: - lpp->n_fts = PORT_AFR_N_FTS_GEN12_DFT; + pci->n_fts[1] = PORT_AFR_N_FTS_GEN12_DFT; break; } - - mask = PORT_AFR_N_FTS_MASK | PORT_AFR_CC_N_FTS_MASK; - val = FIELD_PREP(PORT_AFR_N_FTS_MASK, lpp->n_fts) | - FIELD_PREP(PORT_AFR_CC_N_FTS_MASK, lpp->n_fts); - pcie_rc_cfg_wr_mask(lpp, PCIE_PORT_AFR, mask, val); - - /* Port Link Control Register */ - pcie_rc_cfg_wr_mask(lpp, PCIE_PORT_LINK_CONTROL, PORT_LINK_DLL_LINK_EN, - PORT_LINK_DLL_LINK_EN); + pci->n_fts[0] = PORT_AFR_N_FTS_GEN12_DFT; } static void intel_pcie_rc_setup(struct intel_pcie_port *lpp) { intel_pcie_ltssm_disable(lpp); intel_pcie_link_setup(lpp); + intel_pcie_init_n_fts(&lpp->pci); dw_pcie_setup_rc(&lpp->pci.pp); dw_pcie_upconfig_setup(&lpp->pci); - intel_pcie_port_logic_setup(lpp); - dw_pcie_link_set_max_speed(&lpp->pci, lpp->link_gen); - dw_pcie_link_set_n_fts(&lpp->pci, lpp->n_fts); } static int intel_pcie_ep_rst_init(struct intel_pcie_port *lpp) @@ -275,20 +254,11 @@ static int intel_pcie_get_resources(struct platform_device *pdev) return ret; } - ret = device_property_match_string(dev, "device_type", "pci"); - if (ret) { - dev_err(dev, "Failed to find pci device type: %d\n", ret); - return ret; - } - ret = device_property_read_u32(dev, "reset-assert-ms", &lpp->rst_intrvl); if (ret) lpp->rst_intrvl = RESET_INTERVAL_MS; - ret = of_pci_get_max_link_speed(dev->of_node); - lpp->link_gen = ret < 0 ? 0 : ret; - lpp->app_base = devm_platform_ioremap_resource_byname(pdev, "app"); if (IS_ERR(lpp->app_base)) return PTR_ERR(lpp->app_base); @@ -313,8 +283,9 @@ static int intel_pcie_wait_l2(struct intel_pcie_port *lpp) { u32 value; int ret; + struct dw_pcie *pci = &lpp->pci; - if (pcie_link_speed[lpp->max_speed] < PCIE_SPEED_8_0GT) + if (pci->link_gen < 3) return 0; /* Send PME_TURN_OFF message */ @@ -343,7 +314,6 @@ static void intel_pcie_turn_off(struct intel_pcie_port *lpp) static int intel_pcie_host_setup(struct intel_pcie_port *lpp) { - struct device *dev = lpp->pci.dev; int ret; intel_pcie_core_rst_assert(lpp); @@ -361,17 +331,6 @@ static int intel_pcie_host_setup(struct intel_pcie_port *lpp) goto clk_err; } - if (!lpp->pcie_cap_ofst) { - ret = dw_pcie_find_capability(&lpp->pci, PCI_CAP_ID_EXP); - if (!ret) { - ret = -ENXIO; - dev_err(dev, "Invalid PCIe capability offset\n"); - goto app_init_err; - } - - lpp->pcie_cap_ofst = ret; - } - intel_pcie_rc_setup(lpp); ret = intel_pcie_app_logic_setup(lpp); if (ret) diff --git a/drivers/pci/controller/dwc/pcie-kirin.c b/drivers/pci/controller/dwc/pcie-kirin.c index e496f51e0152..d0a6a2dee6f5 100644 --- a/drivers/pci/controller/dwc/pcie-kirin.c +++ b/drivers/pci/controller/dwc/pcie-kirin.c @@ -330,34 +330,37 @@ static void kirin_pcie_sideband_dbi_r_mode(struct kirin_pcie *kirin_pcie, kirin_apb_ctrl_writel(kirin_pcie, val, SOC_PCIECTRL_CTRL1_ADDR); } -static int kirin_pcie_rd_own_conf(struct pcie_port *pp, +static int kirin_pcie_rd_own_conf(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val) { - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct kirin_pcie *kirin_pcie = to_kirin_pcie(pci); - int ret; + struct dw_pcie *pci = to_dw_pcie_from_pp(bus->sysdata); - kirin_pcie_sideband_dbi_r_mode(kirin_pcie, true); - ret = dw_pcie_read(pci->dbi_base + where, size, val); - kirin_pcie_sideband_dbi_r_mode(kirin_pcie, false); + if (PCI_SLOT(devfn)) { + *val = ~0; + return PCIBIOS_DEVICE_NOT_FOUND; + } - return ret; + *val = dw_pcie_read_dbi(pci, where, size); + return PCIBIOS_SUCCESSFUL; } -static int kirin_pcie_wr_own_conf(struct pcie_port *pp, +static int kirin_pcie_wr_own_conf(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val) { - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct kirin_pcie *kirin_pcie = to_kirin_pcie(pci); - int ret; + struct dw_pcie *pci = to_dw_pcie_from_pp(bus->sysdata); - kirin_pcie_sideband_dbi_w_mode(kirin_pcie, true); - ret = dw_pcie_write(pci->dbi_base + where, size, val); - kirin_pcie_sideband_dbi_w_mode(kirin_pcie, false); + if (PCI_SLOT(devfn)) + return PCIBIOS_DEVICE_NOT_FOUND; - return ret; + dw_pcie_write_dbi(pci, where, size, val); + return PCIBIOS_SUCCESSFUL; } +static struct pci_ops kirin_pci_ops = { + .read = kirin_pcie_rd_own_conf, + .write = kirin_pcie_wr_own_conf, +}; + static u32 kirin_pcie_read_dbi(struct dw_pcie *pci, void __iomem *base, u32 reg, size_t size) { @@ -423,10 +426,10 @@ static int kirin_pcie_establish_link(struct pcie_port *pp) static int kirin_pcie_host_init(struct pcie_port *pp) { - kirin_pcie_establish_link(pp); + pp->bridge->ops = &kirin_pci_ops; - if (IS_ENABLED(CONFIG_PCI_MSI)) - dw_pcie_msi_init(pp); + kirin_pcie_establish_link(pp); + dw_pcie_msi_init(pp); return 0; } @@ -438,8 +441,6 @@ static const struct dw_pcie_ops kirin_dw_pcie_ops = { }; static const struct dw_pcie_host_ops kirin_pcie_host_ops = { - .rd_own_conf = kirin_pcie_rd_own_conf, - .wr_own_conf = kirin_pcie_wr_own_conf, .host_init = kirin_pcie_host_init, }; @@ -507,8 +508,12 @@ static int kirin_pcie_probe(struct platform_device *pdev) kirin_pcie->gpio_id_reset = of_get_named_gpio(dev->of_node, "reset-gpios", 0); - if (kirin_pcie->gpio_id_reset < 0) + if (kirin_pcie->gpio_id_reset == -EPROBE_DEFER) { + return -EPROBE_DEFER; + } else if (!gpio_is_valid(kirin_pcie->gpio_id_reset)) { + dev_err(dev, "unable to get a valid gpio pin\n"); return -ENODEV; + } ret = kirin_pcie_power_on(kirin_pcie); if (ret) diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index 3aac77a295ba..b4761640ffd9 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -67,10 +67,6 @@ #define PCIE20_AXI_MSTR_RESP_COMP_CTRL1 0x81c #define CFG_BRIDGE_SB_INIT BIT(0) -#define PCIE20_CAP 0x70 -#define PCIE20_DEVICE_CONTROL2_STATUS2 (PCIE20_CAP + PCI_EXP_DEVCTL2) -#define PCIE20_CAP_LINK_CAPABILITIES (PCIE20_CAP + PCI_EXP_LNKCAP) -#define PCIE20_CAP_LINK_1 (PCIE20_CAP + 0x14) #define PCIE_CAP_LINK1_VAL 0x2FD7F #define PCIE20_PARF_Q2A_FLUSH 0x1AC @@ -193,7 +189,6 @@ struct qcom_pcie { struct phy *phy; struct gpio_desc *reset; const struct qcom_pcie_ops *ops; - int gen; }; #define to_qcom_pcie(x) dev_get_drvdata((x)->dev) @@ -302,6 +297,9 @@ static void qcom_pcie_deinit_2_1_0(struct qcom_pcie *pcie) reset_control_assert(res->por_reset); reset_control_assert(res->ext_reset); reset_control_assert(res->phy_reset); + + writel(1, pcie->parf + PCIE20_PARF_PHY_CTRL); + regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); } @@ -314,6 +312,16 @@ static int qcom_pcie_init_2_1_0(struct qcom_pcie *pcie) u32 val; int ret; + /* reset the PCIe interface as uboot can leave it undefined state */ + reset_control_assert(res->pci_reset); + reset_control_assert(res->axi_reset); + reset_control_assert(res->ahb_reset); + reset_control_assert(res->por_reset); + reset_control_assert(res->ext_reset); + reset_control_assert(res->phy_reset); + + writel(1, pcie->parf + PCIE20_PARF_PHY_CTRL); + ret = regulator_bulk_enable(ARRAY_SIZE(res->supplies), res->supplies); if (ret < 0) { dev_err(dev, "cannot enable regulators\n"); @@ -394,12 +402,6 @@ static int qcom_pcie_init_2_1_0(struct qcom_pcie *pcie) /* wait for clock acquisition */ usleep_range(1000, 1500); - if (pcie->gen == 1) { - val = readl(pci->dbi_base + PCIE20_LNK_CONTROL2_LINK_STATUS2); - val |= PCI_EXP_LNKSTA_CLS_2_5GB; - writel(val, pci->dbi_base + PCIE20_LNK_CONTROL2_LINK_STATUS2); - } - /* Set the Max TLP size to 2K, instead of using default of 4K */ writel(CFG_REMOTE_RD_REQ_BRIDGE_SIZE_2K, pci->dbi_base + PCIE20_AXI_MSTR_RESP_COMP_CTRL0); @@ -1017,6 +1019,7 @@ static int qcom_pcie_init_2_3_3(struct qcom_pcie *pcie) struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3; struct dw_pcie *pci = pcie->pci; struct device *dev = pci->dev; + u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); int i, ret; u32 val; @@ -1092,14 +1095,14 @@ static int qcom_pcie_init_2_3_3(struct qcom_pcie *pcie) writel(PCI_COMMAND_MASTER, pci->dbi_base + PCI_COMMAND); writel(DBI_RO_WR_EN, pci->dbi_base + PCIE20_MISC_CONTROL_1_REG); - writel(PCIE_CAP_LINK1_VAL, pci->dbi_base + PCIE20_CAP_LINK_1); + writel(PCIE_CAP_LINK1_VAL, pci->dbi_base + offset + PCI_EXP_SLTCAP); - val = readl(pci->dbi_base + PCIE20_CAP_LINK_CAPABILITIES); + val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP); val &= ~PCI_EXP_LNKCAP_ASPMS; - writel(val, pci->dbi_base + PCIE20_CAP_LINK_CAPABILITIES); + writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP); - writel(PCI_EXP_DEVCTL2_COMP_TMOUT_DIS, pci->dbi_base + - PCIE20_DEVICE_CONTROL2_STATUS2); + writel(PCI_EXP_DEVCTL2_COMP_TMOUT_DIS, pci->dbi_base + offset + + PCI_EXP_DEVCTL2); return 0; @@ -1252,7 +1255,8 @@ static void qcom_pcie_post_deinit_2_7_0(struct qcom_pcie *pcie) static int qcom_pcie_link_up(struct dw_pcie *pci) { - u16 val = readw(pci->dbi_base + PCIE20_CAP + PCI_EXP_LNKSTA); + u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + u16 val = readw(pci->dbi_base + offset + PCI_EXP_LNKSTA); return !!(val & PCI_EXP_LNKSTA_DLLLA); } @@ -1280,9 +1284,7 @@ static int qcom_pcie_host_init(struct pcie_port *pp) } dw_pcie_setup_rc(pp); - - if (IS_ENABLED(CONFIG_PCI_MSI)) - dw_pcie_msi_init(pp); + dw_pcie_msi_init(pp); qcom_ep_reset_deassert(pcie); @@ -1399,10 +1401,6 @@ static int qcom_pcie_probe(struct platform_device *pdev) goto err_pm_runtime_put; } - pcie->gen = of_pci_get_max_link_speed(pdev->dev.of_node); - if (pcie->gen < 0) - pcie->gen = 2; - pcie->parf = devm_platform_ioremap_resource_byname(pdev, "parf"); if (IS_ERR(pcie->parf)) { ret = PTR_ERR(pcie->parf); diff --git a/drivers/pci/controller/dwc/pcie-spear13xx.c b/drivers/pci/controller/dwc/pcie-spear13xx.c index 62846562da0b..e348225f651f 100644 --- a/drivers/pci/controller/dwc/pcie-spear13xx.c +++ b/drivers/pci/controller/dwc/pcie-spear13xx.c @@ -26,7 +26,6 @@ struct spear13xx_pcie { void __iomem *app_base; struct phy *phy; struct clk *clk; - bool is_gen1; }; struct pcie_app_reg { @@ -65,8 +64,6 @@ struct pcie_app_reg { /* CR6 */ #define MSI_CTRL_INT (1 << 26) -#define EXP_CAP_ID_OFFSET 0x70 - #define to_spear13xx_pcie(x) dev_get_drvdata((x)->dev) static int spear13xx_pcie_establish_link(struct spear13xx_pcie *spear13xx_pcie) @@ -75,7 +72,7 @@ static int spear13xx_pcie_establish_link(struct spear13xx_pcie *spear13xx_pcie) struct pcie_port *pp = &pci->pp; struct pcie_app_reg *app_reg = spear13xx_pcie->app_base; u32 val; - u32 exp_cap_off = EXP_CAP_ID_OFFSET; + u32 exp_cap_off = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); if (dw_pcie_link_up(pci)) { dev_err(pci->dev, "link already up\n"); @@ -89,36 +86,12 @@ static int spear13xx_pcie_establish_link(struct spear13xx_pcie *spear13xx_pcie) * default value in capability register is 512 bytes. So force * it to 128 here. */ - dw_pcie_read(pci->dbi_base + exp_cap_off + PCI_EXP_DEVCTL, 2, &val); + val = dw_pcie_readw_dbi(pci, exp_cap_off + PCI_EXP_DEVCTL); val &= ~PCI_EXP_DEVCTL_READRQ; - dw_pcie_write(pci->dbi_base + exp_cap_off + PCI_EXP_DEVCTL, 2, val); - - dw_pcie_write(pci->dbi_base + PCI_VENDOR_ID, 2, 0x104A); - dw_pcie_write(pci->dbi_base + PCI_DEVICE_ID, 2, 0xCD80); + dw_pcie_writew_dbi(pci, exp_cap_off + PCI_EXP_DEVCTL, val); - /* - * if is_gen1 is set then handle it, so that some buggy card - * also works - */ - if (spear13xx_pcie->is_gen1) { - dw_pcie_read(pci->dbi_base + exp_cap_off + PCI_EXP_LNKCAP, - 4, &val); - if ((val & PCI_EXP_LNKCAP_SLS) != PCI_EXP_LNKCAP_SLS_2_5GB) { - val &= ~((u32)PCI_EXP_LNKCAP_SLS); - val |= PCI_EXP_LNKCAP_SLS_2_5GB; - dw_pcie_write(pci->dbi_base + exp_cap_off + - PCI_EXP_LNKCAP, 4, val); - } - - dw_pcie_read(pci->dbi_base + exp_cap_off + PCI_EXP_LNKCTL2, - 2, &val); - if ((val & PCI_EXP_LNKCAP_SLS) != PCI_EXP_LNKCAP_SLS_2_5GB) { - val &= ~((u32)PCI_EXP_LNKCAP_SLS); - val |= PCI_EXP_LNKCAP_SLS_2_5GB; - dw_pcie_write(pci->dbi_base + exp_cap_off + - PCI_EXP_LNKCTL2, 2, val); - } - } + dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, 0x104A); + dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, 0xCD80); /* enable ltssm */ writel(DEVICE_TYPE_RC | (1 << MISCTRL_EN_ID) @@ -278,7 +251,7 @@ static int spear13xx_pcie_probe(struct platform_device *pdev) spear13xx_pcie->app_base = pci->dbi_base + 0x2000; if (of_property_read_bool(np, "st,pcie-is-gen1")) - spear13xx_pcie->is_gen1 = true; + pci->link_gen = 1; platform_set_drvdata(pdev, spear13xx_pcie); diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c index 70498689d0c0..f920e7efe118 100644 --- a/drivers/pci/controller/dwc/pcie-tegra194.c +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -183,19 +183,7 @@ #define EVENT_COUNTER_GROUP_SEL_SHIFT 24 #define EVENT_COUNTER_GROUP_5 0x5 -#define PORT_LOGIC_ACK_F_ASPM_CTRL 0x70C -#define ENTER_ASPM BIT(30) -#define L0S_ENTRANCE_LAT_SHIFT 24 -#define L0S_ENTRANCE_LAT_MASK GENMASK(26, 24) -#define L1_ENTRANCE_LAT_SHIFT 27 -#define L1_ENTRANCE_LAT_MASK GENMASK(29, 27) -#define N_FTS_SHIFT 8 -#define N_FTS_MASK GENMASK(7, 0) #define N_FTS_VAL 52 - -#define PORT_LOGIC_GEN2_CTRL 0x80C -#define PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE BIT(17) -#define FTS_MASK GENMASK(7, 0) #define FTS_VAL 52 #define PORT_LOGIC_MSI_CTRL_INT_0_EN 0x828 @@ -296,7 +284,6 @@ struct tegra_pcie_dw { u8 init_link_width; u32 msi_ctrl_int; u32 num_lanes; - u32 max_speed; u32 cid; u32 cfg_link_cap_l1sub; u32 pcie_cap_base; @@ -401,9 +388,9 @@ static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg) val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; appl_writel(pcie, val, APPL_CAR_RESET_OVRD); - val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); - val |= PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE; - dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); + val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); + val |= PORT_LOGIC_SPEED_CHANGE; + dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); } } @@ -568,42 +555,44 @@ static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg) return IRQ_HANDLED; } -static int tegra_pcie_dw_rd_own_conf(struct pcie_port *pp, int where, int size, - u32 *val) +static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where, + int size, u32 *val) { - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - /* * This is an endpoint mode specific register happen to appear even * when controller is operating in root port mode and system hangs * when it is accessed with link being in ASPM-L1 state. * So skip accessing it altogether */ - if (where == PORT_LOGIC_MSIX_DOORBELL) { + if (!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) { *val = 0x00000000; return PCIBIOS_SUCCESSFUL; } - return dw_pcie_read(pci->dbi_base + where, size, val); + return pci_generic_config_read(bus, devfn, where, size, val); } -static int tegra_pcie_dw_wr_own_conf(struct pcie_port *pp, int where, int size, - u32 val) +static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where, + int size, u32 val) { - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - /* * This is an endpoint mode specific register happen to appear even * when controller is operating in root port mode and system hangs * when it is accessed with link being in ASPM-L1 state. * So skip accessing it altogether */ - if (where == PORT_LOGIC_MSIX_DOORBELL) + if (!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) return PCIBIOS_SUCCESSFUL; - return dw_pcie_write(pci->dbi_base + where, size, val); + return pci_generic_config_write(bus, devfn, where, size, val); } +static struct pci_ops tegra_pci_ops = { + .map_bus = dw_pcie_own_conf_map_bus, + .read = tegra_pcie_dw_rd_own_conf, + .write = tegra_pcie_dw_wr_own_conf, +}; + #if defined(CONFIG_PCIEASPM) static void disable_aspm_l11(struct tegra_pcie_dw *pcie) { @@ -692,30 +681,23 @@ static void init_host_aspm(struct tegra_pcie_dw *pcie) dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val); /* Program L0s and L1 entrance latencies */ - val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); - val &= ~L0S_ENTRANCE_LAT_MASK; - val |= (pcie->aspm_l0s_enter_lat << L0S_ENTRANCE_LAT_SHIFT); - val |= ENTER_ASPM; - dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); + val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR); + val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK; + val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT); + val |= PORT_AFR_ENTER_ASPM; + dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val); } -static int init_debugfs(struct tegra_pcie_dw *pcie) +static void init_debugfs(struct tegra_pcie_dw *pcie) { - struct dentry *d; - - d = debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", - pcie->debugfs, aspm_state_cnt); - if (IS_ERR_OR_NULL(d)) - dev_err(pcie->dev, - "Failed to create debugfs file \"aspm_state_cnt\"\n"); - - return 0; + debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", pcie->debugfs, + aspm_state_cnt); } #else static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; } static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; } static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; } -static inline int init_debugfs(struct tegra_pcie_dw *pcie) { return 0; } +static inline void init_debugfs(struct tegra_pcie_dw *pcie) { return; } #endif static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp) @@ -827,26 +809,24 @@ static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie) /* Program init preset */ for (i = 0; i < pcie->num_lanes; i++) { - dw_pcie_read(pci->dbi_base + CAP_SPCIE_CAP_OFF - + (i * 2), 2, &val); + val = dw_pcie_readw_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2)); val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK; val |= GEN3_GEN4_EQ_PRESET_INIT; val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK; val |= (GEN3_GEN4_EQ_PRESET_INIT << CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT); - dw_pcie_write(pci->dbi_base + CAP_SPCIE_CAP_OFF - + (i * 2), 2, val); + dw_pcie_writew_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2), val); offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_PL_16GT) + PCI_PL_16GT_LE_CTRL; - dw_pcie_read(pci->dbi_base + offset + i, 1, &val); + val = dw_pcie_readb_dbi(pci, offset + i); val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK; val |= GEN3_GEN4_EQ_PRESET_INIT; val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK; val |= (GEN3_GEN4_EQ_PRESET_INIT << PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT); - dw_pcie_write(pci->dbi_base + offset + i, 1, val); + dw_pcie_writeb_dbi(pci, offset + i, val); } val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); @@ -892,17 +872,6 @@ static void tegra_pcie_prepare_host(struct pcie_port *pp) dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); - /* Configure FTS */ - val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); - val &= ~(N_FTS_MASK << N_FTS_SHIFT); - val |= N_FTS_VAL << N_FTS_SHIFT; - dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); - - val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); - val &= ~FTS_MASK; - val |= FTS_VAL; - dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); - /* Enable as 0xFFFF0001 response for CRS */ val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT); val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT); @@ -910,16 +879,6 @@ static void tegra_pcie_prepare_host(struct pcie_port *pp) AMBA_ERROR_RESPONSE_CRS_SHIFT); dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val); - /* Configure Max Speed from DT */ - if (pcie->max_speed && pcie->max_speed != -EINVAL) { - val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + - PCI_EXP_LNKCAP); - val &= ~PCI_EXP_LNKCAP_SLS; - val |= pcie->max_speed; - dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, - val); - } - /* Configure Max lane width from DT */ val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP); val &= ~PCI_EXP_LNKCAP_MLW; @@ -970,6 +929,8 @@ static int tegra_pcie_dw_host_init(struct pcie_port *pp) struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); u32 val, tmp, offset, speed; + pp->bridge->ops = &tegra_pci_ops; + tegra_pcie_prepare_host(pp); if (dw_pcie_wait_for_link(pci)) { @@ -1057,8 +1018,6 @@ static const struct dw_pcie_ops tegra_dw_pcie_ops = { }; static struct dw_pcie_host_ops tegra_pcie_dw_host_ops = { - .rd_own_conf = tegra_pcie_dw_rd_own_conf, - .wr_own_conf = tegra_pcie_dw_wr_own_conf, .host_init = tegra_pcie_dw_host_init, .set_num_vectors = tegra_pcie_set_msi_vec_num, }; @@ -1129,8 +1088,6 @@ static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie) return ret; } - pcie->max_speed = of_pci_get_max_link_speed(np); - ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid); if (ret) { dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret); @@ -1262,9 +1219,9 @@ static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie) * 5.2 Link State Power Management (Page #428). */ - list_for_each_entry(child, &pp->root_bus->children, node) { + list_for_each_entry(child, &pp->bridge->bus->children, node) { /* Bring downstream devices to D0 if they are not already in */ - if (child->parent == pp->root_bus) { + if (child->parent == pp->bridge->bus) { root_bus = child; break; } @@ -1641,10 +1598,7 @@ static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie) } pcie->debugfs = debugfs_create_dir(name, NULL); - if (!pcie->debugfs) - dev_err(dev, "Failed to create debugfs\n"); - else - init_debugfs(pcie); + init_debugfs(pcie); return ret; @@ -1817,27 +1771,6 @@ static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); - /* Configure N_FTS & FTS */ - val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); - val &= ~(N_FTS_MASK << N_FTS_SHIFT); - val |= N_FTS_VAL << N_FTS_SHIFT; - dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); - - val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); - val &= ~FTS_MASK; - val |= FTS_VAL; - dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); - - /* Configure Max Speed from DT */ - if (pcie->max_speed && pcie->max_speed != -EINVAL) { - val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + - PCI_EXP_LNKCAP); - val &= ~PCI_EXP_LNKCAP_SLS; - val |= pcie->max_speed; - dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, - val); - } - pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, PCI_CAP_ID_EXP); clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); @@ -2066,6 +1999,9 @@ static int tegra_pcie_dw_probe(struct platform_device *pdev) pci = &pcie->pci; pci->dev = &pdev->dev; pci->ops = &tegra_dw_pcie_ops; + pci->n_fts[0] = N_FTS_VAL; + pci->n_fts[1] = FTS_VAL; + pp = &pci->pp; pcie->dev = &pdev->dev; pcie->mode = (enum dw_pcie_device_mode)data->mode; diff --git a/drivers/pci/controller/dwc/pcie-uniphier.c b/drivers/pci/controller/dwc/pcie-uniphier.c index 3a7f403b57b8..48176265c867 100644 --- a/drivers/pci/controller/dwc/pcie-uniphier.c +++ b/drivers/pci/controller/dwc/pcie-uniphier.c @@ -322,8 +322,7 @@ static int uniphier_pcie_host_init(struct pcie_port *pp) if (ret) return ret; - if (IS_ENABLED(CONFIG_PCI_MSI)) - dw_pcie_msi_init(pp); + dw_pcie_msi_init(pp); return 0; } diff --git a/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c b/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c index 3adec419a45b..a2632d02ce8f 100644 --- a/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c +++ b/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c @@ -480,7 +480,6 @@ static int mobiveil_pcie_init_irq_domain(struct mobiveil_pcie *pcie) struct device *dev = &pcie->pdev->dev; struct device_node *node = dev->of_node; struct mobiveil_root_port *rp = &pcie->rp; - int ret; /* setup INTx */ rp->intx_domain = irq_domain_add_linear(node, PCI_NUM_INTX, @@ -494,11 +493,7 @@ static int mobiveil_pcie_init_irq_domain(struct mobiveil_pcie *pcie) raw_spin_lock_init(&rp->intx_mask_lock); /* setup MSI */ - ret = mobiveil_allocate_msi_domains(pcie); - if (ret) - return ret; - - return 0; + return mobiveil_allocate_msi_domains(pcie); } static int mobiveil_pcie_integrated_interrupt_init(struct mobiveil_pcie *pcie) diff --git a/drivers/pci/controller/pci-aardvark.c b/drivers/pci/controller/pci-aardvark.c index 1559f79e63b6..0be485a25327 100644 --- a/drivers/pci/controller/pci-aardvark.c +++ b/drivers/pci/controller/pci-aardvark.c @@ -9,11 +9,12 @@ */ #include <linux/delay.h> -#include <linux/gpio.h> +#include <linux/gpio/consumer.h> #include <linux/interrupt.h> #include <linux/irq.h> #include <linux/irqdomain.h> #include <linux/kernel.h> +#include <linux/module.h> #include <linux/pci.h> #include <linux/init.h> #include <linux/phy/phy.h> @@ -251,6 +252,25 @@ static void advk_pcie_wait_for_retrain(struct advk_pcie *pcie) } } +static void advk_pcie_issue_perst(struct advk_pcie *pcie) +{ + u32 reg; + + if (!pcie->reset_gpio) + return; + + /* PERST does not work for some cards when link training is enabled */ + reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG); + reg &= ~LINK_TRAINING_EN; + advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG); + + /* 10ms delay is needed for some cards */ + dev_info(&pcie->pdev->dev, "issuing PERST via reset GPIO for 10ms\n"); + gpiod_set_value_cansleep(pcie->reset_gpio, 1); + usleep_range(10000, 11000); + gpiod_set_value_cansleep(pcie->reset_gpio, 0); +} + static int advk_pcie_train_at_gen(struct advk_pcie *pcie, int gen) { int ret, neg_gen; @@ -299,6 +319,21 @@ static void advk_pcie_train_link(struct advk_pcie *pcie) int neg_gen = -1, gen; /* + * Reset PCIe card via PERST# signal. Some cards are not detected + * during link training when they are in some non-initial state. + */ + advk_pcie_issue_perst(pcie); + + /* + * PERST# signal could have been asserted by pinctrl subsystem before + * probe() callback has been called or issued explicitly by reset gpio + * function advk_pcie_issue_perst(), making the endpoint going into + * fundamental reset. As required by PCI Express spec a delay for at + * least 100ms after such a reset before link training is needed. + */ + msleep(PCI_PM_D3COLD_WAIT); + + /* * Try link training at link gen specified by device tree property * 'max-link-speed'. If this fails, iteratively train at lower gen. */ @@ -330,31 +365,10 @@ err: dev_err(dev, "link never came up\n"); } -static void advk_pcie_issue_perst(struct advk_pcie *pcie) -{ - u32 reg; - - if (!pcie->reset_gpio) - return; - - /* PERST does not work for some cards when link training is enabled */ - reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG); - reg &= ~LINK_TRAINING_EN; - advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG); - - /* 10ms delay is needed for some cards */ - dev_info(&pcie->pdev->dev, "issuing PERST via reset GPIO for 10ms\n"); - gpiod_set_value_cansleep(pcie->reset_gpio, 1); - usleep_range(10000, 11000); - gpiod_set_value_cansleep(pcie->reset_gpio, 0); -} - static void advk_pcie_setup_hw(struct advk_pcie *pcie) { u32 reg; - advk_pcie_issue_perst(pcie); - /* Enable TX */ reg = advk_readl(pcie, PCIE_CORE_REF_CLK_REG); reg |= PCIE_CORE_REF_CLK_TX_ENABLE; @@ -431,15 +445,6 @@ static void advk_pcie_setup_hw(struct advk_pcie *pcie) reg |= PIO_CTRL_ADDR_WIN_DISABLE; advk_writel(pcie, reg, PIO_CTRL); - /* - * PERST# signal could have been asserted by pinctrl subsystem before - * probe() callback has been called or issued explicitly by reset gpio - * function advk_pcie_issue_perst(), making the endpoint going into - * fundamental reset. As required by PCI Express spec a delay for at - * least 100ms after such a reset before link training is needed. - */ - msleep(PCI_PM_D3COLD_WAIT); - advk_pcie_train_link(pcie); /* @@ -607,7 +612,7 @@ static struct pci_bridge_emul_ops advk_pci_bridge_emul_ops = { * Initialize the configuration space of the PCI-to-PCI bridge * associated with the given PCIe interface. */ -static void advk_sw_pci_bridge_init(struct advk_pcie *pcie) +static int advk_sw_pci_bridge_init(struct advk_pcie *pcie) { struct pci_bridge_emul *bridge = &pcie->bridge; @@ -633,8 +638,7 @@ static void advk_sw_pci_bridge_init(struct advk_pcie *pcie) bridge->data = pcie; bridge->ops = &advk_pci_bridge_emul_ops; - pci_bridge_emul_init(bridge, 0); - + return pci_bridge_emul_init(bridge, 0); } static bool advk_pcie_valid_device(struct advk_pcie *pcie, struct pci_bus *bus, @@ -1077,7 +1081,9 @@ static int advk_pcie_enable_phy(struct advk_pcie *pcie) } ret = phy_power_on(pcie->phy); - if (ret) { + if (ret == -EOPNOTSUPP) { + dev_warn(&pcie->pdev->dev, "PHY unsupported by firmware\n"); + } else if (ret) { phy_exit(pcie->phy); return ret; } @@ -1122,6 +1128,7 @@ static int advk_pcie_probe(struct platform_device *pdev) pcie = pci_host_bridge_priv(bridge); pcie->pdev = pdev; + platform_set_drvdata(pdev, pcie); pcie->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pcie->base)) @@ -1167,7 +1174,11 @@ static int advk_pcie_probe(struct platform_device *pdev) advk_pcie_setup_hw(pcie); - advk_sw_pci_bridge_init(pcie); + ret = advk_sw_pci_bridge_init(pcie); + if (ret) { + dev_err(dev, "Failed to register emulated root PCI bridge\n"); + return ret; + } ret = advk_pcie_init_irq_domain(pcie); if (ret) { @@ -1195,18 +1206,37 @@ static int advk_pcie_probe(struct platform_device *pdev) return 0; } +static int advk_pcie_remove(struct platform_device *pdev) +{ + struct advk_pcie *pcie = platform_get_drvdata(pdev); + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + + pci_lock_rescan_remove(); + pci_stop_root_bus(bridge->bus); + pci_remove_root_bus(bridge->bus); + pci_unlock_rescan_remove(); + + advk_pcie_remove_msi_irq_domain(pcie); + advk_pcie_remove_irq_domain(pcie); + + return 0; +} + static const struct of_device_id advk_pcie_of_match_table[] = { { .compatible = "marvell,armada-3700-pcie", }, {}, }; +MODULE_DEVICE_TABLE(of, advk_pcie_of_match_table); static struct platform_driver advk_pcie_driver = { .driver = { .name = "advk-pcie", .of_match_table = advk_pcie_of_match_table, - /* Driver unloading/unbinding currently not supported */ - .suppress_bind_attrs = true, }, .probe = advk_pcie_probe, + .remove = advk_pcie_remove, }; -builtin_platform_driver(advk_pcie_driver); +module_platform_driver(advk_pcie_driver); + +MODULE_DESCRIPTION("Aardvark PCIe controller"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index 4e992403fffe..03ed5cb1c4b2 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -1276,11 +1276,25 @@ static void hv_irq_unmask(struct irq_data *data) exit_unlock: spin_unlock_irqrestore(&hbus->retarget_msi_interrupt_lock, flags); - if (res) { + /* + * During hibernation, when a CPU is offlined, the kernel tries + * to move the interrupt to the remaining CPUs that haven't + * been offlined yet. In this case, the below hv_do_hypercall() + * always fails since the vmbus channel has been closed: + * refer to cpu_disable_common() -> fixup_irqs() -> + * irq_migrate_all_off_this_cpu() -> migrate_one_irq(). + * + * Suppress the error message for hibernation because the failure + * during hibernation does not matter (at this time all the devices + * have been frozen). Note: the correct affinity info is still updated + * into the irqdata data structure in migrate_one_irq() -> + * irq_do_set_affinity() -> hv_set_affinity(), so later when the VM + * resumes, hv_pci_restore_msi_state() is able to correctly restore + * the interrupt with the correct affinity. + */ + if (res && hbus->state != hv_pcibus_removing) dev_err(&hbus->hdev->device, "%s() failed: %#llx", __func__, res); - return; - } pci_msi_unmask_irq(data); } @@ -3367,6 +3381,34 @@ static int hv_pci_suspend(struct hv_device *hdev) return 0; } +static int hv_pci_restore_msi_msg(struct pci_dev *pdev, void *arg) +{ + struct msi_desc *entry; + struct irq_data *irq_data; + + for_each_pci_msi_entry(entry, pdev) { + irq_data = irq_get_irq_data(entry->irq); + if (WARN_ON_ONCE(!irq_data)) + return -EINVAL; + + hv_compose_msi_msg(irq_data, &entry->msg); + } + + return 0; +} + +/* + * Upon resume, pci_restore_msi_state() -> ... -> __pci_write_msi_msg() + * directly writes the MSI/MSI-X registers via MMIO, but since Hyper-V + * doesn't trap and emulate the MMIO accesses, here hv_compose_msi_msg() + * must be used to ask Hyper-V to re-create the IOMMU Interrupt Remapping + * Table entries. + */ +static void hv_pci_restore_msi_state(struct hv_pcibus_device *hbus) +{ + pci_walk_bus(hbus->pci_bus, hv_pci_restore_msi_msg, NULL); +} + static int hv_pci_resume(struct hv_device *hdev) { struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); @@ -3400,6 +3442,8 @@ static int hv_pci_resume(struct hv_device *hdev) prepopulate_bars(hbus); + hv_pci_restore_msi_state(hbus); + hbus->state = hv_pcibus_installed; return 0; out: diff --git a/drivers/pci/controller/pci-loongson.c b/drivers/pci/controller/pci-loongson.c index 719c19fe2bfb..48169b1e3817 100644 --- a/drivers/pci/controller/pci-loongson.c +++ b/drivers/pci/controller/pci-loongson.c @@ -183,7 +183,6 @@ static int loongson_pci_probe(struct platform_device *pdev) struct device_node *node = dev->of_node; struct pci_host_bridge *bridge; struct resource *regs; - int err; if (!node) return -ENODEV; @@ -222,11 +221,7 @@ static int loongson_pci_probe(struct platform_device *pdev) bridge->ops = &loongson_pci_ops; bridge->map_irq = loongson_map_irq; - err = pci_host_probe(bridge); - if (err) - return err; - - return 0; + return pci_host_probe(bridge); } static struct platform_driver loongson_pci_driver = { diff --git a/drivers/pci/controller/pci-mvebu.c b/drivers/pci/controller/pci-mvebu.c index c39978b750ec..eee82838f4ba 100644 --- a/drivers/pci/controller/pci-mvebu.c +++ b/drivers/pci/controller/pci-mvebu.c @@ -12,7 +12,6 @@ #include <linux/gpio.h> #include <linux/init.h> #include <linux/mbus.h> -#include <linux/msi.h> #include <linux/slab.h> #include <linux/platform_device.h> #include <linux/of_address.h> @@ -70,7 +69,6 @@ struct mvebu_pcie_port; struct mvebu_pcie { struct platform_device *pdev; struct mvebu_pcie_port *ports; - struct msi_controller *msi; struct resource io; struct resource realio; struct resource mem; @@ -1127,7 +1125,6 @@ static int mvebu_pcie_probe(struct platform_device *pdev) bridge->sysdata = pcie; bridge->ops = &mvebu_pcie_ops; bridge->align_resource = mvebu_pcie_align_resource; - bridge->msi = pcie->msi; return mvebu_pci_host_probe(bridge); } diff --git a/drivers/pci/controller/pci-tegra.c b/drivers/pci/controller/pci-tegra.c index c1d34353c29b..8fcabed7c6a6 100644 --- a/drivers/pci/controller/pci-tegra.c +++ b/drivers/pci/controller/pci-tegra.c @@ -2564,36 +2564,14 @@ static int tegra_pcie_ports_seq_show(struct seq_file *s, void *v) return 0; } -static const struct seq_operations tegra_pcie_ports_seq_ops = { +static const struct seq_operations tegra_pcie_ports_sops = { .start = tegra_pcie_ports_seq_start, .next = tegra_pcie_ports_seq_next, .stop = tegra_pcie_ports_seq_stop, .show = tegra_pcie_ports_seq_show, }; -static int tegra_pcie_ports_open(struct inode *inode, struct file *file) -{ - struct tegra_pcie *pcie = inode->i_private; - struct seq_file *s; - int err; - - err = seq_open(file, &tegra_pcie_ports_seq_ops); - if (err) - return err; - - s = file->private_data; - s->private = pcie; - - return 0; -} - -static const struct file_operations tegra_pcie_ports_ops = { - .owner = THIS_MODULE, - .open = tegra_pcie_ports_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release, -}; +DEFINE_SEQ_ATTRIBUTE(tegra_pcie_ports); static void tegra_pcie_debugfs_exit(struct tegra_pcie *pcie) { @@ -2601,24 +2579,12 @@ static void tegra_pcie_debugfs_exit(struct tegra_pcie *pcie) pcie->debugfs = NULL; } -static int tegra_pcie_debugfs_init(struct tegra_pcie *pcie) +static void tegra_pcie_debugfs_init(struct tegra_pcie *pcie) { - struct dentry *file; - pcie->debugfs = debugfs_create_dir("pcie", NULL); - if (!pcie->debugfs) - return -ENOMEM; - file = debugfs_create_file("ports", S_IFREG | S_IRUGO, pcie->debugfs, - pcie, &tegra_pcie_ports_ops); - if (!file) - goto remove; - - return 0; - -remove: - tegra_pcie_debugfs_exit(pcie); - return -ENOMEM; + debugfs_create_file("ports", S_IFREG | S_IRUGO, pcie->debugfs, pcie, + &tegra_pcie_ports_fops); } static int tegra_pcie_probe(struct platform_device *pdev) @@ -2672,11 +2638,8 @@ static int tegra_pcie_probe(struct platform_device *pdev) goto pm_runtime_put; } - if (IS_ENABLED(CONFIG_DEBUG_FS)) { - err = tegra_pcie_debugfs_init(pcie); - if (err < 0) - dev_err(dev, "failed to setup debugfs: %d\n", err); - } + if (IS_ENABLED(CONFIG_DEBUG_FS)) + tegra_pcie_debugfs_init(pcie); return 0; diff --git a/drivers/pci/controller/pci-v3-semi.c b/drivers/pci/controller/pci-v3-semi.c index 1f54334f09f7..154a5398633c 100644 --- a/drivers/pci/controller/pci-v3-semi.c +++ b/drivers/pci/controller/pci-v3-semi.c @@ -658,7 +658,6 @@ static int v3_get_dma_range_config(struct v3_pci *v3, default: dev_err(v3->dev, "illegal dma memory chunk size\n"); return -EINVAL; - break; } val |= V3_PCI_MAP_M_REG_EN | V3_PCI_MAP_M_ENABLE; *pci_map = val; diff --git a/drivers/pci/controller/pci-xgene-msi.c b/drivers/pci/controller/pci-xgene-msi.c index 02271c6d17a1..2470782cb01a 100644 --- a/drivers/pci/controller/pci-xgene-msi.c +++ b/drivers/pci/controller/pci-xgene-msi.c @@ -493,8 +493,8 @@ static int xgene_msi_probe(struct platform_device *pdev) */ for (irq_index = 0; irq_index < NR_HW_IRQS; irq_index++) { for (msi_idx = 0; msi_idx < IDX_PER_GROUP; msi_idx++) - msi_val = xgene_msi_ir_read(xgene_msi, irq_index, - msi_idx); + xgene_msi_ir_read(xgene_msi, irq_index, msi_idx); + /* Read MSIINTn to confirm */ msi_val = xgene_msi_int_read(xgene_msi, irq_index); if (msi_val) { diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index bac63d04297f..bea86899bd5d 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -23,6 +23,7 @@ #include <linux/of_platform.h> #include <linux/pci.h> #include <linux/printk.h> +#include <linux/reset.h> #include <linux/sizes.h> #include <linux/slab.h> #include <linux/string.h> @@ -52,8 +53,11 @@ #define PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK 0x1000 #define PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK 0x2000 #define PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK 0x300000 -#define PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_128 0x0 + #define PCIE_MISC_MISC_CTRL_SCB0_SIZE_MASK 0xf8000000 +#define PCIE_MISC_MISC_CTRL_SCB1_SIZE_MASK 0x07c00000 +#define PCIE_MISC_MISC_CTRL_SCB2_SIZE_MASK 0x0000001f +#define SCB_SIZE_MASK(x) PCIE_MISC_MISC_CTRL_SCB ## x ## _SIZE_MASK #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO 0x400c #define PCIE_MEM_WIN0_LO(win) \ @@ -77,10 +81,12 @@ #define PCIE_MISC_MSI_BAR_CONFIG_HI 0x4048 #define PCIE_MISC_MSI_DATA_CONFIG 0x404c -#define PCIE_MISC_MSI_DATA_CONFIG_VAL 0xffe06540 +#define PCIE_MISC_MSI_DATA_CONFIG_VAL_32 0xffe06540 +#define PCIE_MISC_MSI_DATA_CONFIG_VAL_8 0xfff86540 #define PCIE_MISC_PCIE_CTRL 0x4064 #define PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK 0x1 +#define PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK 0x4 #define PCIE_MISC_PCIE_STATUS 0x4068 #define PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK 0x80 @@ -88,6 +94,9 @@ #define PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK 0x10 #define PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK 0x40 +#define PCIE_MISC_REVISION 0x406c +#define BRCM_PCIE_HW_REV_33 0x0303 + #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT 0x4070 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK 0xfff00000 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK 0xfff0 @@ -108,10 +117,14 @@ #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK 0x2 #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK 0x08000000 -#define PCIE_MSI_INTR2_STATUS 0x4500 -#define PCIE_MSI_INTR2_CLR 0x4508 -#define PCIE_MSI_INTR2_MASK_SET 0x4510 -#define PCIE_MSI_INTR2_MASK_CLR 0x4514 + +#define PCIE_INTR2_CPU_BASE 0x4300 +#define PCIE_MSI_INTR2_BASE 0x4500 +/* Offsets from PCIE_INTR2_CPU_BASE and PCIE_MSI_INTR2_BASE */ +#define MSI_INT_STATUS 0x0 +#define MSI_INT_CLR 0x8 +#define MSI_INT_MASK_SET 0x10 +#define MSI_INT_MASK_CLR 0x14 #define PCIE_EXT_CFG_DATA 0x8000 @@ -120,13 +133,19 @@ #define PCIE_EXT_SLOT_SHIFT 15 #define PCIE_EXT_FUNC_SHIFT 12 -#define PCIE_RGR1_SW_INIT_1 0x9210 #define PCIE_RGR1_SW_INIT_1_PERST_MASK 0x1 -#define PCIE_RGR1_SW_INIT_1_INIT_MASK 0x2 +#define PCIE_RGR1_SW_INIT_1_PERST_SHIFT 0x0 + +#define RGR1_SW_INIT_1_INIT_GENERIC_MASK 0x2 +#define RGR1_SW_INIT_1_INIT_GENERIC_SHIFT 0x1 +#define RGR1_SW_INIT_1_INIT_7278_MASK 0x1 +#define RGR1_SW_INIT_1_INIT_7278_SHIFT 0x0 /* PCIe parameters */ #define BRCM_NUM_PCIE_OUT_WINS 0x4 #define BRCM_INT_PCI_MSI_NR 32 +#define BRCM_INT_PCI_MSI_LEGACY_NR 8 +#define BRCM_INT_PCI_MSI_SHIFT 0 /* MSI target adresses */ #define BRCM_MSI_TARGET_ADDR_LT_4GB 0x0fffffffcULL @@ -151,6 +170,85 @@ #define SSC_STATUS_OFFSET 0x1 #define SSC_STATUS_SSC_MASK 0x400 #define SSC_STATUS_PLL_LOCK_MASK 0x800 +#define PCIE_BRCM_MAX_MEMC 3 + +#define IDX_ADDR(pcie) (pcie->reg_offsets[EXT_CFG_INDEX]) +#define DATA_ADDR(pcie) (pcie->reg_offsets[EXT_CFG_DATA]) +#define PCIE_RGR1_SW_INIT_1(pcie) (pcie->reg_offsets[RGR1_SW_INIT_1]) + +/* Rescal registers */ +#define PCIE_DVT_PMU_PCIE_PHY_CTRL 0xc700 +#define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS 0x3 +#define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_MASK 0x4 +#define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_SHIFT 0x2 +#define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_MASK 0x2 +#define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_SHIFT 0x1 +#define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_MASK 0x1 +#define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_SHIFT 0x0 + +/* Forward declarations */ +struct brcm_pcie; +static inline void brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 val); +static inline void brcm_pcie_bridge_sw_init_set_generic(struct brcm_pcie *pcie, u32 val); +static inline void brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val); +static inline void brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, u32 val); + +enum { + RGR1_SW_INIT_1, + EXT_CFG_INDEX, + EXT_CFG_DATA, +}; + +enum { + RGR1_SW_INIT_1_INIT_MASK, + RGR1_SW_INIT_1_INIT_SHIFT, +}; + +enum pcie_type { + GENERIC, + BCM7278, + BCM2711, +}; + +struct pcie_cfg_data { + const int *offsets; + const enum pcie_type type; + void (*perst_set)(struct brcm_pcie *pcie, u32 val); + void (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val); +}; + +static const int pcie_offsets[] = { + [RGR1_SW_INIT_1] = 0x9210, + [EXT_CFG_INDEX] = 0x9000, + [EXT_CFG_DATA] = 0x9004, +}; + +static const struct pcie_cfg_data generic_cfg = { + .offsets = pcie_offsets, + .type = GENERIC, + .perst_set = brcm_pcie_perst_set_generic, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, +}; + +static const int pcie_offset_bcm7278[] = { + [RGR1_SW_INIT_1] = 0xc010, + [EXT_CFG_INDEX] = 0x9000, + [EXT_CFG_DATA] = 0x9004, +}; + +static const struct pcie_cfg_data bcm7278_cfg = { + .offsets = pcie_offset_bcm7278, + .type = BCM7278, + .perst_set = brcm_pcie_perst_set_7278, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278, +}; + +static const struct pcie_cfg_data bcm2711_cfg = { + .offsets = pcie_offsets, + .type = BCM2711, + .perst_set = brcm_pcie_perst_set_generic, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, +}; struct brcm_msi { struct device *dev; @@ -163,6 +261,12 @@ struct brcm_msi { int irq; /* used indicates which MSI interrupts have been alloc'd */ unsigned long used; + bool legacy; + /* Some chips have MSIs in bits [31..24] of a shared register. */ + int legacy_shift; + int nr; /* No. of MSI available, depends on chip */ + /* This is the base pointer for interrupt status/set/clr regs */ + void __iomem *intr_base; }; /* Internal PCIe Host Controller Information.*/ @@ -175,6 +279,14 @@ struct brcm_pcie { int gen; u64 msi_target_addr; struct brcm_msi *msi; + const int *reg_offsets; + enum pcie_type type; + struct reset_control *rescal; + int num_memc; + u64 memc_size[PCIE_BRCM_MAX_MEMC]; + u32 hw_rev; + void (*perst_set)(struct brcm_pcie *pcie, u32 val); + void (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val); }; /* @@ -365,8 +477,10 @@ static void brcm_pcie_msi_isr(struct irq_desc *desc) msi = irq_desc_get_handler_data(desc); dev = msi->dev; - status = readl(msi->base + PCIE_MSI_INTR2_STATUS); - for_each_set_bit(bit, &status, BRCM_INT_PCI_MSI_NR) { + status = readl(msi->intr_base + MSI_INT_STATUS); + status >>= msi->legacy_shift; + + for_each_set_bit(bit, &status, msi->nr) { virq = irq_find_mapping(msi->inner_domain, bit); if (virq) generic_handle_irq(virq); @@ -383,7 +497,7 @@ static void brcm_msi_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) msg->address_lo = lower_32_bits(msi->target_addr); msg->address_hi = upper_32_bits(msi->target_addr); - msg->data = (0xffff & PCIE_MISC_MSI_DATA_CONFIG_VAL) | data->hwirq; + msg->data = (0xffff & PCIE_MISC_MSI_DATA_CONFIG_VAL_32) | data->hwirq; } static int brcm_msi_set_affinity(struct irq_data *irq_data, @@ -395,8 +509,9 @@ static int brcm_msi_set_affinity(struct irq_data *irq_data, static void brcm_msi_ack_irq(struct irq_data *data) { struct brcm_msi *msi = irq_data_get_irq_chip_data(data); + const int shift_amt = data->hwirq + msi->legacy_shift; - writel(1 << data->hwirq, msi->base + PCIE_MSI_INTR2_CLR); + writel(1 << shift_amt, msi->intr_base + MSI_INT_CLR); } @@ -412,7 +527,7 @@ static int brcm_msi_alloc(struct brcm_msi *msi) int hwirq; mutex_lock(&msi->lock); - hwirq = bitmap_find_free_region(&msi->used, BRCM_INT_PCI_MSI_NR, 0); + hwirq = bitmap_find_free_region(&msi->used, msi->nr, 0); mutex_unlock(&msi->lock); return hwirq; @@ -461,8 +576,7 @@ static int brcm_allocate_domains(struct brcm_msi *msi) struct fwnode_handle *fwnode = of_node_to_fwnode(msi->np); struct device *dev = msi->dev; - msi->inner_domain = irq_domain_add_linear(NULL, BRCM_INT_PCI_MSI_NR, - &msi_domain_ops, msi); + msi->inner_domain = irq_domain_add_linear(NULL, msi->nr, &msi_domain_ops, msi); if (!msi->inner_domain) { dev_err(dev, "failed to create IRQ domain\n"); return -ENOMEM; @@ -499,7 +613,10 @@ static void brcm_msi_remove(struct brcm_pcie *pcie) static void brcm_msi_set_regs(struct brcm_msi *msi) { - writel(0xffffffff, msi->base + PCIE_MSI_INTR2_MASK_CLR); + u32 val = __GENMASK(31, msi->legacy_shift); + + writel(val, msi->intr_base + MSI_INT_MASK_CLR); + writel(val, msi->intr_base + MSI_INT_CLR); /* * The 0 bit of PCIE_MISC_MSI_BAR_CONFIG_LO is repurposed to MSI @@ -510,8 +627,8 @@ static void brcm_msi_set_regs(struct brcm_msi *msi) writel(upper_32_bits(msi->target_addr), msi->base + PCIE_MISC_MSI_BAR_CONFIG_HI); - writel(PCIE_MISC_MSI_DATA_CONFIG_VAL, - msi->base + PCIE_MISC_MSI_DATA_CONFIG); + val = msi->legacy ? PCIE_MISC_MSI_DATA_CONFIG_VAL_8 : PCIE_MISC_MSI_DATA_CONFIG_VAL_32; + writel(val, msi->base + PCIE_MISC_MSI_DATA_CONFIG); } static int brcm_pcie_enable_msi(struct brcm_pcie *pcie) @@ -536,6 +653,17 @@ static int brcm_pcie_enable_msi(struct brcm_pcie *pcie) msi->np = pcie->np; msi->target_addr = pcie->msi_target_addr; msi->irq = irq; + msi->legacy = pcie->hw_rev < BRCM_PCIE_HW_REV_33; + + if (msi->legacy) { + msi->intr_base = msi->base + PCIE_INTR2_CPU_BASE; + msi->nr = BRCM_INT_PCI_MSI_LEGACY_NR; + msi->legacy_shift = 24; + } else { + msi->intr_base = msi->base + PCIE_MSI_INTR2_BASE; + msi->nr = BRCM_INT_PCI_MSI_NR; + msi->legacy_shift = 0; + } ret = brcm_allocate_domains(msi); if (ret) @@ -599,22 +727,43 @@ static struct pci_ops brcm_pcie_ops = { .write = pci_generic_config_write, }; -static inline void brcm_pcie_bridge_sw_init_set(struct brcm_pcie *pcie, u32 val) +static inline void brcm_pcie_bridge_sw_init_set_generic(struct brcm_pcie *pcie, u32 val) +{ + u32 tmp, mask = RGR1_SW_INIT_1_INIT_GENERIC_MASK; + u32 shift = RGR1_SW_INIT_1_INIT_GENERIC_SHIFT; + + tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); + tmp = (tmp & ~mask) | ((val << shift) & mask); + writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); +} + +static inline void brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 val) +{ + u32 tmp, mask = RGR1_SW_INIT_1_INIT_7278_MASK; + u32 shift = RGR1_SW_INIT_1_INIT_7278_SHIFT; + + tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); + tmp = (tmp & ~mask) | ((val << shift) & mask); + writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); +} + +static inline void brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val) { u32 tmp; - tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1); - u32p_replace_bits(&tmp, val, PCIE_RGR1_SW_INIT_1_INIT_MASK); - writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1); + /* Perst bit has moved and assert value is 0 */ + tmp = readl(pcie->base + PCIE_MISC_PCIE_CTRL); + u32p_replace_bits(&tmp, !val, PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK); + writel(tmp, pcie->base + PCIE_MISC_PCIE_CTRL); } -static inline void brcm_pcie_perst_set(struct brcm_pcie *pcie, u32 val) +static inline void brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, u32 val) { u32 tmp; - tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1); + tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); u32p_replace_bits(&tmp, val, PCIE_RGR1_SW_INIT_1_PERST_MASK); - writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1); + writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); } static inline int brcm_pcie_get_rc_bar2_size_and_offset(struct brcm_pcie *pcie, @@ -622,22 +771,44 @@ static inline int brcm_pcie_get_rc_bar2_size_and_offset(struct brcm_pcie *pcie, u64 *rc_bar2_offset) { struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); - struct device *dev = pcie->dev; struct resource_entry *entry; + struct device *dev = pcie->dev; + u64 lowest_pcie_addr = ~(u64)0; + int ret, i = 0; + u64 size = 0; - entry = resource_list_first_type(&bridge->dma_ranges, IORESOURCE_MEM); - if (!entry) - return -ENODEV; + resource_list_for_each_entry(entry, &bridge->dma_ranges) { + u64 pcie_beg = entry->res->start - entry->offset; + size += entry->res->end - entry->res->start + 1; + if (pcie_beg < lowest_pcie_addr) + lowest_pcie_addr = pcie_beg; + } - /* - * The controller expects the inbound window offset to be calculated as - * the difference between PCIe's address space and CPU's. The offset - * provided by the firmware is calculated the opposite way, so we - * negate it. - */ - *rc_bar2_offset = -entry->offset; - *rc_bar2_size = 1ULL << fls64(entry->res->end - entry->res->start); + if (lowest_pcie_addr == ~(u64)0) { + dev_err(dev, "DT node has no dma-ranges\n"); + return -EINVAL; + } + + ret = of_property_read_variable_u64_array(pcie->np, "brcm,scb-sizes", pcie->memc_size, 1, + PCIE_BRCM_MAX_MEMC); + + if (ret <= 0) { + /* Make an educated guess */ + pcie->num_memc = 1; + pcie->memc_size[0] = 1ULL << fls64(size - 1); + } else { + pcie->num_memc = ret; + } + + /* Each memc is viewed through a "port" that is a power of 2 */ + for (i = 0, size = 0; i < pcie->num_memc; i++) + size += pcie->memc_size[i]; + + /* System memory starts at this address in PCIe-space */ + *rc_bar2_offset = lowest_pcie_addr; + /* The sum of all memc views must also be a power of 2 */ + *rc_bar2_size = 1ULL << fls64(size - 1); /* * We validate the inbound memory view even though we should trust @@ -689,22 +860,19 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) void __iomem *base = pcie->base; struct device *dev = pcie->dev; struct resource_entry *entry; - unsigned int scb_size_val; bool ssc_good = false; struct resource *res; int num_out_wins = 0; u16 nlw, cls, lnksta; - int i, ret; - u32 tmp, aspm_support; + int i, ret, memc; + u32 tmp, burst, aspm_support; /* Reset the bridge */ - brcm_pcie_bridge_sw_init_set(pcie, 1); - brcm_pcie_perst_set(pcie, 1); - + pcie->bridge_sw_init_set(pcie, 1); usleep_range(100, 200); /* Take the bridge out of reset */ - brcm_pcie_bridge_sw_init_set(pcie, 0); + pcie->bridge_sw_init_set(pcie, 0); tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); tmp &= ~PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK; @@ -712,11 +880,22 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) /* Wait for SerDes to be stable */ usleep_range(100, 200); + /* + * SCB_MAX_BURST_SIZE is a two bit field. For GENERIC chips it + * is encoded as 0=128, 1=256, 2=512, 3=Rsvd, for BCM7278 it + * is encoded as 0=Rsvd, 1=128, 2=256, 3=512. + */ + if (pcie->type == BCM2711) + burst = 0x0; /* 128B */ + else if (pcie->type == BCM7278) + burst = 0x3; /* 512 bytes */ + else + burst = 0x2; /* 512 bytes */ + /* Set SCB_MAX_BURST_SIZE, CFG_READ_UR_MODE, SCB_ACCESS_EN */ u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK); u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK); - u32p_replace_bits(&tmp, PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_128, - PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK); + u32p_replace_bits(&tmp, burst, PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK); writel(tmp, base + PCIE_MISC_MISC_CTRL); ret = brcm_pcie_get_rc_bar2_size_and_offset(pcie, &rc_bar2_size, @@ -731,11 +910,17 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) writel(upper_32_bits(rc_bar2_offset), base + PCIE_MISC_RC_BAR2_CONFIG_HI); - scb_size_val = rc_bar2_size ? - ilog2(rc_bar2_size) - 15 : 0xf; /* 0xf is 1GB */ tmp = readl(base + PCIE_MISC_MISC_CTRL); - u32p_replace_bits(&tmp, scb_size_val, - PCIE_MISC_MISC_CTRL_SCB0_SIZE_MASK); + for (memc = 0; memc < pcie->num_memc; memc++) { + u32 scb_size_val = ilog2(pcie->memc_size[memc]) - 15; + + if (memc == 0) + u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(0)); + else if (memc == 1) + u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(1)); + else if (memc == 2) + u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(2)); + } writel(tmp, base + PCIE_MISC_MISC_CTRL); /* @@ -760,17 +945,11 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) tmp &= ~PCIE_MISC_RC_BAR3_CONFIG_LO_SIZE_MASK; writel(tmp, base + PCIE_MISC_RC_BAR3_CONFIG_LO); - /* Mask all interrupts since we are not handling any yet */ - writel(0xffffffff, pcie->base + PCIE_MSI_INTR2_MASK_SET); - - /* clear any interrupts we find on boot */ - writel(0xffffffff, pcie->base + PCIE_MSI_INTR2_CLR); - if (pcie->gen) brcm_pcie_set_gen(pcie, pcie->gen); /* Unassert the fundamental reset */ - brcm_pcie_perst_set(pcie, 0); + pcie->perst_set(pcie, 0); /* * Give the RC/EP time to wake up, before trying to configure RC. @@ -882,6 +1061,52 @@ static void brcm_pcie_enter_l23(struct brcm_pcie *pcie) dev_err(pcie->dev, "failed to enter low-power link state\n"); } +static int brcm_phy_cntl(struct brcm_pcie *pcie, const int start) +{ + static const u32 shifts[PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS] = { + PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_SHIFT, + PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_SHIFT, + PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_SHIFT,}; + static const u32 masks[PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS] = { + PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_MASK, + PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_MASK, + PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_MASK,}; + const int beg = start ? 0 : PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS - 1; + const int end = start ? PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS : -1; + u32 tmp, combined_mask = 0; + u32 val; + void __iomem *base = pcie->base; + int i, ret; + + for (i = beg; i != end; start ? i++ : i--) { + val = start ? BIT_MASK(shifts[i]) : 0; + tmp = readl(base + PCIE_DVT_PMU_PCIE_PHY_CTRL); + tmp = (tmp & ~masks[i]) | (val & masks[i]); + writel(tmp, base + PCIE_DVT_PMU_PCIE_PHY_CTRL); + usleep_range(50, 200); + combined_mask |= masks[i]; + } + + tmp = readl(base + PCIE_DVT_PMU_PCIE_PHY_CTRL); + val = start ? combined_mask : 0; + + ret = (tmp & combined_mask) == val ? 0 : -EIO; + if (ret) + dev_err(pcie->dev, "failed to %s phy\n", (start ? "start" : "stop")); + + return ret; +} + +static inline int brcm_phy_start(struct brcm_pcie *pcie) +{ + return pcie->rescal ? brcm_phy_cntl(pcie, 1) : 0; +} + +static inline int brcm_phy_stop(struct brcm_pcie *pcie) +{ + return pcie->rescal ? brcm_phy_cntl(pcie, 0) : 0; +} + static void brcm_pcie_turn_off(struct brcm_pcie *pcie) { void __iomem *base = pcie->base; @@ -890,7 +1115,7 @@ static void brcm_pcie_turn_off(struct brcm_pcie *pcie) if (brcm_pcie_link_up(pcie)) brcm_pcie_enter_l23(pcie); /* Assert fundamental reset */ - brcm_pcie_perst_set(pcie, 1); + pcie->perst_set(pcie, 1); /* Deassert request for L23 in case it was asserted */ tmp = readl(base + PCIE_MISC_PCIE_CTRL); @@ -903,13 +1128,66 @@ static void brcm_pcie_turn_off(struct brcm_pcie *pcie) writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); /* Shutdown PCIe bridge */ - brcm_pcie_bridge_sw_init_set(pcie, 1); + pcie->bridge_sw_init_set(pcie, 1); +} + +static int brcm_pcie_suspend(struct device *dev) +{ + struct brcm_pcie *pcie = dev_get_drvdata(dev); + int ret; + + brcm_pcie_turn_off(pcie); + ret = brcm_phy_stop(pcie); + clk_disable_unprepare(pcie->clk); + + return ret; +} + +static int brcm_pcie_resume(struct device *dev) +{ + struct brcm_pcie *pcie = dev_get_drvdata(dev); + void __iomem *base; + u32 tmp; + int ret; + + base = pcie->base; + clk_prepare_enable(pcie->clk); + + ret = brcm_phy_start(pcie); + if (ret) + goto err; + + /* Take bridge out of reset so we can access the SERDES reg */ + pcie->bridge_sw_init_set(pcie, 0); + + /* SERDES_IDDQ = 0 */ + tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + u32p_replace_bits(&tmp, 0, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK); + writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + + /* wait for serdes to be stable */ + udelay(100); + + ret = brcm_pcie_setup(pcie); + if (ret) + goto err; + + if (pcie->msi) + brcm_msi_set_regs(pcie->msi); + + return 0; + +err: + clk_disable_unprepare(pcie->clk); + return ret; } static void __brcm_pcie_remove(struct brcm_pcie *pcie) { brcm_msi_remove(pcie); brcm_pcie_turn_off(pcie); + brcm_phy_stop(pcie); + reset_control_assert(pcie->rescal); clk_disable_unprepare(pcie->clk); } @@ -925,10 +1203,20 @@ static int brcm_pcie_remove(struct platform_device *pdev) return 0; } +static const struct of_device_id brcm_pcie_match[] = { + { .compatible = "brcm,bcm2711-pcie", .data = &bcm2711_cfg }, + { .compatible = "brcm,bcm7211-pcie", .data = &generic_cfg }, + { .compatible = "brcm,bcm7278-pcie", .data = &bcm7278_cfg }, + { .compatible = "brcm,bcm7216-pcie", .data = &bcm7278_cfg }, + { .compatible = "brcm,bcm7445-pcie", .data = &generic_cfg }, + {}, +}; + static int brcm_pcie_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node, *msi_np; struct pci_host_bridge *bridge; + const struct pcie_cfg_data *data; struct brcm_pcie *pcie; int ret; @@ -936,9 +1224,19 @@ static int brcm_pcie_probe(struct platform_device *pdev) if (!bridge) return -ENOMEM; + data = of_device_get_match_data(&pdev->dev); + if (!data) { + pr_err("failed to look up compatible string\n"); + return -EINVAL; + } + pcie = pci_host_bridge_priv(bridge); pcie->dev = &pdev->dev; pcie->np = np; + pcie->reg_offsets = data->offsets; + pcie->type = data->type; + pcie->perst_set = data->perst_set; + pcie->bridge_sw_init_set = data->bridge_sw_init_set; pcie->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pcie->base)) @@ -958,11 +1256,29 @@ static int brcm_pcie_probe(struct platform_device *pdev) dev_err(&pdev->dev, "could not enable clock\n"); return ret; } + pcie->rescal = devm_reset_control_get_optional_shared(&pdev->dev, "rescal"); + if (IS_ERR(pcie->rescal)) { + clk_disable_unprepare(pcie->clk); + return PTR_ERR(pcie->rescal); + } + + ret = reset_control_deassert(pcie->rescal); + if (ret) + dev_err(&pdev->dev, "failed to deassert 'rescal'\n"); + + ret = brcm_phy_start(pcie); + if (ret) { + reset_control_assert(pcie->rescal); + clk_disable_unprepare(pcie->clk); + return ret; + } ret = brcm_pcie_setup(pcie); if (ret) goto fail; + pcie->hw_rev = readl(pcie->base + PCIE_MISC_REVISION); + msi_np = of_parse_phandle(pcie->np, "msi-parent", 0); if (pci_msi_enabled() && msi_np == pcie->np) { ret = brcm_pcie_enable_msi(pcie); @@ -983,18 +1299,20 @@ fail: return ret; } -static const struct of_device_id brcm_pcie_match[] = { - { .compatible = "brcm,bcm2711-pcie" }, - {}, -}; MODULE_DEVICE_TABLE(of, brcm_pcie_match); +static const struct dev_pm_ops brcm_pcie_pm_ops = { + .suspend = brcm_pcie_suspend, + .resume = brcm_pcie_resume, +}; + static struct platform_driver brcm_pcie_driver = { .probe = brcm_pcie_probe, .remove = brcm_pcie_remove, .driver = { .name = "brcm-pcie", .of_match_table = brcm_pcie_match, + .pm = &brcm_pcie_pm_ops, }, }; module_platform_driver(brcm_pcie_driver); diff --git a/drivers/pci/controller/pcie-hisi-error.c b/drivers/pci/controller/pcie-hisi-error.c new file mode 100644 index 000000000000..7959c9c8d2bc --- /dev/null +++ b/drivers/pci/controller/pcie-hisi-error.c @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for handling the PCIe controller errors on + * HiSilicon HIP SoCs. + * + * Copyright (c) 2020 HiSilicon Limited. + */ + +#include <linux/acpi.h> +#include <acpi/ghes.h> +#include <linux/bitops.h> +#include <linux/delay.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/kfifo.h> +#include <linux/spinlock.h> + +/* HISI PCIe controller error definitions */ +#define HISI_PCIE_ERR_MISC_REGS 33 + +#define HISI_PCIE_LOCAL_VALID_VERSION BIT(0) +#define HISI_PCIE_LOCAL_VALID_SOC_ID BIT(1) +#define HISI_PCIE_LOCAL_VALID_SOCKET_ID BIT(2) +#define HISI_PCIE_LOCAL_VALID_NIMBUS_ID BIT(3) +#define HISI_PCIE_LOCAL_VALID_SUB_MODULE_ID BIT(4) +#define HISI_PCIE_LOCAL_VALID_CORE_ID BIT(5) +#define HISI_PCIE_LOCAL_VALID_PORT_ID BIT(6) +#define HISI_PCIE_LOCAL_VALID_ERR_TYPE BIT(7) +#define HISI_PCIE_LOCAL_VALID_ERR_SEVERITY BIT(8) +#define HISI_PCIE_LOCAL_VALID_ERR_MISC 9 + +static guid_t hisi_pcie_sec_guid = + GUID_INIT(0xB2889FC9, 0xE7D7, 0x4F9D, + 0xA8, 0x67, 0xAF, 0x42, 0xE9, 0x8B, 0xE7, 0x72); + +/* + * Firmware reports the socket port ID where the error occurred. These + * macros convert that to the core ID and core port ID required by the + * ACPI reset method. + */ +#define HISI_PCIE_PORT_ID(core, v) (((v) >> 1) + ((core) << 3)) +#define HISI_PCIE_CORE_ID(v) ((v) >> 3) +#define HISI_PCIE_CORE_PORT_ID(v) (((v) & 7) << 1) + +struct hisi_pcie_error_data { + u64 val_bits; + u8 version; + u8 soc_id; + u8 socket_id; + u8 nimbus_id; + u8 sub_module_id; + u8 core_id; + u8 port_id; + u8 err_severity; + u16 err_type; + u8 reserv[2]; + u32 err_misc[HISI_PCIE_ERR_MISC_REGS]; +}; + +struct hisi_pcie_error_private { + struct notifier_block nb; + struct device *dev; +}; + +enum hisi_pcie_submodule_id { + HISI_PCIE_SUB_MODULE_ID_AP, + HISI_PCIE_SUB_MODULE_ID_TL, + HISI_PCIE_SUB_MODULE_ID_MAC, + HISI_PCIE_SUB_MODULE_ID_DL, + HISI_PCIE_SUB_MODULE_ID_SDI, +}; + +static const char * const hisi_pcie_sub_module[] = { + [HISI_PCIE_SUB_MODULE_ID_AP] = "AP Layer", + [HISI_PCIE_SUB_MODULE_ID_TL] = "TL Layer", + [HISI_PCIE_SUB_MODULE_ID_MAC] = "MAC Layer", + [HISI_PCIE_SUB_MODULE_ID_DL] = "DL Layer", + [HISI_PCIE_SUB_MODULE_ID_SDI] = "SDI Layer", +}; + +enum hisi_pcie_err_severity { + HISI_PCIE_ERR_SEV_RECOVERABLE, + HISI_PCIE_ERR_SEV_FATAL, + HISI_PCIE_ERR_SEV_CORRECTED, + HISI_PCIE_ERR_SEV_NONE, +}; + +static const char * const hisi_pcie_error_sev[] = { + [HISI_PCIE_ERR_SEV_RECOVERABLE] = "recoverable", + [HISI_PCIE_ERR_SEV_FATAL] = "fatal", + [HISI_PCIE_ERR_SEV_CORRECTED] = "corrected", + [HISI_PCIE_ERR_SEV_NONE] = "none", +}; + +static const char *hisi_pcie_get_string(const char * const *array, + size_t n, u32 id) +{ + u32 index; + + for (index = 0; index < n; index++) { + if (index == id && array[index]) + return array[index]; + } + + return "unknown"; +} + +static int hisi_pcie_port_reset(struct platform_device *pdev, + u32 chip_id, u32 port_id) +{ + struct device *dev = &pdev->dev; + acpi_handle handle = ACPI_HANDLE(dev); + union acpi_object arg[3]; + struct acpi_object_list arg_list; + acpi_status s; + unsigned long long data = 0; + + arg[0].type = ACPI_TYPE_INTEGER; + arg[0].integer.value = chip_id; + arg[1].type = ACPI_TYPE_INTEGER; + arg[1].integer.value = HISI_PCIE_CORE_ID(port_id); + arg[2].type = ACPI_TYPE_INTEGER; + arg[2].integer.value = HISI_PCIE_CORE_PORT_ID(port_id); + + arg_list.count = 3; + arg_list.pointer = arg; + + s = acpi_evaluate_integer(handle, "RST", &arg_list, &data); + if (ACPI_FAILURE(s)) { + dev_err(dev, "No RST method\n"); + return -EIO; + } + + if (data) { + dev_err(dev, "Failed to Reset\n"); + return -EIO; + } + + return 0; +} + +static int hisi_pcie_port_do_recovery(struct platform_device *dev, + u32 chip_id, u32 port_id) +{ + acpi_status s; + struct device *device = &dev->dev; + acpi_handle root_handle = ACPI_HANDLE(device); + struct acpi_pci_root *pci_root; + struct pci_bus *root_bus; + struct pci_dev *pdev; + u32 domain, busnr, devfn; + + s = acpi_get_parent(root_handle, &root_handle); + if (ACPI_FAILURE(s)) + return -ENODEV; + pci_root = acpi_pci_find_root(root_handle); + if (!pci_root) + return -ENODEV; + root_bus = pci_root->bus; + domain = pci_root->segment; + + busnr = root_bus->number; + devfn = PCI_DEVFN(port_id, 0); + pdev = pci_get_domain_bus_and_slot(domain, busnr, devfn); + if (!pdev) { + dev_info(device, "Fail to get root port %04x:%02x:%02x.%d device\n", + domain, busnr, PCI_SLOT(devfn), PCI_FUNC(devfn)); + return -ENODEV; + } + + pci_stop_and_remove_bus_device_locked(pdev); + pci_dev_put(pdev); + + if (hisi_pcie_port_reset(dev, chip_id, port_id)) + return -EIO; + + /* + * The initialization time of subordinate devices after + * hot reset is no more than 1s, which is required by + * the PCI spec v5.0 sec 6.6.1. The time will shorten + * if Readiness Notifications mechanisms are used. But + * wait 1s here to adapt any conditions. + */ + ssleep(1UL); + + /* add root port and downstream devices */ + pci_lock_rescan_remove(); + pci_rescan_bus(root_bus); + pci_unlock_rescan_remove(); + + return 0; +} + +static void hisi_pcie_handle_error(struct platform_device *pdev, + const struct hisi_pcie_error_data *edata) +{ + struct device *dev = &pdev->dev; + int idx, rc; + const unsigned long valid_bits[] = {BITMAP_FROM_U64(edata->val_bits)}; + + if (edata->val_bits == 0) { + dev_warn(dev, "%s: no valid error information\n", __func__); + return; + } + + dev_info(dev, "\nHISI : HIP : PCIe controller error\n"); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_SOC_ID) + dev_info(dev, "Table version = %d\n", edata->version); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_SOCKET_ID) + dev_info(dev, "Socket ID = %d\n", edata->socket_id); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_NIMBUS_ID) + dev_info(dev, "Nimbus ID = %d\n", edata->nimbus_id); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_SUB_MODULE_ID) + dev_info(dev, "Sub Module = %s\n", + hisi_pcie_get_string(hisi_pcie_sub_module, + ARRAY_SIZE(hisi_pcie_sub_module), + edata->sub_module_id)); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_CORE_ID) + dev_info(dev, "Core ID = core%d\n", edata->core_id); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_PORT_ID) + dev_info(dev, "Port ID = port%d\n", edata->port_id); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_ERR_SEVERITY) + dev_info(dev, "Error severity = %s\n", + hisi_pcie_get_string(hisi_pcie_error_sev, + ARRAY_SIZE(hisi_pcie_error_sev), + edata->err_severity)); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_ERR_TYPE) + dev_info(dev, "Error type = 0x%x\n", edata->err_type); + + dev_info(dev, "Reg Dump:\n"); + idx = HISI_PCIE_LOCAL_VALID_ERR_MISC; + for_each_set_bit_from(idx, valid_bits, + HISI_PCIE_LOCAL_VALID_ERR_MISC + HISI_PCIE_ERR_MISC_REGS) + dev_info(dev, "ERR_MISC_%d = 0x%x\n", idx - HISI_PCIE_LOCAL_VALID_ERR_MISC, + edata->err_misc[idx - HISI_PCIE_LOCAL_VALID_ERR_MISC]); + + if (edata->err_severity != HISI_PCIE_ERR_SEV_RECOVERABLE) + return; + + /* Recovery for the PCIe controller errors, try reset + * PCI port for the error recovery + */ + rc = hisi_pcie_port_do_recovery(pdev, edata->socket_id, + HISI_PCIE_PORT_ID(edata->core_id, edata->port_id)); + if (rc) + dev_info(dev, "fail to do hisi pcie port reset\n"); +} + +static int hisi_pcie_notify_error(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct acpi_hest_generic_data *gdata = data; + const struct hisi_pcie_error_data *error_data = acpi_hest_get_payload(gdata); + struct hisi_pcie_error_private *priv; + struct device *dev; + struct platform_device *pdev; + guid_t err_sec_guid; + u8 socket; + + import_guid(&err_sec_guid, gdata->section_type); + if (!guid_equal(&err_sec_guid, &hisi_pcie_sec_guid)) + return NOTIFY_DONE; + + priv = container_of(nb, struct hisi_pcie_error_private, nb); + dev = priv->dev; + + if (device_property_read_u8(dev, "socket", &socket)) + return NOTIFY_DONE; + + if (error_data->socket_id != socket) + return NOTIFY_DONE; + + pdev = container_of(dev, struct platform_device, dev); + hisi_pcie_handle_error(pdev, error_data); + + return NOTIFY_OK; +} + +static int hisi_pcie_error_handler_probe(struct platform_device *pdev) +{ + struct hisi_pcie_error_private *priv; + int ret; + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->nb.notifier_call = hisi_pcie_notify_error; + priv->dev = &pdev->dev; + ret = ghes_register_vendor_record_notifier(&priv->nb); + if (ret) { + dev_err(&pdev->dev, + "Failed to register hisi pcie controller error handler with apei\n"); + return ret; + } + + platform_set_drvdata(pdev, priv); + + return 0; +} + +static int hisi_pcie_error_handler_remove(struct platform_device *pdev) +{ + struct hisi_pcie_error_private *priv = platform_get_drvdata(pdev); + + ghes_unregister_vendor_record_notifier(&priv->nb); + + return 0; +} + +static const struct acpi_device_id hisi_pcie_acpi_match[] = { + { "HISI0361", 0 }, + { } +}; + +static struct platform_driver hisi_pcie_error_handler_driver = { + .driver = { + .name = "hisi-pcie-error-handler", + .acpi_match_table = hisi_pcie_acpi_match, + }, + .probe = hisi_pcie_error_handler_probe, + .remove = hisi_pcie_error_handler_remove, +}; +module_platform_driver(hisi_pcie_error_handler_driver); + +MODULE_DESCRIPTION("HiSilicon HIP PCIe controller error handling driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pcie-iproc-bcma.c b/drivers/pci/controller/pcie-iproc-bcma.c index aa55b064f64d..56b8ee7bf330 100644 --- a/drivers/pci/controller/pcie-iproc-bcma.c +++ b/drivers/pci/controller/pcie-iproc-bcma.c @@ -94,18 +94,7 @@ static struct bcma_driver iproc_pcie_bcma_driver = { .probe = iproc_pcie_bcma_probe, .remove = iproc_pcie_bcma_remove, }; - -static int __init iproc_pcie_bcma_init(void) -{ - return bcma_driver_register(&iproc_pcie_bcma_driver); -} -module_init(iproc_pcie_bcma_init); - -static void __exit iproc_pcie_bcma_exit(void) -{ - bcma_driver_unregister(&iproc_pcie_bcma_driver); -} -module_exit(iproc_pcie_bcma_exit); +module_bcma_driver(iproc_pcie_bcma_driver); MODULE_AUTHOR("Hauke Mehrtens"); MODULE_DESCRIPTION("Broadcom iProc PCIe BCMA driver"); diff --git a/drivers/pci/controller/pcie-iproc-msi.c b/drivers/pci/controller/pcie-iproc-msi.c index 3176ad3ab0e5..908475d27e0e 100644 --- a/drivers/pci/controller/pcie-iproc-msi.c +++ b/drivers/pci/controller/pcie-iproc-msi.c @@ -209,15 +209,20 @@ static int iproc_msi_irq_set_affinity(struct irq_data *data, struct iproc_msi *msi = irq_data_get_irq_chip_data(data); int target_cpu = cpumask_first(mask); int curr_cpu; + int ret; curr_cpu = hwirq_to_cpu(msi, data->hwirq); if (curr_cpu == target_cpu) - return IRQ_SET_MASK_OK_DONE; + ret = IRQ_SET_MASK_OK_DONE; + else { + /* steer MSI to the target CPU */ + data->hwirq = hwirq_to_canonical_hwirq(msi, data->hwirq) + target_cpu; + ret = IRQ_SET_MASK_OK; + } - /* steer MSI to the target CPU */ - data->hwirq = hwirq_to_canonical_hwirq(msi, data->hwirq) + target_cpu; + irq_data_update_effective_affinity(data, cpumask_of(target_cpu)); - return IRQ_SET_MASK_OK; + return ret; } static void iproc_msi_irq_compose_msi_msg(struct irq_data *data, diff --git a/drivers/pci/controller/pcie-iproc-platform.c b/drivers/pci/controller/pcie-iproc-platform.c index a956b0c18bd1..b93e7bda101b 100644 --- a/drivers/pci/controller/pcie-iproc-platform.c +++ b/drivers/pci/controller/pcie-iproc-platform.c @@ -99,7 +99,7 @@ static int iproc_pcie_pltfm_probe(struct platform_device *pdev) switch (pcie->type) { case IPROC_PCIE_PAXC: case IPROC_PCIE_PAXC_V2: - pcie->map_irq = 0; + pcie->map_irq = NULL; break; default: break; diff --git a/drivers/pci/controller/pcie-xilinx-cpm.c b/drivers/pci/controller/pcie-xilinx-cpm.c index f3082de44e8a..f92e0152e65e 100644 --- a/drivers/pci/controller/pcie-xilinx-cpm.c +++ b/drivers/pci/controller/pcie-xilinx-cpm.c @@ -572,12 +572,8 @@ static int xilinx_cpm_pcie_probe(struct platform_device *pdev) goto err_setup_irq; } - bridge->dev.parent = dev; bridge->sysdata = port->cfg; - bridge->busnr = port->cfg->busr.start; bridge->ops = (struct pci_ops *)&pci_generic_ecam_ops.pci_ops; - bridge->map_irq = of_irq_parse_and_map_pci; - bridge->swizzle_irq = pci_common_swizzle; err = pci_host_probe(bridge); if (err < 0) diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c index aa1b12bac9a1..f375c21ceeb1 100644 --- a/drivers/pci/controller/vmd.c +++ b/drivers/pci/controller/vmd.c @@ -298,6 +298,33 @@ static struct msi_domain_info vmd_msi_domain_info = { .chip = &vmd_msi_controller, }; +static int vmd_create_irq_domain(struct vmd_dev *vmd) +{ + struct fwnode_handle *fn; + + fn = irq_domain_alloc_named_id_fwnode("VMD-MSI", vmd->sysdata.domain); + if (!fn) + return -ENODEV; + + vmd->irq_domain = pci_msi_create_irq_domain(fn, &vmd_msi_domain_info, NULL); + if (!vmd->irq_domain) { + irq_domain_free_fwnode(fn); + return -ENODEV; + } + + return 0; +} + +static void vmd_remove_irq_domain(struct vmd_dev *vmd) +{ + if (vmd->irq_domain) { + struct fwnode_handle *fn = vmd->irq_domain->fwnode; + + irq_domain_remove(vmd->irq_domain); + irq_domain_free_fwnode(fn); + } +} + static char __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus, unsigned int devfn, int reg, int len) { @@ -417,97 +444,175 @@ static int vmd_find_free_domain(void) return domain + 1; } -static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) +static int vmd_get_phys_offsets(struct vmd_dev *vmd, bool native_hint, + resource_size_t *offset1, + resource_size_t *offset2) { - struct pci_sysdata *sd = &vmd->sysdata; - struct fwnode_handle *fn; - struct resource *res; - u32 upper_bits; - unsigned long flags; - LIST_HEAD(resources); - resource_size_t offset[2] = {0}; - resource_size_t membar2_offset = 0x2000; - struct pci_bus *child; + struct pci_dev *dev = vmd->dev; + u64 phys1, phys2; - /* - * Shadow registers may exist in certain VMD device ids which allow - * guests to correctly assign host physical addresses to the root ports - * and child devices. These registers will either return the host value - * or 0, depending on an enable bit in the VMD device. - */ - if (features & VMD_FEAT_HAS_MEMBAR_SHADOW) { + if (native_hint) { u32 vmlock; int ret; - membar2_offset = MB2_SHADOW_OFFSET + MB2_SHADOW_SIZE; - ret = pci_read_config_dword(vmd->dev, PCI_REG_VMLOCK, &vmlock); + ret = pci_read_config_dword(dev, PCI_REG_VMLOCK, &vmlock); if (ret || vmlock == ~0) return -ENODEV; if (MB2_SHADOW_EN(vmlock)) { void __iomem *membar2; - membar2 = pci_iomap(vmd->dev, VMD_MEMBAR2, 0); + membar2 = pci_iomap(dev, VMD_MEMBAR2, 0); if (!membar2) return -ENOMEM; - offset[0] = vmd->dev->resource[VMD_MEMBAR1].start - - (readq(membar2 + MB2_SHADOW_OFFSET) & - PCI_BASE_ADDRESS_MEM_MASK); - offset[1] = vmd->dev->resource[VMD_MEMBAR2].start - - (readq(membar2 + MB2_SHADOW_OFFSET + 8) & - PCI_BASE_ADDRESS_MEM_MASK); - pci_iounmap(vmd->dev, membar2); - } - } - - if (features & VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP) { - int pos = pci_find_capability(vmd->dev, PCI_CAP_ID_VNDR); + phys1 = readq(membar2 + MB2_SHADOW_OFFSET); + phys2 = readq(membar2 + MB2_SHADOW_OFFSET + 8); + pci_iounmap(dev, membar2); + } else + return 0; + } else { + /* Hypervisor-Emulated Vendor-Specific Capability */ + int pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); u32 reg, regu; - pci_read_config_dword(vmd->dev, pos + 4, ®); + pci_read_config_dword(dev, pos + 4, ®); /* "SHDW" */ if (pos && reg == 0x53484457) { - pci_read_config_dword(vmd->dev, pos + 8, ®); - pci_read_config_dword(vmd->dev, pos + 12, ®u); - offset[0] = vmd->dev->resource[VMD_MEMBAR1].start - - (((u64) regu << 32 | reg) & - PCI_BASE_ADDRESS_MEM_MASK); - - pci_read_config_dword(vmd->dev, pos + 16, ®); - pci_read_config_dword(vmd->dev, pos + 20, ®u); - offset[1] = vmd->dev->resource[VMD_MEMBAR2].start - - (((u64) regu << 32 | reg) & - PCI_BASE_ADDRESS_MEM_MASK); + pci_read_config_dword(dev, pos + 8, ®); + pci_read_config_dword(dev, pos + 12, ®u); + phys1 = (u64) regu << 32 | reg; + + pci_read_config_dword(dev, pos + 16, ®); + pci_read_config_dword(dev, pos + 20, ®u); + phys2 = (u64) regu << 32 | reg; + } else + return 0; + } + + *offset1 = dev->resource[VMD_MEMBAR1].start - + (phys1 & PCI_BASE_ADDRESS_MEM_MASK); + *offset2 = dev->resource[VMD_MEMBAR2].start - + (phys2 & PCI_BASE_ADDRESS_MEM_MASK); + + return 0; +} + +static int vmd_get_bus_number_start(struct vmd_dev *vmd) +{ + struct pci_dev *dev = vmd->dev; + u16 reg; + + pci_read_config_word(dev, PCI_REG_VMCAP, ®); + if (BUS_RESTRICT_CAP(reg)) { + pci_read_config_word(dev, PCI_REG_VMCONFIG, ®); + + switch (BUS_RESTRICT_CFG(reg)) { + case 0: + vmd->busn_start = 0; + break; + case 1: + vmd->busn_start = 128; + break; + case 2: + vmd->busn_start = 224; + break; + default: + pci_err(dev, "Unknown Bus Offset Setting (%d)\n", + BUS_RESTRICT_CFG(reg)); + return -ENODEV; } } + return 0; +} + +static irqreturn_t vmd_irq(int irq, void *data) +{ + struct vmd_irq_list *irqs = data; + struct vmd_irq *vmdirq; + int idx; + + idx = srcu_read_lock(&irqs->srcu); + list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node) + generic_handle_irq(vmdirq->virq); + srcu_read_unlock(&irqs->srcu, idx); + + return IRQ_HANDLED; +} + +static int vmd_alloc_irqs(struct vmd_dev *vmd) +{ + struct pci_dev *dev = vmd->dev; + int i, err; + + vmd->msix_count = pci_msix_vec_count(dev); + if (vmd->msix_count < 0) + return -ENODEV; + + vmd->msix_count = pci_alloc_irq_vectors(dev, 1, vmd->msix_count, + PCI_IRQ_MSIX); + if (vmd->msix_count < 0) + return vmd->msix_count; + + vmd->irqs = devm_kcalloc(&dev->dev, vmd->msix_count, sizeof(*vmd->irqs), + GFP_KERNEL); + if (!vmd->irqs) + return -ENOMEM; + + for (i = 0; i < vmd->msix_count; i++) { + err = init_srcu_struct(&vmd->irqs[i].srcu); + if (err) + return err; + + INIT_LIST_HEAD(&vmd->irqs[i].irq_list); + err = devm_request_irq(&dev->dev, pci_irq_vector(dev, i), + vmd_irq, IRQF_NO_THREAD, + "vmd", &vmd->irqs[i]); + if (err) + return err; + } + + return 0; +} + +static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) +{ + struct pci_sysdata *sd = &vmd->sysdata; + struct resource *res; + u32 upper_bits; + unsigned long flags; + LIST_HEAD(resources); + resource_size_t offset[2] = {0}; + resource_size_t membar2_offset = 0x2000; + struct pci_bus *child; + int ret; + + /* + * Shadow registers may exist in certain VMD device ids which allow + * guests to correctly assign host physical addresses to the root ports + * and child devices. These registers will either return the host value + * or 0, depending on an enable bit in the VMD device. + */ + if (features & VMD_FEAT_HAS_MEMBAR_SHADOW) { + membar2_offset = MB2_SHADOW_OFFSET + MB2_SHADOW_SIZE; + ret = vmd_get_phys_offsets(vmd, true, &offset[0], &offset[1]); + if (ret) + return ret; + } else if (features & VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP) { + ret = vmd_get_phys_offsets(vmd, false, &offset[0], &offset[1]); + if (ret) + return ret; + } + /* * Certain VMD devices may have a root port configuration option which * limits the bus range to between 0-127, 128-255, or 224-255 */ if (features & VMD_FEAT_HAS_BUS_RESTRICTIONS) { - u16 reg16; - - pci_read_config_word(vmd->dev, PCI_REG_VMCAP, ®16); - if (BUS_RESTRICT_CAP(reg16)) { - pci_read_config_word(vmd->dev, PCI_REG_VMCONFIG, - ®16); - - switch (BUS_RESTRICT_CFG(reg16)) { - case 1: - vmd->busn_start = 128; - break; - case 2: - vmd->busn_start = 224; - break; - case 3: - pci_err(vmd->dev, "Unknown Bus Offset Setting\n"); - return -ENODEV; - default: - break; - } - } + ret = vmd_get_bus_number_start(vmd); + if (ret) + return ret; } res = &vmd->dev->resource[VMD_CFGBAR]; @@ -568,17 +673,9 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) sd->node = pcibus_to_node(vmd->dev->bus); - fn = irq_domain_alloc_named_id_fwnode("VMD-MSI", vmd->sysdata.domain); - if (!fn) - return -ENODEV; - - vmd->irq_domain = pci_msi_create_irq_domain(fn, &vmd_msi_domain_info, - NULL); - - if (!vmd->irq_domain) { - irq_domain_free_fwnode(fn); - return -ENODEV; - } + ret = vmd_create_irq_domain(vmd); + if (ret) + return ret; /* * Override the irq domain bus token so the domain can be distinguished @@ -594,13 +691,13 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) &vmd_ops, sd, &resources); if (!vmd->bus) { pci_free_resource_list(&resources); - irq_domain_remove(vmd->irq_domain); - irq_domain_free_fwnode(fn); + vmd_remove_irq_domain(vmd); return -ENODEV; } vmd_attach_resources(vmd); - dev_set_msi_domain(&vmd->bus->dev, vmd->irq_domain); + if (vmd->irq_domain) + dev_set_msi_domain(&vmd->bus->dev, vmd->irq_domain); pci_scan_child_bus(vmd->bus); pci_assign_unassigned_bus_resources(vmd->bus); @@ -620,24 +717,10 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) return 0; } -static irqreturn_t vmd_irq(int irq, void *data) -{ - struct vmd_irq_list *irqs = data; - struct vmd_irq *vmdirq; - int idx; - - idx = srcu_read_lock(&irqs->srcu); - list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node) - generic_handle_irq(vmdirq->virq); - srcu_read_unlock(&irqs->srcu, idx); - - return IRQ_HANDLED; -} - static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id) { struct vmd_dev *vmd; - int i, err; + int err; if (resource_size(&dev->resource[VMD_CFGBAR]) < (1 << 20)) return -ENOMEM; @@ -660,32 +743,9 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id) dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32))) return -ENODEV; - vmd->msix_count = pci_msix_vec_count(dev); - if (vmd->msix_count < 0) - return -ENODEV; - - vmd->msix_count = pci_alloc_irq_vectors(dev, 1, vmd->msix_count, - PCI_IRQ_MSIX); - if (vmd->msix_count < 0) - return vmd->msix_count; - - vmd->irqs = devm_kcalloc(&dev->dev, vmd->msix_count, sizeof(*vmd->irqs), - GFP_KERNEL); - if (!vmd->irqs) - return -ENOMEM; - - for (i = 0; i < vmd->msix_count; i++) { - err = init_srcu_struct(&vmd->irqs[i].srcu); - if (err) - return err; - - INIT_LIST_HEAD(&vmd->irqs[i].irq_list); - err = devm_request_irq(&dev->dev, pci_irq_vector(dev, i), - vmd_irq, IRQF_NO_THREAD, - "vmd", &vmd->irqs[i]); - if (err) - return err; - } + err = vmd_alloc_irqs(vmd); + if (err) + return err; spin_lock_init(&vmd->cfg_lock); pci_set_drvdata(dev, vmd); @@ -709,15 +769,13 @@ static void vmd_cleanup_srcu(struct vmd_dev *vmd) static void vmd_remove(struct pci_dev *dev) { struct vmd_dev *vmd = pci_get_drvdata(dev); - struct fwnode_handle *fn = vmd->irq_domain->fwnode; sysfs_remove_link(&vmd->dev->dev.kobj, "domain"); pci_stop_root_bus(vmd->bus); pci_remove_root_bus(vmd->bus); vmd_cleanup_srcu(vmd); vmd_detach_resources(vmd); - irq_domain_remove(vmd->irq_domain); - irq_domain_free_fwnode(fn); + vmd_remove_irq_domain(vmd); } #ifdef CONFIG_PM_SLEEP @@ -730,7 +788,6 @@ static int vmd_suspend(struct device *dev) for (i = 0; i < vmd->msix_count; i++) devm_free_irq(dev, pci_irq_vector(pdev, i), &vmd->irqs[i]); - pci_save_state(pdev); return 0; } @@ -748,7 +805,6 @@ static int vmd_resume(struct device *dev) return err; } - pci_restore_state(pdev); return 0; } #endif diff --git a/drivers/pci/ecam.c b/drivers/pci/ecam.c index 8f065a42fc1a..b54d32a31669 100644 --- a/drivers/pci/ecam.c +++ b/drivers/pci/ecam.c @@ -168,4 +168,14 @@ const struct pci_ecam_ops pci_32b_ops = { .write = pci_generic_config_write32, } }; + +/* ECAM ops for 32-bit read only (non-compliant) */ +const struct pci_ecam_ops pci_32b_read_ops = { + .bus_shift = 20, + .pci_ops = { + .map_bus = pci_ecam_map_bus, + .read = pci_generic_config_read32, + .write = pci_generic_config_write, + } +}; #endif diff --git a/drivers/pci/hotplug/pciehp_ctrl.c b/drivers/pci/hotplug/pciehp_ctrl.c index 9f85815b4f53..529c34808440 100644 --- a/drivers/pci/hotplug/pciehp_ctrl.c +++ b/drivers/pci/hotplug/pciehp_ctrl.c @@ -73,10 +73,8 @@ static int board_added(struct controller *ctrl) /* Check link training status */ retval = pciehp_check_link_status(ctrl); - if (retval) { - ctrl_err(ctrl, "Failed to check link status\n"); + if (retval) goto err_exit; - } /* Check for a power fault */ if (ctrl->power_fault_detected || pciehp_query_power_fault(ctrl)) { diff --git a/drivers/pci/hotplug/pciehp_hpc.c b/drivers/pci/hotplug/pciehp_hpc.c index 53433b37e181..fb3840e222ad 100644 --- a/drivers/pci/hotplug/pciehp_hpc.c +++ b/drivers/pci/hotplug/pciehp_hpc.c @@ -283,8 +283,6 @@ static void pcie_wait_for_presence(struct pci_dev *pdev) msleep(10); timeout -= 10; } while (timeout > 0); - - pci_info(pdev, "Timeout waiting for Presence Detect\n"); } int pciehp_check_link_status(struct controller *ctrl) @@ -293,8 +291,10 @@ int pciehp_check_link_status(struct controller *ctrl) bool found; u16 lnk_status; - if (!pcie_wait_for_link(pdev, true)) + if (!pcie_wait_for_link(pdev, true)) { + ctrl_info(ctrl, "Slot(%s): No link\n", slot_name(ctrl)); return -1; + } if (ctrl->inband_presence_disabled) pcie_wait_for_presence(pdev); @@ -311,15 +311,18 @@ int pciehp_check_link_status(struct controller *ctrl) ctrl_dbg(ctrl, "%s: lnk_status = %x\n", __func__, lnk_status); if ((lnk_status & PCI_EXP_LNKSTA_LT) || !(lnk_status & PCI_EXP_LNKSTA_NLW)) { - ctrl_err(ctrl, "link training error: status %#06x\n", - lnk_status); + ctrl_info(ctrl, "Slot(%s): Cannot train link: status %#06x\n", + slot_name(ctrl), lnk_status); return -1; } pcie_update_link_speed(ctrl->pcie->port->subordinate, lnk_status); - if (!found) + if (!found) { + ctrl_info(ctrl, "Slot(%s): No device found\n", + slot_name(ctrl)); return -1; + } return 0; } diff --git a/drivers/pci/hotplug/rpadlpar_core.c b/drivers/pci/hotplug/rpadlpar_core.c index f979b7098acf..0a3c80ba66be 100644 --- a/drivers/pci/hotplug/rpadlpar_core.c +++ b/drivers/pci/hotplug/rpadlpar_core.c @@ -40,13 +40,13 @@ static DEFINE_MUTEX(rpadlpar_mutex); static struct device_node *find_vio_slot_node(char *drc_name) { struct device_node *parent = of_find_node_by_name(NULL, "vdevice"); - struct device_node *dn = NULL; + struct device_node *dn; int rc; if (!parent) return NULL; - while ((dn = of_get_next_child(parent, dn))) { + for_each_child_of_node(parent, dn) { rc = rpaphp_check_drc_props(dn, drc_name, NULL); if (rc == 0) break; @@ -60,10 +60,10 @@ static struct device_node *find_vio_slot_node(char *drc_name) static struct device_node *find_php_slot_pci_node(char *drc_name, char *drc_type) { - struct device_node *np = NULL; + struct device_node *np; int rc; - while ((np = of_find_node_by_name(np, "pci"))) { + for_each_node_by_name(np, "pci") { rc = rpaphp_check_drc_props(np, drc_name, drc_type); if (rc == 0) break; diff --git a/drivers/pci/hotplug/shpchp_ctrl.c b/drivers/pci/hotplug/shpchp_ctrl.c index 65502e3f7b4f..6a6705e0cf17 100644 --- a/drivers/pci/hotplug/shpchp_ctrl.c +++ b/drivers/pci/hotplug/shpchp_ctrl.c @@ -299,7 +299,6 @@ static int board_added(struct slot *p_slot) if (p_slot->status == 0xFF) { /* power fault occurred, but it was benign */ ctrl_dbg(ctrl, "%s: Power fault\n", __func__); - rc = POWER_FAILURE; p_slot->status = 0; goto err_exit; } diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index b37e08c4f9d1..4afd4ee4f7f0 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -180,6 +180,7 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id) virtfn->device = iov->vf_device; virtfn->is_virtfn = 1; virtfn->physfn = pci_dev_get(dev); + virtfn->no_command_memory = 1; if (id == 0) pci_read_vf_config_common(virtfn); diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 9d53c16b7329..de1c331dbed4 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -53,7 +53,7 @@ static ssize_t size_show(struct device *dev, struct device_attribute *attr, if (pdev->p2pdma->pool) size = gen_pool_size(pdev->p2pdma->pool); - return snprintf(buf, PAGE_SIZE, "%zd\n", size); + return scnprintf(buf, PAGE_SIZE, "%zd\n", size); } static DEVICE_ATTR_RO(size); @@ -66,7 +66,7 @@ static ssize_t available_show(struct device *dev, struct device_attribute *attr, if (pdev->p2pdma->pool) avail = gen_pool_avail(pdev->p2pdma->pool); - return snprintf(buf, PAGE_SIZE, "%zd\n", avail); + return scnprintf(buf, PAGE_SIZE, "%zd\n", avail); } static DEVICE_ATTR_RO(available); @@ -75,8 +75,8 @@ static ssize_t published_show(struct device *dev, struct device_attribute *attr, { struct pci_dev *pdev = to_pci_dev(dev); - return snprintf(buf, PAGE_SIZE, "%d\n", - pdev->p2pdma->p2pmem_published); + return scnprintf(buf, PAGE_SIZE, "%d\n", + pdev->p2pdma->p2pmem_published); } static DEVICE_ATTR_RO(published); @@ -762,7 +762,7 @@ struct scatterlist *pci_p2pmem_alloc_sgl(struct pci_dev *pdev, struct scatterlist *sg; void *addr; - sg = kzalloc(sizeof(*sg), GFP_KERNEL); + sg = kmalloc(sizeof(*sg), GFP_KERNEL); if (!sg) return NULL; diff --git a/drivers/pci/pci-acpi.c b/drivers/pci/pci-acpi.c index d9aa551f8423..bf03648c2072 100644 --- a/drivers/pci/pci-acpi.c +++ b/drivers/pci/pci-acpi.c @@ -1177,7 +1177,7 @@ static struct acpi_device *acpi_pci_find_companion(struct device *dev) * @pdev: the PCI device whose delay is to be updated * @handle: ACPI handle of this device * - * Update the d3_delay and d3cold_delay of a PCI device from the ACPI _DSM + * Update the d3hot_delay and d3cold_delay of a PCI device from the ACPI _DSM * control method of either the device itself or the PCI host bridge. * * Function 8, "Reset Delay," applies to the entire hierarchy below a PCI @@ -1216,8 +1216,8 @@ static void pci_acpi_optimize_delay(struct pci_dev *pdev, } if (elements[3].type == ACPI_TYPE_INTEGER) { value = (int)elements[3].integer.value / 1000; - if (value < PCI_PM_D3_WAIT) - pdev->d3_delay = value; + if (value < PCI_PM_D3HOT_WAIT) + pdev->d3hot_delay = value; } } ACPI_FREE(obj); diff --git a/drivers/pci/pci-bridge-emul.c b/drivers/pci/pci-bridge-emul.c index ccf26d12ec61..139869d50eb2 100644 --- a/drivers/pci/pci-bridge-emul.c +++ b/drivers/pci/pci-bridge-emul.c @@ -294,6 +294,7 @@ int pci_bridge_emul_init(struct pci_bridge_emul *bridge, return 0; } +EXPORT_SYMBOL_GPL(pci_bridge_emul_init); /* * Cleanup a pci_bridge_emul structure that was previously initialized @@ -305,6 +306,7 @@ void pci_bridge_emul_cleanup(struct pci_bridge_emul *bridge) kfree(bridge->pcie_cap_regs_behavior); kfree(bridge->pci_regs_behavior); } +EXPORT_SYMBOL_GPL(pci_bridge_emul_cleanup); /* * Should be called by the PCI controller driver when reading the PCI @@ -366,6 +368,7 @@ int pci_bridge_emul_conf_read(struct pci_bridge_emul *bridge, int where, return PCIBIOS_SUCCESSFUL; } +EXPORT_SYMBOL_GPL(pci_bridge_emul_conf_read); /* * Should be called by the PCI controller driver when writing the PCI @@ -430,3 +433,4 @@ int pci_bridge_emul_conf_write(struct pci_bridge_emul *bridge, int where, return PCIBIOS_SUCCESSFUL; } +EXPORT_SYMBOL_GPL(pci_bridge_emul_conf_write); diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index d1b7169c0684..8b587fc97f7b 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -970,12 +970,6 @@ static int pci_pm_resume(struct device *dev) #ifdef CONFIG_HIBERNATE_CALLBACKS -/* - * pcibios_pm_ops - provide arch-specific hooks when a PCI device is doing - * a hibernate transition - */ -struct dev_pm_ops __weak pcibios_pm_ops; - static int pci_pm_freeze(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); @@ -1034,9 +1028,6 @@ static int pci_pm_freeze_noirq(struct device *dev) pci_pm_set_unknown_state(pci_dev); - if (pcibios_pm_ops.freeze_noirq) - return pcibios_pm_ops.freeze_noirq(dev); - return 0; } @@ -1044,13 +1035,6 @@ static int pci_pm_thaw_noirq(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - int error; - - if (pcibios_pm_ops.thaw_noirq) { - error = pcibios_pm_ops.thaw_noirq(dev); - if (error) - return error; - } /* * The pm->thaw_noirq() callback assumes the device has been @@ -1175,9 +1159,6 @@ static int pci_pm_poweroff_noirq(struct device *dev) pci_fixup_device(pci_fixup_suspend_late, pci_dev); - if (pcibios_pm_ops.poweroff_noirq) - return pcibios_pm_ops.poweroff_noirq(dev); - return 0; } @@ -1185,13 +1166,6 @@ static int pci_pm_restore_noirq(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - int error; - - if (pcibios_pm_ops.restore_noirq) { - error = pcibios_pm_ops.restore_noirq(dev); - if (error) - return error; - } pci_pm_default_resume_early(pci_dev); pci_fixup_device(pci_fixup_resume_early, pci_dev); diff --git a/drivers/pci/pci-pf-stub.c b/drivers/pci/pci-pf-stub.c index a0b2bd6c918a..45855a5e9fca 100644 --- a/drivers/pci/pci-pf-stub.c +++ b/drivers/pci/pci-pf-stub.c @@ -37,18 +37,6 @@ static struct pci_driver pf_stub_driver = { .probe = pci_pf_stub_probe, .sriov_configure = pci_sriov_configure_simple, }; - -static int __init pci_pf_stub_init(void) -{ - return pci_register_driver(&pf_stub_driver); -} - -static void __exit pci_pf_stub_exit(void) -{ - pci_unregister_driver(&pf_stub_driver); -} - -module_init(pci_pf_stub_init); -module_exit(pci_pf_stub_exit); +module_pci_driver(pf_stub_driver); MODULE_LICENSE("GPL"); diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index 6d78df981d41..d15c881e2e7e 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -574,7 +574,7 @@ static ssize_t driver_override_show(struct device *dev, ssize_t len; device_lock(dev); - len = snprintf(buf, PAGE_SIZE, "%s\n", pdev->driver_override); + len = scnprintf(buf, PAGE_SIZE, "%s\n", pdev->driver_override); device_unlock(dev); return len; } @@ -708,6 +708,7 @@ static ssize_t pci_read_config(struct file *filp, struct kobject *kobj, data[off - init_off + 3] = (val >> 24) & 0xff; off += 4; size -= 4; + cond_resched(); } if (size >= 2) { @@ -1196,10 +1197,10 @@ static int pci_create_resource_files(struct pci_dev *pdev) } return 0; } -#else /* !HAVE_PCI_MMAP */ +#else /* !(defined(HAVE_PCI_MMAP) || defined(ARCH_GENERIC_PCI_MMAP_RESOURCE)) */ int __weak pci_create_resource_files(struct pci_dev *dev) { return 0; } void __weak pci_remove_resource_files(struct pci_dev *dev) { return; } -#endif /* HAVE_PCI_MMAP */ +#endif /** * pci_write_rom - used to enable access to the PCI ROM display diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e39c5499770f..6d4d5a2f923d 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -15,7 +15,6 @@ #include <linux/init.h> #include <linux/msi.h> #include <linux/of.h> -#include <linux/of_pci.h> #include <linux/pci.h> #include <linux/pm.h> #include <linux/slab.h> @@ -30,8 +29,6 @@ #include <linux/pm_runtime.h> #include <linux/pci_hotplug.h> #include <linux/vmalloc.h> -#include <linux/pci-ats.h> -#include <asm/setup.h> #include <asm/dma.h> #include <linux/aer.h> #include "pci.h" @@ -49,7 +46,7 @@ EXPORT_SYMBOL(isa_dma_bridge_buggy); int pci_pci_problems; EXPORT_SYMBOL(pci_pci_problems); -unsigned int pci_pm_d3_delay; +unsigned int pci_pm_d3hot_delay; static void pci_pme_list_scan(struct work_struct *work); @@ -66,10 +63,10 @@ struct pci_pme_device { static void pci_dev_d3_sleep(struct pci_dev *dev) { - unsigned int delay = dev->d3_delay; + unsigned int delay = dev->d3hot_delay; - if (delay < pci_pm_d3_delay) - delay = pci_pm_d3_delay; + if (delay < pci_pm_d3hot_delay) + delay = pci_pm_d3hot_delay; if (delay) msleep(delay); @@ -101,7 +98,19 @@ unsigned long pci_hotplug_mmio_pref_size = DEFAULT_HOTPLUG_MMIO_PREF_SIZE; #define DEFAULT_HOTPLUG_BUS_SIZE 1 unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE; + +/* PCIe MPS/MRRS strategy; can be overridden by kernel command-line param */ +#ifdef CONFIG_PCIE_BUS_TUNE_OFF +enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_TUNE_OFF; +#elif defined CONFIG_PCIE_BUS_SAFE +enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_SAFE; +#elif defined CONFIG_PCIE_BUS_PERFORMANCE +enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PERFORMANCE; +#elif defined CONFIG_PCIE_BUS_PEER2PEER +enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PEER2PEER; +#else enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT; +#endif /* * The default CLS is used if arch didn't set CLS explicitly and not @@ -876,6 +885,10 @@ static void pci_std_enable_acs(struct pci_dev *dev) /* Upstream Forwarding */ ctrl |= (cap & PCI_ACS_UF); + /* Enable Translation Blocking for external devices */ + if (dev->external_facing || dev->untrusted) + ctrl |= (cap & PCI_ACS_TB); + pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl); } @@ -1065,7 +1078,7 @@ static int pci_raw_set_power_state(struct pci_dev *dev, pci_power_t state) if (state == PCI_D3hot || dev->current_state == PCI_D3hot) pci_dev_d3_sleep(dev); else if (state == PCI_D2 || dev->current_state == PCI_D2) - msleep(PCI_PM_D2_DELAY); + udelay(PCI_PM_D2_DELAY); pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK); @@ -3013,7 +3026,7 @@ void pci_pm_init(struct pci_dev *dev) } dev->pm_cap = pm; - dev->d3_delay = PCI_PM_D3_WAIT; + dev->d3hot_delay = PCI_PM_D3HOT_WAIT; dev->d3cold_delay = PCI_PM_D3COLD_WAIT; dev->bridge_d3 = pci_bridge_d3_possible(dev); dev->d3cold_allowed = true; @@ -3038,7 +3051,7 @@ void pci_pm_init(struct pci_dev *dev) (pmc & PCI_PM_CAP_PME_D0) ? " D0" : "", (pmc & PCI_PM_CAP_PME_D1) ? " D1" : "", (pmc & PCI_PM_CAP_PME_D2) ? " D2" : "", - (pmc & PCI_PM_CAP_PME_D3) ? " D3hot" : "", + (pmc & PCI_PM_CAP_PME_D3hot) ? " D3hot" : "", (pmc & PCI_PM_CAP_PME_D3cold) ? " D3cold" : ""); dev->pme_support = pmc >> PCI_PM_CAP_PME_SHIFT; dev->pme_poll = true; @@ -4621,7 +4634,7 @@ static int pci_af_flr(struct pci_dev *dev, int probe) * * NOTE: This causes the caller to sleep for twice the device power transition * cooldown period, which for the D0->D3hot and D3hot->D0 transitions is 10 ms - * by default (i.e. unless the @dev's d3_delay field has a different value). + * by default (i.e. unless the @dev's d3hot_delay field has a different value). * Moreover, only devices in D0 can be reset by this function. */ static int pci_pm_reset(struct pci_dev *dev, int probe) @@ -4701,9 +4714,7 @@ static bool pcie_wait_for_link_delay(struct pci_dev *pdev, bool active, } if (active && ret) msleep(delay); - else if (ret != active) - pci_info(pdev, "Data Link Layer Link Active not %s in 1000 msec\n", - active ? "set" : "cleared"); + return ret == active; } @@ -4828,6 +4839,7 @@ void pci_bridge_wait_for_secondary_bus(struct pci_dev *dev) delay); if (!pcie_wait_for_link_delay(dev, true, delay)) { /* Did not train, no need to wait any further */ + pci_info(dev, "Data Link Layer Link Active not set in 1000 msec\n"); return; } } @@ -4920,16 +4932,10 @@ static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, int probe) static int pci_dev_reset_slot_function(struct pci_dev *dev, int probe) { - struct pci_dev *pdev; - - if (dev->subordinate || !dev->slot || + if (dev->multifunction || dev->subordinate || !dev->slot || dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET) return -ENOTTY; - list_for_each_entry(pdev, &dev->bus->devices, bus_list) - if (pdev != dev && pdev->slot == dev->slot) - return -ENOTTY; - return pci_reset_hotplug_slot(dev->slot->hotplug, probe); } @@ -6005,7 +6011,7 @@ int pci_set_vga_state(struct pci_dev *dev, bool decode, if (flags & PCI_VGA_STATE_CHANGE_DECODES) { pci_read_config_word(dev, PCI_COMMAND, &cmd); - if (decode == true) + if (decode) cmd |= command_bits; else cmd &= ~command_bits; @@ -6021,7 +6027,7 @@ int pci_set_vga_state(struct pci_dev *dev, bool decode, if (bridge) { pci_read_config_word(bridge, PCI_BRIDGE_CONTROL, &cmd); - if (decode == true) + if (decode) cmd |= PCI_BRIDGE_CTL_VGA; else cmd &= ~PCI_BRIDGE_CTL_VGA; @@ -6350,7 +6356,7 @@ static ssize_t resource_alignment_show(struct bus_type *bus, char *buf) spin_lock(&resource_alignment_lock); if (resource_alignment_param) - count = snprintf(buf, PAGE_SIZE, "%s", resource_alignment_param); + count = scnprintf(buf, PAGE_SIZE, "%s", resource_alignment_param); spin_unlock(&resource_alignment_lock); /* diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index fa12f7cbc1a0..f86cae9aa1f4 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -43,10 +43,9 @@ int pci_probe_reset_function(struct pci_dev *dev); int pci_bridge_secondary_bus_reset(struct pci_dev *dev); int pci_bus_error_reset(struct pci_dev *dev); -#define PCI_PM_D2_DELAY 200 -#define PCI_PM_D3_WAIT 10 -#define PCI_PM_D3COLD_WAIT 100 -#define PCI_PM_BUS_WAIT 50 +#define PCI_PM_D2_DELAY 200 /* usec; see PCIe r4.0, sec 5.9.1 */ +#define PCI_PM_D3HOT_WAIT 10 /* msec */ +#define PCI_PM_D3COLD_WAIT 100 /* msec */ /** * struct pci_platform_pm_ops - Firmware PM callbacks @@ -178,7 +177,7 @@ extern struct mutex pci_slot_mutex; extern raw_spinlock_t pci_lock; -extern unsigned int pci_pm_d3_delay; +extern unsigned int pci_pm_d3hot_delay; #ifdef CONFIG_PCI_MSI void pci_no_msi(void); diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 253c30cc1967..ac0557a305af 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -74,14 +74,6 @@ struct pcie_link_state { * has one slot under it, so at most there are 8 functions. */ struct aspm_latency acceptable[8]; - - /* L1 PM Substate info */ - struct { - u32 up_cap_ptr; /* L1SS cap ptr in upstream dev */ - u32 dw_cap_ptr; /* L1SS cap ptr in downstream dev */ - u32 ctl1; /* value to be programmed in ctl1 */ - u32 ctl2; /* value to be programmed in ctl2 */ - } l1ss; }; static int aspm_disabled, aspm_force; @@ -308,8 +300,10 @@ static void pcie_aspm_configure_common_clock(struct pcie_link_state *link) } /* Convert L0s latency encoding to ns */ -static u32 calc_l0s_latency(u32 encoding) +static u32 calc_l0s_latency(u32 lnkcap) { + u32 encoding = (lnkcap & PCI_EXP_LNKCAP_L0SEL) >> 12; + if (encoding == 0x7) return (5 * 1000); /* > 4us */ return (64 << encoding); @@ -324,8 +318,10 @@ static u32 calc_l0s_acceptable(u32 encoding) } /* Convert L1 latency encoding to ns */ -static u32 calc_l1_latency(u32 encoding) +static u32 calc_l1_latency(u32 lnkcap) { + u32 encoding = (lnkcap & PCI_EXP_LNKCAP_L1EL) >> 15; + if (encoding == 0x7) return (65 * 1000); /* > 64us */ return (1000 << encoding); @@ -380,58 +376,6 @@ static void encode_l12_threshold(u32 threshold_us, u32 *scale, u32 *value) } } -struct aspm_register_info { - u32 support:2; - u32 enabled:2; - u32 latency_encoding_l0s; - u32 latency_encoding_l1; - - /* L1 substates */ - u32 l1ss_cap_ptr; - u32 l1ss_cap; - u32 l1ss_ctl1; - u32 l1ss_ctl2; -}; - -static void pcie_get_aspm_reg(struct pci_dev *pdev, - struct aspm_register_info *info) -{ - u16 reg16; - u32 reg32; - - pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, ®32); - info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10; - info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12; - info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15; - pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, ®16); - info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC; - - /* Read L1 PM substate capabilities */ - info->l1ss_cap = info->l1ss_ctl1 = info->l1ss_ctl2 = 0; - info->l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS); - if (!info->l1ss_cap_ptr) - return; - pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CAP, - &info->l1ss_cap); - if (!(info->l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) { - info->l1ss_cap = 0; - return; - } - - /* - * If we don't have LTR for the entire path from the Root Complex - * to this device, we can't use ASPM L1.2 because it relies on the - * LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18. - */ - if (!pdev->ltr_path) - info->l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2; - - pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL1, - &info->l1ss_ctl1); - pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL2, - &info->l1ss_ctl2); -} - static void pcie_aspm_check_latency(struct pci_dev *endpoint) { u32 latency, l1_switch_latency = 0; @@ -493,39 +437,49 @@ static struct pci_dev *pci_function_0(struct pci_bus *linkbus) return NULL; } +static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos, + u32 clear, u32 set) +{ + u32 val; + + pci_read_config_dword(pdev, pos, &val); + val &= ~clear; + val |= set; + pci_write_config_dword(pdev, pos, val); +} + /* Calculate L1.2 PM substate timing parameters */ static void aspm_calc_l1ss_info(struct pcie_link_state *link, - struct aspm_register_info *upreg, - struct aspm_register_info *dwreg) + u32 parent_l1ss_cap, u32 child_l1ss_cap) { + struct pci_dev *child = link->downstream, *parent = link->pdev; u32 val1, val2, scale1, scale2; u32 t_common_mode, t_power_on, l1_2_threshold, scale, value; - - link->l1ss.up_cap_ptr = upreg->l1ss_cap_ptr; - link->l1ss.dw_cap_ptr = dwreg->l1ss_cap_ptr; - link->l1ss.ctl1 = link->l1ss.ctl2 = 0; + u32 ctl1 = 0, ctl2 = 0; + u32 pctl1, pctl2, cctl1, cctl2; + u32 pl1_2_enables, cl1_2_enables; if (!(link->aspm_support & ASPM_STATE_L1_2_MASK)) return; /* Choose the greater of the two Port Common_Mode_Restore_Times */ - val1 = (upreg->l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8; - val2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8; + val1 = (parent_l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8; + val2 = (child_l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8; t_common_mode = max(val1, val2); /* Choose the greater of the two Port T_POWER_ON times */ - val1 = (upreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19; - scale1 = (upreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16; - val2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19; - scale2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16; - - if (calc_l1ss_pwron(link->pdev, scale1, val1) > - calc_l1ss_pwron(link->downstream, scale2, val2)) { - link->l1ss.ctl2 |= scale1 | (val1 << 3); - t_power_on = calc_l1ss_pwron(link->pdev, scale1, val1); + val1 = (parent_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19; + scale1 = (parent_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16; + val2 = (child_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19; + scale2 = (child_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16; + + if (calc_l1ss_pwron(parent, scale1, val1) > + calc_l1ss_pwron(child, scale2, val2)) { + ctl2 |= scale1 | (val1 << 3); + t_power_on = calc_l1ss_pwron(parent, scale1, val1); } else { - link->l1ss.ctl2 |= scale2 | (val2 << 3); - t_power_on = calc_l1ss_pwron(link->downstream, scale2, val2); + ctl2 |= scale2 | (val2 << 3); + t_power_on = calc_l1ss_pwron(child, scale2, val2); } /* @@ -540,14 +494,60 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link, */ l1_2_threshold = 2 + 4 + t_common_mode + t_power_on; encode_l12_threshold(l1_2_threshold, &scale, &value); - link->l1ss.ctl1 |= t_common_mode << 8 | scale << 29 | value << 16; + ctl1 |= t_common_mode << 8 | scale << 29 | value << 16; + + pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, &pctl1); + pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, &pctl2); + pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, &cctl1); + pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL2, &cctl2); + + if (ctl1 == pctl1 && ctl1 == cctl1 && + ctl2 == pctl2 && ctl2 == cctl2) + return; + + /* Disable L1.2 while updating. See PCIe r5.0, sec 5.5.4, 7.8.3.3 */ + pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK; + cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK; + + if (pl1_2_enables || cl1_2_enables) { + pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, + PCI_L1SS_CTL1_L1_2_MASK, 0); + pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, + PCI_L1SS_CTL1_L1_2_MASK, 0); + } + + /* Program T_POWER_ON times in both ports */ + pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2); + pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2); + + /* Program Common_Mode_Restore_Time in upstream device */ + pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, + PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1); + + /* Program LTR_L1.2_THRESHOLD time in both ports */ + pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, + PCI_L1SS_CTL1_LTR_L12_TH_VALUE | + PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1); + pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, + PCI_L1SS_CTL1_LTR_L12_TH_VALUE | + PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1); + + if (pl1_2_enables || cl1_2_enables) { + pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 0, + pl1_2_enables); + pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 0, + cl1_2_enables); + } } static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) { struct pci_dev *child = link->downstream, *parent = link->pdev; + u32 parent_lnkcap, child_lnkcap; + u16 parent_lnkctl, child_lnkctl; + u32 parent_l1ss_cap, child_l1ss_cap; + u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0; struct pci_bus *linkbus = parent->subordinate; - struct aspm_register_info upreg, dwreg; if (blacklist) { /* Set enabled/disable so that we will disable ASPM later */ @@ -556,26 +556,28 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) return; } - /* Get upstream/downstream components' register state */ - pcie_get_aspm_reg(parent, &upreg); - pcie_get_aspm_reg(child, &dwreg); - /* * If ASPM not supported, don't mess with the clocks and link, * bail out now. */ - if (!(upreg.support & dwreg.support)) + pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap); + pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap); + if (!(parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPMS)) return; /* Configure common clock before checking latencies */ pcie_aspm_configure_common_clock(link); /* - * Re-read upstream/downstream components' register state - * after clock configuration + * Re-read upstream/downstream components' register state after + * clock configuration. L0s & L1 exit latencies in the otherwise + * read-only Link Capabilities may change depending on common clock + * configuration (PCIe r5.0, sec 7.5.3.6). */ - pcie_get_aspm_reg(parent, &upreg); - pcie_get_aspm_reg(child, &dwreg); + pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap); + pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap); + pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &parent_lnkctl); + pcie_capability_read_word(child, PCI_EXP_LNKCTL, &child_lnkctl); /* * Setup L0s state @@ -584,44 +586,71 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) * given link unless components on both sides of the link each * support L0s. */ - if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S) + if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L0S) link->aspm_support |= ASPM_STATE_L0S; - if (dwreg.enabled & PCIE_LINK_STATE_L0S) + + if (child_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S) link->aspm_enabled |= ASPM_STATE_L0S_UP; - if (upreg.enabled & PCIE_LINK_STATE_L0S) + if (parent_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S) link->aspm_enabled |= ASPM_STATE_L0S_DW; - link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s); - link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s); + link->latency_up.l0s = calc_l0s_latency(parent_lnkcap); + link->latency_dw.l0s = calc_l0s_latency(child_lnkcap); /* Setup L1 state */ - if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1) + if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L1) link->aspm_support |= ASPM_STATE_L1; - if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1) + + if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1) link->aspm_enabled |= ASPM_STATE_L1; - link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1); - link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1); + link->latency_up.l1 = calc_l1_latency(parent_lnkcap); + link->latency_dw.l1 = calc_l1_latency(child_lnkcap); /* Setup L1 substate */ - if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1) + pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP, + &parent_l1ss_cap); + pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP, + &child_l1ss_cap); + + if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) + parent_l1ss_cap = 0; + if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) + child_l1ss_cap = 0; + + /* + * If we don't have LTR for the entire path from the Root Complex + * to this device, we can't use ASPM L1.2 because it relies on the + * LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18. + */ + if (!child->ltr_path) + child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2; + + if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1) link->aspm_support |= ASPM_STATE_L1_1; - if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2) + if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2) link->aspm_support |= ASPM_STATE_L1_2; - if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1) + if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1) link->aspm_support |= ASPM_STATE_L1_1_PCIPM; - if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2) + if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2) link->aspm_support |= ASPM_STATE_L1_2_PCIPM; - if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1) + if (parent_l1ss_cap) + pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, + &parent_l1ss_ctl1); + if (child_l1ss_cap) + pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, + &child_l1ss_ctl1); + + if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1) link->aspm_enabled |= ASPM_STATE_L1_1; - if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2) + if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2) link->aspm_enabled |= ASPM_STATE_L1_2; - if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1) + if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1) link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM; - if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2) + if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2) link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; if (link->aspm_support & ASPM_STATE_L1SS) - aspm_calc_l1ss_info(link, &upreg, &dwreg); + aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap); /* Save default state */ link->aspm_default = link->aspm_enabled; @@ -651,24 +680,11 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) } } -static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos, - u32 clear, u32 set) -{ - u32 val; - - pci_read_config_dword(pdev, pos, &val); - val &= ~clear; - val |= set; - pci_write_config_dword(pdev, pos, val); -} - /* Configure the ASPM L1 substates */ static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) { u32 val, enable_req; struct pci_dev *child = link->downstream, *parent = link->pdev; - u32 up_cap_ptr = link->l1ss.up_cap_ptr; - u32 dw_cap_ptr = link->l1ss.dw_cap_ptr; enable_req = (link->aspm_enabled ^ state) & state; @@ -686,9 +702,9 @@ static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) */ /* Disable all L1 substates */ - pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, + pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, PCI_L1SS_CTL1_L1SS_MASK, 0); - pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, + pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, PCI_L1SS_CTL1_L1SS_MASK, 0); /* * If needed, disable L1, and it gets enabled later @@ -701,30 +717,6 @@ static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) PCI_EXP_LNKCTL_ASPM_L1, 0); } - if (enable_req & ASPM_STATE_L1_2_MASK) { - - /* Program T_POWER_ON times in both ports */ - pci_write_config_dword(parent, up_cap_ptr + PCI_L1SS_CTL2, - link->l1ss.ctl2); - pci_write_config_dword(child, dw_cap_ptr + PCI_L1SS_CTL2, - link->l1ss.ctl2); - - /* Program Common_Mode_Restore_Time in upstream device */ - pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, - PCI_L1SS_CTL1_CM_RESTORE_TIME, - link->l1ss.ctl1); - - /* Program LTR_L1.2_THRESHOLD time in both ports */ - pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, - PCI_L1SS_CTL1_LTR_L12_TH_VALUE | - PCI_L1SS_CTL1_LTR_L12_TH_SCALE, - link->l1ss.ctl1); - pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, - PCI_L1SS_CTL1_LTR_L12_TH_VALUE | - PCI_L1SS_CTL1_LTR_L12_TH_SCALE, - link->l1ss.ctl1); - } - val = 0; if (state & ASPM_STATE_L1_1) val |= PCI_L1SS_CTL1_ASPM_L1_1; @@ -736,9 +728,9 @@ static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) val |= PCI_L1SS_CTL1_PCIPM_L1_2; /* Enable what we need to enable */ - pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, + pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, PCI_L1SS_CTL1_L1SS_MASK, val); - pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, + pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, PCI_L1SS_CTL1_L1SS_MASK, val); } diff --git a/drivers/pci/pcie/bw_notification.c b/drivers/pci/pcie/bw_notification.c index 77e685771487..565d23cccb8b 100644 --- a/drivers/pci/pcie/bw_notification.c +++ b/drivers/pci/pcie/bw_notification.c @@ -14,6 +14,8 @@ * and warns when links become degraded in operation. */ +#define dev_fmt(fmt) "bw_notification: " fmt + #include "../pci.h" #include "portdrv.h" @@ -97,6 +99,7 @@ static int pcie_bandwidth_notification_probe(struct pcie_device *srv) return ret; pcie_enable_link_bandwidth_notification(srv->port); + pci_info(srv->port, "enabled with IRQ %d\n", srv->irq); return 0; } diff --git a/drivers/pci/pcie/dpc.c b/drivers/pci/pcie/dpc.c index daa9a4153776..e05aba86a317 100644 --- a/drivers/pci/pcie/dpc.c +++ b/drivers/pci/pcie/dpc.c @@ -103,7 +103,8 @@ pci_ers_result_t dpc_reset_link(struct pci_dev *pdev) * Wait until the Link is inactive, then clear DPC Trigger Status * to allow the Port to leave DPC. */ - pcie_wait_for_link(pdev, false); + if (!pcie_wait_for_link(pdev, false)) + pci_info(pdev, "Data Link Layer Link Active not cleared in 1000 msec\n"); if (pdev->dpc_rp_extensions && dpc_wait_rp_inactive(pdev)) return PCI_ERS_RESULT_DISCONNECT; @@ -111,8 +112,10 @@ pci_ers_result_t dpc_reset_link(struct pci_dev *pdev) pci_write_config_word(pdev, cap + PCI_EXP_DPC_STATUS, PCI_EXP_DPC_STATUS_TRIGGER); - if (!pcie_wait_for_link(pdev, true)) + if (!pcie_wait_for_link(pdev, true)) { + pci_info(pdev, "Data Link Layer Link Active not set in 1000 msec\n"); return PCI_ERS_RESULT_DISCONNECT; + } return PCI_ERS_RESULT_RECOVERED; } diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 03d37128a24f..4289030b0fff 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -941,6 +941,12 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge) pcibios_add_bus(bus); + if (bus->ops->add_bus) { + err = bus->ops->add_bus(bus); + if (WARN_ON(err < 0)) + dev_err(&bus->dev, "failed to add bus: %d\n", err); + } + /* Create legacy_io and legacy_mem files for this bus */ pci_create_legacy_files(bus); @@ -1036,6 +1042,7 @@ static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent, struct pci_dev *bridge, int busnr) { struct pci_bus *child; + struct pci_host_bridge *host; int i; int ret; @@ -1045,11 +1052,16 @@ static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent, return NULL; child->parent = parent; - child->ops = parent->ops; child->msi = parent->msi; child->sysdata = parent->sysdata; child->bus_flags = parent->bus_flags; + host = pci_find_host_bridge(parent); + if (host->child_ops) + child->ops = host->child_ops; + else + child->ops = parent->ops; + /* * Initialize some portions of the bus device, but don't register * it now as the parent is not properly set up yet. @@ -2106,6 +2118,9 @@ static void pci_configure_ltr(struct pci_dev *dev) if (!pci_is_pcie(dev)) return; + /* Read L1 PM substate capabilities */ + dev->l1ss = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_L1SS); + pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, &cap); if (!(cap & PCI_EXP_DEVCAP2_LTR)) return; diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 01f23e30bd8f..f70692ac79c5 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -1846,7 +1846,7 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PXHV, quirk_pci */ static void quirk_intel_pcie_pm(struct pci_dev *dev) { - pci_pm_d3_delay = 120; + pci_pm_d3hot_delay = 120; dev->no_d1d2 = 1; } DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x25e2, quirk_intel_pcie_pm); @@ -1873,12 +1873,12 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x260b, quirk_intel_pcie_pm); static void quirk_d3hot_delay(struct pci_dev *dev, unsigned int delay) { - if (dev->d3_delay >= delay) + if (dev->d3hot_delay >= delay) return; - dev->d3_delay = delay; + dev->d3hot_delay = delay; pci_info(dev, "extending delay after power-on from D3hot to %d msec\n", - dev->d3_delay); + dev->d3hot_delay); } static void quirk_radeon_pm(struct pci_dev *dev) @@ -3387,36 +3387,36 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x0152, disable_igfx_irq); * PCI devices which are on Intel chips can skip the 10ms delay * before entering D3 mode. */ -static void quirk_remove_d3_delay(struct pci_dev *dev) -{ - dev->d3_delay = 0; -} -/* C600 Series devices do not need 10ms d3_delay */ -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x0412, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x0c00, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x0c0c, quirk_remove_d3_delay); -/* Lynxpoint-H PCH devices do not need 10ms d3_delay */ -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c02, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c18, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c1c, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c20, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c22, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c26, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c2d, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c31, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c3a, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c3d, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c4e, quirk_remove_d3_delay); -/* Intel Cherrytrail devices do not need 10ms d3_delay */ -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x2280, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x2298, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x229c, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22b0, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22b5, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22b7, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22b8, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22d8, quirk_remove_d3_delay); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22dc, quirk_remove_d3_delay); +static void quirk_remove_d3hot_delay(struct pci_dev *dev) +{ + dev->d3hot_delay = 0; +} +/* C600 Series devices do not need 10ms d3hot_delay */ +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x0412, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x0c00, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x0c0c, quirk_remove_d3hot_delay); +/* Lynxpoint-H PCH devices do not need 10ms d3hot_delay */ +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c02, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c18, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c1c, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c20, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c22, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c26, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c2d, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c31, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c3a, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c3d, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x8c4e, quirk_remove_d3hot_delay); +/* Intel Cherrytrail devices do not need 10ms d3hot_delay */ +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x2280, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x2298, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x229c, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22b0, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22b5, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22b7, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22b8, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22d8, quirk_remove_d3hot_delay); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22dc, quirk_remove_d3hot_delay); /* * Some devices may pass our check in pci_intx_mask_supported() if @@ -4892,6 +4892,13 @@ static void pci_quirk_enable_intel_rp_mpc_acs(struct pci_dev *dev) } } +/* + * Currently this quirk does the equivalent of + * PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF + * + * TODO: This quirk also needs to do equivalent of PCI_ACS_TB, + * if dev->external_facing || dev->untrusted + */ static int pci_quirk_enable_intel_pch_acs(struct pci_dev *dev) { if (!pci_quirk_intel_pch_acs_match(dev)) @@ -4931,6 +4938,9 @@ static int pci_quirk_enable_intel_spt_pch_acs(struct pci_dev *dev) ctrl |= (cap & PCI_ACS_CR); ctrl |= (cap & PCI_ACS_UF); + if (dev->external_facing || dev->untrusted) + ctrl |= (cap & PCI_ACS_TB); + pci_write_config_dword(dev, pos + INTEL_SPT_ACS_CTRL, ctrl); pci_info(dev, "Intel SPT PCH root port ACS workaround enabled\n"); diff --git a/drivers/phy/marvell/phy-mvebu-a3700-comphy.c b/drivers/phy/marvell/phy-mvebu-a3700-comphy.c index 1a138be8bd6a..810f25a47632 100644 --- a/drivers/phy/marvell/phy-mvebu-a3700-comphy.c +++ b/drivers/phy/marvell/phy-mvebu-a3700-comphy.c @@ -26,7 +26,6 @@ #define COMPHY_SIP_POWER_ON 0x82000001 #define COMPHY_SIP_POWER_OFF 0x82000002 #define COMPHY_SIP_PLL_LOCK 0x82000003 -#define COMPHY_FW_NOT_SUPPORTED (-1) #define COMPHY_FW_MODE_SATA 0x1 #define COMPHY_FW_MODE_SGMII 0x2 @@ -112,10 +111,19 @@ static int mvebu_a3700_comphy_smc(unsigned long function, unsigned long lane, unsigned long mode) { struct arm_smccc_res res; + s32 ret; arm_smccc_smc(function, lane, mode, 0, 0, 0, 0, 0, &res); + ret = res.a0; - return res.a0; + switch (ret) { + case SMCCC_RET_SUCCESS: + return 0; + case SMCCC_RET_NOT_SUPPORTED: + return -EOPNOTSUPP; + default: + return -EINVAL; + } } static int mvebu_a3700_comphy_get_fw_mode(int lane, int port, @@ -220,7 +228,7 @@ static int mvebu_a3700_comphy_power_on(struct phy *phy) } ret = mvebu_a3700_comphy_smc(COMPHY_SIP_POWER_ON, lane->id, fw_param); - if (ret == COMPHY_FW_NOT_SUPPORTED) + if (ret == -EOPNOTSUPP) dev_err(lane->dev, "unsupported SMC call, try updating your firmware\n"); diff --git a/drivers/phy/marvell/phy-mvebu-cp110-comphy.c b/drivers/phy/marvell/phy-mvebu-cp110-comphy.c index e41367f36ee1..53ad127b100f 100644 --- a/drivers/phy/marvell/phy-mvebu-cp110-comphy.c +++ b/drivers/phy/marvell/phy-mvebu-cp110-comphy.c @@ -123,7 +123,6 @@ #define COMPHY_SIP_POWER_ON 0x82000001 #define COMPHY_SIP_POWER_OFF 0x82000002 -#define COMPHY_FW_NOT_SUPPORTED (-1) /* * A lane is described by the following bitfields: @@ -273,10 +272,19 @@ static int mvebu_comphy_smc(unsigned long function, unsigned long phys, unsigned long lane, unsigned long mode) { struct arm_smccc_res res; + s32 ret; arm_smccc_smc(function, phys, lane, mode, 0, 0, 0, 0, &res); + ret = res.a0; - return res.a0; + switch (ret) { + case SMCCC_RET_SUCCESS: + return 0; + case SMCCC_RET_NOT_SUPPORTED: + return -EOPNOTSUPP; + default: + return -EINVAL; + } } static int mvebu_comphy_get_mode(bool fw_mode, int lane, int port, @@ -819,7 +827,7 @@ static int mvebu_comphy_power_on(struct phy *phy) if (!ret) return ret; - if (ret == COMPHY_FW_NOT_SUPPORTED) + if (ret == -EOPNOTSUPP) dev_err(priv->dev, "unsupported SMC call, try updating your firmware\n"); diff --git a/drivers/platform/chrome/Kconfig b/drivers/platform/chrome/Kconfig index a056031dee81..ccc23d8686e8 100644 --- a/drivers/platform/chrome/Kconfig +++ b/drivers/platform/chrome/Kconfig @@ -3,16 +3,6 @@ # Platform support for Chrome OS hardware (Chromebooks and Chromeboxes) # -config MFD_CROS_EC - tristate "Platform support for Chrome hardware (transitional)" - select CHROME_PLATFORMS - select CROS_EC - select MFD_CROS_EC_DEV - depends on X86 || ARM || ARM64 || COMPILE_TEST - help - This is a transitional Kconfig option and will be removed after - everyone enables the parts individually. - menuconfig CHROME_PLATFORMS bool "Platform support for Chrome hardware" depends on X86 || ARM || ARM64 || COMPILE_TEST diff --git a/drivers/platform/chrome/cros_ec_lightbar.c b/drivers/platform/chrome/cros_ec_lightbar.c index b59180bff5a3..de8dfb12e486 100644 --- a/drivers/platform/chrome/cros_ec_lightbar.c +++ b/drivers/platform/chrome/cros_ec_lightbar.c @@ -116,8 +116,10 @@ static int get_lightbar_version(struct cros_ec_dev *ec, param = (struct ec_params_lightbar *)msg->data; param->cmd = LIGHTBAR_CMD_VERSION; + msg->outsize = sizeof(param->cmd); + msg->result = sizeof(resp->version); ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); - if (ret < 0) { + if (ret < 0 && ret != -EINVAL) { ret = 0; goto exit; } @@ -298,11 +300,9 @@ static ssize_t sequence_show(struct device *dev, goto exit; ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); - if (ret == -EPROTO) { - ret = scnprintf(buf, PAGE_SIZE, - "ERROR: EC returned %d\n", msg->result); - goto exit; - } else if (ret < 0) { + if (ret < 0) { + ret = scnprintf(buf, PAGE_SIZE, "XFER / EC ERROR %d / %d\n", + ret, msg->result); goto exit; } diff --git a/drivers/platform/chrome/cros_ec_proto.c b/drivers/platform/chrome/cros_ec_proto.c index 8d52b3b4bd4e..0ecee8b8773d 100644 --- a/drivers/platform/chrome/cros_ec_proto.c +++ b/drivers/platform/chrome/cros_ec_proto.c @@ -15,6 +15,43 @@ #define EC_COMMAND_RETRIES 50 +static const int cros_ec_error_map[] = { + [EC_RES_INVALID_COMMAND] = -EOPNOTSUPP, + [EC_RES_ERROR] = -EIO, + [EC_RES_INVALID_PARAM] = -EINVAL, + [EC_RES_ACCESS_DENIED] = -EACCES, + [EC_RES_INVALID_RESPONSE] = -EPROTO, + [EC_RES_INVALID_VERSION] = -ENOPROTOOPT, + [EC_RES_INVALID_CHECKSUM] = -EBADMSG, + [EC_RES_IN_PROGRESS] = -EINPROGRESS, + [EC_RES_UNAVAILABLE] = -ENODATA, + [EC_RES_TIMEOUT] = -ETIMEDOUT, + [EC_RES_OVERFLOW] = -EOVERFLOW, + [EC_RES_INVALID_HEADER] = -EBADR, + [EC_RES_REQUEST_TRUNCATED] = -EBADR, + [EC_RES_RESPONSE_TOO_BIG] = -EFBIG, + [EC_RES_BUS_ERROR] = -EFAULT, + [EC_RES_BUSY] = -EBUSY, + [EC_RES_INVALID_HEADER_VERSION] = -EBADMSG, + [EC_RES_INVALID_HEADER_CRC] = -EBADMSG, + [EC_RES_INVALID_DATA_CRC] = -EBADMSG, + [EC_RES_DUP_UNAVAILABLE] = -ENODATA, +}; + +static int cros_ec_map_error(uint32_t result) +{ + int ret = 0; + + if (result != EC_RES_SUCCESS) { + if (result < ARRAY_SIZE(cros_ec_error_map) && cros_ec_error_map[result]) + ret = cros_ec_error_map[result]; + else + ret = -EPROTO; + } + + return ret; +} + static int prepare_packet(struct cros_ec_device *ec_dev, struct cros_ec_command *msg) { @@ -512,19 +549,22 @@ exit: EXPORT_SYMBOL(cros_ec_query_all); /** - * cros_ec_cmd_xfer() - Send a command to the ChromeOS EC. + * cros_ec_cmd_xfer_status() - Send a command to the ChromeOS EC. * @ec_dev: EC device. * @msg: Message to write. * - * Call this to send a command to the ChromeOS EC. This should be used - * instead of calling the EC's cmd_xfer() callback directly. + * Call this to send a command to the ChromeOS EC. This should be used instead of calling the EC's + * cmd_xfer() callback directly. It returns success status only if both the command was transmitted + * successfully and the EC replied with success status. * - * Return: 0 on success or negative error code. + * Return: + * >=0 - The number of bytes transferred + * <0 - Linux error code */ -static int cros_ec_cmd_xfer(struct cros_ec_device *ec_dev, +int cros_ec_cmd_xfer_status(struct cros_ec_device *ec_dev, struct cros_ec_command *msg) { - int ret; + int ret, mapped; mutex_lock(&ec_dev->lock); if (ec_dev->proto_version == EC_PROTO_VERSION_UNKNOWN) { @@ -561,42 +601,15 @@ static int cros_ec_cmd_xfer(struct cros_ec_device *ec_dev, return -EMSGSIZE; } } + ret = send_command(ec_dev, msg); mutex_unlock(&ec_dev->lock); - return ret; -} - -/** - * cros_ec_cmd_xfer_status() - Send a command to the ChromeOS EC. - * @ec_dev: EC device. - * @msg: Message to write. - * - * This function is identical to cros_ec_cmd_xfer, except it returns success - * status only if both the command was transmitted successfully and the EC - * replied with success status. It's not necessary to check msg->result when - * using this function. - * - * Return: - * >=0 - The number of bytes transferred - * -ENOTSUPP - Operation not supported - * -EPROTO - Protocol error - */ -int cros_ec_cmd_xfer_status(struct cros_ec_device *ec_dev, - struct cros_ec_command *msg) -{ - int ret; - - ret = cros_ec_cmd_xfer(ec_dev, msg); - if (ret < 0) { - dev_err(ec_dev->dev, "Command xfer error (err:%d)\n", ret); - } else if (msg->result == EC_RES_INVALID_VERSION) { - dev_dbg(ec_dev->dev, "Command invalid version (err:%d)\n", - msg->result); - return -ENOTSUPP; - } else if (msg->result != EC_RES_SUCCESS) { - dev_dbg(ec_dev->dev, "Command result (err: %d)\n", msg->result); - return -EPROTO; + mapped = cros_ec_map_error(msg->result); + if (mapped) { + dev_dbg(ec_dev->dev, "Command result (err: %d [%d])\n", + msg->result, mapped); + ret = mapped; } return ret; @@ -615,7 +628,7 @@ static int get_next_event_xfer(struct cros_ec_device *ec_dev, msg->insize = size; msg->outsize = 0; - ret = cros_ec_cmd_xfer(ec_dev, msg); + ret = cros_ec_cmd_xfer_status(ec_dev, msg); if (ret > 0) { ec_dev->event_size = ret - 1; ec_dev->event_data = *event; @@ -659,7 +672,7 @@ static int get_keyboard_state_event(struct cros_ec_device *ec_dev) msg->insize = sizeof(ec_dev->event_data.data); msg->outsize = 0; - ec_dev->event_size = cros_ec_cmd_xfer(ec_dev, msg); + ec_dev->event_size = cros_ec_cmd_xfer_status(ec_dev, msg); ec_dev->event_data.event_type = EC_MKBP_EVENT_KEY_MATRIX; memcpy(&ec_dev->event_data.data, msg->data, sizeof(ec_dev->event_data.data)); @@ -848,11 +861,9 @@ int cros_ec_get_sensor_count(struct cros_ec_dev *ec) params = (struct ec_params_motion_sense *)msg->data; params->cmd = MOTIONSENSE_CMD_DUMP; - ret = cros_ec_cmd_xfer(ec->ec_dev, msg); + ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); if (ret < 0) { sensor_count = ret; - } else if (msg->result != EC_RES_SUCCESS) { - sensor_count = -EPROTO; } else { resp = (struct ec_response_motion_sense *)msg->data; sensor_count = resp->dump.sensor_count; @@ -863,9 +874,7 @@ int cros_ec_get_sensor_count(struct cros_ec_dev *ec) * Check legacy mode: Let's find out if sensors are accessible * via LPC interface. */ - if (sensor_count == -EPROTO && - ec->cmd_offset == 0 && - ec_dev->cmd_readmem) { + if (sensor_count < 0 && ec->cmd_offset == 0 && ec_dev->cmd_readmem) { ret = ec_dev->cmd_readmem(ec_dev, EC_MEMMAP_ACC_STATUS, 1, &status); if (ret >= 0 && @@ -880,9 +889,6 @@ int cros_ec_get_sensor_count(struct cros_ec_dev *ec) */ sensor_count = 0; } - } else if (sensor_count == -EPROTO) { - /* EC responded, but does not understand DUMP command. */ - sensor_count = 0; } return sensor_count; } diff --git a/drivers/platform/chrome/cros_ec_sysfs.c b/drivers/platform/chrome/cros_ec_sysfs.c index d45ea5d5bfa4..f521a5c65091 100644 --- a/drivers/platform/chrome/cros_ec_sysfs.c +++ b/drivers/platform/chrome/cros_ec_sysfs.c @@ -150,12 +150,10 @@ static ssize_t version_show(struct device *dev, msg->command = EC_CMD_GET_BUILD_INFO + ec->cmd_offset; msg->insize = EC_HOST_PARAM_SIZE; ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); - if (ret == -EPROTO) { - count += scnprintf(buf + count, PAGE_SIZE - count, - "Build info: EC error %d\n", msg->result); - } else if (ret < 0) { + if (ret < 0) { count += scnprintf(buf + count, PAGE_SIZE - count, - "Build info: XFER ERROR %d\n", ret); + "Build info: XFER / EC ERROR %d / %d\n", + ret, msg->result); } else { msg->data[EC_HOST_PARAM_SIZE - 1] = '\0'; count += scnprintf(buf + count, PAGE_SIZE - count, @@ -166,12 +164,10 @@ static ssize_t version_show(struct device *dev, msg->command = EC_CMD_GET_CHIP_INFO + ec->cmd_offset; msg->insize = sizeof(*r_chip); ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); - if (ret == -EPROTO) { - count += scnprintf(buf + count, PAGE_SIZE - count, - "Chip info: EC error %d\n", msg->result); - } else if (ret < 0) { + if (ret < 0) { count += scnprintf(buf + count, PAGE_SIZE - count, - "Chip info: XFER ERROR %d\n", ret); + "Chip info: XFER / EC ERROR %d / %d\n", + ret, msg->result); } else { r_chip = (struct ec_response_get_chip_info *)msg->data; @@ -190,12 +186,10 @@ static ssize_t version_show(struct device *dev, msg->command = EC_CMD_GET_BOARD_VERSION + ec->cmd_offset; msg->insize = sizeof(*r_board); ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); - if (ret == -EPROTO) { - count += scnprintf(buf + count, PAGE_SIZE - count, - "Board version: EC error %d\n", msg->result); - } else if (ret < 0) { + if (ret < 0) { count += scnprintf(buf + count, PAGE_SIZE - count, - "Board version: XFER ERROR %d\n", ret); + "Board version: XFER / EC ERROR %d / %d\n", + ret, msg->result); } else { r_board = (struct ec_response_board_version *)msg->data; @@ -326,7 +320,7 @@ static struct attribute *__ec_attrs[] = { static umode_t cros_ec_ctrl_visible(struct kobject *kobj, struct attribute *a, int n) { - struct device *dev = container_of(kobj, struct device, kobj); + struct device *dev = kobj_to_dev(kobj); struct cros_ec_dev *ec = to_cros_ec_dev(dev); if (a == &dev_attr_kb_wake_angle.attr && !ec->has_kb_wake_angle) diff --git a/drivers/platform/chrome/cros_ec_trace.h b/drivers/platform/chrome/cros_ec_trace.h index e9fb05f89ef0..f744b21bc655 100644 --- a/drivers/platform/chrome/cros_ec_trace.h +++ b/drivers/platform/chrome/cros_ec_trace.h @@ -23,14 +23,22 @@ TRACE_EVENT(cros_ec_request_start, TP_ARGS(cmd), TP_STRUCT__entry( __field(uint32_t, version) + __field(uint32_t, offset) __field(uint32_t, command) + __field(uint32_t, outsize) + __field(uint32_t, insize) ), TP_fast_assign( __entry->version = cmd->version; - __entry->command = cmd->command; + __entry->offset = cmd->command / EC_CMD_PASSTHRU_OFFSET(1); + __entry->command = cmd->command % EC_CMD_PASSTHRU_OFFSET(1); + __entry->outsize = cmd->outsize; + __entry->insize = cmd->insize; ), - TP_printk("version: %u, command: %s", __entry->version, - __print_symbolic(__entry->command, EC_CMDS)) + TP_printk("version: %u, offset: %d, command: %s, outsize: %u, insize: %u", + __entry->version, __entry->offset, + __print_symbolic(__entry->command, EC_CMDS), + __entry->outsize, __entry->insize) ); TRACE_EVENT(cros_ec_request_done, @@ -38,19 +46,26 @@ TRACE_EVENT(cros_ec_request_done, TP_ARGS(cmd, retval), TP_STRUCT__entry( __field(uint32_t, version) + __field(uint32_t, offset) __field(uint32_t, command) + __field(uint32_t, outsize) + __field(uint32_t, insize) __field(uint32_t, result) __field(int, retval) ), TP_fast_assign( __entry->version = cmd->version; - __entry->command = cmd->command; + __entry->offset = cmd->command / EC_CMD_PASSTHRU_OFFSET(1); + __entry->command = cmd->command % EC_CMD_PASSTHRU_OFFSET(1); + __entry->outsize = cmd->outsize; + __entry->insize = cmd->insize; __entry->result = cmd->result; __entry->retval = retval; ), - TP_printk("version: %u, command: %s, ec result: %s, retval: %d", - __entry->version, + TP_printk("version: %u, offset: %d, command: %s, outsize: %u, insize: %u, ec result: %s, retval: %u", + __entry->version, __entry->offset, __print_symbolic(__entry->command, EC_CMDS), + __entry->outsize, __entry->insize, __print_symbolic(__entry->result, EC_RESULT), __entry->retval) ); diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c index 3fcd27ec9ad8..31be31161350 100644 --- a/drivers/platform/chrome/cros_ec_typec.c +++ b/drivers/platform/chrome/cros_ec_typec.c @@ -13,6 +13,7 @@ #include <linux/platform_data/cros_ec_proto.h> #include <linux/platform_data/cros_usbpd_notify.h> #include <linux/platform_device.h> +#include <linux/usb/pd.h> #include <linux/usb/typec.h> #include <linux/usb/typec_altmode.h> #include <linux/usb/typec_dp.h> @@ -496,6 +497,34 @@ static int cros_typec_enable_dp(struct cros_typec_data *typec, return typec_mux_set(port->mux, &port->state); } +static int cros_typec_enable_usb4(struct cros_typec_data *typec, + int port_num, + struct ec_response_usb_pd_control_v2 *pd_ctrl) +{ + struct cros_typec_port *port = typec->ports[port_num]; + struct enter_usb_data data; + + data.eudo = EUDO_USB_MODE_USB4 << EUDO_USB_MODE_SHIFT; + + /* Cable Speed */ + data.eudo |= pd_ctrl->cable_speed << EUDO_CABLE_SPEED_SHIFT; + + /* Cable Type */ + if (pd_ctrl->control_flags & USB_PD_CTRL_OPTICAL_CABLE) + data.eudo |= EUDO_CABLE_TYPE_OPTICAL << EUDO_CABLE_TYPE_SHIFT; + else if (pd_ctrl->control_flags & USB_PD_CTRL_ACTIVE_CABLE) + data.eudo |= EUDO_CABLE_TYPE_RE_TIMER << EUDO_CABLE_TYPE_SHIFT; + + data.active_link_training = !!(pd_ctrl->control_flags & + USB_PD_CTRL_ACTIVE_LINK_UNIDIR); + + port->state.alt = NULL; + port->state.data = &data; + port->state.mode = TYPEC_MODE_USB4; + + return typec_mux_set(port->mux, &port->state); +} + static int cros_typec_configure_mux(struct cros_typec_data *typec, int port_num, uint8_t mux_flags, struct ec_response_usb_pd_control_v2 *pd_ctrl) @@ -516,7 +545,15 @@ static int cros_typec_configure_mux(struct cros_typec_data *typec, int port_num, if (ret) return ret; - if (mux_flags & USB_PD_MUX_TBT_COMPAT_ENABLED) { + ret = usb_role_switch_set_role(typec->ports[port_num]->role_sw, + pd_ctrl->role & PD_CTRL_RESP_ROLE_DATA + ? USB_ROLE_HOST : USB_ROLE_DEVICE); + if (ret) + return ret; + + if (mux_flags & USB_PD_MUX_USB4_ENABLED) { + ret = cros_typec_enable_usb4(typec, port_num, pd_ctrl); + } else if (mux_flags & USB_PD_MUX_TBT_COMPAT_ENABLED) { ret = cros_typec_enable_tbt(typec, port_num, pd_ctrl); } else if (mux_flags & USB_PD_MUX_DP_ENABLED) { ret = cros_typec_enable_dp(typec, port_num, pd_ctrl); @@ -590,8 +627,7 @@ static int cros_typec_port_update(struct cros_typec_data *typec, int port_num) if (ret) dev_warn(typec->dev, "Configure muxes failed, err = %d\n", ret); - return usb_role_switch_set_role(typec->ports[port_num]->role_sw, - !!(resp.role & PD_CTRL_RESP_ROLE_DATA)); + return ret; } static int cros_typec_get_cmd_version(struct cros_typec_data *typec) diff --git a/drivers/platform/chrome/cros_ec_vbc.c b/drivers/platform/chrome/cros_ec_vbc.c index 46482d12cffe..f3a70a312b43 100644 --- a/drivers/platform/chrome/cros_ec_vbc.c +++ b/drivers/platform/chrome/cros_ec_vbc.c @@ -17,7 +17,7 @@ static ssize_t vboot_context_read(struct file *filp, struct kobject *kobj, struct bin_attribute *att, char *buf, loff_t pos, size_t count) { - struct device *dev = container_of(kobj, struct device, kobj); + struct device *dev = kobj_to_dev(kobj); struct cros_ec_dev *ec = to_cros_ec_dev(dev); struct cros_ec_device *ecdev = ec->ec_dev; struct ec_params_vbnvcontext *params; @@ -57,7 +57,7 @@ static ssize_t vboot_context_write(struct file *filp, struct kobject *kobj, struct bin_attribute *attr, char *buf, loff_t pos, size_t count) { - struct device *dev = container_of(kobj, struct device, kobj); + struct device *dev = kobj_to_dev(kobj); struct cros_ec_dev *ec = to_cros_ec_dev(dev); struct cros_ec_device *ecdev = ec->ec_dev; struct ec_params_vbnvcontext *params; diff --git a/drivers/platform/x86/thinkpad_acpi.c b/drivers/platform/x86/thinkpad_acpi.c index eae3579f106f..e3810675090a 100644 --- a/drivers/platform/x86/thinkpad_acpi.c +++ b/drivers/platform/x86/thinkpad_acpi.c @@ -1913,6 +1913,10 @@ enum { /* hot key scan codes (derived from ACPI DSDT) */ TP_ACPI_HOTKEYSCAN_CALCULATOR, TP_ACPI_HOTKEYSCAN_BLUETOOTH, TP_ACPI_HOTKEYSCAN_KEYBOARD, + TP_ACPI_HOTKEYSCAN_FN_RIGHT_SHIFT, /* Used by "Lenovo Quick Clean" */ + TP_ACPI_HOTKEYSCAN_NOTIFICATION_CENTER, + TP_ACPI_HOTKEYSCAN_PICKUP_PHONE, + TP_ACPI_HOTKEYSCAN_HANGUP_PHONE, /* Hotkey keymap size */ TPACPI_HOTKEY_MAP_LEN @@ -3429,11 +3433,15 @@ static int __init hotkey_init(struct ibm_init_struct *iibm) KEY_UNKNOWN, KEY_UNKNOWN, KEY_UNKNOWN, KEY_UNKNOWN, KEY_UNKNOWN, - KEY_BOOKMARKS, /* Favorite app, 0x311 */ - KEY_RESERVED, /* Clipping tool */ - KEY_CALC, /* Calculator (above numpad, P52) */ - KEY_BLUETOOTH, /* Bluetooth */ - KEY_KEYBOARD /* Keyboard, 0x315 */ + KEY_BOOKMARKS, /* Favorite app, 0x311 */ + KEY_SELECTIVE_SCREENSHOT, /* Clipping tool */ + KEY_CALC, /* Calculator (above numpad, P52) */ + KEY_BLUETOOTH, /* Bluetooth */ + KEY_KEYBOARD, /* Keyboard, 0x315 */ + KEY_FN_RIGHT_SHIFT, /* Fn + right Shift */ + KEY_NOTIFICATION_CENTER, /* Notification Center */ + KEY_PICKUP_PHONE, /* Answer incoming call */ + KEY_HANGUP_PHONE, /* Decline incoming call */ }, }; diff --git a/drivers/power/Kconfig b/drivers/power/Kconfig index ff0350ca3b74..696bf77a7042 100644 --- a/drivers/power/Kconfig +++ b/drivers/power/Kconfig @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -source "drivers/power/avs/Kconfig" source "drivers/power/reset/Kconfig" source "drivers/power/supply/Kconfig" diff --git a/drivers/power/Makefile b/drivers/power/Makefile index b7c2e372186b..effbf0377f32 100644 --- a/drivers/power/Makefile +++ b/drivers/power/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_POWER_AVS) += avs/ obj-$(CONFIG_POWER_RESET) += reset/ obj-$(CONFIG_POWER_SUPPLY) += supply/ diff --git a/drivers/power/avs/Kconfig b/drivers/power/avs/Kconfig deleted file mode 100644 index cdb4237bfd02..000000000000 --- a/drivers/power/avs/Kconfig +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -menuconfig POWER_AVS - bool "Adaptive Voltage Scaling class support" - help - AVS is a power management technique which finely controls the - operating voltage of a device in order to optimize (i.e. reduce) - its power consumption. - At a given operating point the voltage is adapted depending on - static factors (chip manufacturing process) and dynamic factors - (temperature depending performance). - AVS is also called SmartReflex on OMAP devices. - - Say Y here to enable Adaptive Voltage Scaling class support. - -config QCOM_CPR - tristate "QCOM Core Power Reduction (CPR) support" - depends on POWER_AVS && HAS_IOMEM - select PM_OPP - select REGMAP - help - Say Y here to enable support for the CPR hardware found on Qualcomm - SoCs like QCS404. - - This driver populates CPU OPPs tables and makes adjustments to the - tables based on feedback from the CPR hardware. If you want to do - CPUfrequency scaling say Y here. - - To compile this driver as a module, choose M here: the module will - be called qcom-cpr - -config ROCKCHIP_IODOMAIN - tristate "Rockchip IO domain support" - depends on POWER_AVS && ARCH_ROCKCHIP && OF - help - Say y here to enable support io domains on Rockchip SoCs. It is - necessary for the io domain setting of the SoC to match the - voltage supplied by the regulators. diff --git a/drivers/power/avs/Makefile b/drivers/power/avs/Makefile deleted file mode 100644 index 9007d05853e2..000000000000 --- a/drivers/power/avs/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_POWER_AVS_OMAP) += smartreflex.o -obj-$(CONFIG_QCOM_CPR) += qcom-cpr.o -obj-$(CONFIG_ROCKCHIP_IODOMAIN) += rockchip-io-domain.o diff --git a/drivers/power/reset/Kconfig b/drivers/power/reset/Kconfig index 0a1fb5c74f83..d55b3727e00e 100644 --- a/drivers/power/reset/Kconfig +++ b/drivers/power/reset/Kconfig @@ -129,10 +129,10 @@ config POWER_RESET_QCOM_PON config POWER_RESET_OCELOT_RESET bool "Microsemi Ocelot reset driver" - depends on MSCC_OCELOT || COMPILE_TEST + depends on MSCC_OCELOT || ARCH_SPARX5 || COMPILE_TEST select MFD_SYSCON help - This driver supports restart for Microsemi Ocelot SoC. + This driver supports restart for Microsemi Ocelot SoC and similar. config POWER_RESET_OXNAS bool "OXNAS SoC restart driver" diff --git a/drivers/power/reset/ocelot-reset.c b/drivers/power/reset/ocelot-reset.c index 419952c61fd0..f74e1dbb4ba3 100644 --- a/drivers/power/reset/ocelot-reset.c +++ b/drivers/power/reset/ocelot-reset.c @@ -15,15 +15,20 @@ #include <linux/reboot.h> #include <linux/regmap.h> +struct reset_props { + const char *syscon; + u32 protect_reg; + u32 vcore_protect; + u32 if_si_owner_bit; +}; + struct ocelot_reset_context { void __iomem *base; struct regmap *cpu_ctrl; + const struct reset_props *props; struct notifier_block restart_handler; }; -#define ICPU_CFG_CPU_SYSTEM_CTRL_RESET 0x20 -#define CORE_RST_PROTECT BIT(2) - #define SOFT_CHIP_RST BIT(0) #define ICPU_CFG_CPU_SYSTEM_CTRL_GENERAL_CTRL 0x24 @@ -31,7 +36,6 @@ struct ocelot_reset_context { #define IF_SI_OWNER_SISL 0 #define IF_SI_OWNER_SIBM 1 #define IF_SI_OWNER_SIMC 2 -#define IF_SI_OWNER_OFFSET 4 static int ocelot_restart_handle(struct notifier_block *this, unsigned long mode, void *cmd) @@ -39,15 +43,18 @@ static int ocelot_restart_handle(struct notifier_block *this, struct ocelot_reset_context *ctx = container_of(this, struct ocelot_reset_context, restart_handler); + u32 if_si_owner_bit = ctx->props->if_si_owner_bit; /* Make sure the core is not protected from reset */ - regmap_update_bits(ctx->cpu_ctrl, ICPU_CFG_CPU_SYSTEM_CTRL_RESET, - CORE_RST_PROTECT, 0); + regmap_update_bits(ctx->cpu_ctrl, ctx->props->protect_reg, + ctx->props->vcore_protect, 0); /* Make the SI back to boot mode */ regmap_update_bits(ctx->cpu_ctrl, ICPU_CFG_CPU_SYSTEM_CTRL_GENERAL_CTRL, - IF_SI_OWNER_MASK << IF_SI_OWNER_OFFSET, - IF_SI_OWNER_SIBM << IF_SI_OWNER_OFFSET); + IF_SI_OWNER_MASK << if_si_owner_bit, + IF_SI_OWNER_SIBM << if_si_owner_bit); + + pr_emerg("Resetting SoC\n"); writel(SOFT_CHIP_RST, ctx->base); @@ -72,9 +79,13 @@ static int ocelot_reset_probe(struct platform_device *pdev) if (IS_ERR(ctx->base)) return PTR_ERR(ctx->base); - ctx->cpu_ctrl = syscon_regmap_lookup_by_compatible("mscc,ocelot-cpu-syscon"); - if (IS_ERR(ctx->cpu_ctrl)) + ctx->props = device_get_match_data(dev); + + ctx->cpu_ctrl = syscon_regmap_lookup_by_compatible(ctx->props->syscon); + if (IS_ERR(ctx->cpu_ctrl)) { + dev_err(dev, "No syscon map: %s\n", ctx->props->syscon); return PTR_ERR(ctx->cpu_ctrl); + } ctx->restart_handler.notifier_call = ocelot_restart_handle; ctx->restart_handler.priority = 192; @@ -85,9 +96,29 @@ static int ocelot_reset_probe(struct platform_device *pdev) return err; } +static const struct reset_props reset_props_ocelot = { + .syscon = "mscc,ocelot-cpu-syscon", + .protect_reg = 0x20, + .vcore_protect = BIT(2), + .if_si_owner_bit = 4, +}; + +static const struct reset_props reset_props_sparx5 = { + .syscon = "microchip,sparx5-cpu-syscon", + .protect_reg = 0x84, + .vcore_protect = BIT(10), + .if_si_owner_bit = 6, +}; + static const struct of_device_id ocelot_reset_of_match[] = { - { .compatible = "mscc,ocelot-chip-reset" }, - {} + { + .compatible = "mscc,ocelot-chip-reset", + .data = &reset_props_ocelot + }, { + .compatible = "microchip,sparx5-chip-reset", + .data = &reset_props_sparx5 + }, + { /*sentinel*/ } }; static struct platform_driver ocelot_reset_driver = { diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig index faf2830aa152..eec646c568b7 100644 --- a/drivers/power/supply/Kconfig +++ b/drivers/power/supply/Kconfig @@ -164,7 +164,7 @@ config BATTERY_DS2782 config BATTERY_LEGO_EV3 tristate "LEGO MINDSTORMS EV3 battery" - depends on OF && IIO && GPIOLIB + depends on OF && IIO && GPIOLIB && (ARCH_DAVINCI_DA850 || COMPILE_TEST) help Say Y here to enable support for the LEGO MINDSTORMS EV3 battery. @@ -367,10 +367,15 @@ config AXP288_FUEL_GAUGE config BATTERY_MAX17040 tristate "Maxim MAX17040 Fuel Gauge" depends on I2C + select REGMAP_I2C help - MAX17040 is fuel-gauge systems for lithium-ion (Li+) batteries - in handheld and portable equipment. The MAX17040 is configured - to operate with a single lithium cell + Maxim models with ModelGauge are fuel-gauge systems for lithium-ion + (Li+) batteries in handheld and portable equipment, including + max17040, max17041, max17043, max17044, max17048, max17049, max17058, + max17059. It is also included in some batteries like max77836. + + Driver supports reporting SOC (State of Charge, i.e capacity), + voltage and configurable low-SOC wakeup interrupt. config BATTERY_MAX17042 tristate "Maxim MAX17042/17047/17050/8997/8966 Fuel Gauge" @@ -631,13 +636,22 @@ config CHARGER_BQ25890 help Say Y to enable support for the TI BQ25890 battery charger. +config CHARGER_BQ25980 + tristate "TI BQ25980 battery charger driver" + depends on I2C + depends on GPIOLIB || COMPILE_TEST + select REGMAP_I2C + help + Say Y to enable support for the TI BQ25980, BQ25975 and BQ25960 + series of fast battery chargers. + config CHARGER_SMB347 - tristate "Summit Microelectronics SMB347 Battery Charger" + tristate "Summit Microelectronics SMB3XX Battery Charger" depends on I2C select REGMAP_I2C help - Say Y to include support for Summit Microelectronics SMB347 - Battery Charger. + Say Y to include support for Summit Microelectronics SMB345, + SMB347 or SMB358 Battery Charger. config CHARGER_TPS65090 tristate "TPS65090 battery charger driver" @@ -752,4 +766,12 @@ config CHARGER_WILCO information can be found in Documentation/ABI/testing/sysfs-class-power-wilco +config RN5T618_POWER + tristate "RN5T618 charger/fuel gauge support" + depends on MFD_RN5T618 + help + Say Y here to have support for RN5T618 PMIC family fuel gauge and charger. + This driver can also be built as a module. If so, the module will be + called rn5t618_power. + endif # POWER_SUPPLY diff --git a/drivers/power/supply/Makefile b/drivers/power/supply/Makefile index b3c694a65114..dd4b86318cd9 100644 --- a/drivers/power/supply/Makefile +++ b/drivers/power/supply/Makefile @@ -84,6 +84,7 @@ obj-$(CONFIG_CHARGER_BQ24257) += bq24257_charger.o obj-$(CONFIG_CHARGER_BQ24735) += bq24735-charger.o obj-$(CONFIG_CHARGER_BQ2515X) += bq2515x_charger.o obj-$(CONFIG_CHARGER_BQ25890) += bq25890_charger.o +obj-$(CONFIG_CHARGER_BQ25980) += bq25980_charger.o obj-$(CONFIG_CHARGER_SMB347) += smb347-charger.o obj-$(CONFIG_CHARGER_TPS65090) += tps65090-charger.o obj-$(CONFIG_CHARGER_TPS65217) += tps65217_charger.o @@ -96,3 +97,4 @@ obj-$(CONFIG_CHARGER_UCS1002) += ucs1002_power.o obj-$(CONFIG_CHARGER_BD70528) += bd70528-charger.o obj-$(CONFIG_CHARGER_BD99954) += bd99954-charger.o obj-$(CONFIG_CHARGER_WILCO) += wilco-charger.o +obj-$(CONFIG_RN5T618_POWER) += rn5t618_power.o diff --git a/drivers/power/supply/ab8500_fg.c b/drivers/power/supply/ab8500_fg.c index 7eec415c82a3..592a73d4dde6 100644 --- a/drivers/power/supply/ab8500_fg.c +++ b/drivers/power/supply/ab8500_fg.c @@ -653,7 +653,7 @@ int ab8500_fg_inst_curr_finalize(struct ab8500_fg *di, int *res) /* * negative value for Discharging - * convert 2's compliment into decimal + * convert 2's complement into decimal */ if (high & 0x10) val = (low | (high << 8) | 0xFFFFE000); @@ -781,7 +781,7 @@ static void ab8500_fg_acc_cur_work(struct work_struct *work) if (ret < 0) goto exit; - /* Check for sign bit in case of negative value, 2's compliment */ + /* Check for sign bit in case of negative value, 2's complement */ if (high & 0x10) val = (low | (med << 8) | (high << 16) | 0xFFE00000); else diff --git a/drivers/power/supply/bq24257_charger.c b/drivers/power/supply/bq24257_charger.c index 8e60cb0f3c3f..96cb3290bcaa 100644 --- a/drivers/power/supply/bq24257_charger.c +++ b/drivers/power/supply/bq24257_charger.c @@ -1152,6 +1152,7 @@ static const struct of_device_id bq24257_of_match[] = { }; MODULE_DEVICE_TABLE(of, bq24257_of_match); +#ifdef CONFIG_ACPI static const struct acpi_device_id bq24257_acpi_match[] = { { "BQ242500", BQ24250 }, { "BQ242510", BQ24251 }, @@ -1159,6 +1160,7 @@ static const struct acpi_device_id bq24257_acpi_match[] = { {}, }; MODULE_DEVICE_TABLE(acpi, bq24257_acpi_match); +#endif static struct i2c_driver bq24257_driver = { .driver = { diff --git a/drivers/power/supply/bq2515x_charger.c b/drivers/power/supply/bq2515x_charger.c index 36b0c8c98d40..374b112f712a 100644 --- a/drivers/power/supply/bq2515x_charger.c +++ b/drivers/power/supply/bq2515x_charger.c @@ -168,7 +168,7 @@ enum bq2515x_id { * @device_id: value of device_id * @mains_online: boolean value indicating power supply online * - * @bq2515x_init_data init_data: charger initialization data structure + * @init_data: charger initialization data structure */ struct bq2515x_device { struct power_supply *mains; @@ -188,7 +188,7 @@ struct bq2515x_device { struct bq2515x_init_data init_data; }; -static struct reg_default bq25150_reg_defaults[] = { +static const struct reg_default bq25150_reg_defaults[] = { {BQ2515X_FLAG0, 0x0}, {BQ2515X_FLAG1, 0x0}, {BQ2515X_FLAG2, 0x0}, @@ -227,7 +227,7 @@ static struct reg_default bq25150_reg_defaults[] = { {BQ2515X_DEVICE_ID, 0x20}, }; -static struct reg_default bq25155_reg_defaults[] = { +static const struct reg_default bq25155_reg_defaults[] = { {BQ2515X_FLAG0, 0x0}, {BQ2515X_FLAG1, 0x0}, {BQ2515X_FLAG2, 0x0}, @@ -886,14 +886,14 @@ static int bq2515x_battery_get_property(struct power_supply *psy, return 0; } -static enum power_supply_property bq2515x_battery_properties[] = { +static const enum power_supply_property bq2515x_battery_properties[] = { POWER_SUPPLY_PROP_VOLTAGE_NOW, POWER_SUPPLY_PROP_CURRENT_NOW, POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX, POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX, }; -static enum power_supply_property bq2515x_mains_properties[] = { +static const enum power_supply_property bq2515x_mains_properties[] = { POWER_SUPPLY_PROP_ONLINE, POWER_SUPPLY_PROP_STATUS, POWER_SUPPLY_PROP_HEALTH, @@ -905,7 +905,7 @@ static enum power_supply_property bq2515x_mains_properties[] = { POWER_SUPPLY_PROP_PRECHARGE_CURRENT, }; -static struct power_supply_desc bq2515x_mains_desc = { +static const struct power_supply_desc bq2515x_mains_desc = { .name = "bq2515x-mains", .type = POWER_SUPPLY_TYPE_MAINS, .get_property = bq2515x_mains_get_property, @@ -915,7 +915,7 @@ static struct power_supply_desc bq2515x_mains_desc = { .property_is_writeable = bq2515x_power_supply_property_is_writeable, }; -static struct power_supply_desc bq2515x_battery_desc = { +static const struct power_supply_desc bq2515x_battery_desc = { .name = "bq2515x-battery", .type = POWER_SUPPLY_TYPE_BATTERY, .get_property = bq2515x_battery_get_property, diff --git a/drivers/power/supply/bq25890_charger.c b/drivers/power/supply/bq25890_charger.c index 77150667e36b..34c21c51bac1 100644 --- a/drivers/power/supply/bq25890_charger.c +++ b/drivers/power/supply/bq25890_charger.c @@ -83,6 +83,8 @@ struct bq25890_init_data { u8 boostf; /* boost frequency */ u8 ilim_en; /* enable ILIM pin */ u8 treg; /* thermal regulation threshold */ + u8 rbatcomp; /* IBAT sense resistor value */ + u8 vclamp; /* IBAT compensation voltage limit */ }; struct bq25890_state { @@ -258,6 +260,8 @@ enum bq25890_table_ids { TBL_VREG, TBL_BOOSTV, TBL_SYSVMIN, + TBL_VBATCOMP, + TBL_RBATCOMP, /* lookup tables */ TBL_TREG, @@ -299,6 +303,8 @@ static const union { [TBL_VREG] = { .rt = {3840000, 4608000, 16000} }, /* uV */ [TBL_BOOSTV] = { .rt = {4550000, 5510000, 64000} }, /* uV */ [TBL_SYSVMIN] = { .rt = {3000000, 3700000, 100000} }, /* uV */ + [TBL_VBATCOMP] ={ .rt = {0, 224000, 32000} }, /* uV */ + [TBL_RBATCOMP] ={ .rt = {0, 140000, 20000} }, /* uOhm */ /* lookup tables */ [TBL_TREG] = { .lt = {bq25890_treg_tbl, BQ25890_TREG_TBL_SIZE} }, @@ -648,7 +654,9 @@ static int bq25890_hw_init(struct bq25890_device *bq) {F_BOOSTI, bq->init_data.boosti}, {F_BOOSTF, bq->init_data.boostf}, {F_EN_ILIM, bq->init_data.ilim_en}, - {F_TREG, bq->init_data.treg} + {F_TREG, bq->init_data.treg}, + {F_BATCMP, bq->init_data.rbatcomp}, + {F_VCLAMP, bq->init_data.vclamp}, }; ret = bq25890_chip_reset(bq); @@ -859,11 +867,14 @@ static int bq25890_fw_read_u32_props(struct bq25890_device *bq) {"ti,boost-max-current", false, TBL_BOOSTI, &init->boosti}, /* optional properties */ - {"ti,thermal-regulation-threshold", true, TBL_TREG, &init->treg} + {"ti,thermal-regulation-threshold", true, TBL_TREG, &init->treg}, + {"ti,ibatcomp-micro-ohms", true, TBL_RBATCOMP, &init->rbatcomp}, + {"ti,ibatcomp-clamp-microvolt", true, TBL_VBATCOMP, &init->vclamp}, }; /* initialize data for optional properties */ init->treg = 3; /* 120 degrees Celsius */ + init->rbatcomp = init->vclamp = 0; /* IBAT compensation disabled */ for (i = 0; i < ARRAY_SIZE(props); i++) { ret = device_property_read_u32(bq->dev, props[i].name, @@ -1073,11 +1084,13 @@ static const struct of_device_id bq25890_of_match[] = { }; MODULE_DEVICE_TABLE(of, bq25890_of_match); +#ifdef CONFIG_ACPI static const struct acpi_device_id bq25890_acpi_match[] = { {"BQ258900", 0}, {}, }; MODULE_DEVICE_TABLE(acpi, bq25890_acpi_match); +#endif static struct i2c_driver bq25890_driver = { .driver = { diff --git a/drivers/power/supply/bq25980_charger.c b/drivers/power/supply/bq25980_charger.c new file mode 100644 index 000000000000..c936f311eb4f --- /dev/null +++ b/drivers/power/supply/bq25980_charger.c @@ -0,0 +1,1314 @@ +// SPDX-License-Identifier: GPL-2.0 +// BQ25980 Battery Charger Driver +// Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com/ + +#include <linux/err.h> +#include <linux/i2c.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/gpio/consumer.h> +#include <linux/power_supply.h> +#include <linux/regmap.h> +#include <linux/types.h> +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/moduleparam.h> +#include <linux/slab.h> + +#include "bq25980_charger.h" + +struct bq25980_state { + bool dischg; + bool ovp; + bool ocp; + bool wdt; + bool tflt; + bool online; + bool ce; + bool hiz; + bool bypass; + + u32 vbat_adc; + u32 vsys_adc; + u32 ibat_adc; +}; + +enum bq25980_id { + BQ25980, + BQ25975, + BQ25960, +}; + +struct bq25980_chip_info { + + int model_id; + + const struct regmap_config *regmap_config; + + int busocp_def; + int busocp_sc_max; + int busocp_byp_max; + int busocp_sc_min; + int busocp_byp_min; + + int busovp_sc_def; + int busovp_byp_def; + int busovp_sc_step; + + int busovp_sc_offset; + int busovp_byp_step; + int busovp_byp_offset; + int busovp_sc_min; + int busovp_sc_max; + int busovp_byp_min; + int busovp_byp_max; + + int batovp_def; + int batovp_max; + int batovp_min; + int batovp_step; + int batovp_offset; + + int batocp_def; + int batocp_max; +}; + +struct bq25980_init_data { + u32 ichg; + u32 bypass_ilim; + u32 sc_ilim; + u32 vreg; + u32 iterm; + u32 iprechg; + u32 bypass_vlim; + u32 sc_vlim; + u32 ichg_max; + u32 vreg_max; +}; + +struct bq25980_device { + struct i2c_client *client; + struct device *dev; + struct power_supply *charger; + struct power_supply *battery; + struct mutex lock; + struct regmap *regmap; + + char model_name[I2C_NAME_SIZE]; + + struct bq25980_init_data init_data; + const struct bq25980_chip_info *chip_info; + struct bq25980_state state; + int watchdog_timer; +}; + +static struct reg_default bq25980_reg_defs[] = { + {BQ25980_BATOVP, 0x5A}, + {BQ25980_BATOVP_ALM, 0x46}, + {BQ25980_BATOCP, 0x51}, + {BQ25980_BATOCP_ALM, 0x50}, + {BQ25980_BATUCP_ALM, 0x28}, + {BQ25980_CHRGR_CTRL_1, 0x0}, + {BQ25980_BUSOVP, 0x26}, + {BQ25980_BUSOVP_ALM, 0x22}, + {BQ25980_BUSOCP, 0xD}, + {BQ25980_BUSOCP_ALM, 0xC}, + {BQ25980_TEMP_CONTROL, 0x30}, + {BQ25980_TDIE_ALM, 0xC8}, + {BQ25980_TSBUS_FLT, 0x15}, + {BQ25980_TSBAT_FLG, 0x15}, + {BQ25980_VAC_CONTROL, 0x0}, + {BQ25980_CHRGR_CTRL_2, 0x0}, + {BQ25980_CHRGR_CTRL_3, 0x20}, + {BQ25980_CHRGR_CTRL_4, 0x1D}, + {BQ25980_CHRGR_CTRL_5, 0x18}, + {BQ25980_STAT1, 0x0}, + {BQ25980_STAT2, 0x0}, + {BQ25980_STAT3, 0x0}, + {BQ25980_STAT4, 0x0}, + {BQ25980_STAT5, 0x0}, + {BQ25980_FLAG1, 0x0}, + {BQ25980_FLAG2, 0x0}, + {BQ25980_FLAG3, 0x0}, + {BQ25980_FLAG4, 0x0}, + {BQ25980_FLAG5, 0x0}, + {BQ25980_MASK1, 0x0}, + {BQ25980_MASK2, 0x0}, + {BQ25980_MASK3, 0x0}, + {BQ25980_MASK4, 0x0}, + {BQ25980_MASK5, 0x0}, + {BQ25980_DEVICE_INFO, 0x8}, + {BQ25980_ADC_CONTROL1, 0x0}, + {BQ25980_ADC_CONTROL2, 0x0}, + {BQ25980_IBUS_ADC_LSB, 0x0}, + {BQ25980_IBUS_ADC_MSB, 0x0}, + {BQ25980_VBUS_ADC_LSB, 0x0}, + {BQ25980_VBUS_ADC_MSB, 0x0}, + {BQ25980_VAC1_ADC_LSB, 0x0}, + {BQ25980_VAC2_ADC_LSB, 0x0}, + {BQ25980_VOUT_ADC_LSB, 0x0}, + {BQ25980_VBAT_ADC_LSB, 0x0}, + {BQ25980_IBAT_ADC_MSB, 0x0}, + {BQ25980_IBAT_ADC_LSB, 0x0}, + {BQ25980_TSBUS_ADC_LSB, 0x0}, + {BQ25980_TSBAT_ADC_LSB, 0x0}, + {BQ25980_TDIE_ADC_LSB, 0x0}, + {BQ25980_DEGLITCH_TIME, 0x0}, + {BQ25980_CHRGR_CTRL_6, 0x0}, +}; + +static struct reg_default bq25975_reg_defs[] = { + {BQ25980_BATOVP, 0x5A}, + {BQ25980_BATOVP_ALM, 0x46}, + {BQ25980_BATOCP, 0x51}, + {BQ25980_BATOCP_ALM, 0x50}, + {BQ25980_BATUCP_ALM, 0x28}, + {BQ25980_CHRGR_CTRL_1, 0x0}, + {BQ25980_BUSOVP, 0x26}, + {BQ25980_BUSOVP_ALM, 0x22}, + {BQ25980_BUSOCP, 0xD}, + {BQ25980_BUSOCP_ALM, 0xC}, + {BQ25980_TEMP_CONTROL, 0x30}, + {BQ25980_TDIE_ALM, 0xC8}, + {BQ25980_TSBUS_FLT, 0x15}, + {BQ25980_TSBAT_FLG, 0x15}, + {BQ25980_VAC_CONTROL, 0x0}, + {BQ25980_CHRGR_CTRL_2, 0x0}, + {BQ25980_CHRGR_CTRL_3, 0x20}, + {BQ25980_CHRGR_CTRL_4, 0x1D}, + {BQ25980_CHRGR_CTRL_5, 0x18}, + {BQ25980_STAT1, 0x0}, + {BQ25980_STAT2, 0x0}, + {BQ25980_STAT3, 0x0}, + {BQ25980_STAT4, 0x0}, + {BQ25980_STAT5, 0x0}, + {BQ25980_FLAG1, 0x0}, + {BQ25980_FLAG2, 0x0}, + {BQ25980_FLAG3, 0x0}, + {BQ25980_FLAG4, 0x0}, + {BQ25980_FLAG5, 0x0}, + {BQ25980_MASK1, 0x0}, + {BQ25980_MASK2, 0x0}, + {BQ25980_MASK3, 0x0}, + {BQ25980_MASK4, 0x0}, + {BQ25980_MASK5, 0x0}, + {BQ25980_DEVICE_INFO, 0x8}, + {BQ25980_ADC_CONTROL1, 0x0}, + {BQ25980_ADC_CONTROL2, 0x0}, + {BQ25980_IBUS_ADC_LSB, 0x0}, + {BQ25980_IBUS_ADC_MSB, 0x0}, + {BQ25980_VBUS_ADC_LSB, 0x0}, + {BQ25980_VBUS_ADC_MSB, 0x0}, + {BQ25980_VAC1_ADC_LSB, 0x0}, + {BQ25980_VAC2_ADC_LSB, 0x0}, + {BQ25980_VOUT_ADC_LSB, 0x0}, + {BQ25980_VBAT_ADC_LSB, 0x0}, + {BQ25980_IBAT_ADC_MSB, 0x0}, + {BQ25980_IBAT_ADC_LSB, 0x0}, + {BQ25980_TSBUS_ADC_LSB, 0x0}, + {BQ25980_TSBAT_ADC_LSB, 0x0}, + {BQ25980_TDIE_ADC_LSB, 0x0}, + {BQ25980_DEGLITCH_TIME, 0x0}, + {BQ25980_CHRGR_CTRL_6, 0x0}, +}; + +static struct reg_default bq25960_reg_defs[] = { + {BQ25980_BATOVP, 0x5A}, + {BQ25980_BATOVP_ALM, 0x46}, + {BQ25980_BATOCP, 0x51}, + {BQ25980_BATOCP_ALM, 0x50}, + {BQ25980_BATUCP_ALM, 0x28}, + {BQ25980_CHRGR_CTRL_1, 0x0}, + {BQ25980_BUSOVP, 0x26}, + {BQ25980_BUSOVP_ALM, 0x22}, + {BQ25980_BUSOCP, 0xD}, + {BQ25980_BUSOCP_ALM, 0xC}, + {BQ25980_TEMP_CONTROL, 0x30}, + {BQ25980_TDIE_ALM, 0xC8}, + {BQ25980_TSBUS_FLT, 0x15}, + {BQ25980_TSBAT_FLG, 0x15}, + {BQ25980_VAC_CONTROL, 0x0}, + {BQ25980_CHRGR_CTRL_2, 0x0}, + {BQ25980_CHRGR_CTRL_3, 0x20}, + {BQ25980_CHRGR_CTRL_4, 0x1D}, + {BQ25980_CHRGR_CTRL_5, 0x18}, + {BQ25980_STAT1, 0x0}, + {BQ25980_STAT2, 0x0}, + {BQ25980_STAT3, 0x0}, + {BQ25980_STAT4, 0x0}, + {BQ25980_STAT5, 0x0}, + {BQ25980_FLAG1, 0x0}, + {BQ25980_FLAG2, 0x0}, + {BQ25980_FLAG3, 0x0}, + {BQ25980_FLAG4, 0x0}, + {BQ25980_FLAG5, 0x0}, + {BQ25980_MASK1, 0x0}, + {BQ25980_MASK2, 0x0}, + {BQ25980_MASK3, 0x0}, + {BQ25980_MASK4, 0x0}, + {BQ25980_MASK5, 0x0}, + {BQ25980_DEVICE_INFO, 0x8}, + {BQ25980_ADC_CONTROL1, 0x0}, + {BQ25980_ADC_CONTROL2, 0x0}, + {BQ25980_IBUS_ADC_LSB, 0x0}, + {BQ25980_IBUS_ADC_MSB, 0x0}, + {BQ25980_VBUS_ADC_LSB, 0x0}, + {BQ25980_VBUS_ADC_MSB, 0x0}, + {BQ25980_VAC1_ADC_LSB, 0x0}, + {BQ25980_VAC2_ADC_LSB, 0x0}, + {BQ25980_VOUT_ADC_LSB, 0x0}, + {BQ25980_VBAT_ADC_LSB, 0x0}, + {BQ25980_IBAT_ADC_MSB, 0x0}, + {BQ25980_IBAT_ADC_LSB, 0x0}, + {BQ25980_TSBUS_ADC_LSB, 0x0}, + {BQ25980_TSBAT_ADC_LSB, 0x0}, + {BQ25980_TDIE_ADC_LSB, 0x0}, + {BQ25980_DEGLITCH_TIME, 0x0}, + {BQ25980_CHRGR_CTRL_6, 0x0}, +}; + +static int bq25980_watchdog_time[BQ25980_NUM_WD_VAL] = {5000, 10000, 50000, + 300000}; + +static int bq25980_get_input_curr_lim(struct bq25980_device *bq) +{ + unsigned int busocp_reg_code; + int ret; + + ret = regmap_read(bq->regmap, BQ25980_BUSOCP, &busocp_reg_code); + if (ret) + return ret; + + return (busocp_reg_code * BQ25980_BUSOCP_STEP_uA) + BQ25980_BUSOCP_OFFSET_uA; +} + +static int bq25980_set_hiz(struct bq25980_device *bq, int setting) +{ + return regmap_update_bits(bq->regmap, BQ25980_CHRGR_CTRL_2, + BQ25980_EN_HIZ, setting); +} + +static int bq25980_set_input_curr_lim(struct bq25980_device *bq, int busocp) +{ + unsigned int busocp_reg_code; + int ret; + + if (!busocp) + return bq25980_set_hiz(bq, BQ25980_ENABLE_HIZ); + + bq25980_set_hiz(bq, BQ25980_DISABLE_HIZ); + + if (busocp < BQ25980_BUSOCP_MIN_uA) + busocp = BQ25980_BUSOCP_MIN_uA; + + if (bq->state.bypass) + busocp = min(busocp, bq->chip_info->busocp_sc_max); + else + busocp = min(busocp, bq->chip_info->busocp_byp_max); + + busocp_reg_code = (busocp - BQ25980_BUSOCP_OFFSET_uA) + / BQ25980_BUSOCP_STEP_uA; + + ret = regmap_write(bq->regmap, BQ25980_BUSOCP, busocp_reg_code); + if (ret) + return ret; + + return regmap_write(bq->regmap, BQ25980_BUSOCP_ALM, busocp_reg_code); +} + +static int bq25980_get_input_volt_lim(struct bq25980_device *bq) +{ + unsigned int busovp_reg_code; + unsigned int busovp_offset; + unsigned int busovp_step; + int ret; + + if (bq->state.bypass) { + busovp_step = bq->chip_info->busovp_byp_step; + busovp_offset = bq->chip_info->busovp_byp_offset; + } else { + busovp_step = bq->chip_info->busovp_sc_step; + busovp_offset = bq->chip_info->busovp_sc_offset; + } + + ret = regmap_read(bq->regmap, BQ25980_BUSOVP, &busovp_reg_code); + if (ret) + return ret; + + return (busovp_reg_code * busovp_step) + busovp_offset; +} + +static int bq25980_set_input_volt_lim(struct bq25980_device *bq, int busovp) +{ + unsigned int busovp_reg_code; + unsigned int busovp_step; + unsigned int busovp_offset; + int ret; + + if (bq->state.bypass) { + busovp_step = bq->chip_info->busovp_byp_step; + busovp_offset = bq->chip_info->busovp_byp_offset; + if (busovp > bq->chip_info->busovp_byp_max) + busovp = bq->chip_info->busovp_byp_max; + else if (busovp < bq->chip_info->busovp_byp_min) + busovp = bq->chip_info->busovp_byp_min; + } else { + busovp_step = bq->chip_info->busovp_sc_step; + busovp_offset = bq->chip_info->busovp_sc_offset; + if (busovp > bq->chip_info->busovp_sc_max) + busovp = bq->chip_info->busovp_sc_max; + else if (busovp < bq->chip_info->busovp_sc_min) + busovp = bq->chip_info->busovp_sc_min; + } + + busovp_reg_code = (busovp - busovp_offset) / busovp_step; + + ret = regmap_write(bq->regmap, BQ25980_BUSOVP, busovp_reg_code); + if (ret) + return ret; + + return regmap_write(bq->regmap, BQ25980_BUSOVP_ALM, busovp_reg_code); +} + +static int bq25980_get_const_charge_curr(struct bq25980_device *bq) +{ + unsigned int batocp_reg_code; + int ret; + + ret = regmap_read(bq->regmap, BQ25980_BATOCP, &batocp_reg_code); + if (ret) + return ret; + + return (batocp_reg_code & BQ25980_BATOCP_MASK) * + BQ25980_BATOCP_STEP_uA; +} + +static int bq25980_set_const_charge_curr(struct bq25980_device *bq, int batocp) +{ + unsigned int batocp_reg_code; + int ret; + + batocp = max(batocp, BQ25980_BATOCP_MIN_uA); + batocp = min(batocp, bq->chip_info->batocp_max); + + batocp_reg_code = batocp / BQ25980_BATOCP_STEP_uA; + + ret = regmap_update_bits(bq->regmap, BQ25980_BATOCP, + BQ25980_BATOCP_MASK, batocp_reg_code); + if (ret) + return ret; + + return regmap_update_bits(bq->regmap, BQ25980_BATOCP_ALM, + BQ25980_BATOCP_MASK, batocp_reg_code); +} + +static int bq25980_get_const_charge_volt(struct bq25980_device *bq) +{ + unsigned int batovp_reg_code; + int ret; + + ret = regmap_read(bq->regmap, BQ25980_BATOVP, &batovp_reg_code); + if (ret) + return ret; + + return ((batovp_reg_code * bq->chip_info->batovp_step) + + bq->chip_info->batovp_offset); +} + +static int bq25980_set_const_charge_volt(struct bq25980_device *bq, int batovp) +{ + unsigned int batovp_reg_code; + int ret; + + if (batovp < bq->chip_info->batovp_min) + batovp = bq->chip_info->batovp_min; + + if (batovp > bq->chip_info->batovp_max) + batovp = bq->chip_info->batovp_max; + + batovp_reg_code = (batovp - bq->chip_info->batovp_offset) / + bq->chip_info->batovp_step; + + ret = regmap_write(bq->regmap, BQ25980_BATOVP, batovp_reg_code); + if (ret) + return ret; + + return regmap_write(bq->regmap, BQ25980_BATOVP_ALM, batovp_reg_code); +} + +static int bq25980_set_bypass(struct bq25980_device *bq, bool en_bypass) +{ + int ret; + + if (en_bypass) + ret = regmap_update_bits(bq->regmap, BQ25980_CHRGR_CTRL_2, + BQ25980_EN_BYPASS, BQ25980_EN_BYPASS); + else + ret = regmap_update_bits(bq->regmap, BQ25980_CHRGR_CTRL_2, + BQ25980_EN_BYPASS, en_bypass); + if (ret) + return ret; + + bq->state.bypass = en_bypass; + + return bq->state.bypass; +} + +static int bq25980_set_chg_en(struct bq25980_device *bq, bool en_chg) +{ + int ret; + + if (en_chg) + ret = regmap_update_bits(bq->regmap, BQ25980_CHRGR_CTRL_2, + BQ25980_CHG_EN, BQ25980_CHG_EN); + else + ret = regmap_update_bits(bq->regmap, BQ25980_CHRGR_CTRL_2, + BQ25980_CHG_EN, en_chg); + if (ret) + return ret; + + bq->state.ce = en_chg; + + return 0; +} + +static int bq25980_get_adc_ibus(struct bq25980_device *bq) +{ + int ibus_adc_lsb, ibus_adc_msb; + u16 ibus_adc; + int ret; + + ret = regmap_read(bq->regmap, BQ25980_IBUS_ADC_MSB, &ibus_adc_msb); + if (ret) + return ret; + + ret = regmap_read(bq->regmap, BQ25980_IBUS_ADC_LSB, &ibus_adc_lsb); + if (ret) + return ret; + + ibus_adc = (ibus_adc_msb << 8) | ibus_adc_lsb; + + if (ibus_adc_msb & BQ25980_ADC_POLARITY_BIT) + return ((ibus_adc ^ 0xffff) + 1) * BQ25980_ADC_CURR_STEP_uA; + + return ibus_adc * BQ25980_ADC_CURR_STEP_uA; +} + +static int bq25980_get_adc_vbus(struct bq25980_device *bq) +{ + int vbus_adc_lsb, vbus_adc_msb; + u16 vbus_adc; + int ret; + + ret = regmap_read(bq->regmap, BQ25980_VBUS_ADC_MSB, &vbus_adc_msb); + if (ret) + return ret; + + ret = regmap_read(bq->regmap, BQ25980_VBUS_ADC_LSB, &vbus_adc_lsb); + if (ret) + return ret; + + vbus_adc = (vbus_adc_msb << 8) | vbus_adc_lsb; + + return vbus_adc * BQ25980_ADC_VOLT_STEP_uV; +} + +static int bq25980_get_ibat_adc(struct bq25980_device *bq) +{ + int ret; + int ibat_adc_lsb, ibat_adc_msb; + int ibat_adc; + + ret = regmap_read(bq->regmap, BQ25980_IBAT_ADC_MSB, &ibat_adc_msb); + if (ret) + return ret; + + ret = regmap_read(bq->regmap, BQ25980_IBAT_ADC_LSB, &ibat_adc_lsb); + if (ret) + return ret; + + ibat_adc = (ibat_adc_msb << 8) | ibat_adc_lsb; + + if (ibat_adc_msb & BQ25980_ADC_POLARITY_BIT) + return ((ibat_adc ^ 0xffff) + 1) * BQ25980_ADC_CURR_STEP_uA; + + return ibat_adc * BQ25980_ADC_CURR_STEP_uA; +} + +static int bq25980_get_adc_vbat(struct bq25980_device *bq) +{ + int vsys_adc_lsb, vsys_adc_msb; + u16 vsys_adc; + int ret; + + ret = regmap_read(bq->regmap, BQ25980_VBAT_ADC_MSB, &vsys_adc_msb); + if (ret) + return ret; + + ret = regmap_read(bq->regmap, BQ25980_VBAT_ADC_LSB, &vsys_adc_lsb); + if (ret) + return ret; + + vsys_adc = (vsys_adc_msb << 8) | vsys_adc_lsb; + + return vsys_adc * BQ25980_ADC_VOLT_STEP_uV; +} + +static int bq25980_get_state(struct bq25980_device *bq, + struct bq25980_state *state) +{ + unsigned int chg_ctrl_2; + unsigned int stat1; + unsigned int stat2; + unsigned int stat3; + unsigned int stat4; + unsigned int ibat_adc_msb; + int ret; + + ret = regmap_read(bq->regmap, BQ25980_STAT1, &stat1); + if (ret) + return ret; + + ret = regmap_read(bq->regmap, BQ25980_STAT2, &stat2); + if (ret) + return ret; + + ret = regmap_read(bq->regmap, BQ25980_STAT3, &stat3); + if (ret) + return ret; + + ret = regmap_read(bq->regmap, BQ25980_STAT4, &stat4); + if (ret) + return ret; + + ret = regmap_read(bq->regmap, BQ25980_CHRGR_CTRL_2, &chg_ctrl_2); + if (ret) + return ret; + + ret = regmap_read(bq->regmap, BQ25980_IBAT_ADC_MSB, &ibat_adc_msb); + if (ret) + return ret; + + state->dischg = ibat_adc_msb & BQ25980_ADC_POLARITY_BIT; + state->ovp = (stat1 & BQ25980_STAT1_OVP_MASK) | + (stat3 & BQ25980_STAT3_OVP_MASK); + state->ocp = (stat1 & BQ25980_STAT1_OCP_MASK) | + (stat2 & BQ25980_STAT2_OCP_MASK); + state->tflt = stat4 & BQ25980_STAT4_TFLT_MASK; + state->wdt = stat4 & BQ25980_WD_STAT; + state->online = stat3 & BQ25980_PRESENT_MASK; + state->ce = chg_ctrl_2 & BQ25980_CHG_EN; + state->hiz = chg_ctrl_2 & BQ25980_EN_HIZ; + state->bypass = chg_ctrl_2 & BQ25980_EN_BYPASS; + + return 0; +} + +static int bq25980_set_battery_property(struct power_supply *psy, + enum power_supply_property psp, + const union power_supply_propval *val) +{ + struct bq25980_device *bq = power_supply_get_drvdata(psy); + int ret = 0; + + switch (psp) { + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT: + ret = bq25980_set_const_charge_curr(bq, val->intval); + if (ret) + return ret; + break; + + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE: + ret = bq25980_set_const_charge_volt(bq, val->intval); + if (ret) + return ret; + break; + + default: + return -EINVAL; + } + + return ret; +} + +static int bq25980_get_battery_property(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + struct bq25980_device *bq = power_supply_get_drvdata(psy); + int ret = 0; + + switch (psp) { + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX: + val->intval = bq->init_data.ichg_max; + break; + + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX: + val->intval = bq->init_data.vreg_max; + break; + + case POWER_SUPPLY_PROP_CURRENT_NOW: + ret = bq25980_get_ibat_adc(bq); + val->intval = ret; + break; + + case POWER_SUPPLY_PROP_VOLTAGE_NOW: + ret = bq25980_get_adc_vbat(bq); + if (ret < 0) + return ret; + + val->intval = ret; + break; + + default: + return -EINVAL; + } + + return ret; +} + +static int bq25980_set_charger_property(struct power_supply *psy, + enum power_supply_property prop, + const union power_supply_propval *val) +{ + struct bq25980_device *bq = power_supply_get_drvdata(psy); + int ret = -EINVAL; + + switch (prop) { + case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT: + ret = bq25980_set_input_curr_lim(bq, val->intval); + if (ret) + return ret; + break; + + case POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT: + ret = bq25980_set_input_volt_lim(bq, val->intval); + if (ret) + return ret; + break; + + case POWER_SUPPLY_PROP_CHARGE_TYPE: + ret = bq25980_set_bypass(bq, val->intval); + if (ret) + return ret; + break; + + case POWER_SUPPLY_PROP_STATUS: + ret = bq25980_set_chg_en(bq, val->intval); + if (ret) + return ret; + break; + + default: + return -EINVAL; + } + + return ret; +} + +static int bq25980_get_charger_property(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + struct bq25980_device *bq = power_supply_get_drvdata(psy); + struct bq25980_state state; + int ret = 0; + + mutex_lock(&bq->lock); + ret = bq25980_get_state(bq, &state); + mutex_unlock(&bq->lock); + if (ret) + return ret; + + switch (psp) { + case POWER_SUPPLY_PROP_MANUFACTURER: + val->strval = BQ25980_MANUFACTURER; + break; + case POWER_SUPPLY_PROP_MODEL_NAME: + val->strval = bq->model_name; + break; + case POWER_SUPPLY_PROP_ONLINE: + val->intval = state.online; + break; + + case POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT: + ret = bq25980_get_input_volt_lim(bq); + if (ret < 0) + return ret; + val->intval = ret; + break; + + case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT: + ret = bq25980_get_input_curr_lim(bq); + if (ret < 0) + return ret; + + val->intval = ret; + break; + + case POWER_SUPPLY_PROP_HEALTH: + val->intval = POWER_SUPPLY_HEALTH_GOOD; + + if (state.tflt) + val->intval = POWER_SUPPLY_HEALTH_OVERHEAT; + else if (state.ovp) + val->intval = POWER_SUPPLY_HEALTH_OVERVOLTAGE; + else if (state.ocp) + val->intval = POWER_SUPPLY_HEALTH_OVERCURRENT; + else if (state.wdt) + val->intval = + POWER_SUPPLY_HEALTH_WATCHDOG_TIMER_EXPIRE; + break; + + case POWER_SUPPLY_PROP_STATUS: + val->intval = POWER_SUPPLY_STATUS_UNKNOWN; + + if ((state.ce) && (!state.hiz)) + val->intval = POWER_SUPPLY_STATUS_CHARGING; + else if (state.dischg) + val->intval = POWER_SUPPLY_STATUS_DISCHARGING; + else if (!state.ce) + val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; + break; + + case POWER_SUPPLY_PROP_CHARGE_TYPE: + val->intval = POWER_SUPPLY_CHARGE_TYPE_UNKNOWN; + + if (!state.ce) + val->intval = POWER_SUPPLY_CHARGE_TYPE_NONE; + else if (state.bypass) + val->intval = POWER_SUPPLY_CHARGE_TYPE_FAST; + else if (!state.bypass) + val->intval = POWER_SUPPLY_CHARGE_TYPE_STANDARD; + break; + + case POWER_SUPPLY_PROP_CURRENT_NOW: + ret = bq25980_get_adc_ibus(bq); + if (ret < 0) + return ret; + + val->intval = ret; + break; + + case POWER_SUPPLY_PROP_VOLTAGE_NOW: + ret = bq25980_get_adc_vbus(bq); + if (ret < 0) + return ret; + + val->intval = ret; + break; + + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT: + ret = bq25980_get_const_charge_curr(bq); + if (ret < 0) + return ret; + + val->intval = ret; + break; + + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE: + ret = bq25980_get_const_charge_volt(bq); + if (ret < 0) + return ret; + + val->intval = ret; + break; + + default: + return -EINVAL; + } + + return ret; +} + +static bool bq25980_state_changed(struct bq25980_device *bq, + struct bq25980_state *new_state) +{ + struct bq25980_state old_state; + + mutex_lock(&bq->lock); + old_state = bq->state; + mutex_unlock(&bq->lock); + + return (old_state.dischg != new_state->dischg || + old_state.ovp != new_state->ovp || + old_state.ocp != new_state->ocp || + old_state.online != new_state->online || + old_state.wdt != new_state->wdt || + old_state.tflt != new_state->tflt || + old_state.ce != new_state->ce || + old_state.hiz != new_state->hiz || + old_state.bypass != new_state->bypass); +} + +static irqreturn_t bq25980_irq_handler_thread(int irq, void *private) +{ + struct bq25980_device *bq = private; + struct bq25980_state state; + int ret; + + ret = bq25980_get_state(bq, &state); + if (ret < 0) + goto irq_out; + + if (!bq25980_state_changed(bq, &state)) + goto irq_out; + + mutex_lock(&bq->lock); + bq->state = state; + mutex_unlock(&bq->lock); + + power_supply_changed(bq->charger); + +irq_out: + return IRQ_HANDLED; +} + +static enum power_supply_property bq25980_power_supply_props[] = { + POWER_SUPPLY_PROP_MANUFACTURER, + POWER_SUPPLY_PROP_MODEL_NAME, + POWER_SUPPLY_PROP_STATUS, + POWER_SUPPLY_PROP_ONLINE, + POWER_SUPPLY_PROP_HEALTH, + POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT, + POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, + POWER_SUPPLY_PROP_CHARGE_TYPE, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE, + POWER_SUPPLY_PROP_CURRENT_NOW, + POWER_SUPPLY_PROP_VOLTAGE_NOW, +}; + +static enum power_supply_property bq25980_battery_props[] = { + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX, + POWER_SUPPLY_PROP_CURRENT_NOW, + POWER_SUPPLY_PROP_VOLTAGE_NOW, +}; + +static char *bq25980_charger_supplied_to[] = { + "main-battery", +}; + +static int bq25980_property_is_writeable(struct power_supply *psy, + enum power_supply_property prop) +{ + switch (prop) { + case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT: + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE: + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT: + case POWER_SUPPLY_PROP_CHARGE_TYPE: + case POWER_SUPPLY_PROP_STATUS: + case POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT: + return true; + default: + return false; + } +} + +static const struct power_supply_desc bq25980_power_supply_desc = { + .name = "bq25980-charger", + .type = POWER_SUPPLY_TYPE_MAINS, + .properties = bq25980_power_supply_props, + .num_properties = ARRAY_SIZE(bq25980_power_supply_props), + .get_property = bq25980_get_charger_property, + .set_property = bq25980_set_charger_property, + .property_is_writeable = bq25980_property_is_writeable, +}; + +static struct power_supply_desc bq25980_battery_desc = { + .name = "bq25980-battery", + .type = POWER_SUPPLY_TYPE_BATTERY, + .get_property = bq25980_get_battery_property, + .set_property = bq25980_set_battery_property, + .properties = bq25980_battery_props, + .num_properties = ARRAY_SIZE(bq25980_battery_props), + .property_is_writeable = bq25980_property_is_writeable, +}; + + +static bool bq25980_is_volatile_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case BQ25980_CHRGR_CTRL_2: + case BQ25980_STAT1...BQ25980_FLAG5: + case BQ25980_ADC_CONTROL1...BQ25980_TDIE_ADC_LSB: + return true; + default: + return false; + } +} + +static const struct regmap_config bq25980_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + + .max_register = BQ25980_CHRGR_CTRL_6, + .reg_defaults = bq25980_reg_defs, + .num_reg_defaults = ARRAY_SIZE(bq25980_reg_defs), + .cache_type = REGCACHE_RBTREE, + .volatile_reg = bq25980_is_volatile_reg, +}; + +static const struct regmap_config bq25975_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + + .max_register = BQ25980_CHRGR_CTRL_6, + .reg_defaults = bq25975_reg_defs, + .num_reg_defaults = ARRAY_SIZE(bq25975_reg_defs), + .cache_type = REGCACHE_RBTREE, + .volatile_reg = bq25980_is_volatile_reg, +}; + +static const struct regmap_config bq25960_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + + .max_register = BQ25980_CHRGR_CTRL_6, + .reg_defaults = bq25960_reg_defs, + .num_reg_defaults = ARRAY_SIZE(bq25960_reg_defs), + .cache_type = REGCACHE_RBTREE, + .volatile_reg = bq25980_is_volatile_reg, +}; + +static const struct bq25980_chip_info bq25980_chip_info_tbl[] = { + [BQ25980] = { + .model_id = BQ25980, + .regmap_config = &bq25980_regmap_config, + + .busocp_def = BQ25980_BUSOCP_DFLT_uA, + .busocp_sc_min = BQ25960_BUSOCP_SC_MAX_uA, + .busocp_sc_max = BQ25980_BUSOCP_SC_MAX_uA, + .busocp_byp_max = BQ25980_BUSOCP_BYP_MAX_uA, + .busocp_byp_min = BQ25980_BUSOCP_MIN_uA, + + .busovp_sc_def = BQ25980_BUSOVP_DFLT_uV, + .busovp_byp_def = BQ25980_BUSOVP_BYPASS_DFLT_uV, + .busovp_sc_step = BQ25980_BUSOVP_SC_STEP_uV, + .busovp_sc_offset = BQ25980_BUSOVP_SC_OFFSET_uV, + .busovp_byp_step = BQ25980_BUSOVP_BYP_STEP_uV, + .busovp_byp_offset = BQ25980_BUSOVP_BYP_OFFSET_uV, + .busovp_sc_min = BQ25980_BUSOVP_SC_MIN_uV, + .busovp_sc_max = BQ25980_BUSOVP_SC_MAX_uV, + .busovp_byp_min = BQ25980_BUSOVP_BYP_MIN_uV, + .busovp_byp_max = BQ25980_BUSOVP_BYP_MAX_uV, + + .batovp_def = BQ25980_BATOVP_DFLT_uV, + .batovp_max = BQ25980_BATOVP_MAX_uV, + .batovp_min = BQ25980_BATOVP_MIN_uV, + .batovp_step = BQ25980_BATOVP_STEP_uV, + .batovp_offset = BQ25980_BATOVP_OFFSET_uV, + + .batocp_def = BQ25980_BATOCP_DFLT_uA, + .batocp_max = BQ25980_BATOCP_MAX_uA, + }, + + [BQ25975] = { + .model_id = BQ25975, + .regmap_config = &bq25975_regmap_config, + + .busocp_def = BQ25975_BUSOCP_DFLT_uA, + .busocp_sc_min = BQ25975_BUSOCP_SC_MAX_uA, + .busocp_sc_max = BQ25975_BUSOCP_SC_MAX_uA, + .busocp_byp_min = BQ25980_BUSOCP_MIN_uA, + .busocp_byp_max = BQ25975_BUSOCP_BYP_MAX_uA, + + .busovp_sc_def = BQ25975_BUSOVP_DFLT_uV, + .busovp_byp_def = BQ25975_BUSOVP_BYPASS_DFLT_uV, + .busovp_sc_step = BQ25975_BUSOVP_SC_STEP_uV, + .busovp_sc_offset = BQ25975_BUSOVP_SC_OFFSET_uV, + .busovp_byp_step = BQ25975_BUSOVP_BYP_STEP_uV, + .busovp_byp_offset = BQ25975_BUSOVP_BYP_OFFSET_uV, + .busovp_sc_min = BQ25975_BUSOVP_SC_MIN_uV, + .busovp_sc_max = BQ25975_BUSOVP_SC_MAX_uV, + .busovp_byp_min = BQ25975_BUSOVP_BYP_MIN_uV, + .busovp_byp_max = BQ25975_BUSOVP_BYP_MAX_uV, + + .batovp_def = BQ25975_BATOVP_DFLT_uV, + .batovp_max = BQ25975_BATOVP_MAX_uV, + .batovp_min = BQ25975_BATOVP_MIN_uV, + .batovp_step = BQ25975_BATOVP_STEP_uV, + .batovp_offset = BQ25975_BATOVP_OFFSET_uV, + + .batocp_def = BQ25980_BATOCP_DFLT_uA, + .batocp_max = BQ25980_BATOCP_MAX_uA, + }, + + [BQ25960] = { + .model_id = BQ25960, + .regmap_config = &bq25960_regmap_config, + + .busocp_def = BQ25960_BUSOCP_DFLT_uA, + .busocp_sc_min = BQ25960_BUSOCP_SC_MAX_uA, + .busocp_sc_max = BQ25960_BUSOCP_SC_MAX_uA, + .busocp_byp_min = BQ25960_BUSOCP_SC_MAX_uA, + .busocp_byp_max = BQ25960_BUSOCP_BYP_MAX_uA, + + .busovp_sc_def = BQ25975_BUSOVP_DFLT_uV, + .busovp_byp_def = BQ25975_BUSOVP_BYPASS_DFLT_uV, + .busovp_sc_step = BQ25960_BUSOVP_SC_STEP_uV, + .busovp_sc_offset = BQ25960_BUSOVP_SC_OFFSET_uV, + .busovp_byp_step = BQ25960_BUSOVP_BYP_STEP_uV, + .busovp_byp_offset = BQ25960_BUSOVP_BYP_OFFSET_uV, + .busovp_sc_min = BQ25960_BUSOVP_SC_MIN_uV, + .busovp_sc_max = BQ25960_BUSOVP_SC_MAX_uV, + .busovp_byp_min = BQ25960_BUSOVP_BYP_MIN_uV, + .busovp_byp_max = BQ25960_BUSOVP_BYP_MAX_uV, + + .batovp_def = BQ25960_BATOVP_DFLT_uV, + .batovp_max = BQ25960_BATOVP_MAX_uV, + .batovp_min = BQ25960_BATOVP_MIN_uV, + .batovp_step = BQ25960_BATOVP_STEP_uV, + .batovp_offset = BQ25960_BATOVP_OFFSET_uV, + + .batocp_def = BQ25960_BATOCP_DFLT_uA, + .batocp_max = BQ25960_BATOCP_MAX_uA, + }, +}; + +static int bq25980_power_supply_init(struct bq25980_device *bq, + struct device *dev) +{ + struct power_supply_config psy_cfg = { .drv_data = bq, + .of_node = dev->of_node, }; + + psy_cfg.supplied_to = bq25980_charger_supplied_to; + psy_cfg.num_supplicants = ARRAY_SIZE(bq25980_charger_supplied_to); + + bq->charger = devm_power_supply_register(bq->dev, + &bq25980_power_supply_desc, + &psy_cfg); + if (IS_ERR(bq->charger)) + return -EINVAL; + + bq->battery = devm_power_supply_register(bq->dev, + &bq25980_battery_desc, + &psy_cfg); + if (IS_ERR(bq->battery)) + return -EINVAL; + + return 0; +} + +static int bq25980_hw_init(struct bq25980_device *bq) +{ + struct power_supply_battery_info bat_info = { }; + int wd_reg_val = BQ25980_WATCHDOG_DIS; + int wd_max_val = BQ25980_NUM_WD_VAL - 1; + int ret = 0; + int curr_val; + int volt_val; + int i; + + if (bq->watchdog_timer) { + if (bq->watchdog_timer >= bq25980_watchdog_time[wd_max_val]) + wd_reg_val = wd_max_val; + else { + for (i = 0; i < wd_max_val; i++) { + if (bq->watchdog_timer > bq25980_watchdog_time[i] && + bq->watchdog_timer < bq25980_watchdog_time[i + 1]) { + wd_reg_val = i; + break; + } + } + } + } + + ret = regmap_update_bits(bq->regmap, BQ25980_CHRGR_CTRL_3, + BQ25980_WATCHDOG_MASK, wd_reg_val); + if (ret) + return ret; + + ret = power_supply_get_battery_info(bq->charger, &bat_info); + if (ret) { + dev_warn(bq->dev, "battery info missing\n"); + return -EINVAL; + } + + bq->init_data.ichg_max = bat_info.constant_charge_current_max_ua; + bq->init_data.vreg_max = bat_info.constant_charge_voltage_max_uv; + + if (bq->state.bypass) { + ret = regmap_update_bits(bq->regmap, BQ25980_CHRGR_CTRL_2, + BQ25980_EN_BYPASS, BQ25980_EN_BYPASS); + if (ret) + return ret; + + curr_val = bq->init_data.bypass_ilim; + volt_val = bq->init_data.bypass_vlim; + } else { + curr_val = bq->init_data.sc_ilim; + volt_val = bq->init_data.sc_vlim; + } + + ret = bq25980_set_input_curr_lim(bq, curr_val); + if (ret) + return ret; + + ret = bq25980_set_input_volt_lim(bq, volt_val); + if (ret) + return ret; + + return regmap_update_bits(bq->regmap, BQ25980_ADC_CONTROL1, + BQ25980_ADC_EN, BQ25980_ADC_EN); +} + +static int bq25980_parse_dt(struct bq25980_device *bq) +{ + int ret; + + ret = device_property_read_u32(bq->dev, "ti,watchdog-timeout-ms", + &bq->watchdog_timer); + if (ret) + bq->watchdog_timer = BQ25980_WATCHDOG_MIN; + + if (bq->watchdog_timer > BQ25980_WATCHDOG_MAX || + bq->watchdog_timer < BQ25980_WATCHDOG_MIN) + return -EINVAL; + + ret = device_property_read_u32(bq->dev, + "ti,sc-ovp-limit-microvolt", + &bq->init_data.sc_vlim); + if (ret) + bq->init_data.sc_vlim = bq->chip_info->busovp_sc_def; + + if (bq->init_data.sc_vlim > bq->chip_info->busovp_sc_max || + bq->init_data.sc_vlim < bq->chip_info->busovp_sc_min) { + dev_err(bq->dev, "SC ovp limit is out of range\n"); + return -EINVAL; + } + + ret = device_property_read_u32(bq->dev, + "ti,sc-ocp-limit-microamp", + &bq->init_data.sc_ilim); + if (ret) + bq->init_data.sc_ilim = bq->chip_info->busocp_def; + + if (bq->init_data.sc_ilim > bq->chip_info->busocp_sc_max || + bq->init_data.sc_ilim < bq->chip_info->busocp_sc_min) { + dev_err(bq->dev, "SC ocp limit is out of range\n"); + return -EINVAL; + } + + ret = device_property_read_u32(bq->dev, + "ti,bypass-ovp-limit-microvolt", + &bq->init_data.bypass_vlim); + if (ret) + bq->init_data.bypass_vlim = bq->chip_info->busovp_byp_def; + + if (bq->init_data.bypass_vlim > bq->chip_info->busovp_byp_max || + bq->init_data.bypass_vlim < bq->chip_info->busovp_byp_min) { + dev_err(bq->dev, "Bypass ovp limit is out of range\n"); + return -EINVAL; + } + + ret = device_property_read_u32(bq->dev, + "ti,bypass-ocp-limit-microamp", + &bq->init_data.bypass_ilim); + if (ret) + bq->init_data.bypass_ilim = bq->chip_info->busocp_def; + + if (bq->init_data.bypass_ilim > bq->chip_info->busocp_byp_max || + bq->init_data.bypass_ilim < bq->chip_info->busocp_byp_min) { + dev_err(bq->dev, "Bypass ocp limit is out of range\n"); + return -EINVAL; + } + + + bq->state.bypass = device_property_read_bool(bq->dev, + "ti,bypass-enable"); + return 0; +} + +static int bq25980_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + struct device *dev = &client->dev; + struct bq25980_device *bq; + int ret; + + bq = devm_kzalloc(dev, sizeof(*bq), GFP_KERNEL); + if (!bq) + return -ENOMEM; + + bq->client = client; + bq->dev = dev; + + mutex_init(&bq->lock); + + strncpy(bq->model_name, id->name, I2C_NAME_SIZE); + bq->chip_info = &bq25980_chip_info_tbl[id->driver_data]; + + bq->regmap = devm_regmap_init_i2c(client, + bq->chip_info->regmap_config); + if (IS_ERR(bq->regmap)) { + dev_err(dev, "Failed to allocate register map\n"); + return PTR_ERR(bq->regmap); + } + + i2c_set_clientdata(client, bq); + + ret = bq25980_parse_dt(bq); + if (ret) { + dev_err(dev, "Failed to read device tree properties%d\n", ret); + return ret; + } + + if (client->irq) { + ret = devm_request_threaded_irq(dev, client->irq, NULL, + bq25980_irq_handler_thread, + IRQF_TRIGGER_FALLING | + IRQF_ONESHOT, + dev_name(&client->dev), bq); + if (ret) + return ret; + } + + ret = bq25980_power_supply_init(bq, dev); + if (ret) { + dev_err(dev, "Failed to register power supply\n"); + return ret; + } + + ret = bq25980_hw_init(bq); + if (ret) { + dev_err(dev, "Cannot initialize the chip.\n"); + return ret; + } + + return 0; +} + +static const struct i2c_device_id bq25980_i2c_ids[] = { + { "bq25980", BQ25980 }, + { "bq25975", BQ25975 }, + { "bq25975", BQ25975 }, + {}, +}; +MODULE_DEVICE_TABLE(i2c, bq25980_i2c_ids); + +static const struct of_device_id bq25980_of_match[] = { + { .compatible = "ti,bq25980", .data = (void *)BQ25980 }, + { .compatible = "ti,bq25975", .data = (void *)BQ25975 }, + { .compatible = "ti,bq25960", .data = (void *)BQ25960 }, + { }, +}; +MODULE_DEVICE_TABLE(of, bq25980_of_match); + +static struct i2c_driver bq25980_driver = { + .driver = { + .name = "bq25980-charger", + .of_match_table = bq25980_of_match, + }, + .probe = bq25980_probe, + .id_table = bq25980_i2c_ids, +}; +module_i2c_driver(bq25980_driver); + +MODULE_AUTHOR("Dan Murphy <dmurphy@ti.com>"); +MODULE_AUTHOR("Ricardo Rivera-Matos <r-rivera-matos@ti.com>"); +MODULE_DESCRIPTION("bq25980 charger driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/power/supply/bq25980_charger.h b/drivers/power/supply/bq25980_charger.h new file mode 100644 index 000000000000..39f94eba5f6c --- /dev/null +++ b/drivers/power/supply/bq25980_charger.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com/ */ + +#ifndef BQ25980_CHARGER_H +#define BQ25980_CHARGER_H + +#define BQ25980_MANUFACTURER "Texas Instruments" + +#define BQ25980_BATOVP 0x0 +#define BQ25980_BATOVP_ALM 0x1 +#define BQ25980_BATOCP 0x2 +#define BQ25980_BATOCP_ALM 0x3 +#define BQ25980_BATUCP_ALM 0x4 +#define BQ25980_CHRGR_CTRL_1 0x5 +#define BQ25980_BUSOVP 0x6 +#define BQ25980_BUSOVP_ALM 0x7 +#define BQ25980_BUSOCP 0x8 +#define BQ25980_BUSOCP_ALM 0x9 +#define BQ25980_TEMP_CONTROL 0xA +#define BQ25980_TDIE_ALM 0xB +#define BQ25980_TSBUS_FLT 0xC +#define BQ25980_TSBAT_FLG 0xD +#define BQ25980_VAC_CONTROL 0xE +#define BQ25980_CHRGR_CTRL_2 0xF +#define BQ25980_CHRGR_CTRL_3 0x10 +#define BQ25980_CHRGR_CTRL_4 0x11 +#define BQ25980_CHRGR_CTRL_5 0x12 +#define BQ25980_STAT1 0x13 +#define BQ25980_STAT2 0x14 +#define BQ25980_STAT3 0x15 +#define BQ25980_STAT4 0x16 +#define BQ25980_STAT5 0x17 +#define BQ25980_FLAG1 0x18 +#define BQ25980_FLAG2 0x19 +#define BQ25980_FLAG3 0x1A +#define BQ25980_FLAG4 0x1B +#define BQ25980_FLAG5 0x1C +#define BQ25980_MASK1 0x1D +#define BQ25980_MASK2 0x1E +#define BQ25980_MASK3 0x1F +#define BQ25980_MASK4 0x20 +#define BQ25980_MASK5 0x21 +#define BQ25980_DEVICE_INFO 0x22 +#define BQ25980_ADC_CONTROL1 0x23 +#define BQ25980_ADC_CONTROL2 0x24 +#define BQ25980_IBUS_ADC_MSB 0x25 +#define BQ25980_IBUS_ADC_LSB 0x26 +#define BQ25980_VBUS_ADC_MSB 0x27 +#define BQ25980_VBUS_ADC_LSB 0x28 +#define BQ25980_VAC1_ADC_MSB 0x29 +#define BQ25980_VAC1_ADC_LSB 0x2A +#define BQ25980_VAC2_ADC_MSB 0x2B +#define BQ25980_VAC2_ADC_LSB 0x2C +#define BQ25980_VOUT_ADC_MSB 0x2D +#define BQ25980_VOUT_ADC_LSB 0x2E +#define BQ25980_VBAT_ADC_MSB 0x2F +#define BQ25980_VBAT_ADC_LSB 0x30 +#define BQ25980_IBAT_ADC_MSB 0x31 +#define BQ25980_IBAT_ADC_LSB 0x32 +#define BQ25980_TSBUS_ADC_MSB 0x33 +#define BQ25980_TSBUS_ADC_LSB 0x34 +#define BQ25980_TSBAT_ADC_MSB 0x35 +#define BQ25980_TSBAT_ADC_LSB 0x36 +#define BQ25980_TDIE_ADC_MSB 0x37 +#define BQ25980_TDIE_ADC_LSB 0x38 +#define BQ25980_DEGLITCH_TIME 0x39 +#define BQ25980_CHRGR_CTRL_6 0x3A + +#define BQ25980_BUSOCP_STEP_uA 250000 +#define BQ25980_BUSOCP_OFFSET_uA 1000000 + +#define BQ25980_BUSOCP_DFLT_uA 4250000 +#define BQ25975_BUSOCP_DFLT_uA 4250000 +#define BQ25960_BUSOCP_DFLT_uA 3250000 + +#define BQ25980_BUSOCP_MIN_uA 1000000 + +#define BQ25980_BUSOCP_SC_MAX_uA 5750000 +#define BQ25975_BUSOCP_SC_MAX_uA 5750000 +#define BQ25960_BUSOCP_SC_MAX_uA 3750000 + +#define BQ25980_BUSOCP_BYP_MAX_uA 8500000 +#define BQ25975_BUSOCP_BYP_MAX_uA 8500000 +#define BQ25960_BUSOCP_BYP_MAX_uA 5750000 + +#define BQ25980_BUSOVP_SC_STEP_uV 100000 +#define BQ25975_BUSOVP_SC_STEP_uV 50000 +#define BQ25960_BUSOVP_SC_STEP_uV 50000 +#define BQ25980_BUSOVP_SC_OFFSET_uV 14000000 +#define BQ25975_BUSOVP_SC_OFFSET_uV 7000000 +#define BQ25960_BUSOVP_SC_OFFSET_uV 7000000 + +#define BQ25980_BUSOVP_BYP_STEP_uV 50000 +#define BQ25975_BUSOVP_BYP_STEP_uV 25000 +#define BQ25960_BUSOVP_BYP_STEP_uV 25000 +#define BQ25980_BUSOVP_BYP_OFFSET_uV 7000000 +#define BQ25975_BUSOVP_BYP_OFFSET_uV 3500000 +#define BQ25960_BUSOVP_BYP_OFFSET_uV 3500000 + +#define BQ25980_BUSOVP_DFLT_uV 17800000 +#define BQ25980_BUSOVP_BYPASS_DFLT_uV 8900000 +#define BQ25975_BUSOVP_DFLT_uV 8900000 +#define BQ25975_BUSOVP_BYPASS_DFLT_uV 4450000 +#define BQ25960_BUSOVP_DFLT_uV 8900000 + +#define BQ25980_BUSOVP_SC_MIN_uV 14000000 +#define BQ25975_BUSOVP_SC_MIN_uV 7000000 +#define BQ25960_BUSOVP_SC_MIN_uV 7000000 +#define BQ25980_BUSOVP_BYP_MIN_uV 7000000 +#define BQ25975_BUSOVP_BYP_MIN_uV 3500000 +#define BQ25960_BUSOVP_BYP_MIN_uV 3500000 + +#define BQ25980_BUSOVP_SC_MAX_uV 22000000 +#define BQ25975_BUSOVP_SC_MAX_uV 12750000 +#define BQ25960_BUSOVP_SC_MAX_uV 12750000 + +#define BQ25980_BUSOVP_BYP_MAX_uV 12750000 +#define BQ25975_BUSOVP_BYP_MAX_uV 6500000 +#define BQ25960_BUSOVP_BYP_MAX_uV 6500000 + +#define BQ25980_BATOVP_STEP_uV 20000 +#define BQ25975_BATOVP_STEP_uV 10000 +#define BQ25960_BATOVP_STEP_uV 10000 + +#define BQ25980_BATOVP_OFFSET_uV 7000000 +#define BQ25975_BATOVP_OFFSET_uV 3500000 +#define BQ25960_BATOVP_OFFSET_uV 3500000 + +#define BQ25980_BATOVP_DFLT_uV 14000000 +#define BQ25975_BATOVP_DFLT_uV 8900000 +#define BQ25960_BATOVP_DFLT_uV 8900000 + +#define BQ25980_BATOVP_MIN_uV 7000000 +#define BQ25975_BATOVP_MIN_uV 3500000 +#define BQ25960_BATOVP_MIN_uV 3500000 + +#define BQ25980_BATOVP_MAX_uV 9540000 +#define BQ25975_BATOVP_MAX_uV 4770000 +#define BQ25960_BATOVP_MAX_uV 4770000 + +#define BQ25980_BATOCP_STEP_uA 100000 + +#define BQ25980_BATOCP_MASK GENMASK(6, 0) + +#define BQ25980_BATOCP_DFLT_uA 8100000 +#define BQ25960_BATOCP_DFLT_uA 6100000 + +#define BQ25980_BATOCP_MIN_uA 2000000 + +#define BQ25980_BATOCP_MAX_uA 11000000 +#define BQ25975_BATOCP_MAX_uA 11000000 +#define BQ25960_BATOCP_MAX_uA 7000000 + +#define BQ25980_ENABLE_HIZ 0xff +#define BQ25980_DISABLE_HIZ 0x0 +#define BQ25980_EN_BYPASS BIT(3) +#define BQ25980_STAT1_OVP_MASK (BIT(6) | BIT(5) | BIT(0)) +#define BQ25980_STAT3_OVP_MASK (BIT(7) | BIT(6)) +#define BQ25980_STAT1_OCP_MASK BIT(3) +#define BQ25980_STAT2_OCP_MASK (BIT(6) | BIT(1)) +#define BQ25980_STAT4_TFLT_MASK GENMASK(5, 1) +#define BQ25980_WD_STAT BIT(0) +#define BQ25980_PRESENT_MASK GENMASK(4, 2) +#define BQ25980_CHG_EN BIT(4) +#define BQ25980_EN_HIZ BIT(6) +#define BQ25980_ADC_EN BIT(7) + +#define BQ25980_ADC_VOLT_STEP_uV 1000 +#define BQ25980_ADC_CURR_STEP_uA 1000 +#define BQ25980_ADC_POLARITY_BIT BIT(7) + +#define BQ25980_WATCHDOG_MASK GENMASK(4, 3) +#define BQ25980_WATCHDOG_DIS BIT(2) +#define BQ25980_WATCHDOG_MAX 300000 +#define BQ25980_WATCHDOG_MIN 0 +#define BQ25980_NUM_WD_VAL 4 + +#endif /* BQ25980_CHARGER_H */ diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index a123f6e21f08..315e0909e6a4 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * BQ27xxx battery driver * @@ -9,14 +10,6 @@ * * Based on a previous work by Copyright (C) 2008 Texas Instruments, Inc. * - * This package is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * THIS PACKAGE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED - * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. - * * Datasheets: * https://www.ti.com/product/bq27000 * https://www.ti.com/product/bq27200 @@ -45,6 +38,7 @@ * https://www.ti.com/product/bq27621-g1 * https://www.ti.com/product/bq27z561 * https://www.ti.com/product/bq28z610 + * https://www.ti.com/product/bq34z100-g1 */ #include <linux/device.h> @@ -83,7 +77,7 @@ /* BQ27Z561 has different layout for Flags register */ #define BQ27Z561_FLAG_FDC BIT(4) /* Battery fully discharged */ -#define BQ27Z561_FLAG_FC BIT(5) /* Battery fully charged */ +#define BQ27Z561_FLAG_FC BIT(5) /* Battery fully charged */ #define BQ27Z561_FLAG_DIS_CH BIT(6) /* Battery is discharging */ /* control register params */ @@ -483,6 +477,26 @@ static u8 [BQ27XXX_REG_DCAP] = 0x3c, [BQ27XXX_REG_AP] = 0x22, BQ27XXX_DM_REG_ROWS, + }, + bq34z100_regs[BQ27XXX_REG_MAX] = { + [BQ27XXX_REG_CTRL] = 0x00, + [BQ27XXX_REG_TEMP] = 0x0c, + [BQ27XXX_REG_INT_TEMP] = 0x2a, + [BQ27XXX_REG_VOLT] = 0x08, + [BQ27XXX_REG_AI] = 0x0a, + [BQ27XXX_REG_FLAGS] = 0x0e, + [BQ27XXX_REG_TTE] = 0x18, + [BQ27XXX_REG_TTF] = 0x1a, + [BQ27XXX_REG_TTES] = 0x1e, + [BQ27XXX_REG_TTECP] = INVALID_REG_ADDR, + [BQ27XXX_REG_NAC] = INVALID_REG_ADDR, + [BQ27XXX_REG_FCC] = 0x06, + [BQ27XXX_REG_CYCT] = 0x2c, + [BQ27XXX_REG_AE] = 0x24, + [BQ27XXX_REG_SOC] = 0x02, + [BQ27XXX_REG_DCAP] = 0x3c, + [BQ27XXX_REG_AP] = 0x22, + BQ27XXX_DM_REG_ROWS, }; static enum power_supply_property bq27000_props[] = { @@ -757,6 +771,27 @@ static enum power_supply_property bq28z610_props[] = { POWER_SUPPLY_PROP_MANUFACTURER, }; +static enum power_supply_property bq34z100_props[] = { + POWER_SUPPLY_PROP_STATUS, + POWER_SUPPLY_PROP_PRESENT, + POWER_SUPPLY_PROP_VOLTAGE_NOW, + POWER_SUPPLY_PROP_CURRENT_NOW, + POWER_SUPPLY_PROP_CAPACITY, + POWER_SUPPLY_PROP_CAPACITY_LEVEL, + POWER_SUPPLY_PROP_TEMP, + POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW, + POWER_SUPPLY_PROP_TIME_TO_EMPTY_AVG, + POWER_SUPPLY_PROP_TIME_TO_FULL_NOW, + POWER_SUPPLY_PROP_TECHNOLOGY, + POWER_SUPPLY_PROP_CHARGE_FULL, + POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN, + POWER_SUPPLY_PROP_CYCLE_COUNT, + POWER_SUPPLY_PROP_ENERGY_NOW, + POWER_SUPPLY_PROP_POWER_AVG, + POWER_SUPPLY_PROP_HEALTH, + POWER_SUPPLY_PROP_MANUFACTURER, +}; + struct bq27xxx_dm_reg { u8 subclass_id; u8 offset; @@ -854,13 +889,17 @@ static struct bq27xxx_dm_reg bq27621_dm_regs[] = { #define bq27z561_dm_regs 0 #define bq28z610_dm_regs 0 - -#define BQ27XXX_O_ZERO 0x00000001 -#define BQ27XXX_O_OTDC 0x00000002 /* has OTC/OTD overtemperature flags */ -#define BQ27XXX_O_UTOT 0x00000004 /* has OT overtemperature flag */ -#define BQ27XXX_O_CFGUP 0x00000008 -#define BQ27XXX_O_RAM 0x00000010 -#define BQ27Z561_O_BITS 0x00000020 +#define bq34z100_dm_regs 0 + +#define BQ27XXX_O_ZERO BIT(0) +#define BQ27XXX_O_OTDC BIT(1) /* has OTC/OTD overtemperature flags */ +#define BQ27XXX_O_UTOT BIT(2) /* has OT overtemperature flag */ +#define BQ27XXX_O_CFGUP BIT(3) +#define BQ27XXX_O_RAM BIT(4) +#define BQ27Z561_O_BITS BIT(5) +#define BQ27XXX_O_SOC_SI BIT(6) /* SoC is single register */ +#define BQ27XXX_O_HAS_CI BIT(7) /* has Capacity Inaccurate flag */ +#define BQ27XXX_O_MUL_CHEM BIT(8) /* multiple chemistries supported */ #define BQ27XXX_DATA(ref, key, opt) { \ .opts = (opt), \ @@ -878,8 +917,8 @@ static struct { enum power_supply_property *props; size_t props_size; } bq27xxx_chip_data[] = { - [BQ27000] = BQ27XXX_DATA(bq27000, 0 , BQ27XXX_O_ZERO), - [BQ27010] = BQ27XXX_DATA(bq27010, 0 , BQ27XXX_O_ZERO), + [BQ27000] = BQ27XXX_DATA(bq27000, 0 , BQ27XXX_O_ZERO | BQ27XXX_O_SOC_SI | BQ27XXX_O_HAS_CI), + [BQ27010] = BQ27XXX_DATA(bq27010, 0 , BQ27XXX_O_ZERO | BQ27XXX_O_SOC_SI | BQ27XXX_O_HAS_CI), [BQ2750X] = BQ27XXX_DATA(bq2750x, 0 , BQ27XXX_O_OTDC), [BQ2751X] = BQ27XXX_DATA(bq2751x, 0 , BQ27XXX_O_OTDC), [BQ2752X] = BQ27XXX_DATA(bq2752x, 0 , BQ27XXX_O_OTDC), @@ -907,6 +946,8 @@ static struct { [BQ27621] = BQ27XXX_DATA(bq27621, 0x80008000, BQ27XXX_O_UTOT | BQ27XXX_O_CFGUP | BQ27XXX_O_RAM), [BQ27Z561] = BQ27XXX_DATA(bq27z561, 0 , BQ27Z561_O_BITS), [BQ28Z610] = BQ27XXX_DATA(bq28z610, 0 , BQ27Z561_O_BITS), + [BQ34Z100] = BQ27XXX_DATA(bq34z100, 0 , BQ27XXX_O_OTDC | BQ27XXX_O_SOC_SI | \ + BQ27XXX_O_HAS_CI | BQ27XXX_O_MUL_CHEM), }; static DEFINE_MUTEX(bq27xxx_list_lock); @@ -1426,7 +1467,7 @@ static int bq27xxx_battery_read_soc(struct bq27xxx_device_info *di) { int soc; - if (di->opts & BQ27XXX_O_ZERO) + if (di->opts & BQ27XXX_O_SOC_SI) soc = bq27xxx_read(di, BQ27XXX_REG_SOC, true); else soc = bq27xxx_read(di, BQ27XXX_REG_SOC, false); @@ -1664,7 +1705,7 @@ static int bq27xxx_battery_read_health(struct bq27xxx_device_info *di) void bq27xxx_battery_update(struct bq27xxx_device_info *di) { struct bq27xxx_reg_cache cache = {0, }; - bool has_ci_flag = di->opts & BQ27XXX_O_ZERO; + bool has_ci_flag = di->opts & BQ27XXX_O_HAS_CI; bool has_singe_flag = di->opts & BQ27XXX_O_ZERO; cache.flags = bq27xxx_read(di, BQ27XXX_REG_FLAGS, has_singe_flag); @@ -1772,8 +1813,6 @@ static int bq27xxx_battery_status(struct bq27xxx_device_info *di, status = POWER_SUPPLY_STATUS_FULL; else if (di->cache.flags & BQ27000_FLAG_CHGS) status = POWER_SUPPLY_STATUS_CHARGING; - else if (power_supply_am_i_supplied(di->bat) > 0) - status = POWER_SUPPLY_STATUS_NOT_CHARGING; else status = POWER_SUPPLY_STATUS_DISCHARGING; } else if (di->opts & BQ27Z561_O_BITS) { @@ -1792,6 +1831,10 @@ static int bq27xxx_battery_status(struct bq27xxx_device_info *di, status = POWER_SUPPLY_STATUS_CHARGING; } + if ((status == POWER_SUPPLY_STATUS_DISCHARGING) && + (power_supply_am_i_supplied(di->bat) > 0)) + status = POWER_SUPPLY_STATUS_NOT_CHARGING; + val->intval = status; return 0; @@ -1916,7 +1959,10 @@ static int bq27xxx_battery_get_property(struct power_supply *psy, ret = bq27xxx_simple_value(di->cache.time_to_full, val); break; case POWER_SUPPLY_PROP_TECHNOLOGY: - val->intval = POWER_SUPPLY_TECHNOLOGY_LION; + if (di->opts & BQ27XXX_O_MUL_CHEM) + val->intval = POWER_SUPPLY_TECHNOLOGY_UNKNOWN; + else + val->intval = POWER_SUPPLY_TECHNOLOGY_LION; break; case POWER_SUPPLY_PROP_CHARGE_NOW: ret = bq27xxx_simple_value(bq27xxx_battery_read_nac(di), val); @@ -1992,13 +2038,9 @@ int bq27xxx_battery_setup(struct bq27xxx_device_info *di) psy_desc->external_power_changed = bq27xxx_external_power_changed; di->bat = power_supply_register_no_ws(di->dev, psy_desc, &psy_cfg); - if (IS_ERR(di->bat)) { - if (PTR_ERR(di->bat) == -EPROBE_DEFER) - dev_dbg(di->dev, "failed to register battery, deferring probe\n"); - else - dev_err(di->dev, "failed to register battery\n"); - return PTR_ERR(di->bat); - } + if (IS_ERR(di->bat)) + return dev_err_probe(di->dev, PTR_ERR(di->bat), + "failed to register battery\n"); bq27xxx_battery_settings(di); bq27xxx_battery_update(di); diff --git a/drivers/power/supply/bq27xxx_battery_hdq.c b/drivers/power/supply/bq27xxx_battery_hdq.c index d56b3e19e996..922759ab2e04 100644 --- a/drivers/power/supply/bq27xxx_battery_hdq.c +++ b/drivers/power/supply/bq27xxx_battery_hdq.c @@ -1,16 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * BQ27xxx battery monitor HDQ/1-wire driver * * Copyright (C) 2007-2017 Texas Instruments Incorporated - https://www.ti.com/ * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include <linux/kernel.h> diff --git a/drivers/power/supply/bq27xxx_battery_i2c.c b/drivers/power/supply/bq27xxx_battery_i2c.c index ab02456d69e5..eb4f4284982f 100644 --- a/drivers/power/supply/bq27xxx_battery_i2c.c +++ b/drivers/power/supply/bq27xxx_battery_i2c.c @@ -1,17 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * BQ27xxx battery monitor I2C driver * * Copyright (C) 2015 Texas Instruments Incorporated - https://www.ti.com/ * Andrew F. Davis <afd@ti.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include <linux/i2c.h> @@ -255,6 +247,7 @@ static const struct i2c_device_id bq27xxx_i2c_id_table[] = { { "bq27621", BQ27621 }, { "bq27z561", BQ27Z561 }, { "bq28z610", BQ28Z610 }, + { "bq34z100", BQ34Z100 }, {}, }; MODULE_DEVICE_TABLE(i2c, bq27xxx_i2c_id_table); @@ -290,6 +283,7 @@ static const struct of_device_id bq27xxx_battery_i2c_of_match_table[] = { { .compatible = "ti,bq27621" }, { .compatible = "ti,bq27z561" }, { .compatible = "ti,bq28z610" }, + { .compatible = "ti,bq34z100" }, {}, }; MODULE_DEVICE_TABLE(of, bq27xxx_battery_i2c_of_match_table); diff --git a/drivers/power/supply/charger-manager.c b/drivers/power/supply/charger-manager.c index 2ef53dc1f2fb..6fcebe441552 100644 --- a/drivers/power/supply/charger-manager.c +++ b/drivers/power/supply/charger-manager.c @@ -26,6 +26,29 @@ #include <linux/of.h> #include <linux/thermal.h> +static struct { + const char *name; + u64 extcon_type; +} extcon_mapping[] = { + /* Current textual representations */ + { "USB", EXTCON_USB }, + { "USB-HOST", EXTCON_USB_HOST }, + { "SDP", EXTCON_CHG_USB_SDP }, + { "DCP", EXTCON_CHG_USB_DCP }, + { "CDP", EXTCON_CHG_USB_CDP }, + { "ACA", EXTCON_CHG_USB_ACA }, + { "FAST-CHARGER", EXTCON_CHG_USB_FAST }, + { "SLOW-CHARGER", EXTCON_CHG_USB_SLOW }, + { "WPT", EXTCON_CHG_WPT }, + { "PD", EXTCON_CHG_USB_PD }, + { "DOCK", EXTCON_DOCK }, + { "JIG", EXTCON_JIG }, + { "MECHANICAL", EXTCON_MECHANICAL }, + /* Deprecated textual representations */ + { "TA", EXTCON_CHG_USB_SDP }, + { "CHARGE-DOWNSTREAM", EXTCON_CHG_USB_CDP }, +}; + /* * Default temperature threshold for charging. * Every temperature units are in tenth of centigrade. @@ -33,18 +56,6 @@ #define CM_DEFAULT_RECHARGE_TEMP_DIFF 50 #define CM_DEFAULT_CHARGE_TEMP_MAX 500 -static const char * const default_event_names[] = { - [CM_EVENT_UNKNOWN] = "Unknown", - [CM_EVENT_BATT_FULL] = "Battery Full", - [CM_EVENT_BATT_IN] = "Battery Inserted", - [CM_EVENT_BATT_OUT] = "Battery Pulled Out", - [CM_EVENT_BATT_OVERHEAT] = "Battery Overheat", - [CM_EVENT_BATT_COLD] = "Battery Cold", - [CM_EVENT_EXT_PWR_IN_OUT] = "External Power Attach/Detach", - [CM_EVENT_CHG_START_STOP] = "Charging Start/Stop", - [CM_EVENT_OTHERS] = "Other battery events" -}; - /* * Regard CM_JIFFIES_SMALL jiffies is small enough to ignore for * delayed works so that we can run delayed works with CM_JIFFIES_SMALL @@ -61,8 +72,6 @@ static const char * const default_event_names[] = { */ #define CM_RTC_SMALL (2) -#define UEVENT_BUF_SIZE 32 - static LIST_HEAD(cm_list); static DEFINE_MUTEX(cm_list_mtx); @@ -285,6 +294,19 @@ static bool is_full_charged(struct charger_manager *cm) if (!fuel_gauge) return false; + /* Full, if it's over the fullbatt voltage */ + if (desc->fullbatt_uV > 0) { + ret = get_batt_uV(cm, &uV); + if (!ret) { + /* Battery is already full, checks voltage drop. */ + if (cm->battery_status == POWER_SUPPLY_STATUS_FULL + && desc->fullbatt_vchkdrop_uV) + uV += desc->fullbatt_vchkdrop_uV; + if (uV >= desc->fullbatt_uV) + return true; + } + } + if (desc->fullbatt_full_capacity > 0) { val.intval = 0; @@ -297,15 +319,6 @@ static bool is_full_charged(struct charger_manager *cm) } } - /* Full, if it's over the fullbatt voltage */ - if (desc->fullbatt_uV > 0) { - ret = get_batt_uV(cm, &uV); - if (!ret && uV >= desc->fullbatt_uV) { - is_full = true; - goto out; - } - } - /* Full, if the capacity is more than fullbatt_soc */ if (desc->fullbatt_soc > 0) { val.intval = 0; @@ -427,122 +440,6 @@ static int try_charger_enable(struct charger_manager *cm, bool enable) } /** - * try_charger_restart - Restart charging. - * @cm: the Charger Manager representing the battery. - * - * Restart charging by turning off and on the charger. - */ -static int try_charger_restart(struct charger_manager *cm) -{ - int err; - - if (cm->emergency_stop) - return -EAGAIN; - - err = try_charger_enable(cm, false); - if (err) - return err; - - return try_charger_enable(cm, true); -} - -/** - * uevent_notify - Let users know something has changed. - * @cm: the Charger Manager representing the battery. - * @event: the event string. - * - * If @event is null, it implies that uevent_notify is called - * by resume function. When called in the resume function, cm_suspended - * should be already reset to false in order to let uevent_notify - * notify the recent event during the suspend to users. While - * suspended, uevent_notify does not notify users, but tracks - * events so that uevent_notify can notify users later after resumed. - */ -static void uevent_notify(struct charger_manager *cm, const char *event) -{ - static char env_str[UEVENT_BUF_SIZE + 1] = ""; - static char env_str_save[UEVENT_BUF_SIZE + 1] = ""; - - if (cm_suspended) { - /* Nothing in suspended-event buffer */ - if (env_str_save[0] == 0) { - if (!strncmp(env_str, event, UEVENT_BUF_SIZE)) - return; /* status not changed */ - strncpy(env_str_save, event, UEVENT_BUF_SIZE); - return; - } - - if (!strncmp(env_str_save, event, UEVENT_BUF_SIZE)) - return; /* Duplicated. */ - strncpy(env_str_save, event, UEVENT_BUF_SIZE); - return; - } - - if (event == NULL) { - /* No messages pending */ - if (!env_str_save[0]) - return; - - strncpy(env_str, env_str_save, UEVENT_BUF_SIZE); - kobject_uevent(&cm->dev->kobj, KOBJ_CHANGE); - env_str_save[0] = 0; - - return; - } - - /* status not changed */ - if (!strncmp(env_str, event, UEVENT_BUF_SIZE)) - return; - - /* save the status and notify the update */ - strncpy(env_str, event, UEVENT_BUF_SIZE); - kobject_uevent(&cm->dev->kobj, KOBJ_CHANGE); - - dev_info(cm->dev, "%s\n", event); -} - -/** - * fullbatt_vchk - Check voltage drop some times after "FULL" event. - * @work: the work_struct appointing the function - * - * If a user has designated "fullbatt_vchkdrop_ms/uV" values with - * charger_desc, Charger Manager checks voltage drop after the battery - * "FULL" event. It checks whether the voltage has dropped more than - * fullbatt_vchkdrop_uV by calling this function after fullbatt_vchkrop_ms. - */ -static void fullbatt_vchk(struct work_struct *work) -{ - struct delayed_work *dwork = to_delayed_work(work); - struct charger_manager *cm = container_of(dwork, - struct charger_manager, fullbatt_vchk_work); - struct charger_desc *desc = cm->desc; - int batt_uV, err, diff; - - /* remove the appointment for fullbatt_vchk */ - cm->fullbatt_vchk_jiffies_at = 0; - - if (!desc->fullbatt_vchkdrop_uV || !desc->fullbatt_vchkdrop_ms) - return; - - err = get_batt_uV(cm, &batt_uV); - if (err) { - dev_err(cm->dev, "%s: get_batt_uV error(%d)\n", __func__, err); - return; - } - - diff = desc->fullbatt_uV - batt_uV; - if (diff < 0) - return; - - dev_info(cm->dev, "VBATT dropped %duV after full-batt\n", diff); - - if (diff > desc->fullbatt_vchkdrop_uV) { - try_charger_restart(cm); - uevent_notify(cm, "Recharging"); - } -} - -/** * check_charging_duration - Monitor charging/discharging duration * @cm: the Charger Manager representing the battery. * @@ -569,19 +466,14 @@ static int check_charging_duration(struct charger_manager *cm) if (duration > desc->charging_max_duration_ms) { dev_info(cm->dev, "Charging duration exceed %ums\n", desc->charging_max_duration_ms); - uevent_notify(cm, "Discharging"); - try_charger_enable(cm, false); ret = true; } - } else if (is_ext_pwr_online(cm) && !cm->charger_enabled) { + } else if (cm->battery_status == POWER_SUPPLY_STATUS_NOT_CHARGING) { duration = curr - cm->charging_end_time; - if (duration > desc->discharging_max_duration_ms && - is_ext_pwr_online(cm)) { + if (duration > desc->discharging_max_duration_ms) { dev_info(cm->dev, "Discharging duration exceed %ums\n", desc->discharging_max_duration_ms); - uevent_notify(cm, "Recharging"); - try_charger_enable(cm, true); ret = true; } } @@ -657,14 +549,53 @@ static int cm_check_thermal_status(struct charger_manager *cm) } if (temp > upper_limit) - ret = CM_EVENT_BATT_OVERHEAT; + ret = CM_BATT_OVERHEAT; else if (temp < lower_limit) - ret = CM_EVENT_BATT_COLD; + ret = CM_BATT_COLD; + else + ret = CM_BATT_OK; + + cm->emergency_stop = ret; return ret; } /** + * cm_get_target_status - Check current status and get next target status. + * @cm: the Charger Manager representing the battery. + */ +static int cm_get_target_status(struct charger_manager *cm) +{ + if (!is_ext_pwr_online(cm)) + return POWER_SUPPLY_STATUS_DISCHARGING; + + if (cm_check_thermal_status(cm)) { + /* Check if discharging duration exeeds limit. */ + if (check_charging_duration(cm)) + goto charging_ok; + return POWER_SUPPLY_STATUS_NOT_CHARGING; + } + + switch (cm->battery_status) { + case POWER_SUPPLY_STATUS_CHARGING: + /* Check if charging duration exeeds limit. */ + if (check_charging_duration(cm)) + return POWER_SUPPLY_STATUS_FULL; + fallthrough; + case POWER_SUPPLY_STATUS_FULL: + if (is_full_charged(cm)) + return POWER_SUPPLY_STATUS_FULL; + fallthrough; + default: + break; + } + +charging_ok: + /* Charging is allowed. */ + return POWER_SUPPLY_STATUS_CHARGING; +} + +/** * _cm_monitor - Monitor the temperature and return true for exceptions. * @cm: the Charger Manager representing the battery. * @@ -673,60 +604,18 @@ static int cm_check_thermal_status(struct charger_manager *cm) */ static bool _cm_monitor(struct charger_manager *cm) { - int temp_alrt; - - temp_alrt = cm_check_thermal_status(cm); - - /* It has been stopped already */ - if (temp_alrt && cm->emergency_stop) - return false; - - /* - * Check temperature whether overheat or cold. - * If temperature is out of range normal state, stop charging. - */ - if (temp_alrt) { - cm->emergency_stop = temp_alrt; - if (!try_charger_enable(cm, false)) - uevent_notify(cm, default_event_names[temp_alrt]); - - /* - * Check whole charging duration and discharging duration - * after full-batt. - */ - } else if (!cm->emergency_stop && check_charging_duration(cm)) { - dev_dbg(cm->dev, - "Charging/Discharging duration is out of range\n"); - /* - * Check dropped voltage of battery. If battery voltage is more - * dropped than fullbatt_vchkdrop_uV after fully charged state, - * charger-manager have to recharge battery. - */ - } else if (!cm->emergency_stop && is_ext_pwr_online(cm) && - !cm->charger_enabled) { - fullbatt_vchk(&cm->fullbatt_vchk_work.work); + int target; - /* - * Check whether fully charged state to protect overcharge - * if charger-manager is charging for battery. - */ - } else if (!cm->emergency_stop && is_full_charged(cm) && - cm->charger_enabled) { - dev_info(cm->dev, "EVENT_HANDLE: Battery Fully Charged\n"); - uevent_notify(cm, default_event_names[CM_EVENT_BATT_FULL]); + target = cm_get_target_status(cm); - try_charger_enable(cm, false); + try_charger_enable(cm, (target == POWER_SUPPLY_STATUS_CHARGING)); - fullbatt_vchk(&cm->fullbatt_vchk_work.work); - } else { - cm->emergency_stop = 0; - if (is_ext_pwr_online(cm)) { - if (!try_charger_enable(cm, true)) - uevent_notify(cm, "CHARGING"); - } + if (cm->battery_status != target) { + cm->battery_status = target; + power_supply_changed(cm->charger_psy); } - return true; + return (cm->battery_status == POWER_SUPPLY_STATUS_NOT_CHARGING); } /** @@ -819,66 +708,6 @@ static void cm_monitor_poller(struct work_struct *work) schedule_work(&setup_polling); } -/** - * fullbatt_handler - Event handler for CM_EVENT_BATT_FULL - * @cm: the Charger Manager representing the battery. - */ -static void fullbatt_handler(struct charger_manager *cm) -{ - struct charger_desc *desc = cm->desc; - - if (!desc->fullbatt_vchkdrop_uV || !desc->fullbatt_vchkdrop_ms) - goto out; - - if (cm_suspended) - device_set_wakeup_capable(cm->dev, true); - - mod_delayed_work(cm_wq, &cm->fullbatt_vchk_work, - msecs_to_jiffies(desc->fullbatt_vchkdrop_ms)); - cm->fullbatt_vchk_jiffies_at = jiffies + msecs_to_jiffies( - desc->fullbatt_vchkdrop_ms); - - if (cm->fullbatt_vchk_jiffies_at == 0) - cm->fullbatt_vchk_jiffies_at = 1; - -out: - dev_info(cm->dev, "EVENT_HANDLE: Battery Fully Charged\n"); - uevent_notify(cm, default_event_names[CM_EVENT_BATT_FULL]); -} - -/** - * battout_handler - Event handler for CM_EVENT_BATT_OUT - * @cm: the Charger Manager representing the battery. - */ -static void battout_handler(struct charger_manager *cm) -{ - if (cm_suspended) - device_set_wakeup_capable(cm->dev, true); - - if (!is_batt_present(cm)) { - dev_emerg(cm->dev, "Battery Pulled Out!\n"); - uevent_notify(cm, default_event_names[CM_EVENT_BATT_OUT]); - } else { - uevent_notify(cm, "Battery Reinserted?"); - } -} - -/** - * misc_event_handler - Handler for other events - * @cm: the Charger Manager representing the battery. - * @type: the Charger Manager representing the battery. - */ -static void misc_event_handler(struct charger_manager *cm, - enum cm_event_types type) -{ - if (cm_suspended) - device_set_wakeup_capable(cm->dev, true); - - if (is_polling_required(cm) && cm->desc->polling_interval_ms) - schedule_work(&setup_polling); - uevent_notify(cm, default_event_names[type]); -} - static int charger_get_property(struct power_supply *psy, enum power_supply_property psp, union power_supply_propval *val) @@ -891,12 +720,7 @@ static int charger_get_property(struct power_supply *psy, switch (psp) { case POWER_SUPPLY_PROP_STATUS: - if (is_charging(cm)) - val->intval = POWER_SUPPLY_STATUS_CHARGING; - else if (is_ext_pwr_online(cm)) - val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; - else - val->intval = POWER_SUPPLY_STATUS_DISCHARGING; + val->intval = cm->battery_status; break; case POWER_SUPPLY_PROP_HEALTH: if (cm->emergency_stop > 0) @@ -925,7 +749,6 @@ static int charger_get_property(struct power_supply *psy, POWER_SUPPLY_PROP_CURRENT_NOW, val); break; case POWER_SUPPLY_PROP_TEMP: - case POWER_SUPPLY_PROP_TEMP_AMBIENT: return cm_get_battery_temperature(cm, &val->intval); case POWER_SUPPLY_PROP_CAPACITY: if (!is_batt_present(cm)) { @@ -981,35 +804,13 @@ static int charger_get_property(struct power_supply *psy, val->intval = 0; break; case POWER_SUPPLY_PROP_CHARGE_FULL: - if (is_full_charged(cm)) - val->intval = 1; - else - val->intval = 0; - ret = 0; - break; case POWER_SUPPLY_PROP_CHARGE_NOW: - if (is_charging(cm)) { - fuel_gauge = power_supply_get_by_name( - cm->desc->psy_fuel_gauge); - if (!fuel_gauge) { - ret = -ENODEV; - break; - } - - ret = power_supply_get_property(fuel_gauge, - POWER_SUPPLY_PROP_CHARGE_NOW, - val); - if (ret) { - val->intval = 1; - ret = 0; - } else { - /* If CHARGE_NOW is supplied, use it */ - val->intval = (val->intval > 0) ? - val->intval : 1; - } - } else { - val->intval = 0; + fuel_gauge = power_supply_get_by_name(cm->desc->psy_fuel_gauge); + if (!fuel_gauge) { + ret = -ENODEV; + break; } + ret = power_supply_get_property(fuel_gauge, psp, val); break; default: return -EINVAL; @@ -1028,13 +829,12 @@ static enum power_supply_property default_charger_props[] = { POWER_SUPPLY_PROP_VOLTAGE_NOW, POWER_SUPPLY_PROP_CAPACITY, POWER_SUPPLY_PROP_ONLINE, - POWER_SUPPLY_PROP_CHARGE_FULL, /* * Optional properties are: + * POWER_SUPPLY_PROP_CHARGE_FULL, * POWER_SUPPLY_PROP_CHARGE_NOW, * POWER_SUPPLY_PROP_CURRENT_NOW, - * POWER_SUPPLY_PROP_TEMP, and - * POWER_SUPPLY_PROP_TEMP_AMBIENT, + * POWER_SUPPLY_PROP_TEMP, */ }; @@ -1069,21 +869,6 @@ static bool cm_setup_timer(void) mutex_lock(&cm_list_mtx); list_for_each_entry(cm, &cm_list, entry) { - unsigned int fbchk_ms = 0; - - /* fullbatt_vchk is required. setup timer for that */ - if (cm->fullbatt_vchk_jiffies_at) { - fbchk_ms = jiffies_to_msecs(cm->fullbatt_vchk_jiffies_at - - jiffies); - if (time_is_before_eq_jiffies( - cm->fullbatt_vchk_jiffies_at) || - msecs_to_jiffies(fbchk_ms) < CM_JIFFIES_SMALL) { - fullbatt_vchk(&cm->fullbatt_vchk_work.work); - fbchk_ms = 0; - } - } - CM_MIN_VALID(wakeup_ms, fbchk_ms); - /* Skip if polling is not required for this CM */ if (!is_polling_required(cm) && !cm->emergency_stop) continue; @@ -1145,7 +930,8 @@ static void charger_extcon_work(struct work_struct *work) cable->min_uA, cable->max_uA); } - try_charger_enable(cable->cm, cable->attached); + cancel_delayed_work(&cm_monitor_work); + queue_delayed_work(cm_wq, &cm_monitor_work, 0); } /** @@ -1169,15 +955,6 @@ static int charger_extcon_notifier(struct notifier_block *self, cable->attached = event; /* - * Setup monitoring to check battery state - * when charger cable is attached. - */ - if (cable->attached && is_polling_required(cable->cm)) { - cancel_work_sync(&setup_polling); - schedule_work(&setup_polling); - } - - /* * Setup work for controlling charger(regulator) * according to charger cable. */ @@ -1196,7 +973,8 @@ static int charger_extcon_notifier(struct notifier_block *self, static int charger_extcon_init(struct charger_manager *cm, struct charger_cable *cable) { - int ret; + int ret, i; + u64 extcon_type = EXTCON_NONE; /* * Charger manager use Extcon framework to identify @@ -1205,14 +983,39 @@ static int charger_extcon_init(struct charger_manager *cm, */ INIT_WORK(&cable->wq, charger_extcon_work); cable->nb.notifier_call = charger_extcon_notifier; - ret = extcon_register_interest(&cable->extcon_dev, - cable->extcon_name, cable->name, &cable->nb); + + cable->extcon_dev = extcon_get_extcon_dev(cable->extcon_name); + if (IS_ERR_OR_NULL(cable->extcon_dev)) { + pr_err("Cannot find extcon_dev for %s (cable: %s)\n", + cable->extcon_name, cable->name); + if (cable->extcon_dev == NULL) + return -EPROBE_DEFER; + else + return PTR_ERR(cable->extcon_dev); + } + + for (i = 0; i < ARRAY_SIZE(extcon_mapping); i++) { + if (!strcmp(cable->name, extcon_mapping[i].name)) { + extcon_type = extcon_mapping[i].extcon_type; + break; + } + } + if (extcon_type == EXTCON_NONE) { + pr_err("Cannot find cable for type %s", cable->name); + return -EINVAL; + } + + cable->extcon_type = extcon_type; + + ret = devm_extcon_register_notifier(cm->dev, cable->extcon_dev, + cable->extcon_type, &cable->nb); if (ret < 0) { - pr_info("Cannot register extcon_dev for %s(cable: %s)\n", + pr_err("Cannot register extcon_dev for %s (cable: %s)\n", cable->extcon_name, cable->name); + return ret; } - return ret; + return 0; } /** @@ -1229,6 +1032,7 @@ static int charger_manager_register_extcon(struct charger_manager *cm) { struct charger_desc *desc = cm->desc; struct charger_regulator *charger; + unsigned long event; int ret; int i; int j; @@ -1256,6 +1060,11 @@ static int charger_manager_register_extcon(struct charger_manager *cm) } cable->charger = charger; cable->cm = cm; + + event = extcon_get_state(cable->extcon_dev, + cable->extcon_type); + charger_extcon_notifier(&cable->nb, + event, NULL); } } @@ -1447,7 +1256,7 @@ static int cm_init_thermal_data(struct charger_manager *cm, return PTR_ERR(cm->tzd_batt); /* Use external thermometer */ - properties[*num_properties] = POWER_SUPPLY_PROP_TEMP_AMBIENT; + properties[*num_properties] = POWER_SUPPLY_PROP_TEMP; (*num_properties)++; cm->desc->measure_battery_temp = true; ret = 0; @@ -1491,8 +1300,6 @@ static struct charger_desc *of_cm_parse_desc(struct device *dev) of_property_read_u32(np, "cm-poll-interval", &desc->polling_interval_ms); - of_property_read_u32(np, "cm-fullbatt-vchkdrop-ms", - &desc->fullbatt_vchkdrop_ms); of_property_read_u32(np, "cm-fullbatt-vchkdrop-volt", &desc->fullbatt_vchkdrop_uV); of_property_read_u32(np, "cm-fullbatt-voltage", &desc->fullbatt_uV); @@ -1504,8 +1311,8 @@ static struct charger_desc *of_cm_parse_desc(struct device *dev) desc->battery_present = battery_stat; /* chargers */ - of_property_read_u32(np, "cm-num-chargers", &num_chgs); - if (num_chgs) { + num_chgs = of_property_count_strings(np, "cm-chargers"); + if (num_chgs > 0) { int i; /* Allocate empty bin at the tail of array */ @@ -1618,7 +1425,6 @@ static int charger_manager_probe(struct platform_device *pdev) struct charger_desc *desc = cm_get_drv_data(pdev); struct charger_manager *cm; int ret, i = 0; - int j = 0; union power_supply_propval val; struct power_supply *fuel_gauge; enum power_supply_property *properties; @@ -1654,9 +1460,8 @@ static int charger_manager_probe(struct platform_device *pdev) if (desc->fullbatt_uV == 0) { dev_info(&pdev->dev, "Ignoring full-battery voltage threshold as it is not supplied\n"); } - if (!desc->fullbatt_vchkdrop_ms || !desc->fullbatt_vchkdrop_uV) { + if (!desc->fullbatt_vchkdrop_uV) { dev_info(&pdev->dev, "Disabling full-battery voltage drop checking mechanism as it is not supplied\n"); - desc->fullbatt_vchkdrop_ms = 0; desc->fullbatt_vchkdrop_uV = 0; } if (desc->fullbatt_soc == 0) { @@ -1739,6 +1544,12 @@ static int charger_manager_probe(struct platform_device *pdev) return -ENODEV; } if (!power_supply_get_property(fuel_gauge, + POWER_SUPPLY_PROP_CHARGE_FULL, &val)) { + properties[num_properties] = + POWER_SUPPLY_PROP_CHARGE_FULL; + num_properties++; + } + if (!power_supply_get_property(fuel_gauge, POWER_SUPPLY_PROP_CHARGE_NOW, &val)) { properties[num_properties] = POWER_SUPPLY_PROP_CHARGE_NOW; @@ -1762,8 +1573,6 @@ static int charger_manager_probe(struct platform_device *pdev) cm->charger_psy_desc.properties = properties; cm->charger_psy_desc.num_properties = num_properties; - INIT_DELAYED_WORK(&cm->fullbatt_vchk_work, fullbatt_vchk); - /* Register sysfs entry for charger(regulator) */ ret = charger_manager_prepare_sysfs(cm); if (ret < 0) { @@ -1813,19 +1622,8 @@ static int charger_manager_probe(struct platform_device *pdev) return 0; err_reg_extcon: - for (i = 0; i < desc->num_charger_regulators; i++) { - struct charger_regulator *charger; - - charger = &desc->charger_regulators[i]; - for (j = 0; j < charger->num_cables; j++) { - struct charger_cable *cable = &charger->cables[j]; - /* Remove notifier block if only edev exists */ - if (cable->extcon_dev.edev) - extcon_unregister_interest(&cable->extcon_dev); - } - + for (i = 0; i < desc->num_charger_regulators; i++) regulator_put(desc->charger_regulators[i].consumer); - } power_supply_unregister(cm->charger_psy); @@ -1837,7 +1635,6 @@ static int charger_manager_remove(struct platform_device *pdev) struct charger_manager *cm = platform_get_drvdata(pdev); struct charger_desc *desc = cm->desc; int i = 0; - int j = 0; /* Remove from the list */ mutex_lock(&cm_list_mtx); @@ -1847,15 +1644,6 @@ static int charger_manager_remove(struct platform_device *pdev) cancel_work_sync(&setup_polling); cancel_delayed_work_sync(&cm_monitor_work); - for (i = 0 ; i < desc->num_charger_regulators ; i++) { - struct charger_regulator *charger - = &desc->charger_regulators[i]; - for (j = 0 ; j < charger->num_cables ; j++) { - struct charger_cable *cable = &charger->cables[j]; - extcon_unregister_interest(&cable->extcon_dev); - } - } - for (i = 0 ; i < desc->num_charger_regulators ; i++) regulator_put(desc->charger_regulators[i].consumer); @@ -1903,8 +1691,6 @@ static bool cm_need_to_awake(void) static int cm_suspend_prepare(struct device *dev) { - struct charger_manager *cm = dev_get_drvdata(dev); - if (cm_need_to_awake()) return -EBUSY; @@ -1916,7 +1702,6 @@ static int cm_suspend_prepare(struct device *dev) if (cm_timer_set) { cancel_work_sync(&setup_polling); cancel_delayed_work_sync(&cm_monitor_work); - cancel_delayed_work(&cm->fullbatt_vchk_work); } return 0; @@ -1941,31 +1726,6 @@ static void cm_suspend_complete(struct device *dev) _cm_monitor(cm); - /* Re-enqueue delayed work (fullbatt_vchk_work) */ - if (cm->fullbatt_vchk_jiffies_at) { - unsigned long delay = 0; - unsigned long now = jiffies + CM_JIFFIES_SMALL; - - if (time_after_eq(now, cm->fullbatt_vchk_jiffies_at)) { - delay = (unsigned long)((long)now - - (long)(cm->fullbatt_vchk_jiffies_at)); - delay = jiffies_to_msecs(delay); - } else { - delay = 0; - } - - /* - * Account for cm_suspend_duration_ms with assuming that - * timer stops in suspend. - */ - if (delay > cm_suspend_duration_ms) - delay -= cm_suspend_duration_ms; - else - delay = 0; - - queue_delayed_work(cm_wq, &cm->fullbatt_vchk_work, - msecs_to_jiffies(delay)); - } device_set_wakeup_capable(cm->dev, false); } @@ -2007,56 +1767,6 @@ static void __exit charger_manager_cleanup(void) } module_exit(charger_manager_cleanup); -/** - * cm_notify_event - charger driver notify Charger Manager of charger event - * @psy: pointer to instance of charger's power_supply - * @type: type of charger event - * @msg: optional message passed to uevent_notify function - */ -void cm_notify_event(struct power_supply *psy, enum cm_event_types type, - char *msg) -{ - struct charger_manager *cm; - bool found_power_supply = false; - - if (psy == NULL) - return; - - mutex_lock(&cm_list_mtx); - list_for_each_entry(cm, &cm_list, entry) { - if (match_string(cm->desc->psy_charger_stat, -1, - psy->desc->name) >= 0) { - found_power_supply = true; - break; - } - } - mutex_unlock(&cm_list_mtx); - - if (!found_power_supply) - return; - - switch (type) { - case CM_EVENT_BATT_FULL: - fullbatt_handler(cm); - break; - case CM_EVENT_BATT_OUT: - battout_handler(cm); - break; - case CM_EVENT_BATT_IN: - case CM_EVENT_EXT_PWR_IN_OUT ... CM_EVENT_CHG_START_STOP: - misc_event_handler(cm, type); - break; - case CM_EVENT_UNKNOWN: - case CM_EVENT_OTHERS: - uevent_notify(cm, msg ? msg : default_event_names[type]); - break; - default: - dev_err(cm->dev, "%s: type not specified\n", __func__); - break; - } -} -EXPORT_SYMBOL_GPL(cm_notify_event); - MODULE_AUTHOR("MyungJoo Ham <myungjoo.ham@samsung.com>"); MODULE_DESCRIPTION("Charger Manager"); MODULE_LICENSE("GPL"); diff --git a/drivers/power/supply/cpcap-battery.c b/drivers/power/supply/cpcap-battery.c index 90eba364664b..295611b3b15e 100644 --- a/drivers/power/supply/cpcap-battery.c +++ b/drivers/power/supply/cpcap-battery.c @@ -747,11 +747,8 @@ static int cpcap_battery_init_iio(struct cpcap_battery_ddata *ddata) return 0; out_err: - if (error != -EPROBE_DEFER) - dev_err(ddata->dev, "could not initialize VBUS or ID IIO: %i\n", - error); - - return error; + return dev_err_probe(ddata->dev, error, + "could not initialize VBUS or ID IIO\n"); } /* Calibrate coulomb counter */ diff --git a/drivers/power/supply/ds2780_battery.c b/drivers/power/supply/ds2780_battery.c index db3a25404c9f..dd57a472e878 100644 --- a/drivers/power/supply/ds2780_battery.c +++ b/drivers/power/supply/ds2780_battery.c @@ -160,7 +160,7 @@ static int ds2780_get_voltage(struct ds2780_device_info *dev_info, /* * The voltage value is located in 10 bits across the voltage MSB - * and LSB registers in two's compliment form + * and LSB registers in two's complement form * Sign bit of the voltage value is in bit 7 of the voltage MSB register * Bits 9 - 3 of the voltage value are in bits 6 - 0 of the * voltage MSB register @@ -188,7 +188,7 @@ static int ds2780_get_temperature(struct ds2780_device_info *dev_info, /* * The temperature value is located in 10 bits across the temperature - * MSB and LSB registers in two's compliment form + * MSB and LSB registers in two's complement form * Sign bit of the temperature value is in bit 7 of the temperature * MSB register * Bits 9 - 3 of the temperature value are in bits 6 - 0 of the @@ -241,7 +241,7 @@ static int ds2780_get_current(struct ds2780_device_info *dev_info, /* * The current value is located in 16 bits across the current MSB - * and LSB registers in two's compliment form + * and LSB registers in two's complement form * Sign bit of the current value is in bit 7 of the current MSB register * Bits 14 - 8 of the current value are in bits 6 - 0 of the current * MSB register diff --git a/drivers/power/supply/ds2781_battery.c b/drivers/power/supply/ds2781_battery.c index 130cbdfc14eb..3df3c820b38c 100644 --- a/drivers/power/supply/ds2781_battery.c +++ b/drivers/power/supply/ds2781_battery.c @@ -168,7 +168,7 @@ static int ds2781_get_voltage(struct ds2781_device_info *dev_info, return ret; /* * The voltage value is located in 10 bits across the voltage MSB - * and LSB registers in two's compliment form + * and LSB registers in two's complement form * Sign bit of the voltage value is in bit 7 of the voltage MSB register * Bits 9 - 3 of the voltage value are in bits 6 - 0 of the * voltage MSB register @@ -197,7 +197,7 @@ static int ds2781_get_temperature(struct ds2781_device_info *dev_info, return ret; /* * The temperature value is located in 10 bits across the temperature - * MSB and LSB registers in two's compliment form + * MSB and LSB registers in two's complement form * Sign bit of the temperature value is in bit 7 of the temperature * MSB register * Bits 9 - 3 of the temperature value are in bits 6 - 0 of the @@ -242,7 +242,7 @@ static int ds2781_get_current(struct ds2781_device_info *dev_info, /* * The current value is located in 16 bits across the current MSB - * and LSB registers in two's compliment form + * and LSB registers in two's complement form * Sign bit of the current value is in bit 7 of the current MSB register * Bits 14 - 8 of the current value are in bits 6 - 0 of the current * MSB register diff --git a/drivers/power/supply/goldfish_battery.c b/drivers/power/supply/goldfish_battery.c index c2644a9fe80f..bf1754355c9f 100644 --- a/drivers/power/supply/goldfish_battery.c +++ b/drivers/power/supply/goldfish_battery.c @@ -266,11 +266,13 @@ static const struct of_device_id goldfish_battery_of_match[] = { }; MODULE_DEVICE_TABLE(of, goldfish_battery_of_match); +#ifdef CONFIG_ACPI static const struct acpi_device_id goldfish_battery_acpi_match[] = { { "GFSH0001", 0 }, { }, }; MODULE_DEVICE_TABLE(acpi, goldfish_battery_acpi_match); +#endif static struct platform_driver goldfish_battery_device = { .probe = goldfish_battery_probe, diff --git a/drivers/power/supply/gpio-charger.c b/drivers/power/supply/gpio-charger.c index 875735d50716..68212b39785b 100644 --- a/drivers/power/supply/gpio-charger.c +++ b/drivers/power/supply/gpio-charger.c @@ -5,7 +5,6 @@ */ #include <linux/device.h> -#include <linux/gpio.h> /* For legacy platform data */ #include <linux/init.h> #include <linux/interrupt.h> #include <linux/kernel.h> @@ -18,7 +17,13 @@ #include <linux/power/gpio-charger.h> +struct gpio_mapping { + u32 limit_ua; + u32 gpiodata; +} __packed; + struct gpio_charger { + struct device *dev; unsigned int irq; unsigned int charge_status_irq; bool wakeup_enabled; @@ -27,6 +32,11 @@ struct gpio_charger { struct power_supply_desc charger_desc; struct gpio_desc *gpiod; struct gpio_desc *charge_status; + + struct gpio_descs *current_limit_gpios; + struct gpio_mapping *current_limit_map; + u32 current_limit_map_size; + u32 charge_current_limit; }; static irqreturn_t gpio_charger_irq(int irq, void *devid) @@ -43,6 +53,35 @@ static inline struct gpio_charger *psy_to_gpio_charger(struct power_supply *psy) return power_supply_get_drvdata(psy); } +static int set_charge_current_limit(struct gpio_charger *gpio_charger, int val) +{ + struct gpio_mapping mapping; + int ndescs = gpio_charger->current_limit_gpios->ndescs; + struct gpio_desc **gpios = gpio_charger->current_limit_gpios->desc; + int i; + + if (!gpio_charger->current_limit_map_size) + return -EINVAL; + + for (i = 0; i < gpio_charger->current_limit_map_size; i++) { + if (gpio_charger->current_limit_map[i].limit_ua <= val) + break; + } + mapping = gpio_charger->current_limit_map[i]; + + for (i = 0; i < ndescs; i++) { + bool val = (mapping.gpiodata >> i) & 1; + gpiod_set_value_cansleep(gpios[ndescs-i-1], val); + } + + gpio_charger->charge_current_limit = mapping.limit_ua; + + dev_dbg(gpio_charger->dev, "set charge current limit to %d (requested: %d)\n", + gpio_charger->charge_current_limit, val); + + return 0; +} + static int gpio_charger_get_property(struct power_supply *psy, enum power_supply_property psp, union power_supply_propval *val) { @@ -58,6 +97,24 @@ static int gpio_charger_get_property(struct power_supply *psy, else val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; break; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX: + val->intval = gpio_charger->charge_current_limit; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int gpio_charger_set_property(struct power_supply *psy, + enum power_supply_property psp, const union power_supply_propval *val) +{ + struct gpio_charger *gpio_charger = psy_to_gpio_charger(psy); + + switch (psp) { + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX: + return set_charge_current_limit(gpio_charger, val->intval); default: return -EINVAL; } @@ -65,6 +122,19 @@ static int gpio_charger_get_property(struct power_supply *psy, return 0; } +static int gpio_charger_property_is_writeable(struct power_supply *psy, + enum power_supply_property psp) +{ + switch (psp) { + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX: + return 1; + default: + break; + } + + return 0; +} + static enum power_supply_type gpio_charger_get_type(struct device *dev) { const char *chargetype; @@ -112,6 +182,61 @@ static int gpio_charger_get_irq(struct device *dev, void *dev_id, return irq; } +static int init_charge_current_limit(struct device *dev, + struct gpio_charger *gpio_charger) +{ + int i, len; + u32 cur_limit = U32_MAX; + + gpio_charger->current_limit_gpios = devm_gpiod_get_array_optional(dev, + "charge-current-limit", GPIOD_OUT_LOW); + if (IS_ERR(gpio_charger->current_limit_gpios)) { + dev_err(dev, "error getting current-limit GPIOs\n"); + return PTR_ERR(gpio_charger->current_limit_gpios); + } + + if (!gpio_charger->current_limit_gpios) + return 0; + + len = device_property_read_u32_array(dev, "charge-current-limit-mapping", + NULL, 0); + if (len < 0) + return len; + + if (len == 0 || len % 2) { + dev_err(dev, "invalid charge-current-limit-mapping length\n"); + return -EINVAL; + } + + gpio_charger->current_limit_map = devm_kmalloc_array(dev, + len / 2, sizeof(*gpio_charger->current_limit_map), GFP_KERNEL); + if (!gpio_charger->current_limit_map) + return -ENOMEM; + + gpio_charger->current_limit_map_size = len / 2; + + len = device_property_read_u32_array(dev, "charge-current-limit-mapping", + (u32*) gpio_charger->current_limit_map, len); + if (len < 0) + return len; + + for (i=0; i < gpio_charger->current_limit_map_size; i++) { + if (gpio_charger->current_limit_map[i].limit_ua > cur_limit) { + dev_err(dev, "charge-current-limit-mapping not sorted by current in descending order\n"); + return -EINVAL; + } + + cur_limit = gpio_charger->current_limit_map[i].limit_ua; + } + + /* default to smallest current limitation for safety reasons */ + len = gpio_charger->current_limit_map_size - 1; + set_charge_current_limit(gpio_charger, + gpio_charger->current_limit_map[len].limit_ua); + + return 0; +} + /* * The entries will be overwritten by driver's probe routine depending * on the available features. This list ensures, that the array is big @@ -120,6 +245,7 @@ static int gpio_charger_get_irq(struct device *dev, void *dev_id, static enum power_supply_property gpio_charger_properties[] = { POWER_SUPPLY_PROP_ONLINE, POWER_SUPPLY_PROP_STATUS, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX, }; static int gpio_charger_probe(struct platform_device *pdev) @@ -131,7 +257,6 @@ static int gpio_charger_probe(struct platform_device *pdev) struct power_supply_desc *charger_desc; struct gpio_desc *charge_status; int charge_status_irq; - unsigned long flags; int ret; int num_props = 0; @@ -143,40 +268,17 @@ static int gpio_charger_probe(struct platform_device *pdev) gpio_charger = devm_kzalloc(dev, sizeof(*gpio_charger), GFP_KERNEL); if (!gpio_charger) return -ENOMEM; + gpio_charger->dev = dev; /* * This will fetch a GPIO descriptor from device tree, ACPI or * boardfile descriptor tables. It's good to try this first. */ gpio_charger->gpiod = devm_gpiod_get_optional(dev, NULL, GPIOD_IN); - - /* - * Fallback to legacy platform data method, if no GPIO is specified - * using boardfile descriptor tables. - */ - if (!gpio_charger->gpiod && pdata) { - /* Non-DT: use legacy GPIO numbers */ - if (!gpio_is_valid(pdata->gpio)) { - dev_err(dev, "Invalid gpio pin in pdata\n"); - return -EINVAL; - } - flags = GPIOF_IN; - if (pdata->gpio_active_low) - flags |= GPIOF_ACTIVE_LOW; - ret = devm_gpio_request_one(dev, pdata->gpio, flags, - dev_name(dev)); - if (ret) { - dev_err(dev, "Failed to request gpio pin: %d\n", ret); - return ret; - } - /* Then convert this to gpiod for now */ - gpio_charger->gpiod = gpio_to_desc(pdata->gpio); - } else if (IS_ERR(gpio_charger->gpiod)) { + if (IS_ERR(gpio_charger->gpiod)) { /* Just try again if this happens */ - if (PTR_ERR(gpio_charger->gpiod) == -EPROBE_DEFER) - return -EPROBE_DEFER; - dev_err(dev, "error getting GPIO descriptor\n"); - return PTR_ERR(gpio_charger->gpiod); + return dev_err_probe(dev, PTR_ERR(gpio_charger->gpiod), + "error getting GPIO descriptor\n"); } if (gpio_charger->gpiod) { @@ -193,10 +295,22 @@ static int gpio_charger_probe(struct platform_device *pdev) num_props++; } + ret = init_charge_current_limit(dev, gpio_charger); + if (ret < 0) + return ret; + if (gpio_charger->current_limit_map) { + gpio_charger_properties[num_props] = + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX; + num_props++; + } + charger_desc = &gpio_charger->charger_desc; charger_desc->properties = gpio_charger_properties; charger_desc->num_properties = num_props; charger_desc->get_property = gpio_charger_get_property; + charger_desc->set_property = gpio_charger_set_property; + charger_desc->property_is_writeable = + gpio_charger_property_is_writeable; psy_cfg.of_node = dev->of_node; psy_cfg.drv_data = gpio_charger; diff --git a/drivers/power/supply/ingenic-battery.c b/drivers/power/supply/ingenic-battery.c index dd3d93dfe3eb..32dc77fd9a95 100644 --- a/drivers/power/supply/ingenic-battery.c +++ b/drivers/power/supply/ingenic-battery.c @@ -147,11 +147,9 @@ static int ingenic_battery_probe(struct platform_device *pdev) psy_cfg.of_node = dev->of_node; bat->battery = devm_power_supply_register(dev, desc, &psy_cfg); - if (IS_ERR(bat->battery)) { - if (PTR_ERR(bat->battery) != -EPROBE_DEFER) - dev_err(dev, "Unable to register battery\n"); - return PTR_ERR(bat->battery); - } + if (IS_ERR(bat->battery)) + return dev_err_probe(dev, PTR_ERR(bat->battery), + "Unable to register battery\n"); ret = power_supply_get_battery_info(bat->battery, &bat->info); if (ret) { diff --git a/drivers/power/supply/lego_ev3_battery.c b/drivers/power/supply/lego_ev3_battery.c index 1ae3710909b7..ccb00be38e2c 100644 --- a/drivers/power/supply/lego_ev3_battery.c +++ b/drivers/power/supply/lego_ev3_battery.c @@ -166,27 +166,21 @@ static int lego_ev3_battery_probe(struct platform_device *pdev) batt->iio_v = devm_iio_channel_get(dev, "voltage"); err = PTR_ERR_OR_ZERO(batt->iio_v); - if (err) { - if (err != -EPROBE_DEFER) - dev_err(dev, "Failed to get voltage iio channel\n"); - return err; - } + if (err) + return dev_err_probe(dev, err, + "Failed to get voltage iio channel\n"); batt->iio_i = devm_iio_channel_get(dev, "current"); err = PTR_ERR_OR_ZERO(batt->iio_i); - if (err) { - if (err != -EPROBE_DEFER) - dev_err(dev, "Failed to get current iio channel\n"); - return err; - } + if (err) + return dev_err_probe(dev, err, + "Failed to get current iio channel\n"); batt->rechargeable_gpio = devm_gpiod_get(dev, "rechargeable", GPIOD_IN); err = PTR_ERR_OR_ZERO(batt->rechargeable_gpio); - if (err) { - if (err != -EPROBE_DEFER) - dev_err(dev, "Failed to get rechargeable gpio\n"); - return err; - } + if (err) + return dev_err_probe(dev, err, + "Failed to get rechargeable gpio\n"); /* * The rechargeable battery indication switch cannot be changed without diff --git a/drivers/power/supply/ltc2941-battery-gauge.c b/drivers/power/supply/ltc2941-battery-gauge.c index 30a9014b2f95..10cd617516ec 100644 --- a/drivers/power/supply/ltc2941-battery-gauge.c +++ b/drivers/power/supply/ltc2941-battery-gauge.c @@ -473,7 +473,8 @@ static int ltc294x_i2c_probe(struct i2c_client *client, np = of_node_get(client->dev.of_node); - info->id = (enum ltc294x_id)of_device_get_match_data(&client->dev); + info->id = (enum ltc294x_id) (uintptr_t) of_device_get_match_data( + &client->dev); info->supply_desc.name = np->name; /* r_sense can be negative, when sense+ is connected to the battery diff --git a/drivers/power/supply/max17040_battery.c b/drivers/power/supply/max17040_battery.c index 6cb31b9a958d..d956c67d5155 100644 --- a/drivers/power/supply/max17040_battery.c +++ b/drivers/power/supply/max17040_battery.c @@ -15,196 +15,289 @@ #include <linux/delay.h> #include <linux/interrupt.h> #include <linux/power_supply.h> +#include <linux/of_device.h> #include <linux/max17040_battery.h> +#include <linux/regmap.h> #include <linux/slab.h> #define MAX17040_VCELL 0x02 #define MAX17040_SOC 0x04 #define MAX17040_MODE 0x06 #define MAX17040_VER 0x08 -#define MAX17040_RCOMP 0x0C +#define MAX17040_CONFIG 0x0C +#define MAX17040_STATUS 0x1A #define MAX17040_CMD 0xFE #define MAX17040_DELAY 1000 #define MAX17040_BATTERY_FULL 95 +#define MAX17040_RCOMP_DEFAULT 0x9700 -#define MAX17040_ATHD_MASK 0xFFC0 +#define MAX17040_ATHD_MASK 0x3f +#define MAX17040_ALSC_MASK 0x40 #define MAX17040_ATHD_DEFAULT_POWER_UP 4 +#define MAX17040_STATUS_HD_MASK 0x1000 +#define MAX17040_STATUS_SC_MASK 0x2000 +#define MAX17040_CFG_RCOMP_MASK 0xff00 + +enum chip_id { + ID_MAX17040, + ID_MAX17041, + ID_MAX17043, + ID_MAX17044, + ID_MAX17048, + ID_MAX17049, + ID_MAX17058, + ID_MAX17059, +}; + +/* values that differ by chip_id */ +struct chip_data { + u16 reset_val; + u16 vcell_shift; + u16 vcell_mul; + u16 vcell_div; + u8 has_low_soc_alert; + u8 rcomp_bytes; + u8 has_soc_alert; +}; + +static struct chip_data max17040_family[] = { + [ID_MAX17040] = { + .reset_val = 0x0054, + .vcell_shift = 4, + .vcell_mul = 1250, + .vcell_div = 1, + .has_low_soc_alert = 0, + .rcomp_bytes = 2, + .has_soc_alert = 0, + }, + [ID_MAX17041] = { + .reset_val = 0x0054, + .vcell_shift = 4, + .vcell_mul = 2500, + .vcell_div = 1, + .has_low_soc_alert = 0, + .rcomp_bytes = 2, + .has_soc_alert = 0, + }, + [ID_MAX17043] = { + .reset_val = 0x0054, + .vcell_shift = 4, + .vcell_mul = 1250, + .vcell_div = 1, + .has_low_soc_alert = 1, + .rcomp_bytes = 1, + .has_soc_alert = 0, + }, + [ID_MAX17044] = { + .reset_val = 0x0054, + .vcell_shift = 4, + .vcell_mul = 2500, + .vcell_div = 1, + .has_low_soc_alert = 1, + .rcomp_bytes = 1, + .has_soc_alert = 0, + }, + [ID_MAX17048] = { + .reset_val = 0x5400, + .vcell_shift = 0, + .vcell_mul = 625, + .vcell_div = 8, + .has_low_soc_alert = 1, + .rcomp_bytes = 1, + .has_soc_alert = 1, + }, + [ID_MAX17049] = { + .reset_val = 0x5400, + .vcell_shift = 0, + .vcell_mul = 625, + .vcell_div = 4, + .has_low_soc_alert = 1, + .rcomp_bytes = 1, + .has_soc_alert = 1, + }, + [ID_MAX17058] = { + .reset_val = 0x5400, + .vcell_shift = 0, + .vcell_mul = 625, + .vcell_div = 8, + .has_low_soc_alert = 1, + .rcomp_bytes = 1, + .has_soc_alert = 0, + }, + [ID_MAX17059] = { + .reset_val = 0x5400, + .vcell_shift = 0, + .vcell_mul = 625, + .vcell_div = 4, + .has_low_soc_alert = 1, + .rcomp_bytes = 1, + .has_soc_alert = 0, + }, +}; struct max17040_chip { struct i2c_client *client; + struct regmap *regmap; struct delayed_work work; struct power_supply *battery; struct max17040_platform_data *pdata; + struct chip_data data; - /* State Of Connect */ - int online; - /* battery voltage */ - int vcell; /* battery capacity */ int soc; /* State Of Charge */ int status; /* Low alert threshold from 32% to 1% of the State of Charge */ u32 low_soc_alert; + /* some devices return twice the capacity */ + bool quirk_double_soc; + /* higher 8 bits for 17043+, 16 bits for 17040,41 */ + u16 rcomp; }; -static int max17040_get_property(struct power_supply *psy, - enum power_supply_property psp, - union power_supply_propval *val) +static int max17040_reset(struct max17040_chip *chip) { - struct max17040_chip *chip = power_supply_get_drvdata(psy); - - switch (psp) { - case POWER_SUPPLY_PROP_STATUS: - val->intval = chip->status; - break; - case POWER_SUPPLY_PROP_ONLINE: - val->intval = chip->online; - break; - case POWER_SUPPLY_PROP_VOLTAGE_NOW: - val->intval = chip->vcell; - break; - case POWER_SUPPLY_PROP_CAPACITY: - val->intval = chip->soc; - break; - case POWER_SUPPLY_PROP_CAPACITY_ALERT_MIN: - val->intval = chip->low_soc_alert; - break; - default: - return -EINVAL; - } - return 0; + return regmap_write(chip->regmap, MAX17040_CMD, chip->data.reset_val); } -static int max17040_write_reg(struct i2c_client *client, int reg, u16 value) +static int max17040_set_low_soc_alert(struct max17040_chip *chip, u32 level) { - int ret; - - ret = i2c_smbus_write_word_swapped(client, reg, value); - - if (ret < 0) - dev_err(&client->dev, "%s: err %d\n", __func__, ret); - - return ret; + level = 32 - level * (chip->quirk_double_soc ? 2 : 1); + return regmap_update_bits(chip->regmap, MAX17040_CONFIG, + MAX17040_ATHD_MASK, level); } -static int max17040_read_reg(struct i2c_client *client, int reg) +static int max17040_set_soc_alert(struct max17040_chip *chip, bool enable) { - int ret; - - ret = i2c_smbus_read_word_swapped(client, reg); - - if (ret < 0) - dev_err(&client->dev, "%s: err %d\n", __func__, ret); - - return ret; + return regmap_update_bits(chip->regmap, MAX17040_CONFIG, + MAX17040_ALSC_MASK, enable ? MAX17040_ALSC_MASK : 0); } -static void max17040_reset(struct i2c_client *client) +static int max17040_set_rcomp(struct max17040_chip *chip, u16 rcomp) { - max17040_write_reg(client, MAX17040_CMD, 0x0054); + u16 mask = chip->data.rcomp_bytes == 2 ? + 0xffff : MAX17040_CFG_RCOMP_MASK; + + return regmap_update_bits(chip->regmap, MAX17040_CONFIG, mask, rcomp); } -static int max17040_set_low_soc_alert(struct i2c_client *client, u32 level) +static int max17040_raw_vcell_to_uvolts(struct max17040_chip *chip, u16 vcell) { - int ret; - u16 data; + struct chip_data *d = &chip->data; - level = 32 - level; - data = max17040_read_reg(client, MAX17040_RCOMP); - /* clear the alrt bit and set LSb 5 bits */ - data &= MAX17040_ATHD_MASK; - data |= level; - ret = max17040_write_reg(client, MAX17040_RCOMP, data); - - return ret; + return (vcell >> d->vcell_shift) * d->vcell_mul / d->vcell_div; } -static void max17040_get_vcell(struct i2c_client *client) + +static int max17040_get_vcell(struct max17040_chip *chip) { - struct max17040_chip *chip = i2c_get_clientdata(client); - u16 vcell; + u32 vcell; - vcell = max17040_read_reg(client, MAX17040_VCELL); + regmap_read(chip->regmap, MAX17040_VCELL, &vcell); - chip->vcell = (vcell >> 4) * 1250; + return max17040_raw_vcell_to_uvolts(chip, vcell); } -static void max17040_get_soc(struct i2c_client *client) +static int max17040_get_soc(struct max17040_chip *chip) { - struct max17040_chip *chip = i2c_get_clientdata(client); - u16 soc; + u32 soc; - soc = max17040_read_reg(client, MAX17040_SOC); + regmap_read(chip->regmap, MAX17040_SOC, &soc); - chip->soc = (soc >> 8); + return soc >> (chip->quirk_double_soc ? 9 : 8); } -static void max17040_get_version(struct i2c_client *client) +static int max17040_get_version(struct max17040_chip *chip) { - u16 version; + int ret; + u32 version; - version = max17040_read_reg(client, MAX17040_VER); + ret = regmap_read(chip->regmap, MAX17040_VER, &version); - dev_info(&client->dev, "MAX17040 Fuel-Gauge Ver 0x%x\n", version); + return ret ? ret : version; } -static void max17040_get_online(struct i2c_client *client) +static int max17040_get_online(struct max17040_chip *chip) { - struct max17040_chip *chip = i2c_get_clientdata(client); - - if (chip->pdata && chip->pdata->battery_online) - chip->online = chip->pdata->battery_online(); - else - chip->online = 1; + return chip->pdata && chip->pdata->battery_online ? + chip->pdata->battery_online() : 1; } -static void max17040_get_status(struct i2c_client *client) +static int max17040_get_status(struct max17040_chip *chip) { - struct max17040_chip *chip = i2c_get_clientdata(client); - if (!chip->pdata || !chip->pdata->charger_online - || !chip->pdata->charger_enable) { - chip->status = POWER_SUPPLY_STATUS_UNKNOWN; - return; - } + || !chip->pdata->charger_enable) + return POWER_SUPPLY_STATUS_UNKNOWN; - if (chip->pdata->charger_online()) { + if (max17040_get_soc(chip) > MAX17040_BATTERY_FULL) + return POWER_SUPPLY_STATUS_FULL; + + if (chip->pdata->charger_online()) if (chip->pdata->charger_enable()) - chip->status = POWER_SUPPLY_STATUS_CHARGING; + return POWER_SUPPLY_STATUS_CHARGING; else - chip->status = POWER_SUPPLY_STATUS_NOT_CHARGING; - } else { - chip->status = POWER_SUPPLY_STATUS_DISCHARGING; - } - - if (chip->soc > MAX17040_BATTERY_FULL) - chip->status = POWER_SUPPLY_STATUS_FULL; + return POWER_SUPPLY_STATUS_NOT_CHARGING; + else + return POWER_SUPPLY_STATUS_DISCHARGING; } static int max17040_get_of_data(struct max17040_chip *chip) { struct device *dev = &chip->client->dev; + struct chip_data *data = &max17040_family[ + (uintptr_t) of_device_get_match_data(dev)]; + int rcomp_len; + u8 rcomp[2]; + + chip->quirk_double_soc = device_property_read_bool(dev, + "maxim,double-soc"); chip->low_soc_alert = MAX17040_ATHD_DEFAULT_POWER_UP; device_property_read_u32(dev, "maxim,alert-low-soc-level", &chip->low_soc_alert); - if (chip->low_soc_alert <= 0 || chip->low_soc_alert >= 33) + if (chip->low_soc_alert <= 0 || + chip->low_soc_alert > (chip->quirk_double_soc ? 16 : 32)) { + dev_err(dev, "maxim,alert-low-soc-level out of bounds\n"); return -EINVAL; + } + + rcomp_len = device_property_count_u8(dev, "maxim,rcomp"); + chip->rcomp = MAX17040_RCOMP_DEFAULT; + if (rcomp_len == data->rcomp_bytes) { + device_property_read_u8_array(dev, "maxim,rcomp", + rcomp, rcomp_len); + chip->rcomp = rcomp_len == 2 ? + rcomp[0] << 8 | rcomp[1] : + rcomp[0] << 8; + } else if (rcomp_len > 0) { + dev_err(dev, "maxim,rcomp has incorrect length\n"); + return -EINVAL; + } return 0; } -static void max17040_check_changes(struct i2c_client *client) +static void max17040_check_changes(struct max17040_chip *chip) { - max17040_get_vcell(client); - max17040_get_soc(client); - max17040_get_online(client); - max17040_get_status(client); + chip->soc = max17040_get_soc(chip); + chip->status = max17040_get_status(chip); +} + +static void max17040_queue_work(struct max17040_chip *chip) +{ + queue_delayed_work(system_power_efficient_wq, &chip->work, + MAX17040_DELAY); +} + +static void max17040_stop_work(void *data) +{ + struct max17040_chip *chip = data; + + cancel_delayed_work_sync(&chip->work); } static void max17040_work(struct work_struct *work) @@ -217,30 +310,51 @@ static void max17040_work(struct work_struct *work) /* store SOC and status to check changes */ last_soc = chip->soc; last_status = chip->status; - max17040_check_changes(chip->client); + max17040_check_changes(chip); /* check changes and send uevent */ if (last_soc != chip->soc || last_status != chip->status) power_supply_changed(chip->battery); - queue_delayed_work(system_power_efficient_wq, &chip->work, - MAX17040_DELAY); + max17040_queue_work(chip); +} + +/* Returns true if alert cause was SOC change, not low SOC */ +static bool max17040_handle_soc_alert(struct max17040_chip *chip) +{ + bool ret = true; + u32 data; + + regmap_read(chip->regmap, MAX17040_STATUS, &data); + + if (data & MAX17040_STATUS_HD_MASK) { + // this alert was caused by low soc + ret = false; + } + if (data & MAX17040_STATUS_SC_MASK) { + // soc change bit -- deassert to mark as handled + regmap_write(chip->regmap, MAX17040_STATUS, + data & ~MAX17040_STATUS_SC_MASK); + } + + return ret; } static irqreturn_t max17040_thread_handler(int id, void *dev) { struct max17040_chip *chip = dev; - struct i2c_client *client = chip->client; - dev_warn(&client->dev, "IRQ: Alert battery low level"); + if (!(chip->data.has_soc_alert && max17040_handle_soc_alert(chip))) + dev_warn(&chip->client->dev, "IRQ: Alert battery low level\n"); + /* read registers */ - max17040_check_changes(chip->client); + max17040_check_changes(chip); /* send uevent */ power_supply_changed(chip->battery); /* reset alert bit */ - max17040_set_low_soc_alert(client, chip->low_soc_alert); + max17040_set_low_soc_alert(chip, chip->low_soc_alert); return IRQ_HANDLED; } @@ -279,12 +393,13 @@ static int max17040_set_property(struct power_supply *psy, switch (psp) { case POWER_SUPPLY_PROP_CAPACITY_ALERT_MIN: - /* alert threshold can be programmed from 1% up to 32% */ - if ((val->intval < 1) || (val->intval > 32)) { + /* alert threshold can be programmed from 1% up to 16/32% */ + if ((val->intval < 1) || + (val->intval > (chip->quirk_double_soc ? 16 : 32))) { ret = -EINVAL; break; } - ret = max17040_set_low_soc_alert(chip->client, val->intval); + ret = max17040_set_low_soc_alert(chip, val->intval); chip->low_soc_alert = val->intval; break; default: @@ -294,6 +409,41 @@ static int max17040_set_property(struct power_supply *psy, return ret; } +static int max17040_get_property(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + struct max17040_chip *chip = power_supply_get_drvdata(psy); + + switch (psp) { + case POWER_SUPPLY_PROP_STATUS: + val->intval = max17040_get_status(chip); + break; + case POWER_SUPPLY_PROP_ONLINE: + val->intval = max17040_get_online(chip); + break; + case POWER_SUPPLY_PROP_VOLTAGE_NOW: + val->intval = max17040_get_vcell(chip); + break; + case POWER_SUPPLY_PROP_CAPACITY: + val->intval = max17040_get_soc(chip); + break; + case POWER_SUPPLY_PROP_CAPACITY_ALERT_MIN: + val->intval = chip->low_soc_alert; + break; + default: + return -EINVAL; + } + return 0; +} + +static const struct regmap_config max17040_regmap = { + .reg_bits = 8, + .reg_stride = 2, + .val_bits = 16, + .val_format_endian = REGMAP_ENDIAN_BIG, +}; + static enum power_supply_property max17040_battery_props[] = { POWER_SUPPLY_PROP_STATUS, POWER_SUPPLY_PROP_ONLINE, @@ -318,6 +468,8 @@ static int max17040_probe(struct i2c_client *client, struct i2c_adapter *adapter = client->adapter; struct power_supply_config psy_cfg = {}; struct max17040_chip *chip; + enum chip_id chip_id; + bool enable_irq = false; int ret; if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE)) @@ -328,37 +480,68 @@ static int max17040_probe(struct i2c_client *client, return -ENOMEM; chip->client = client; + chip->regmap = devm_regmap_init_i2c(client, &max17040_regmap); chip->pdata = client->dev.platform_data; - ret = max17040_get_of_data(chip); - if (ret) { - dev_err(&client->dev, - "failed: low SOC alert OF data out of bounds\n"); - return ret; + chip_id = (enum chip_id) id->driver_data; + if (client->dev.of_node) { + ret = max17040_get_of_data(chip); + if (ret) + return ret; + chip_id = (enum chip_id) (uintptr_t) + of_device_get_match_data(&client->dev); } + chip->data = max17040_family[chip_id]; i2c_set_clientdata(client, chip); psy_cfg.drv_data = chip; - chip->battery = power_supply_register(&client->dev, + chip->battery = devm_power_supply_register(&client->dev, &max17040_battery_desc, &psy_cfg); if (IS_ERR(chip->battery)) { dev_err(&client->dev, "failed: power supply register\n"); return PTR_ERR(chip->battery); } - max17040_reset(client); - max17040_get_version(client); + ret = max17040_get_version(chip); + if (ret < 0) + return ret; + dev_dbg(&chip->client->dev, "MAX17040 Fuel-Gauge Ver 0x%x\n", ret); + + if (chip_id == ID_MAX17040 || chip_id == ID_MAX17041) + max17040_reset(chip); + + max17040_set_rcomp(chip, chip->rcomp); /* check interrupt */ - if (client->irq && of_device_is_compatible(client->dev.of_node, - "maxim,max77836-battery")) { - ret = max17040_set_low_soc_alert(client, chip->low_soc_alert); + if (client->irq && chip->data.has_low_soc_alert) { + ret = max17040_set_low_soc_alert(chip, chip->low_soc_alert); if (ret) { dev_err(&client->dev, "Failed to set low SOC alert: err %d\n", ret); return ret; } + enable_irq = true; + } + + if (client->irq && chip->data.has_soc_alert) { + ret = max17040_set_soc_alert(chip, 1); + if (ret) { + dev_err(&client->dev, + "Failed to set SOC alert: err %d\n", ret); + return ret; + } + enable_irq = true; + } else { + /* soc alerts negate the need for polling */ + INIT_DEFERRABLE_WORK(&chip->work, max17040_work); + ret = devm_add_action(&client->dev, max17040_stop_work, chip); + if (ret) + return ret; + max17040_queue_work(chip); + } + + if (enable_irq) { ret = max17040_enable_alert_irq(chip); if (ret) { client->irq = 0; @@ -367,19 +550,6 @@ static int max17040_probe(struct i2c_client *client, } } - INIT_DEFERRABLE_WORK(&chip->work, max17040_work); - queue_delayed_work(system_power_efficient_wq, &chip->work, - MAX17040_DELAY); - - return 0; -} - -static int max17040_remove(struct i2c_client *client) -{ - struct max17040_chip *chip = i2c_get_clientdata(client); - - power_supply_unregister(chip->battery); - cancel_delayed_work(&chip->work); return 0; } @@ -390,7 +560,11 @@ static int max17040_suspend(struct device *dev) struct i2c_client *client = to_i2c_client(dev); struct max17040_chip *chip = i2c_get_clientdata(client); - cancel_delayed_work(&chip->work); + if (client->irq && chip->data.has_soc_alert) + // disable soc alert to prevent wakeup + max17040_set_soc_alert(chip, 0); + else + cancel_delayed_work(&chip->work); if (client->irq && device_may_wakeup(dev)) enable_irq_wake(client->irq); @@ -403,12 +577,14 @@ static int max17040_resume(struct device *dev) struct i2c_client *client = to_i2c_client(dev); struct max17040_chip *chip = i2c_get_clientdata(client); - queue_delayed_work(system_power_efficient_wq, &chip->work, - MAX17040_DELAY); - if (client->irq && device_may_wakeup(dev)) disable_irq_wake(client->irq); + if (client->irq && chip->data.has_soc_alert) + max17040_set_soc_alert(chip, 1); + else + max17040_queue_work(chip); + return 0; } @@ -422,16 +598,30 @@ static SIMPLE_DEV_PM_OPS(max17040_pm_ops, max17040_suspend, max17040_resume); #endif /* CONFIG_PM_SLEEP */ static const struct i2c_device_id max17040_id[] = { - { "max17040" }, - { "max77836-battery" }, - { } + { "max17040", ID_MAX17040 }, + { "max17041", ID_MAX17041 }, + { "max17043", ID_MAX17043 }, + { "max77836-battery", ID_MAX17043 }, + { "max17044", ID_MAX17044 }, + { "max17048", ID_MAX17048 }, + { "max17049", ID_MAX17049 }, + { "max17058", ID_MAX17058 }, + { "max17059", ID_MAX17059 }, + { /* sentinel */ } }; MODULE_DEVICE_TABLE(i2c, max17040_id); static const struct of_device_id max17040_of_match[] = { - { .compatible = "maxim,max17040" }, - { .compatible = "maxim,max77836-battery" }, - { }, + { .compatible = "maxim,max17040", .data = (void *) ID_MAX17040 }, + { .compatible = "maxim,max17041", .data = (void *) ID_MAX17041 }, + { .compatible = "maxim,max17043", .data = (void *) ID_MAX17043 }, + { .compatible = "maxim,max77836-battery", .data = (void *) ID_MAX17043 }, + { .compatible = "maxim,max17044", .data = (void *) ID_MAX17044 }, + { .compatible = "maxim,max17048", .data = (void *) ID_MAX17048 }, + { .compatible = "maxim,max17049", .data = (void *) ID_MAX17049 }, + { .compatible = "maxim,max17058", .data = (void *) ID_MAX17058 }, + { .compatible = "maxim,max17059", .data = (void *) ID_MAX17059 }, + { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, max17040_of_match); @@ -442,7 +632,6 @@ static struct i2c_driver max17040_i2c_driver = { .pm = MAX17040_PM_OPS, }, .probe = max17040_probe, - .remove = max17040_remove, .id_table = max17040_id, }; module_i2c_driver(max17040_i2c_driver); diff --git a/drivers/power/supply/pm2301_charger.c b/drivers/power/supply/pm2301_charger.c index 17749fc90e16..2df6a2459d1f 100644 --- a/drivers/power/supply/pm2301_charger.c +++ b/drivers/power/supply/pm2301_charger.c @@ -104,11 +104,6 @@ static int pm2xxx_charger_current_map[] = { 3000, }; -static const struct i2c_device_id pm2xxx_ident[] = { - { "pm2301", 0 }, - { } -}; - static void set_lpn_pin(struct pm2xxx_charger *pm2) { if (!pm2->ac.charger_connected && gpio_is_valid(pm2->lpn_pin)) { @@ -396,7 +391,7 @@ static int pm2_int_reg3(void *pm2_data, int val) if (val & (PM2XXX_INT4_ITCHARGINGON)) { dev_dbg(pm2->dev , - "chargind operation has started\n"); + "charging operation has started\n"); } if (val & (PM2XXX_INT4_ITVRESUME)) { diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c index ccbad435ed12..38e3aa642131 100644 --- a/drivers/power/supply/power_supply_core.c +++ b/drivers/power/supply/power_supply_core.c @@ -579,6 +579,12 @@ int power_supply_get_battery_info(struct power_supply *psy, info->charge_term_current_ua = -EINVAL; info->constant_charge_current_max_ua = -EINVAL; info->constant_charge_voltage_max_uv = -EINVAL; + info->temp_ambient_alert_min = INT_MIN; + info->temp_ambient_alert_max = INT_MAX; + info->temp_alert_min = INT_MIN; + info->temp_alert_max = INT_MAX; + info->temp_min = INT_MIN; + info->temp_max = INT_MAX; info->factory_internal_resistance_uohm = -EINVAL; info->resist_table = NULL; @@ -639,6 +645,19 @@ int power_supply_get_battery_info(struct power_supply *psy, of_property_read_u32(battery_np, "factory-internal-resistance-micro-ohms", &info->factory_internal_resistance_uohm); + of_property_read_u32_index(battery_np, "ambient-celsius", + 0, &info->temp_ambient_alert_min); + of_property_read_u32_index(battery_np, "ambient-celsius", + 1, &info->temp_ambient_alert_max); + of_property_read_u32_index(battery_np, "alert-celsius", + 0, &info->temp_alert_min); + of_property_read_u32_index(battery_np, "alert-celsius", + 1, &info->temp_alert_max); + of_property_read_u32_index(battery_np, "operating-range-celsius", + 0, &info->temp_min); + of_property_read_u32_index(battery_np, "operating-range-celsius", + 1, &info->temp_max); + len = of_property_count_u32_elems(battery_np, "ocv-capacity-celsius"); if (len < 0 && len != -EINVAL) { err = len; diff --git a/drivers/power/supply/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c index 3d383086018c..a616b9d8f43c 100644 --- a/drivers/power/supply/power_supply_sysfs.c +++ b/drivers/power/supply/power_supply_sysfs.c @@ -56,6 +56,7 @@ static const char * const POWER_SUPPLY_TYPE_TEXT[] = { [POWER_SUPPLY_TYPE_USB_PD] = "USB_PD", [POWER_SUPPLY_TYPE_USB_PD_DRP] = "USB_PD_DRP", [POWER_SUPPLY_TYPE_APPLE_BRICK_ID] = "BrickID", + [POWER_SUPPLY_TYPE_WIRELESS] = "Wireless", }; static const char * const POWER_SUPPLY_USB_TYPE_TEXT[] = { diff --git a/drivers/power/supply/rn5t618_power.c b/drivers/power/supply/rn5t618_power.c new file mode 100644 index 000000000000..dee520f0fdf5 --- /dev/null +++ b/drivers/power/supply/rn5t618_power.c @@ -0,0 +1,556 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Power supply driver for the RICOH RN5T618 power management chip family + * + * Copyright (C) 2020 Andreas Kemnade + */ + +#include <linux/kernel.h> +#include <linux/device.h> +#include <linux/bitops.h> +#include <linux/errno.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/mfd/rn5t618.h> +#include <linux/platform_device.h> +#include <linux/power_supply.h> +#include <linux/regmap.h> +#include <linux/slab.h> + +#define CHG_STATE_ADP_INPUT 0x40 +#define CHG_STATE_USB_INPUT 0x80 +#define CHG_STATE_MASK 0x1f +#define CHG_STATE_CHG_OFF 0 +#define CHG_STATE_CHG_READY_VADP 1 +#define CHG_STATE_CHG_TRICKLE 2 +#define CHG_STATE_CHG_RAPID 3 +#define CHG_STATE_CHG_COMPLETE 4 +#define CHG_STATE_SUSPEND 5 +#define CHG_STATE_VCHG_OVER_VOL 6 +#define CHG_STATE_BAT_ERROR 7 +#define CHG_STATE_NO_BAT 8 +#define CHG_STATE_BAT_OVER_VOL 9 +#define CHG_STATE_BAT_TEMP_ERR 10 +#define CHG_STATE_DIE_ERR 11 +#define CHG_STATE_DIE_SHUTDOWN 12 +#define CHG_STATE_NO_BAT2 13 +#define CHG_STATE_CHG_READY_VUSB 14 + +#define FG_ENABLE 1 + +struct rn5t618_power_info { + struct rn5t618 *rn5t618; + struct platform_device *pdev; + struct power_supply *battery; + struct power_supply *usb; + struct power_supply *adp; + int irq; +}; + +static enum power_supply_property rn5t618_usb_props[] = { + POWER_SUPPLY_PROP_STATUS, + POWER_SUPPLY_PROP_ONLINE, +}; + +static enum power_supply_property rn5t618_adp_props[] = { + POWER_SUPPLY_PROP_STATUS, + POWER_SUPPLY_PROP_ONLINE, +}; + + +static enum power_supply_property rn5t618_battery_props[] = { + POWER_SUPPLY_PROP_STATUS, + POWER_SUPPLY_PROP_PRESENT, + POWER_SUPPLY_PROP_VOLTAGE_NOW, + POWER_SUPPLY_PROP_CURRENT_NOW, + POWER_SUPPLY_PROP_CAPACITY, + POWER_SUPPLY_PROP_TEMP, + POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW, + POWER_SUPPLY_PROP_TIME_TO_FULL_NOW, + POWER_SUPPLY_PROP_TECHNOLOGY, + POWER_SUPPLY_PROP_CHARGE_FULL, + POWER_SUPPLY_PROP_CHARGE_NOW, +}; + +static int rn5t618_battery_read_doublereg(struct rn5t618_power_info *info, + u8 reg, u16 *result) +{ + int ret, i; + u8 data[2]; + u16 old, new; + + old = 0; + /* Prevent races when registers are changing. */ + for (i = 0; i < 3; i++) { + ret = regmap_bulk_read(info->rn5t618->regmap, + reg, data, sizeof(data)); + if (ret) + return ret; + + new = data[0] << 8; + new |= data[1]; + if (new == old) + break; + + old = new; + } + + *result = new; + + return 0; +} + +static int rn5t618_decode_status(unsigned int status) +{ + switch (status & CHG_STATE_MASK) { + case CHG_STATE_CHG_OFF: + case CHG_STATE_SUSPEND: + case CHG_STATE_VCHG_OVER_VOL: + case CHG_STATE_DIE_SHUTDOWN: + return POWER_SUPPLY_STATUS_DISCHARGING; + + case CHG_STATE_CHG_TRICKLE: + case CHG_STATE_CHG_RAPID: + return POWER_SUPPLY_STATUS_CHARGING; + + case CHG_STATE_CHG_COMPLETE: + return POWER_SUPPLY_STATUS_FULL; + + default: + return POWER_SUPPLY_STATUS_NOT_CHARGING; + } +} + +static int rn5t618_battery_status(struct rn5t618_power_info *info, + union power_supply_propval *val) +{ + unsigned int v; + int ret; + + ret = regmap_read(info->rn5t618->regmap, RN5T618_CHGSTATE, &v); + if (ret) + return ret; + + val->intval = POWER_SUPPLY_STATUS_UNKNOWN; + + if (v & 0xc0) { /* USB or ADP plugged */ + val->intval = rn5t618_decode_status(v); + } else + val->intval = POWER_SUPPLY_STATUS_DISCHARGING; + + return ret; +} + +static int rn5t618_battery_present(struct rn5t618_power_info *info, + union power_supply_propval *val) +{ + unsigned int v; + int ret; + + ret = regmap_read(info->rn5t618->regmap, RN5T618_CHGSTATE, &v); + if (ret) + return ret; + + v &= CHG_STATE_MASK; + if ((v == CHG_STATE_NO_BAT) || (v == CHG_STATE_NO_BAT2)) + val->intval = 0; + else + val->intval = 1; + + return ret; +} + +static int rn5t618_battery_voltage_now(struct rn5t618_power_info *info, + union power_supply_propval *val) +{ + u16 res; + int ret; + + ret = rn5t618_battery_read_doublereg(info, RN5T618_VOLTAGE_1, &res); + if (ret) + return ret; + + val->intval = res * 2 * 2500 / 4095 * 1000; + + return 0; +} + +static int rn5t618_battery_current_now(struct rn5t618_power_info *info, + union power_supply_propval *val) +{ + u16 res; + int ret; + + ret = rn5t618_battery_read_doublereg(info, RN5T618_CC_AVEREG1, &res); + if (ret) + return ret; + + /* current is negative when discharging */ + val->intval = sign_extend32(res, 13) * 1000; + + return 0; +} + +static int rn5t618_battery_capacity(struct rn5t618_power_info *info, + union power_supply_propval *val) +{ + unsigned int v; + int ret; + + ret = regmap_read(info->rn5t618->regmap, RN5T618_SOC, &v); + if (ret) + return ret; + + val->intval = v; + + return 0; +} + +static int rn5t618_battery_temp(struct rn5t618_power_info *info, + union power_supply_propval *val) +{ + u16 res; + int ret; + + ret = rn5t618_battery_read_doublereg(info, RN5T618_TEMP_1, &res); + if (ret) + return ret; + + val->intval = sign_extend32(res, 11) * 10 / 16; + + return 0; +} + +static int rn5t618_battery_tte(struct rn5t618_power_info *info, + union power_supply_propval *val) +{ + u16 res; + int ret; + + ret = rn5t618_battery_read_doublereg(info, RN5T618_TT_EMPTY_H, &res); + if (ret) + return ret; + + if (res == 65535) + return -ENODATA; + + val->intval = res * 60; + + return 0; +} + +static int rn5t618_battery_ttf(struct rn5t618_power_info *info, + union power_supply_propval *val) +{ + u16 res; + int ret; + + ret = rn5t618_battery_read_doublereg(info, RN5T618_TT_FULL_H, &res); + if (ret) + return ret; + + if (res == 65535) + return -ENODATA; + + val->intval = res * 60; + + return 0; +} + +static int rn5t618_battery_charge_full(struct rn5t618_power_info *info, + union power_supply_propval *val) +{ + u16 res; + int ret; + + ret = rn5t618_battery_read_doublereg(info, RN5T618_FA_CAP_H, &res); + if (ret) + return ret; + + val->intval = res * 1000; + + return 0; +} + +static int rn5t618_battery_charge_now(struct rn5t618_power_info *info, + union power_supply_propval *val) +{ + u16 res; + int ret; + + ret = rn5t618_battery_read_doublereg(info, RN5T618_RE_CAP_H, &res); + if (ret) + return ret; + + val->intval = res * 1000; + + return 0; +} + +static int rn5t618_battery_get_property(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + int ret = 0; + struct rn5t618_power_info *info = power_supply_get_drvdata(psy); + + switch (psp) { + case POWER_SUPPLY_PROP_STATUS: + ret = rn5t618_battery_status(info, val); + break; + case POWER_SUPPLY_PROP_PRESENT: + ret = rn5t618_battery_present(info, val); + break; + case POWER_SUPPLY_PROP_VOLTAGE_NOW: + ret = rn5t618_battery_voltage_now(info, val); + break; + case POWER_SUPPLY_PROP_CURRENT_NOW: + ret = rn5t618_battery_current_now(info, val); + break; + case POWER_SUPPLY_PROP_CAPACITY: + ret = rn5t618_battery_capacity(info, val); + break; + case POWER_SUPPLY_PROP_TEMP: + ret = rn5t618_battery_temp(info, val); + break; + case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW: + ret = rn5t618_battery_tte(info, val); + break; + case POWER_SUPPLY_PROP_TIME_TO_FULL_NOW: + ret = rn5t618_battery_ttf(info, val); + break; + case POWER_SUPPLY_PROP_TECHNOLOGY: + val->intval = POWER_SUPPLY_TECHNOLOGY_LION; + break; + case POWER_SUPPLY_PROP_CHARGE_FULL: + ret = rn5t618_battery_charge_full(info, val); + break; + case POWER_SUPPLY_PROP_CHARGE_NOW: + ret = rn5t618_battery_charge_now(info, val); + break; + default: + return -EINVAL; + } + + return ret; +} + +static int rn5t618_adp_get_property(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + struct rn5t618_power_info *info = power_supply_get_drvdata(psy); + unsigned int chgstate; + bool online; + int ret; + + ret = regmap_read(info->rn5t618->regmap, RN5T618_CHGSTATE, &chgstate); + if (ret) + return ret; + + online = !!(chgstate & CHG_STATE_ADP_INPUT); + + switch (psp) { + case POWER_SUPPLY_PROP_ONLINE: + val->intval = online; + break; + case POWER_SUPPLY_PROP_STATUS: + if (!online) { + val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; + break; + } + val->intval = rn5t618_decode_status(chgstate); + if (val->intval != POWER_SUPPLY_STATUS_CHARGING) + val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; + + break; + default: + return -EINVAL; + } + + return 0; +} + +static int rn5t618_usb_get_property(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + struct rn5t618_power_info *info = power_supply_get_drvdata(psy); + unsigned int chgstate; + bool online; + int ret; + + ret = regmap_read(info->rn5t618->regmap, RN5T618_CHGSTATE, &chgstate); + if (ret) + return ret; + + online = !!(chgstate & CHG_STATE_USB_INPUT); + + switch (psp) { + case POWER_SUPPLY_PROP_ONLINE: + val->intval = online; + break; + case POWER_SUPPLY_PROP_STATUS: + if (!online) { + val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; + break; + } + val->intval = rn5t618_decode_status(chgstate); + if (val->intval != POWER_SUPPLY_STATUS_CHARGING) + val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; + + break; + default: + return -EINVAL; + } + + return 0; +} + +static const struct power_supply_desc rn5t618_battery_desc = { + .name = "rn5t618-battery", + .type = POWER_SUPPLY_TYPE_BATTERY, + .properties = rn5t618_battery_props, + .num_properties = ARRAY_SIZE(rn5t618_battery_props), + .get_property = rn5t618_battery_get_property, +}; + +static const struct power_supply_desc rn5t618_adp_desc = { + .name = "rn5t618-adp", + .type = POWER_SUPPLY_TYPE_MAINS, + .properties = rn5t618_adp_props, + .num_properties = ARRAY_SIZE(rn5t618_adp_props), + .get_property = rn5t618_adp_get_property, +}; + +static const struct power_supply_desc rn5t618_usb_desc = { + .name = "rn5t618-usb", + .type = POWER_SUPPLY_TYPE_USB, + .properties = rn5t618_usb_props, + .num_properties = ARRAY_SIZE(rn5t618_usb_props), + .get_property = rn5t618_usb_get_property, +}; + +static irqreturn_t rn5t618_charger_irq(int irq, void *data) +{ + struct device *dev = data; + struct rn5t618_power_info *info = dev_get_drvdata(dev); + + unsigned int ctrl, stat1, stat2, err; + + regmap_read(info->rn5t618->regmap, RN5T618_CHGERR_IRR, &err); + regmap_read(info->rn5t618->regmap, RN5T618_CHGCTRL_IRR, &ctrl); + regmap_read(info->rn5t618->regmap, RN5T618_CHGSTAT_IRR1, &stat1); + regmap_read(info->rn5t618->regmap, RN5T618_CHGSTAT_IRR2, &stat2); + + regmap_write(info->rn5t618->regmap, RN5T618_CHGERR_IRR, 0); + regmap_write(info->rn5t618->regmap, RN5T618_CHGCTRL_IRR, 0); + regmap_write(info->rn5t618->regmap, RN5T618_CHGSTAT_IRR1, 0); + regmap_write(info->rn5t618->regmap, RN5T618_CHGSTAT_IRR2, 0); + + dev_dbg(dev, "chgerr: %x chgctrl: %x chgstat: %x chgstat2: %x\n", + err, ctrl, stat1, stat2); + + power_supply_changed(info->usb); + power_supply_changed(info->adp); + power_supply_changed(info->battery); + + return IRQ_HANDLED; +} + +static int rn5t618_power_probe(struct platform_device *pdev) +{ + int ret = 0; + unsigned int v; + struct power_supply_config psy_cfg = {}; + struct rn5t618_power_info *info; + + info = devm_kzalloc(&pdev->dev, sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + info->pdev = pdev; + info->rn5t618 = dev_get_drvdata(pdev->dev.parent); + info->irq = -1; + + platform_set_drvdata(pdev, info); + + ret = regmap_read(info->rn5t618->regmap, RN5T618_CONTROL, &v); + if (ret) + return ret; + + if (!(v & FG_ENABLE)) { + /* E.g. the vendor kernels of various Kobo and Tolino Ebook + * readers disable the fuel gauge on shutdown. If a kernel + * without fuel gauge support is booted after that, the fuel + * gauge will get decalibrated. + */ + dev_info(&pdev->dev, "Fuel gauge not enabled, enabling now\n"); + dev_info(&pdev->dev, "Expect imprecise results\n"); + regmap_update_bits(info->rn5t618->regmap, RN5T618_CONTROL, + FG_ENABLE, FG_ENABLE); + } + + psy_cfg.drv_data = info; + info->battery = devm_power_supply_register(&pdev->dev, + &rn5t618_battery_desc, + &psy_cfg); + if (IS_ERR(info->battery)) { + ret = PTR_ERR(info->battery); + dev_err(&pdev->dev, "failed to register battery: %d\n", ret); + return ret; + } + + info->adp = devm_power_supply_register(&pdev->dev, + &rn5t618_adp_desc, + &psy_cfg); + if (IS_ERR(info->adp)) { + ret = PTR_ERR(info->adp); + dev_err(&pdev->dev, "failed to register adp: %d\n", ret); + return ret; + } + + info->usb = devm_power_supply_register(&pdev->dev, + &rn5t618_usb_desc, + &psy_cfg); + if (IS_ERR(info->usb)) { + ret = PTR_ERR(info->usb); + dev_err(&pdev->dev, "failed to register usb: %d\n", ret); + return ret; + } + + if (info->rn5t618->irq_data) + info->irq = regmap_irq_get_virq(info->rn5t618->irq_data, + RN5T618_IRQ_CHG); + + if (info->irq < 0) + info->irq = -1; + else { + ret = devm_request_threaded_irq(&pdev->dev, info->irq, NULL, + rn5t618_charger_irq, + IRQF_ONESHOT, + "rn5t618_power", + &pdev->dev); + + if (ret < 0) { + dev_err(&pdev->dev, "request IRQ:%d fail\n", + info->irq); + info->irq = -1; + } + } + + return 0; +} + +static struct platform_driver rn5t618_power_driver = { + .driver = { + .name = "rn5t618-power", + }, + .probe = rn5t618_power_probe, +}; + +module_platform_driver(rn5t618_power_driver); +MODULE_ALIAS("platform:rn5t618-power"); +MODULE_DESCRIPTION("Power supply driver for RICOH RN5T618"); +MODULE_LICENSE("GPL"); diff --git a/drivers/power/supply/rt9455_charger.c b/drivers/power/supply/rt9455_charger.c index 29161ae90245..594bb3b8a4d1 100644 --- a/drivers/power/supply/rt9455_charger.c +++ b/drivers/power/supply/rt9455_charger.c @@ -1731,11 +1731,13 @@ static const struct of_device_id rt9455_of_match[] = { }; MODULE_DEVICE_TABLE(of, rt9455_of_match); +#ifdef CONFIG_ACPI static const struct acpi_device_id rt9455_i2c_acpi_match[] = { { "RT945500", 0 }, { } }; MODULE_DEVICE_TABLE(acpi, rt9455_i2c_acpi_match); +#endif static struct i2c_driver rt9455_driver = { .probe = rt9455_probe, diff --git a/drivers/power/supply/s3c_adc_battery.c b/drivers/power/supply/s3c_adc_battery.c index 3d00b35cafc9..60b7f41ab063 100644 --- a/drivers/power/supply/s3c_adc_battery.c +++ b/drivers/power/supply/s3c_adc_battery.c @@ -22,7 +22,7 @@ #include <linux/init.h> #include <linux/module.h> -#include <plat/adc.h> +#include <linux/soc/samsung/s3c-adc.h> #define BAT_POLL_INTERVAL 10000 /* ms */ #define JITTER_DELAY 500 /* ms */ diff --git a/drivers/power/supply/sbs-battery.c b/drivers/power/supply/sbs-battery.c index 7439753fac87..b6a538ebb378 100644 --- a/drivers/power/supply/sbs-battery.c +++ b/drivers/power/supply/sbs-battery.c @@ -193,7 +193,6 @@ struct sbs_info { struct power_supply *power_supply; bool is_present; struct gpio_desc *gpio_detect; - bool enable_detection; bool charger_broadcasts; int last_state; int poll_time; @@ -480,37 +479,6 @@ static bool sbs_bat_needs_calibration(struct i2c_client *client) return !!(ret & BIT(7)); } -static int sbs_get_battery_presence_and_health( - struct i2c_client *client, enum power_supply_property psp, - union power_supply_propval *val) -{ - int ret; - - /* Dummy command; if it succeeds, battery is present. */ - ret = sbs_read_word_data(client, sbs_data[REG_STATUS].addr); - - if (ret < 0) { /* battery not present*/ - if (psp == POWER_SUPPLY_PROP_PRESENT) { - val->intval = 0; - return 0; - } - return ret; - } - - if (psp == POWER_SUPPLY_PROP_PRESENT) - val->intval = 1; /* battery present */ - else { /* POWER_SUPPLY_PROP_HEALTH */ - if (sbs_bat_needs_calibration(client)) { - val->intval = POWER_SUPPLY_HEALTH_CALIBRATION_REQUIRED; - } else { - /* SBS spec doesn't have a general health command. */ - val->intval = POWER_SUPPLY_HEALTH_UNKNOWN; - } - } - - return 0; -} - static int sbs_get_ti_battery_presence_and_health( struct i2c_client *client, enum power_supply_property psp, union power_supply_propval *val) @@ -569,6 +537,41 @@ static int sbs_get_ti_battery_presence_and_health( return 0; } +static int sbs_get_battery_presence_and_health( + struct i2c_client *client, enum power_supply_property psp, + union power_supply_propval *val) +{ + struct sbs_info *chip = i2c_get_clientdata(client); + int ret; + + if (chip->flags & SBS_FLAGS_TI_BQ20ZX5) + return sbs_get_ti_battery_presence_and_health(client, psp, val); + + /* Dummy command; if it succeeds, battery is present. */ + ret = sbs_read_word_data(client, sbs_data[REG_STATUS].addr); + + if (ret < 0) { /* battery not present*/ + if (psp == POWER_SUPPLY_PROP_PRESENT) { + val->intval = 0; + return 0; + } + return ret; + } + + if (psp == POWER_SUPPLY_PROP_PRESENT) + val->intval = 1; /* battery present */ + else { /* POWER_SUPPLY_PROP_HEALTH */ + if (sbs_bat_needs_calibration(client)) { + val->intval = POWER_SUPPLY_HEALTH_CALIBRATION_REQUIRED; + } else { + /* SBS spec doesn't have a general health command. */ + val->intval = POWER_SUPPLY_HEALTH_UNKNOWN; + } + } + + return 0; +} + static int sbs_get_battery_property(struct i2c_client *client, int reg_offset, enum power_supply_property psp, union power_supply_propval *val) @@ -871,12 +874,7 @@ static int sbs_get_property(struct power_supply *psy, switch (psp) { case POWER_SUPPLY_PROP_PRESENT: case POWER_SUPPLY_PROP_HEALTH: - if (chip->flags & SBS_FLAGS_TI_BQ20ZX5) - ret = sbs_get_ti_battery_presence_and_health(client, - psp, val); - else - ret = sbs_get_battery_presence_and_health(client, psp, - val); + ret = sbs_get_battery_presence_and_health(client, psp, val); /* this can only be true if no gpio is used */ if (psp == POWER_SUPPLY_PROP_PRESENT) @@ -967,32 +965,30 @@ static int sbs_get_property(struct power_supply *psy, return -EINVAL; } - if (!chip->enable_detection) - goto done; + if (!chip->gpio_detect && chip->is_present != (ret >= 0)) { + bool old_present = chip->is_present; + union power_supply_propval val; + int err = sbs_get_battery_presence_and_health( + client, POWER_SUPPLY_PROP_PRESENT, &val); - if (!chip->gpio_detect && - chip->is_present != (ret >= 0)) { - sbs_update_presence(chip, (ret >= 0)); - power_supply_changed(chip->power_supply); + sbs_update_presence(chip, !err && val.intval); + + if (old_present != chip->is_present) + power_supply_changed(chip->power_supply); } done: if (!ret) { /* Convert units to match requirements for power supply class */ sbs_unit_adjustment(client, psp, val); + dev_dbg(&client->dev, + "%s: property = %d, value = %x\n", __func__, + psp, val->intval); + } else if (!chip->is_present) { + /* battery not present, so return NODATA for properties */ + ret = -ENODATA; } - - dev_dbg(&client->dev, - "%s: property = %d, value = %x\n", __func__, psp, val->intval); - - if (ret && chip->is_present) - return ret; - - /* battery not present, so return NODATA for properties */ - if (ret) - return -ENODATA; - - return 0; + return ret; } static void sbs_supply_changed(struct sbs_info *chip) @@ -1098,7 +1094,6 @@ static int sbs_probe(struct i2c_client *client) chip->flags = (u32)(uintptr_t)device_get_match_data(&client->dev); chip->client = client; - chip->enable_detection = false; psy_cfg.of_node = client->dev.of_node; psy_cfg.drv_data = chip; chip->last_state = POWER_SUPPLY_STATUS_UNKNOWN; @@ -1159,15 +1154,19 @@ skip_gpio: * to the battery. */ if (!(force_load || chip->gpio_detect)) { - rc = sbs_read_word_data(client, sbs_data[REG_STATUS].addr); + union power_supply_propval val; - if (rc < 0) { - dev_err(&client->dev, "%s: Failed to get device status\n", - __func__); + rc = sbs_get_battery_presence_and_health( + client, POWER_SUPPLY_PROP_PRESENT, &val); + if (rc < 0 || !val.intval) { + dev_err(&client->dev, "Failed to get present status\n"); + rc = -ENODEV; goto exit_psupply; } } + INIT_DELAYED_WORK(&chip->work, sbs_delayed_work); + chip->power_supply = devm_power_supply_register(&client->dev, sbs_desc, &psy_cfg); if (IS_ERR(chip->power_supply)) { @@ -1180,10 +1179,6 @@ skip_gpio: dev_info(&client->dev, "%s: battery gas gauge device registered\n", client->name); - INIT_DELAYED_WORK(&chip->work, sbs_delayed_work); - - chip->enable_detection = true; - return 0; exit_psupply: diff --git a/drivers/power/supply/smb347-charger.c b/drivers/power/supply/smb347-charger.c index f99026d81f2a..d3bf35ed12ce 100644 --- a/drivers/power/supply/smb347-charger.c +++ b/drivers/power/supply/smb347-charger.c @@ -16,11 +16,18 @@ #include <linux/init.h> #include <linux/interrupt.h> #include <linux/i2c.h> -#include <linux/mutex.h> #include <linux/power_supply.h> -#include <linux/power/smb347-charger.h> +#include <linux/property.h> #include <linux/regmap.h> +#include <dt-bindings/power/summit,smb347-charger.h> + +/* Use the default compensation method */ +#define SMB3XX_SOFT_TEMP_COMPENSATE_DEFAULT -1 + +/* Use default factory programmed value for hard/soft temperature limit */ +#define SMB3XX_TEMP_USE_DEFAULT -273 + /* * Configuration registers. These are mirrored to volatile RAM and can be * written once %CMD_A_ALLOW_WRITE is set in %CMD_A register. They will be @@ -122,82 +129,140 @@ /** * struct smb347_charger - smb347 charger instance - * @lock: protects concurrent access to online variables * @dev: pointer to device * @regmap: pointer to driver regmap * @mains: power_supply instance for AC/DC power * @usb: power_supply instance for USB power - * @battery: power_supply instance for battery + * @id: SMB charger ID * @mains_online: is AC/DC input connected * @usb_online: is USB input connected * @charging_enabled: is charging enabled - * @pdata: pointer to platform data + * @max_charge_current: maximum current (in uA) the battery can be charged + * @max_charge_voltage: maximum voltage (in uV) the battery can be charged + * @pre_charge_current: current (in uA) to use in pre-charging phase + * @termination_current: current (in uA) used to determine when the + * charging cycle terminates + * @pre_to_fast_voltage: voltage (in uV) treshold used for transitioning to + * pre-charge to fast charge mode + * @mains_current_limit: maximum input current drawn from AC/DC input (in uA) + * @usb_hc_current_limit: maximum input high current (in uA) drawn from USB + * input + * @chip_temp_threshold: die temperature where device starts limiting charge + * current [%100 - %130] (in degree C) + * @soft_cold_temp_limit: soft cold temperature limit [%0 - %15] (in degree C), + * granularity is 5 deg C. + * @soft_hot_temp_limit: soft hot temperature limit [%40 - %55] (in degree C), + * granularity is 5 deg C. + * @hard_cold_temp_limit: hard cold temperature limit [%-5 - %10] (in degree C), + * granularity is 5 deg C. + * @hard_hot_temp_limit: hard hot temperature limit [%50 - %65] (in degree C), + * granularity is 5 deg C. + * @suspend_on_hard_temp_limit: suspend charging when hard limit is hit + * @soft_temp_limit_compensation: compensation method when soft temperature + * limit is hit + * @charge_current_compensation: current (in uA) for charging compensation + * current when temperature hits soft limits + * @use_mains: AC/DC input can be used + * @use_usb: USB input can be used + * @use_usb_otg: USB OTG output can be used (not implemented yet) + * @enable_control: how charging enable/disable is controlled + * (driver/pin controls) + * + * @use_main, @use_usb, and @use_usb_otg are means to enable/disable + * hardware support for these. This is useful when we want to have for + * example OTG charging controlled via OTG transceiver driver and not by + * the SMB347 hardware. + * + * Hard and soft temperature limit values are given as described in the + * device data sheet and assuming NTC beta value is %3750. Even if this is + * not the case, these values should be used. They can be mapped to the + * corresponding NTC beta values with the help of table %2 in the data + * sheet. So for example if NTC beta is %3375 and we want to program hard + * hot limit to be %53 deg C, @hard_hot_temp_limit should be set to %50. + * + * If zero value is given in any of the current and voltage values, the + * factory programmed default will be used. For soft/hard temperature + * values, pass in %SMB3XX_TEMP_USE_DEFAULT instead. */ struct smb347_charger { - struct mutex lock; struct device *dev; struct regmap *regmap; struct power_supply *mains; struct power_supply *usb; - struct power_supply *battery; + unsigned int id; bool mains_online; bool usb_online; bool charging_enabled; - const struct smb347_charger_platform_data *pdata; + + unsigned int max_charge_current; + unsigned int max_charge_voltage; + unsigned int pre_charge_current; + unsigned int termination_current; + unsigned int pre_to_fast_voltage; + unsigned int mains_current_limit; + unsigned int usb_hc_current_limit; + unsigned int chip_temp_threshold; + int soft_cold_temp_limit; + int soft_hot_temp_limit; + int hard_cold_temp_limit; + int hard_hot_temp_limit; + bool suspend_on_hard_temp_limit; + unsigned int soft_temp_limit_compensation; + unsigned int charge_current_compensation; + bool use_mains; + bool use_usb; + bool use_usb_otg; + unsigned int enable_control; }; -/* Fast charge current in uA */ -static const unsigned int fcc_tbl[] = { - 700000, - 900000, - 1200000, - 1500000, - 1800000, - 2000000, - 2200000, - 2500000, +enum smb_charger_chipid { + SMB345, + SMB347, + SMB358, + NUM_CHIP_TYPES, }; +/* Fast charge current in uA */ +static const unsigned int fcc_tbl[NUM_CHIP_TYPES][8] = { + [SMB345] = { 200000, 450000, 600000, 900000, + 1300000, 1500000, 1800000, 2000000 }, + [SMB347] = { 700000, 900000, 1200000, 1500000, + 1800000, 2000000, 2200000, 2500000 }, + [SMB358] = { 200000, 450000, 600000, 900000, + 1300000, 1500000, 1800000, 2000000 }, +}; /* Pre-charge current in uA */ -static const unsigned int pcc_tbl[] = { - 100000, - 150000, - 200000, - 250000, +static const unsigned int pcc_tbl[NUM_CHIP_TYPES][4] = { + [SMB345] = { 150000, 250000, 350000, 450000 }, + [SMB347] = { 100000, 150000, 200000, 250000 }, + [SMB358] = { 150000, 250000, 350000, 450000 }, }; /* Termination current in uA */ -static const unsigned int tc_tbl[] = { - 37500, - 50000, - 100000, - 150000, - 200000, - 250000, - 500000, - 600000, +static const unsigned int tc_tbl[NUM_CHIP_TYPES][8] = { + [SMB345] = { 30000, 40000, 60000, 80000, + 100000, 125000, 150000, 200000 }, + [SMB347] = { 37500, 50000, 100000, 150000, + 200000, 250000, 500000, 600000 }, + [SMB358] = { 30000, 40000, 60000, 80000, + 100000, 125000, 150000, 200000 }, }; /* Input current limit in uA */ -static const unsigned int icl_tbl[] = { - 300000, - 500000, - 700000, - 900000, - 1200000, - 1500000, - 1800000, - 2000000, - 2200000, - 2500000, +static const unsigned int icl_tbl[NUM_CHIP_TYPES][10] = { + [SMB345] = { 300000, 500000, 700000, 1000000, 1500000, + 1800000, 2000000, 2000000, 2000000, 2000000 }, + [SMB347] = { 300000, 500000, 700000, 900000, 1200000, + 1500000, 1800000, 2000000, 2200000, 2500000 }, + [SMB358] = { 300000, 500000, 700000, 1000000, 1500000, + 1800000, 2000000, 2000000, 2000000, 2000000 }, }; /* Charge current compensation in uA */ -static const unsigned int ccc_tbl[] = { - 250000, - 700000, - 900000, - 1200000, +static const unsigned int ccc_tbl[NUM_CHIP_TYPES][4] = { + [SMB345] = { 200000, 450000, 600000, 900000 }, + [SMB347] = { 250000, 700000, 900000, 1200000 }, + [SMB358] = { 200000, 450000, 600000, 900000 }, }; /* Convert register value to current using lookup table */ @@ -242,16 +307,14 @@ static int smb347_update_ps_status(struct smb347_charger *smb) * Dc and usb are set depending on whether they are enabled in * platform data _and_ whether corresponding undervoltage is set. */ - if (smb->pdata->use_mains) + if (smb->use_mains) dc = !(val & IRQSTAT_E_DCIN_UV_STAT); - if (smb->pdata->use_usb) + if (smb->use_usb) usb = !(val & IRQSTAT_E_USBIN_UV_STAT); - mutex_lock(&smb->lock); ret = smb->mains_online != dc || smb->usb_online != usb; smb->mains_online = dc; smb->usb_online = usb; - mutex_unlock(&smb->lock); return ret; } @@ -267,13 +330,7 @@ static int smb347_update_ps_status(struct smb347_charger *smb) */ static bool smb347_is_ps_online(struct smb347_charger *smb) { - bool ret; - - mutex_lock(&smb->lock); - ret = smb->usb_online || smb->mains_online; - mutex_unlock(&smb->lock); - - return ret; + return smb->usb_online || smb->mains_online; } /** @@ -302,19 +359,18 @@ static int smb347_charging_set(struct smb347_charger *smb, bool enable) { int ret = 0; - if (smb->pdata->enable_control != SMB347_CHG_ENABLE_SW) { + if (smb->enable_control != SMB3XX_CHG_ENABLE_SW) { dev_dbg(smb->dev, "charging enable/disable in SW disabled\n"); return 0; } - mutex_lock(&smb->lock); if (smb->charging_enabled != enable) { ret = regmap_update_bits(smb->regmap, CMD_A, CMD_A_CHG_ENABLED, enable ? CMD_A_CHG_ENABLED : 0); if (!ret) smb->charging_enabled = enable; } - mutex_unlock(&smb->lock); + return ret; } @@ -352,11 +408,12 @@ static int smb347_start_stop_charging(struct smb347_charger *smb) static int smb347_set_charge_current(struct smb347_charger *smb) { + unsigned int id = smb->id; int ret; - if (smb->pdata->max_charge_current) { - ret = current_to_hw(fcc_tbl, ARRAY_SIZE(fcc_tbl), - smb->pdata->max_charge_current); + if (smb->max_charge_current) { + ret = current_to_hw(fcc_tbl[id], ARRAY_SIZE(fcc_tbl[id]), + smb->max_charge_current); if (ret < 0) return ret; @@ -367,9 +424,9 @@ static int smb347_set_charge_current(struct smb347_charger *smb) return ret; } - if (smb->pdata->pre_charge_current) { - ret = current_to_hw(pcc_tbl, ARRAY_SIZE(pcc_tbl), - smb->pdata->pre_charge_current); + if (smb->pre_charge_current) { + ret = current_to_hw(pcc_tbl[id], ARRAY_SIZE(pcc_tbl[id]), + smb->pre_charge_current); if (ret < 0) return ret; @@ -380,9 +437,9 @@ static int smb347_set_charge_current(struct smb347_charger *smb) return ret; } - if (smb->pdata->termination_current) { - ret = current_to_hw(tc_tbl, ARRAY_SIZE(tc_tbl), - smb->pdata->termination_current); + if (smb->termination_current) { + ret = current_to_hw(tc_tbl[id], ARRAY_SIZE(tc_tbl[id]), + smb->termination_current); if (ret < 0) return ret; @@ -397,11 +454,12 @@ static int smb347_set_charge_current(struct smb347_charger *smb) static int smb347_set_current_limits(struct smb347_charger *smb) { + unsigned int id = smb->id; int ret; - if (smb->pdata->mains_current_limit) { - ret = current_to_hw(icl_tbl, ARRAY_SIZE(icl_tbl), - smb->pdata->mains_current_limit); + if (smb->mains_current_limit) { + ret = current_to_hw(icl_tbl[id], ARRAY_SIZE(icl_tbl[id]), + smb->mains_current_limit); if (ret < 0) return ret; @@ -412,9 +470,9 @@ static int smb347_set_current_limits(struct smb347_charger *smb) return ret; } - if (smb->pdata->usb_hc_current_limit) { - ret = current_to_hw(icl_tbl, ARRAY_SIZE(icl_tbl), - smb->pdata->usb_hc_current_limit); + if (smb->usb_hc_current_limit) { + ret = current_to_hw(icl_tbl[id], ARRAY_SIZE(icl_tbl[id]), + smb->usb_hc_current_limit); if (ret < 0) return ret; @@ -431,8 +489,8 @@ static int smb347_set_voltage_limits(struct smb347_charger *smb) { int ret; - if (smb->pdata->pre_to_fast_voltage) { - ret = smb->pdata->pre_to_fast_voltage; + if (smb->pre_to_fast_voltage) { + ret = smb->pre_to_fast_voltage; /* uV */ ret = clamp_val(ret, 2400000, 3000000) - 2400000; @@ -445,8 +503,8 @@ static int smb347_set_voltage_limits(struct smb347_charger *smb) return ret; } - if (smb->pdata->max_charge_voltage) { - ret = smb->pdata->max_charge_voltage; + if (smb->max_charge_voltage) { + ret = smb->max_charge_voltage; /* uV */ ret = clamp_val(ret, 3500000, 4500000) - 3500000; @@ -463,12 +521,13 @@ static int smb347_set_voltage_limits(struct smb347_charger *smb) static int smb347_set_temp_limits(struct smb347_charger *smb) { + unsigned int id = smb->id; bool enable_therm_monitor = false; int ret = 0; int val; - if (smb->pdata->chip_temp_threshold) { - val = smb->pdata->chip_temp_threshold; + if (smb->chip_temp_threshold) { + val = smb->chip_temp_threshold; /* degree C */ val = clamp_val(val, 100, 130) - 100; @@ -481,8 +540,8 @@ static int smb347_set_temp_limits(struct smb347_charger *smb) return ret; } - if (smb->pdata->soft_cold_temp_limit != SMB347_TEMP_USE_DEFAULT) { - val = smb->pdata->soft_cold_temp_limit; + if (smb->soft_cold_temp_limit != SMB3XX_TEMP_USE_DEFAULT) { + val = smb->soft_cold_temp_limit; val = clamp_val(val, 0, 15); val /= 5; @@ -498,8 +557,8 @@ static int smb347_set_temp_limits(struct smb347_charger *smb) enable_therm_monitor = true; } - if (smb->pdata->soft_hot_temp_limit != SMB347_TEMP_USE_DEFAULT) { - val = smb->pdata->soft_hot_temp_limit; + if (smb->soft_hot_temp_limit != SMB3XX_TEMP_USE_DEFAULT) { + val = smb->soft_hot_temp_limit; val = clamp_val(val, 40, 55) - 40; val /= 5; @@ -513,8 +572,8 @@ static int smb347_set_temp_limits(struct smb347_charger *smb) enable_therm_monitor = true; } - if (smb->pdata->hard_cold_temp_limit != SMB347_TEMP_USE_DEFAULT) { - val = smb->pdata->hard_cold_temp_limit; + if (smb->hard_cold_temp_limit != SMB3XX_TEMP_USE_DEFAULT) { + val = smb->hard_cold_temp_limit; val = clamp_val(val, -5, 10) + 5; val /= 5; @@ -530,8 +589,8 @@ static int smb347_set_temp_limits(struct smb347_charger *smb) enable_therm_monitor = true; } - if (smb->pdata->hard_hot_temp_limit != SMB347_TEMP_USE_DEFAULT) { - val = smb->pdata->hard_hot_temp_limit; + if (smb->hard_hot_temp_limit != SMB3XX_TEMP_USE_DEFAULT) { + val = smb->hard_hot_temp_limit; val = clamp_val(val, 50, 65) - 50; val /= 5; @@ -562,16 +621,16 @@ static int smb347_set_temp_limits(struct smb347_charger *smb) return ret; } - if (smb->pdata->suspend_on_hard_temp_limit) { + if (smb->suspend_on_hard_temp_limit) { ret = regmap_update_bits(smb->regmap, CFG_SYSOK, CFG_SYSOK_SUSPEND_HARD_LIMIT_DISABLED, 0); if (ret < 0) return ret; } - if (smb->pdata->soft_temp_limit_compensation != - SMB347_SOFT_TEMP_COMPENSATE_DEFAULT) { - val = smb->pdata->soft_temp_limit_compensation & 0x3; + if (smb->soft_temp_limit_compensation != + SMB3XX_SOFT_TEMP_COMPENSATE_DEFAULT) { + val = smb->soft_temp_limit_compensation & 0x3; ret = regmap_update_bits(smb->regmap, CFG_THERM, CFG_THERM_SOFT_HOT_COMPENSATION_MASK, @@ -586,9 +645,9 @@ static int smb347_set_temp_limits(struct smb347_charger *smb) return ret; } - if (smb->pdata->charge_current_compensation) { - val = current_to_hw(ccc_tbl, ARRAY_SIZE(ccc_tbl), - smb->pdata->charge_current_compensation); + if (smb->charge_current_compensation) { + val = current_to_hw(ccc_tbl[id], ARRAY_SIZE(ccc_tbl[id]), + smb->charge_current_compensation); if (val < 0) return val; @@ -647,7 +706,7 @@ static int smb347_hw_init(struct smb347_charger *smb) goto fail; /* If USB charging is disabled we put the USB in suspend mode */ - if (!smb->pdata->use_usb) { + if (!smb->use_usb) { ret = regmap_update_bits(smb->regmap, CMD_A, CMD_A_SUSPEND_ENABLED, CMD_A_SUSPEND_ENABLED); @@ -660,7 +719,7 @@ static int smb347_hw_init(struct smb347_charger *smb) * support for driving VBUS. Otherwise we disable it. */ ret = regmap_update_bits(smb->regmap, CFG_OTHER, CFG_OTHER_RID_MASK, - smb->pdata->use_usb_otg ? CFG_OTHER_RID_ENABLED_AUTO_OTG : 0); + smb->use_usb_otg ? CFG_OTHER_RID_ENABLED_AUTO_OTG : 0); if (ret < 0) goto fail; @@ -669,11 +728,11 @@ static int smb347_hw_init(struct smb347_charger *smb) * command register unless pin control is specified in the platform * data. */ - switch (smb->pdata->enable_control) { - case SMB347_CHG_ENABLE_PIN_ACTIVE_LOW: + switch (smb->enable_control) { + case SMB3XX_CHG_ENABLE_PIN_ACTIVE_LOW: val = CFG_PIN_EN_CTRL_ACTIVE_LOW; break; - case SMB347_CHG_ENABLE_PIN_ACTIVE_HIGH: + case SMB3XX_CHG_ENABLE_PIN_ACTIVE_HIGH: val = CFG_PIN_EN_CTRL_ACTIVE_HIGH; break; default: @@ -742,7 +801,10 @@ static irqreturn_t smb347_interrupt(int irq, void *data) */ if (stat_c & STAT_C_CHARGER_ERROR) { dev_err(smb->dev, "charging stopped due to charger error\n"); - power_supply_changed(smb->battery); + if (smb->use_mains) + power_supply_changed(smb->mains); + if (smb->use_usb) + power_supply_changed(smb->usb); handled = true; } @@ -752,8 +814,12 @@ static irqreturn_t smb347_interrupt(int irq, void *data) * disabled by the hardware. */ if (irqstat_c & (IRQSTAT_C_TERMINATION_IRQ | IRQSTAT_C_TAPER_IRQ)) { - if (irqstat_c & IRQSTAT_C_TERMINATION_STAT) - power_supply_changed(smb->battery); + if (irqstat_c & IRQSTAT_C_TERMINATION_STAT) { + if (smb->use_mains) + power_supply_changed(smb->mains); + if (smb->use_usb) + power_supply_changed(smb->usb); + } dev_dbg(smb->dev, "going to HW maintenance mode\n"); handled = true; } @@ -767,7 +833,10 @@ static irqreturn_t smb347_interrupt(int irq, void *data) if (irqstat_d & IRQSTAT_D_CHARGE_TIMEOUT_STAT) dev_warn(smb->dev, "charging stopped due to timeout\n"); - power_supply_changed(smb->battery); + if (smb->use_mains) + power_supply_changed(smb->mains); + if (smb->use_usb) + power_supply_changed(smb->usb); handled = true; } @@ -778,9 +847,9 @@ static irqreturn_t smb347_interrupt(int irq, void *data) if (irqstat_e & (IRQSTAT_E_USBIN_UV_IRQ | IRQSTAT_E_DCIN_UV_IRQ)) { if (smb347_update_ps_status(smb) > 0) { smb347_start_stop_charging(smb); - if (smb->pdata->use_mains) + if (smb->use_mains) power_supply_changed(smb->mains); - if (smb->pdata->use_usb) + if (smb->use_usb) power_supply_changed(smb->usb); } handled = true; @@ -835,22 +904,17 @@ static inline int smb347_irq_disable(struct smb347_charger *smb) static int smb347_irq_init(struct smb347_charger *smb, struct i2c_client *client) { - const struct smb347_charger_platform_data *pdata = smb->pdata; - int ret, irq = gpio_to_irq(pdata->irq_gpio); - - ret = gpio_request_one(pdata->irq_gpio, GPIOF_IN, client->name); - if (ret < 0) - goto fail; + int ret; - ret = request_threaded_irq(irq, NULL, smb347_interrupt, - IRQF_TRIGGER_FALLING | IRQF_ONESHOT, - client->name, smb); + ret = devm_request_threaded_irq(smb->dev, client->irq, NULL, + smb347_interrupt, IRQF_ONESHOT, + client->name, smb); if (ret < 0) - goto fail_gpio; + return ret; ret = smb347_set_writable(smb, true); if (ret < 0) - goto fail_irq; + return ret; /* * Configure the STAT output to be suitable for interrupts: disable @@ -860,20 +924,10 @@ static int smb347_irq_init(struct smb347_charger *smb, CFG_STAT_ACTIVE_HIGH | CFG_STAT_DISABLED, CFG_STAT_DISABLED); if (ret < 0) - goto fail_readonly; + client->irq = 0; smb347_set_writable(smb, false); - client->irq = irq; - return 0; -fail_readonly: - smb347_set_writable(smb, false); -fail_irq: - free_irq(irq, smb); -fail_gpio: - gpio_free(pdata->irq_gpio); -fail: - client->irq = 0; return ret; } @@ -883,6 +937,7 @@ fail: */ static int get_const_charge_current(struct smb347_charger *smb) { + unsigned int id = smb->id; int ret, intval; unsigned int v; @@ -898,10 +953,12 @@ static int get_const_charge_current(struct smb347_charger *smb) * and we can detect which table to use from bit 5. */ if (v & 0x20) { - intval = hw_to_current(fcc_tbl, ARRAY_SIZE(fcc_tbl), v & 7); + intval = hw_to_current(fcc_tbl[id], + ARRAY_SIZE(fcc_tbl[id]), v & 7); } else { v >>= 3; - intval = hw_to_current(pcc_tbl, ARRAY_SIZE(pcc_tbl), v & 7); + intval = hw_to_current(pcc_tbl[id], + ARRAY_SIZE(pcc_tbl[id]), v & 7); } return intval; @@ -932,95 +989,19 @@ static int get_const_charge_voltage(struct smb347_charger *smb) return intval; } -static int smb347_mains_get_property(struct power_supply *psy, - enum power_supply_property prop, - union power_supply_propval *val) -{ - struct smb347_charger *smb = power_supply_get_drvdata(psy); - int ret; - - switch (prop) { - case POWER_SUPPLY_PROP_ONLINE: - val->intval = smb->mains_online; - break; - - case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE: - ret = get_const_charge_voltage(smb); - if (ret < 0) - return ret; - else - val->intval = ret; - break; - - case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT: - ret = get_const_charge_current(smb); - if (ret < 0) - return ret; - else - val->intval = ret; - break; - - default: - return -EINVAL; - } - - return 0; -} - -static enum power_supply_property smb347_mains_properties[] = { - POWER_SUPPLY_PROP_ONLINE, - POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT, - POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE, -}; - -static int smb347_usb_get_property(struct power_supply *psy, - enum power_supply_property prop, - union power_supply_propval *val) -{ - struct smb347_charger *smb = power_supply_get_drvdata(psy); - int ret; - - switch (prop) { - case POWER_SUPPLY_PROP_ONLINE: - val->intval = smb->usb_online; - break; - - case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE: - ret = get_const_charge_voltage(smb); - if (ret < 0) - return ret; - else - val->intval = ret; - break; - - case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT: - ret = get_const_charge_current(smb); - if (ret < 0) - return ret; - else - val->intval = ret; - break; - - default: - return -EINVAL; - } - - return 0; -} - -static enum power_supply_property smb347_usb_properties[] = { - POWER_SUPPLY_PROP_ONLINE, - POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT, - POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE, -}; - -static int smb347_get_charging_status(struct smb347_charger *smb) +static int smb347_get_charging_status(struct smb347_charger *smb, + struct power_supply *psy) { int ret, status; unsigned int val; - if (!smb347_is_ps_online(smb)) - return POWER_SUPPLY_STATUS_DISCHARGING; + if (psy->desc->type == POWER_SUPPLY_TYPE_USB) { + if (!smb->usb_online) + return POWER_SUPPLY_STATUS_DISCHARGING; + } else { + if (!smb->mains_online) + return POWER_SUPPLY_STATUS_DISCHARGING; + } ret = regmap_read(smb->regmap, STAT_C, &val); if (ret < 0) @@ -1059,29 +1040,29 @@ static int smb347_get_charging_status(struct smb347_charger *smb) return status; } -static int smb347_battery_get_property(struct power_supply *psy, - enum power_supply_property prop, - union power_supply_propval *val) +static int smb347_get_property_locked(struct power_supply *psy, + enum power_supply_property prop, + union power_supply_propval *val) { struct smb347_charger *smb = power_supply_get_drvdata(psy); - const struct smb347_charger_platform_data *pdata = smb->pdata; int ret; - ret = smb347_update_ps_status(smb); - if (ret < 0) - return ret; - switch (prop) { case POWER_SUPPLY_PROP_STATUS: - ret = smb347_get_charging_status(smb); + ret = smb347_get_charging_status(smb, psy); if (ret < 0) return ret; val->intval = ret; break; case POWER_SUPPLY_PROP_CHARGE_TYPE: - if (!smb347_is_ps_online(smb)) - return -ENODATA; + if (psy->desc->type == POWER_SUPPLY_TYPE_USB) { + if (!smb->usb_online) + return -ENODATA; + } else { + if (!smb->mains_online) + return -ENODATA; + } /* * We handle trickle and pre-charging the same, and taper @@ -1100,24 +1081,25 @@ static int smb347_battery_get_property(struct power_supply *psy, } break; - case POWER_SUPPLY_PROP_TECHNOLOGY: - val->intval = pdata->battery_info.technology; - break; - - case POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN: - val->intval = pdata->battery_info.voltage_min_design; - break; - - case POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN: - val->intval = pdata->battery_info.voltage_max_design; + case POWER_SUPPLY_PROP_ONLINE: + if (psy->desc->type == POWER_SUPPLY_TYPE_USB) + val->intval = smb->usb_online; + else + val->intval = smb->mains_online; break; - case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN: - val->intval = pdata->battery_info.charge_full_design; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE: + ret = get_const_charge_voltage(smb); + if (ret < 0) + return ret; + val->intval = ret; break; - case POWER_SUPPLY_PROP_MODEL_NAME: - val->strval = pdata->battery_info.name; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT: + ret = get_const_charge_current(smb); + if (ret < 0) + return ret; + val->intval = ret; break; default: @@ -1127,14 +1109,27 @@ static int smb347_battery_get_property(struct power_supply *psy, return 0; } -static enum power_supply_property smb347_battery_properties[] = { +static int smb347_get_property(struct power_supply *psy, + enum power_supply_property prop, + union power_supply_propval *val) +{ + struct smb347_charger *smb = power_supply_get_drvdata(psy); + struct i2c_client *client = to_i2c_client(smb->dev); + int ret; + + disable_irq(client->irq); + ret = smb347_get_property_locked(psy, prop, val); + enable_irq(client->irq); + + return ret; +} + +static enum power_supply_property smb347_properties[] = { POWER_SUPPLY_PROP_STATUS, POWER_SUPPLY_PROP_CHARGE_TYPE, - POWER_SUPPLY_PROP_TECHNOLOGY, - POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN, - POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN, - POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN, - POWER_SUPPLY_PROP_MODEL_NAME, + POWER_SUPPLY_PROP_ONLINE, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE, }; static bool smb347_volatile_reg(struct device *dev, unsigned int reg) @@ -1180,6 +1175,96 @@ static bool smb347_readable_reg(struct device *dev, unsigned int reg) return smb347_volatile_reg(dev, reg); } +static void smb347_dt_parse_dev_info(struct smb347_charger *smb) +{ + struct device *dev = smb->dev; + + smb->soft_temp_limit_compensation = + SMB3XX_SOFT_TEMP_COMPENSATE_DEFAULT; + /* + * These properties come from the battery info, still we need to + * pre-initialize the values. See smb347_get_battery_info() below. + */ + smb->soft_cold_temp_limit = SMB3XX_TEMP_USE_DEFAULT; + smb->hard_cold_temp_limit = SMB3XX_TEMP_USE_DEFAULT; + smb->soft_hot_temp_limit = SMB3XX_TEMP_USE_DEFAULT; + smb->hard_hot_temp_limit = SMB3XX_TEMP_USE_DEFAULT; + + /* Charging constraints */ + device_property_read_u32(dev, "summit,fast-voltage-threshold-microvolt", + &smb->pre_to_fast_voltage); + device_property_read_u32(dev, "summit,mains-current-limit-microamp", + &smb->mains_current_limit); + device_property_read_u32(dev, "summit,usb-current-limit-microamp", + &smb->usb_hc_current_limit); + + /* For thermometer monitoring */ + device_property_read_u32(dev, "summit,chip-temperature-threshold-celsius", + &smb->chip_temp_threshold); + device_property_read_u32(dev, "summit,soft-compensation-method", + &smb->soft_temp_limit_compensation); + device_property_read_u32(dev, "summit,charge-current-compensation-microamp", + &smb->charge_current_compensation); + + /* Supported charging mode */ + smb->use_mains = device_property_read_bool(dev, "summit,enable-mains-charging"); + smb->use_usb = device_property_read_bool(dev, "summit,enable-usb-charging"); + smb->use_usb_otg = device_property_read_bool(dev, "summit,enable-otg-charging"); + + /* Select charging control */ + device_property_read_u32(dev, "summit,enable-charge-control", + &smb->enable_control); +} + +static int smb347_get_battery_info(struct smb347_charger *smb) +{ + struct power_supply_battery_info info = {}; + struct power_supply *supply; + int err; + + if (smb->mains) + supply = smb->mains; + else + supply = smb->usb; + + err = power_supply_get_battery_info(supply, &info); + if (err == -ENXIO || err == -ENODEV) + return 0; + if (err) + return err; + + if (info.constant_charge_current_max_ua != -EINVAL) + smb->max_charge_current = info.constant_charge_current_max_ua; + + if (info.constant_charge_voltage_max_uv != -EINVAL) + smb->max_charge_voltage = info.constant_charge_voltage_max_uv; + + if (info.precharge_current_ua != -EINVAL) + smb->pre_charge_current = info.precharge_current_ua; + + if (info.charge_term_current_ua != -EINVAL) + smb->termination_current = info.charge_term_current_ua; + + if (info.temp_alert_min != INT_MIN) + smb->soft_cold_temp_limit = info.temp_alert_min; + + if (info.temp_alert_max != INT_MAX) + smb->soft_hot_temp_limit = info.temp_alert_max; + + if (info.temp_min != INT_MIN) + smb->hard_cold_temp_limit = info.temp_min; + + if (info.temp_max != INT_MAX) + smb->hard_hot_temp_limit = info.temp_max; + + /* Suspend when battery temperature is outside hard limits */ + if (smb->hard_cold_temp_limit != SMB3XX_TEMP_USE_DEFAULT || + smb->hard_hot_temp_limit != SMB3XX_TEMP_USE_DEFAULT) + smb->suspend_on_hard_temp_limit = true; + + return 0; +} + static const struct regmap_config smb347_regmap = { .reg_bits = 8, .val_bits = 8, @@ -1191,98 +1276,71 @@ static const struct regmap_config smb347_regmap = { static const struct power_supply_desc smb347_mains_desc = { .name = "smb347-mains", .type = POWER_SUPPLY_TYPE_MAINS, - .get_property = smb347_mains_get_property, - .properties = smb347_mains_properties, - .num_properties = ARRAY_SIZE(smb347_mains_properties), + .get_property = smb347_get_property, + .properties = smb347_properties, + .num_properties = ARRAY_SIZE(smb347_properties), }; static const struct power_supply_desc smb347_usb_desc = { .name = "smb347-usb", .type = POWER_SUPPLY_TYPE_USB, - .get_property = smb347_usb_get_property, - .properties = smb347_usb_properties, - .num_properties = ARRAY_SIZE(smb347_usb_properties), -}; - -static const struct power_supply_desc smb347_battery_desc = { - .name = "smb347-battery", - .type = POWER_SUPPLY_TYPE_BATTERY, - .get_property = smb347_battery_get_property, - .properties = smb347_battery_properties, - .num_properties = ARRAY_SIZE(smb347_battery_properties), + .get_property = smb347_get_property, + .properties = smb347_properties, + .num_properties = ARRAY_SIZE(smb347_properties), }; static int smb347_probe(struct i2c_client *client, const struct i2c_device_id *id) { - static char *battery[] = { "smb347-battery" }; - const struct smb347_charger_platform_data *pdata; - struct power_supply_config mains_usb_cfg = {}, battery_cfg = {}; + struct power_supply_config mains_usb_cfg = {}; struct device *dev = &client->dev; struct smb347_charger *smb; int ret; - pdata = dev->platform_data; - if (!pdata) - return -EINVAL; - - if (!pdata->use_mains && !pdata->use_usb) - return -EINVAL; - smb = devm_kzalloc(dev, sizeof(*smb), GFP_KERNEL); if (!smb) return -ENOMEM; - + smb->dev = &client->dev; + smb->id = id->driver_data; i2c_set_clientdata(client, smb); - mutex_init(&smb->lock); - smb->dev = &client->dev; - smb->pdata = pdata; + smb347_dt_parse_dev_info(smb); + if (!smb->use_mains && !smb->use_usb) + return -EINVAL; smb->regmap = devm_regmap_init_i2c(client, &smb347_regmap); if (IS_ERR(smb->regmap)) return PTR_ERR(smb->regmap); - ret = smb347_hw_init(smb); - if (ret < 0) - return ret; - - mains_usb_cfg.supplied_to = battery; - mains_usb_cfg.num_supplicants = ARRAY_SIZE(battery); mains_usb_cfg.drv_data = smb; - if (smb->pdata->use_mains) { - smb->mains = power_supply_register(dev, &smb347_mains_desc, - &mains_usb_cfg); + mains_usb_cfg.of_node = dev->of_node; + if (smb->use_mains) { + smb->mains = devm_power_supply_register(dev, &smb347_mains_desc, + &mains_usb_cfg); if (IS_ERR(smb->mains)) return PTR_ERR(smb->mains); } - if (smb->pdata->use_usb) { - smb->usb = power_supply_register(dev, &smb347_usb_desc, - &mains_usb_cfg); - if (IS_ERR(smb->usb)) { - if (smb->pdata->use_mains) - power_supply_unregister(smb->mains); + if (smb->use_usb) { + smb->usb = devm_power_supply_register(dev, &smb347_usb_desc, + &mains_usb_cfg); + if (IS_ERR(smb->usb)) return PTR_ERR(smb->usb); - } } - battery_cfg.drv_data = smb; - smb->battery = power_supply_register(dev, &smb347_battery_desc, - &battery_cfg); - if (IS_ERR(smb->battery)) { - if (smb->pdata->use_usb) - power_supply_unregister(smb->usb); - if (smb->pdata->use_mains) - power_supply_unregister(smb->mains); - return PTR_ERR(smb->battery); - } + ret = smb347_get_battery_info(smb); + if (ret) + return ret; + + ret = smb347_hw_init(smb); + if (ret < 0) + return ret; /* * Interrupt pin is optional. If it is connected, we setup the * interrupt support here. */ - if (pdata->irq_gpio >= 0) { + if (client->irq) { ret = smb347_irq_init(smb, client); if (ret < 0) { dev_warn(dev, "failed to initialize IRQ: %d\n", ret); @@ -1299,29 +1357,31 @@ static int smb347_remove(struct i2c_client *client) { struct smb347_charger *smb = i2c_get_clientdata(client); - if (client->irq) { + if (client->irq) smb347_irq_disable(smb); - free_irq(client->irq, smb); - gpio_free(smb->pdata->irq_gpio); - } - - power_supply_unregister(smb->battery); - if (smb->pdata->use_usb) - power_supply_unregister(smb->usb); - if (smb->pdata->use_mains) - power_supply_unregister(smb->mains); return 0; } static const struct i2c_device_id smb347_id[] = { - { "smb347", 0 }, - { } + { "smb345", SMB345 }, + { "smb347", SMB347 }, + { "smb358", SMB358 }, + { }, }; MODULE_DEVICE_TABLE(i2c, smb347_id); +static const struct of_device_id smb3xx_of_match[] = { + { .compatible = "summit,smb345" }, + { .compatible = "summit,smb347" }, + { .compatible = "summit,smb358" }, + { }, +}; +MODULE_DEVICE_TABLE(of, smb3xx_of_match); + static struct i2c_driver smb347_driver = { .driver = { .name = "smb347", + .of_match_table = smb3xx_of_match, }, .probe = smb347_probe, .remove = smb347_remove, diff --git a/drivers/power/supply/test_power.c b/drivers/power/supply/test_power.c index 04acd76bbaa1..5f510ddc946d 100644 --- a/drivers/power/supply/test_power.c +++ b/drivers/power/supply/test_power.c @@ -352,8 +352,8 @@ static int param_set_ac_online(const char *key, const struct kernel_param *kp) static int param_get_ac_online(char *buffer, const struct kernel_param *kp) { - strcpy(buffer, map_get_key(map_ac_online, ac_online, "unknown")); - return strlen(buffer); + return sprintf(buffer, "%s\n", + map_get_key(map_ac_online, ac_online, "unknown")); } static int param_set_usb_online(const char *key, const struct kernel_param *kp) @@ -365,8 +365,8 @@ static int param_set_usb_online(const char *key, const struct kernel_param *kp) static int param_get_usb_online(char *buffer, const struct kernel_param *kp) { - strcpy(buffer, map_get_key(map_ac_online, usb_online, "unknown")); - return strlen(buffer); + return sprintf(buffer, "%s\n", + map_get_key(map_ac_online, usb_online, "unknown")); } static int param_set_battery_status(const char *key, @@ -379,8 +379,8 @@ static int param_set_battery_status(const char *key, static int param_get_battery_status(char *buffer, const struct kernel_param *kp) { - strcpy(buffer, map_get_key(map_status, battery_status, "unknown")); - return strlen(buffer); + return sprintf(buffer, "%s\n", + map_get_key(map_ac_online, battery_status, "unknown")); } static int param_set_battery_health(const char *key, @@ -393,8 +393,8 @@ static int param_set_battery_health(const char *key, static int param_get_battery_health(char *buffer, const struct kernel_param *kp) { - strcpy(buffer, map_get_key(map_health, battery_health, "unknown")); - return strlen(buffer); + return sprintf(buffer, "%s\n", + map_get_key(map_ac_online, battery_health, "unknown")); } static int param_set_battery_present(const char *key, @@ -408,8 +408,8 @@ static int param_set_battery_present(const char *key, static int param_get_battery_present(char *buffer, const struct kernel_param *kp) { - strcpy(buffer, map_get_key(map_present, battery_present, "unknown")); - return strlen(buffer); + return sprintf(buffer, "%s\n", + map_get_key(map_ac_online, battery_present, "unknown")); } static int param_set_battery_technology(const char *key, @@ -424,9 +424,9 @@ static int param_set_battery_technology(const char *key, static int param_get_battery_technology(char *buffer, const struct kernel_param *kp) { - strcpy(buffer, - map_get_key(map_technology, battery_technology, "unknown")); - return strlen(buffer); + return sprintf(buffer, "%s\n", + map_get_key(map_ac_online, battery_technology, + "unknown")); } static int param_set_battery_capacity(const char *key, diff --git a/drivers/power/supply/ucs1002_power.c b/drivers/power/supply/ucs1002_power.c index cdb9a23d825f..ef673ec3db56 100644 --- a/drivers/power/supply/ucs1002_power.c +++ b/drivers/power/supply/ucs1002_power.c @@ -38,6 +38,7 @@ /* Interrupt Status */ #define UCS1002_REG_INTERRUPT_STATUS 0x10 +# define F_ERR BIT(7) # define F_DISCHARGE_ERR BIT(6) # define F_RESET BIT(5) # define F_MIN_KEEP_OUT BIT(4) @@ -103,6 +104,9 @@ struct ucs1002_info { struct regulator_dev *rdev; bool present; bool output_disable; + struct delayed_work health_poll; + int health; + }; static enum power_supply_property ucs1002_props[] = { @@ -362,32 +366,6 @@ static int ucs1002_get_usb_type(struct ucs1002_info *info, return 0; } -static int ucs1002_get_health(struct ucs1002_info *info, - union power_supply_propval *val) -{ - unsigned int reg; - int ret, health; - - ret = regmap_read(info->regmap, UCS1002_REG_INTERRUPT_STATUS, ®); - if (ret) - return ret; - - if (reg & F_TSD) - health = POWER_SUPPLY_HEALTH_OVERHEAT; - else if (reg & (F_OVER_VOLT | F_BACK_VOLT)) - health = POWER_SUPPLY_HEALTH_OVERVOLTAGE; - else if (reg & F_OVER_ILIM) - health = POWER_SUPPLY_HEALTH_OVERCURRENT; - else if (reg & (F_DISCHARGE_ERR | F_MIN_KEEP_OUT)) - health = POWER_SUPPLY_HEALTH_UNSPEC_FAILURE; - else - health = POWER_SUPPLY_HEALTH_GOOD; - - val->intval = health; - - return 0; -} - static int ucs1002_get_property(struct power_supply *psy, enum power_supply_property psp, union power_supply_propval *val) @@ -406,7 +384,7 @@ static int ucs1002_get_property(struct power_supply *psy, case POWER_SUPPLY_PROP_USB_TYPE: return ucs1002_get_usb_type(info, val); case POWER_SUPPLY_PROP_HEALTH: - return ucs1002_get_health(info, val); + return val->intval = info->health; case POWER_SUPPLY_PROP_PRESENT: val->intval = info->present; return 0; @@ -458,6 +436,38 @@ static const struct power_supply_desc ucs1002_charger_desc = { .num_properties = ARRAY_SIZE(ucs1002_props), }; +static void ucs1002_health_poll(struct work_struct *work) +{ + struct ucs1002_info *info = container_of(work, struct ucs1002_info, + health_poll.work); + int ret; + u32 reg; + + ret = regmap_read(info->regmap, UCS1002_REG_INTERRUPT_STATUS, ®); + if (ret) + return; + + /* bad health and no status change, just schedule us again in a while */ + if ((reg & F_ERR) && info->health != POWER_SUPPLY_HEALTH_GOOD) { + schedule_delayed_work(&info->health_poll, + msecs_to_jiffies(2000)); + return; + } + + if (reg & F_TSD) + info->health = POWER_SUPPLY_HEALTH_OVERHEAT; + else if (reg & (F_OVER_VOLT | F_BACK_VOLT)) + info->health = POWER_SUPPLY_HEALTH_OVERVOLTAGE; + else if (reg & F_OVER_ILIM) + info->health = POWER_SUPPLY_HEALTH_OVERCURRENT; + else if (reg & (F_DISCHARGE_ERR | F_MIN_KEEP_OUT)) + info->health = POWER_SUPPLY_HEALTH_UNSPEC_FAILURE; + else + info->health = POWER_SUPPLY_HEALTH_GOOD; + + sysfs_notify(&info->charger->dev.kobj, NULL, "health"); +} + static irqreturn_t ucs1002_charger_irq(int irq, void *data) { int ret, regval; @@ -484,7 +494,7 @@ static irqreturn_t ucs1002_alert_irq(int irq, void *data) { struct ucs1002_info *info = data; - power_supply_changed(info->charger); + mod_delayed_work(system_wq, &info->health_poll, 0); return IRQ_HANDLED; } @@ -632,6 +642,9 @@ static int ucs1002_probe(struct i2c_client *client, return ret; } + info->health = POWER_SUPPLY_HEALTH_GOOD; + INIT_DELAYED_WORK(&info->health_poll, ucs1002_health_poll); + if (irq_a_det > 0) { ret = devm_request_threaded_irq(dev, irq_a_det, NULL, ucs1002_charger_irq, @@ -645,10 +658,8 @@ static int ucs1002_probe(struct i2c_client *client, } if (irq_alert > 0) { - ret = devm_request_threaded_irq(dev, irq_alert, NULL, - ucs1002_alert_irq, - IRQF_ONESHOT, - "ucs1002-alert", info); + ret = devm_request_irq(dev, irq_alert, ucs1002_alert_irq, + 0,"ucs1002-alert", info); if (ret) { dev_err(dev, "Failed to request ALERT threaded irq: %d\n", ret); diff --git a/drivers/powercap/Kconfig b/drivers/powercap/Kconfig index ebc4d4578339..bc228725346b 100644 --- a/drivers/powercap/Kconfig +++ b/drivers/powercap/Kconfig @@ -30,7 +30,7 @@ config INTEL_RAPL In RAPL, the platform level settings are divided into domains for fine grained control. These domains include processor package, DRAM - controller, CPU core (Power Plance 0), graphics uncore (Power Plane + controller, CPU core (Power Plane 0), graphics uncore (Power Plane 1), etc. config IDLE_INJECT diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 983d75bd5bd1..0b2830efc574 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -544,7 +544,14 @@ static void rapl_init_domains(struct rapl_package *rp) continue; rd->rp = rp; - rd->name = rapl_domain_names[i]; + + if (i == RAPL_DOMAIN_PLATFORM && rp->id > 0) { + snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "psys-%d", + cpu_data(rp->lead_cpu).phys_proc_id); + } else + snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "%s", + rapl_domain_names[i]); + rd->id = i; rd->rpl[0].prim_id = PL1_ENABLE; rd->rpl[0].name = pl1_name; @@ -1112,13 +1119,17 @@ static int rapl_package_register_powercap(struct rapl_package *rp) } /* now register domains as children of the socket/package */ for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { + struct powercap_zone *parent = rp->power_zone; + if (rd->id == RAPL_DOMAIN_PACKAGE) continue; + if (rd->id == RAPL_DOMAIN_PLATFORM) + parent = NULL; /* number of power limits per domain varies */ nr_pl = find_nr_power_limit(rd); power_zone = powercap_register_zone(&rd->power_zone, rp->priv->control_type, - rd->name, rp->power_zone, + rd->name, parent, &zone_ops[rd->id], nr_pl, &constraint_ops); @@ -1145,67 +1156,6 @@ err_cleanup: return ret; } -int rapl_add_platform_domain(struct rapl_if_priv *priv) -{ - struct rapl_domain *rd; - struct powercap_zone *power_zone; - struct reg_action ra; - int ret; - - ra.reg = priv->regs[RAPL_DOMAIN_PLATFORM][RAPL_DOMAIN_REG_STATUS]; - ra.mask = ~0; - ret = priv->read_raw(0, &ra); - if (ret || !ra.value) - return -ENODEV; - - ra.reg = priv->regs[RAPL_DOMAIN_PLATFORM][RAPL_DOMAIN_REG_LIMIT]; - ra.mask = ~0; - ret = priv->read_raw(0, &ra); - if (ret || !ra.value) - return -ENODEV; - - rd = kzalloc(sizeof(*rd), GFP_KERNEL); - if (!rd) - return -ENOMEM; - - rd->name = rapl_domain_names[RAPL_DOMAIN_PLATFORM]; - rd->id = RAPL_DOMAIN_PLATFORM; - rd->regs[RAPL_DOMAIN_REG_LIMIT] = - priv->regs[RAPL_DOMAIN_PLATFORM][RAPL_DOMAIN_REG_LIMIT]; - rd->regs[RAPL_DOMAIN_REG_STATUS] = - priv->regs[RAPL_DOMAIN_PLATFORM][RAPL_DOMAIN_REG_STATUS]; - rd->rpl[0].prim_id = PL1_ENABLE; - rd->rpl[0].name = pl1_name; - rd->rpl[1].prim_id = PL2_ENABLE; - rd->rpl[1].name = pl2_name; - rd->rp = rapl_find_package_domain(0, priv); - - power_zone = powercap_register_zone(&rd->power_zone, priv->control_type, - "psys", NULL, - &zone_ops[RAPL_DOMAIN_PLATFORM], - 2, &constraint_ops); - - if (IS_ERR(power_zone)) { - kfree(rd); - return PTR_ERR(power_zone); - } - - priv->platform_rapl_domain = rd; - - return 0; -} -EXPORT_SYMBOL_GPL(rapl_add_platform_domain); - -void rapl_remove_platform_domain(struct rapl_if_priv *priv) -{ - if (priv->platform_rapl_domain) { - powercap_unregister_zone(priv->control_type, - &priv->platform_rapl_domain->power_zone); - kfree(priv->platform_rapl_domain); - } -} -EXPORT_SYMBOL_GPL(rapl_remove_platform_domain); - static int rapl_check_domain(int cpu, int domain, struct rapl_package *rp) { struct reg_action ra; @@ -1215,11 +1165,9 @@ static int rapl_check_domain(int cpu, int domain, struct rapl_package *rp) case RAPL_DOMAIN_PP0: case RAPL_DOMAIN_PP1: case RAPL_DOMAIN_DRAM: + case RAPL_DOMAIN_PLATFORM: ra.reg = rp->priv->regs[domain][RAPL_DOMAIN_REG_STATUS]; break; - case RAPL_DOMAIN_PLATFORM: - /* PSYS(PLATFORM) is not a CPU domain, so avoid printng error */ - return -EINVAL; default: pr_err("invalid domain id %d\n", domain); return -EINVAL; @@ -1228,7 +1176,7 @@ static int rapl_check_domain(int cpu, int domain, struct rapl_package *rp) * values, otherwise skip it. */ - ra.mask = ~0; + ra.mask = ENERGY_STATUS_MASK; if (rp->priv->read_raw(cpu, &ra) || !ra.value) return -ENODEV; diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index d2a2627507a9..1646808d354c 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -44,6 +44,7 @@ static struct rapl_if_priv rapl_msr_priv = { .regs[RAPL_DOMAIN_PLATFORM] = { MSR_PLATFORM_POWER_LIMIT, MSR_PLATFORM_ENERGY_STATUS, 0, 0, 0}, .limits[RAPL_DOMAIN_PACKAGE] = 2, + .limits[RAPL_DOMAIN_PLATFORM] = 2, }; /* Handles CPU hotplug on multi-socket systems. @@ -157,9 +158,6 @@ static int rapl_msr_probe(struct platform_device *pdev) goto out; rapl_msr_priv.pcap_rapl_online = ret; - /* Don't bail out if PSys is not supported */ - rapl_add_platform_domain(&rapl_msr_priv); - return 0; out: @@ -171,7 +169,6 @@ out: static int rapl_msr_remove(struct platform_device *pdev) { cpuhp_remove_state(rapl_msr_priv.pcap_rapl_online); - rapl_remove_platform_domain(&rapl_msr_priv); powercap_unregister_control_type(rapl_msr_priv.control_type); return 0; } diff --git a/drivers/pwm/Kconfig b/drivers/pwm/Kconfig index 78ddc127e45e..63be5362fd3a 100644 --- a/drivers/pwm/Kconfig +++ b/drivers/pwm/Kconfig @@ -410,7 +410,7 @@ config PWM_ROCKCHIP config PWM_SAMSUNG tristate "Samsung PWM support" - depends on PLAT_SAMSUNG || ARCH_EXYNOS || COMPILE_TEST + depends on PLAT_SAMSUNG || ARCH_S5PV210 || ARCH_EXYNOS || COMPILE_TEST help Generic PWM framework driver for Samsung. diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index 276e939a5684..1f16f5365d3c 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -1327,30 +1327,19 @@ static int pwm_seq_show(struct seq_file *s, void *v) return 0; } -static const struct seq_operations pwm_seq_ops = { +static const struct seq_operations pwm_debugfs_sops = { .start = pwm_seq_start, .next = pwm_seq_next, .stop = pwm_seq_stop, .show = pwm_seq_show, }; -static int pwm_seq_open(struct inode *inode, struct file *file) -{ - return seq_open(file, &pwm_seq_ops); -} - -static const struct file_operations pwm_debugfs_ops = { - .owner = THIS_MODULE, - .open = pwm_seq_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release, -}; +DEFINE_SEQ_ATTRIBUTE(pwm_debugfs); static int __init pwm_debugfs_init(void) { debugfs_create_file("pwm", S_IFREG | S_IRUGO, NULL, NULL, - &pwm_debugfs_ops); + &pwm_debugfs_fops); return 0; } diff --git a/drivers/pwm/pwm-bcm2835.c b/drivers/pwm/pwm-bcm2835.c index d78f86f8e462..6841dcfe27fc 100644 --- a/drivers/pwm/pwm-bcm2835.c +++ b/drivers/pwm/pwm-bcm2835.c @@ -152,13 +152,9 @@ static int bcm2835_pwm_probe(struct platform_device *pdev) return PTR_ERR(pc->base); pc->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(pc->clk)) { - ret = PTR_ERR(pc->clk); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "clock not found: %d\n", ret); - - return ret; - } + if (IS_ERR(pc->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(pc->clk), + "clock not found\n"); ret = clk_prepare_enable(pc->clk); if (ret) diff --git a/drivers/pwm/pwm-cros-ec.c b/drivers/pwm/pwm-cros-ec.c index 09c08dee099e..c1c337969e4e 100644 --- a/drivers/pwm/pwm-cros-ec.c +++ b/drivers/pwm/pwm-cros-ec.c @@ -81,8 +81,7 @@ static int cros_ec_pwm_set_duty(struct cros_ec_device *ec, u8 index, u16 duty) return cros_ec_cmd_xfer_status(ec, msg); } -static int __cros_ec_pwm_get_duty(struct cros_ec_device *ec, u8 index, - u32 *result) +static int cros_ec_pwm_get_duty(struct cros_ec_device *ec, u8 index) { struct { struct cros_ec_command msg; @@ -107,19 +106,12 @@ static int __cros_ec_pwm_get_duty(struct cros_ec_device *ec, u8 index, params->index = index; ret = cros_ec_cmd_xfer_status(ec, msg); - if (result) - *result = msg->result; if (ret < 0) return ret; return resp->duty; } -static int cros_ec_pwm_get_duty(struct cros_ec_device *ec, u8 index) -{ - return __cros_ec_pwm_get_duty(ec, index, NULL); -} - static int cros_ec_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { @@ -204,29 +196,34 @@ static const struct pwm_ops cros_ec_pwm_ops = { .owner = THIS_MODULE, }; +/* + * Determine the number of supported PWMs. The EC does not return the number + * of PWMs it supports directly, so we have to read the pwm duty cycle for + * subsequent channels until we get an error. + */ static int cros_ec_num_pwms(struct cros_ec_device *ec) { int i, ret; /* The index field is only 8 bits */ for (i = 0; i <= U8_MAX; i++) { - u32 result = 0; - - ret = __cros_ec_pwm_get_duty(ec, i, &result); - /* We want to parse EC protocol errors */ - if (ret < 0 && !(ret == -EPROTO && result)) - return ret; - + ret = cros_ec_pwm_get_duty(ec, i); /* * We look for SUCCESS, INVALID_COMMAND, or INVALID_PARAM * responses; everything else is treated as an error. + * The EC error codes map to -EOPNOTSUPP and -EINVAL, + * so check for those. */ - if (result == EC_RES_INVALID_COMMAND) + switch (ret) { + case -EOPNOTSUPP: /* invalid command */ return -ENODEV; - else if (result == EC_RES_INVALID_PARAM) + case -EINVAL: /* invalid parameter */ return i; - else if (result) - return -EPROTO; + default: + if (ret < 0) + return ret; + break; + } } return U8_MAX; diff --git a/drivers/pwm/pwm-img.c b/drivers/pwm/pwm-img.c index 599a0f66a384..a34d95ed70b2 100644 --- a/drivers/pwm/pwm-img.c +++ b/drivers/pwm/pwm-img.c @@ -277,6 +277,8 @@ static int img_pwm_probe(struct platform_device *pdev) return PTR_ERR(pwm->pwm_clk); } + platform_set_drvdata(pdev, pwm); + pm_runtime_set_autosuspend_delay(&pdev->dev, IMG_PWM_PM_TIMEOUT); pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_enable(&pdev->dev); @@ -313,7 +315,6 @@ static int img_pwm_probe(struct platform_device *pdev) goto err_suspend; } - platform_set_drvdata(pdev, pwm); return 0; err_suspend: diff --git a/drivers/pwm/pwm-jz4740.c b/drivers/pwm/pwm-jz4740.c index 5830ac2bdf6a..00c642fa2eed 100644 --- a/drivers/pwm/pwm-jz4740.c +++ b/drivers/pwm/pwm-jz4740.c @@ -60,12 +60,9 @@ static int jz4740_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) snprintf(name, sizeof(name), "timer%u", pwm->hwpwm); clk = clk_get(chip->dev, name); - if (IS_ERR(clk)) { - if (PTR_ERR(clk) != -EPROBE_DEFER) - dev_err(chip->dev, "Failed to get clock: %pe", clk); - - return PTR_ERR(clk); - } + if (IS_ERR(clk)) + return dev_err_probe(chip->dev, PTR_ERR(clk), + "Failed to get clock\n"); err = clk_prepare_enable(clk); if (err < 0) { diff --git a/drivers/pwm/pwm-pca9685.c b/drivers/pwm/pwm-pca9685.c index 76cd22bd6614..4a55dc18656c 100644 --- a/drivers/pwm/pwm-pca9685.c +++ b/drivers/pwm/pwm-pca9685.c @@ -57,10 +57,14 @@ #define PCA9685_NUMREGS 0xFF #define PCA9685_MAXCHAN 0x10 -#define LED_FULL (1 << 4) -#define MODE1_SLEEP (1 << 4) -#define MODE2_INVRT (1 << 4) -#define MODE2_OUTDRV (1 << 2) +#define LED_FULL BIT(4) +#define MODE1_ALLCALL BIT(0) +#define MODE1_SUB3 BIT(1) +#define MODE1_SUB2 BIT(2) +#define MODE1_SUB1 BIT(3) +#define MODE1_SLEEP BIT(4) +#define MODE2_INVRT BIT(4) +#define MODE2_OUTDRV BIT(2) #define LED_N_ON_H(N) (PCA9685_LEDX_ON_H + (4 * (N))) #define LED_N_ON_L(N) (PCA9685_LEDX_ON_L + (4 * (N))) @@ -91,7 +95,7 @@ static bool pca9685_pwm_test_and_set_inuse(struct pca9685 *pca, int pwm_idx) mutex_lock(&pca->lock); if (pwm_idx >= PCA9685_MAXCHAN) { /* - * "all LEDs" channel: + * "All LEDs" channel: * pretend already in use if any of the PWMs are requested */ if (!bitmap_empty(pca->pwms_inuse, PCA9685_MAXCHAN)) { @@ -100,7 +104,7 @@ static bool pca9685_pwm_test_and_set_inuse(struct pca9685 *pca, int pwm_idx) } } else { /* - * regular channel: + * Regular channel: * pretend already in use if the "all LEDs" channel is requested */ if (test_bit(PCA9685_MAXCHAN, pca->pwms_inuse)) { @@ -257,7 +261,7 @@ static int pca9685_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (prescale >= PCA9685_PRESCALE_MIN && prescale <= PCA9685_PRESCALE_MAX) { /* - * putting the chip briefly into SLEEP mode + * Putting the chip briefly into SLEEP mode * at this point won't interfere with the * pm_runtime framework, because the pm_runtime * state is guaranteed active here. @@ -443,8 +447,8 @@ static int pca9685_pwm_probe(struct i2c_client *client, const struct i2c_device_id *id) { struct pca9685 *pca; + unsigned int reg; int ret; - int mode2; pca = devm_kzalloc(&client->dev, sizeof(*pca), GFP_KERNEL); if (!pca) @@ -461,26 +465,31 @@ static int pca9685_pwm_probe(struct i2c_client *client, i2c_set_clientdata(client, pca); - regmap_read(pca->regmap, PCA9685_MODE2, &mode2); + regmap_read(pca->regmap, PCA9685_MODE2, ®); if (device_property_read_bool(&client->dev, "invert")) - mode2 |= MODE2_INVRT; + reg |= MODE2_INVRT; else - mode2 &= ~MODE2_INVRT; + reg &= ~MODE2_INVRT; if (device_property_read_bool(&client->dev, "open-drain")) - mode2 &= ~MODE2_OUTDRV; + reg &= ~MODE2_OUTDRV; else - mode2 |= MODE2_OUTDRV; + reg |= MODE2_OUTDRV; + + regmap_write(pca->regmap, PCA9685_MODE2, reg); - regmap_write(pca->regmap, PCA9685_MODE2, mode2); + /* Disable all LED ALLCALL and SUBx addresses to avoid bus collisions */ + regmap_read(pca->regmap, PCA9685_MODE1, ®); + reg &= ~(MODE1_ALLCALL | MODE1_SUB1 | MODE1_SUB2 | MODE1_SUB3); + regmap_write(pca->regmap, PCA9685_MODE1, reg); - /* clear all "full off" bits */ + /* Clear all "full off" bits */ regmap_write(pca->regmap, PCA9685_ALL_LED_OFF_L, 0); regmap_write(pca->regmap, PCA9685_ALL_LED_OFF_H, 0); pca->chip.ops = &pca9685_pwm_ops; - /* add an extra channel for ALL_LED */ + /* Add an extra channel for ALL_LED */ pca->chip.npwm = PCA9685_MAXCHAN + 1; pca->chip.dev = &client->dev; @@ -496,10 +505,10 @@ static int pca9685_pwm_probe(struct i2c_client *client, return ret; } - /* the chip comes out of power-up in the active state */ + /* The chip comes out of power-up in the active state */ pm_runtime_set_active(&client->dev); /* - * enable will put the chip into suspend, which is what we + * Enable will put the chip into suspend, which is what we * want as all outputs are disabled at this point */ pm_runtime_enable(&client->dev); diff --git a/drivers/pwm/pwm-rockchip.c b/drivers/pwm/pwm-rockchip.c index eb8c9cb645a6..77c23a2c6d71 100644 --- a/drivers/pwm/pwm-rockchip.c +++ b/drivers/pwm/pwm-rockchip.c @@ -288,6 +288,7 @@ static int rockchip_pwm_probe(struct platform_device *pdev) const struct of_device_id *id; struct rockchip_pwm_chip *pc; struct resource *r; + u32 enable_conf, ctrl; int ret, count; id = of_match_device(rockchip_pwm_dt_ids, &pdev->dev); @@ -306,13 +307,9 @@ static int rockchip_pwm_probe(struct platform_device *pdev) pc->clk = devm_clk_get(&pdev->dev, "pwm"); if (IS_ERR(pc->clk)) { pc->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(pc->clk)) { - ret = PTR_ERR(pc->clk); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "Can't get bus clk: %d\n", - ret); - return ret; - } + if (IS_ERR(pc->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(pc->clk), + "Can't get bus clk\n"); } count = of_count_phandle_with_args(pdev->dev.of_node, @@ -362,7 +359,9 @@ static int rockchip_pwm_probe(struct platform_device *pdev) } /* Keep the PWM clk enabled if the PWM appears to be up and running. */ - if (!pwm_is_enabled(pc->chip.pwms)) + enable_conf = pc->data->enable_conf; + ctrl = readl_relaxed(pc->base + pc->data->regs.ctrl); + if ((ctrl & enable_conf) != enable_conf) clk_disable(pc->clk); return 0; diff --git a/drivers/pwm/pwm-sifive.c b/drivers/pwm/pwm-sifive.c index 62de0bb85921..2485fbaaead2 100644 --- a/drivers/pwm/pwm-sifive.c +++ b/drivers/pwm/pwm-sifive.c @@ -254,11 +254,9 @@ static int pwm_sifive_probe(struct platform_device *pdev) return PTR_ERR(ddata->regs); ddata->clk = devm_clk_get(dev, NULL); - if (IS_ERR(ddata->clk)) { - if (PTR_ERR(ddata->clk) != -EPROBE_DEFER) - dev_err(dev, "Unable to find controller clock\n"); - return PTR_ERR(ddata->clk); - } + if (IS_ERR(ddata->clk)) + return dev_err_probe(dev, PTR_ERR(ddata->clk), + "Unable to find controller clock\n"); ret = clk_prepare_enable(ddata->clk); if (ret) { diff --git a/drivers/pwm/pwm-sprd.c b/drivers/pwm/pwm-sprd.c index be2394227423..5123d948efd6 100644 --- a/drivers/pwm/pwm-sprd.c +++ b/drivers/pwm/pwm-sprd.c @@ -228,11 +228,8 @@ static int sprd_pwm_clk_init(struct sprd_pwm_chip *spc) if (ret == -ENOENT) break; - if (ret != -EPROBE_DEFER) - dev_err(spc->dev, - "failed to get channel clocks\n"); - - return ret; + return dev_err_probe(spc->dev, ret, + "failed to get channel clocks\n"); } clk_pwm = chn->clks[SPRD_PWM_CHN_OUTPUT_CLK].clk; diff --git a/drivers/pwm/pwm-sun4i.c b/drivers/pwm/pwm-sun4i.c index 961c59c99bb3..38a4c5c1317b 100644 --- a/drivers/pwm/pwm-sun4i.c +++ b/drivers/pwm/pwm-sun4i.c @@ -423,38 +423,26 @@ static int sun4i_pwm_probe(struct platform_device *pdev) * back to the first clock of the PWM. */ pwm->clk = devm_clk_get_optional(&pdev->dev, "mod"); - if (IS_ERR(pwm->clk)) { - if (PTR_ERR(pwm->clk) != -EPROBE_DEFER) - dev_err(&pdev->dev, "get mod clock failed %pe\n", - pwm->clk); - return PTR_ERR(pwm->clk); - } + if (IS_ERR(pwm->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(pwm->clk), + "get mod clock failed\n"); if (!pwm->clk) { pwm->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(pwm->clk)) { - if (PTR_ERR(pwm->clk) != -EPROBE_DEFER) - dev_err(&pdev->dev, "get unnamed clock failed %pe\n", - pwm->clk); - return PTR_ERR(pwm->clk); - } + if (IS_ERR(pwm->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(pwm->clk), + "get unnamed clock failed\n"); } pwm->bus_clk = devm_clk_get_optional(&pdev->dev, "bus"); - if (IS_ERR(pwm->bus_clk)) { - if (PTR_ERR(pwm->bus_clk) != -EPROBE_DEFER) - dev_err(&pdev->dev, "get bus clock failed %pe\n", - pwm->bus_clk); - return PTR_ERR(pwm->bus_clk); - } + if (IS_ERR(pwm->bus_clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(pwm->bus_clk), + "get bus clock failed\n"); pwm->rst = devm_reset_control_get_optional_shared(&pdev->dev, NULL); - if (IS_ERR(pwm->rst)) { - if (PTR_ERR(pwm->rst) != -EPROBE_DEFER) - dev_err(&pdev->dev, "get reset failed %pe\n", - pwm->rst); - return PTR_ERR(pwm->rst); - } + if (IS_ERR(pwm->rst)) + return dev_err_probe(&pdev->dev, PTR_ERR(pwm->rst), + "get reset failed\n"); /* Deassert reset */ ret = reset_control_deassert(pwm->rst); diff --git a/drivers/pwm/sysfs.c b/drivers/pwm/sysfs.c index 449dbc0f49ed..9903c3a7eced 100644 --- a/drivers/pwm/sysfs.c +++ b/drivers/pwm/sysfs.c @@ -87,10 +87,10 @@ static ssize_t duty_cycle_store(struct device *child, struct pwm_export *export = child_to_pwm_export(child); struct pwm_device *pwm = export->pwm; struct pwm_state state; - unsigned int val; + u64 val; int ret; - ret = kstrtouint(buf, 0, &val); + ret = kstrtou64(buf, 0, &val); if (ret) return ret; diff --git a/drivers/rapidio/devices/rio_mport_cdev.c b/drivers/rapidio/devices/rio_mport_cdev.c index a30342942e26..94331d999d27 100644 --- a/drivers/rapidio/devices/rio_mport_cdev.c +++ b/drivers/rapidio/devices/rio_mport_cdev.c @@ -871,15 +871,16 @@ rio_dma_transfer(struct file *filp, u32 transfer_mode, rmcd_error("pin_user_pages_fast err=%ld", pinned); nr_pages = 0; - } else + } else { rmcd_error("pinned %ld out of %ld pages", pinned, nr_pages); + /* + * Set nr_pages up to mean "how many pages to unpin, in + * the error handler: + */ + nr_pages = pinned; + } ret = -EFAULT; - /* - * Set nr_pages up to mean "how many pages to unpin, in - * the error handler: - */ - nr_pages = pinned; goto err_pg; } @@ -1679,6 +1680,7 @@ static int rio_mport_add_riodev(struct mport_cdev_priv *priv, struct rio_dev *rdev; struct rio_switch *rswitch = NULL; struct rio_mport *mport; + struct device *dev; size_t size; u32 rval; u32 swpinfo = 0; @@ -1693,8 +1695,10 @@ static int rio_mport_add_riodev(struct mport_cdev_priv *priv, rmcd_debug(RDEV, "name:%s ct:0x%x did:0x%x hc:0x%x", dev_info.name, dev_info.comptag, dev_info.destid, dev_info.hopcount); - if (bus_find_device_by_name(&rio_bus_type, NULL, dev_info.name)) { + dev = bus_find_device_by_name(&rio_bus_type, NULL, dev_info.name); + if (dev) { rmcd_debug(RDEV, "device %s already exists", dev_info.name); + put_device(dev); return -EEXIST; } diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig index d1fcada71017..d99548fb5dde 100644 --- a/drivers/remoteproc/Kconfig +++ b/drivers/remoteproc/Kconfig @@ -275,6 +275,19 @@ config TI_K3_DSP_REMOTEPROC It's safe to say N here if you're not interested in utilizing the DSP slave processors. +config TI_K3_R5_REMOTEPROC + tristate "TI K3 R5 remoteproc support" + depends on ARCH_K3 + select MAILBOX + select OMAP2PLUS_MBOX + help + Say m here to support TI's R5F remote processor subsystems + on various TI K3 family of SoCs through the remote processor + framework. + + It's safe to say N here if you're not interested in utilizing + a slave processor. + endif # REMOTEPROC endmenu diff --git a/drivers/remoteproc/Makefile b/drivers/remoteproc/Makefile index 3dfa28e6c701..da2ace4ec86c 100644 --- a/drivers/remoteproc/Makefile +++ b/drivers/remoteproc/Makefile @@ -33,3 +33,4 @@ obj-$(CONFIG_ST_REMOTEPROC) += st_remoteproc.o obj-$(CONFIG_ST_SLIM_REMOTEPROC) += st_slim_rproc.o obj-$(CONFIG_STM32_RPROC) += stm32_rproc.o obj-$(CONFIG_TI_K3_DSP_REMOTEPROC) += ti_k3_dsp_remoteproc.o +obj-$(CONFIG_TI_K3_R5_REMOTEPROC) += ti_k3_r5_remoteproc.o diff --git a/drivers/remoteproc/mtk_common.h b/drivers/remoteproc/mtk_common.h index 0066c83636d0..47b4561443a9 100644 --- a/drivers/remoteproc/mtk_common.h +++ b/drivers/remoteproc/mtk_common.h @@ -32,6 +32,23 @@ #define MT8183_SCP_CACHESIZE_8KB BIT(8) #define MT8183_SCP_CACHE_CON_WAYEN BIT(10) +#define MT8192_L2TCM_SRAM_PD_0 0x210C0 +#define MT8192_L2TCM_SRAM_PD_1 0x210C4 +#define MT8192_L2TCM_SRAM_PD_2 0x210C8 +#define MT8192_L1TCM_SRAM_PDN 0x2102C +#define MT8192_CPU0_SRAM_PD 0x21080 + +#define MT8192_SCP2APMCU_IPC_SET 0x24080 +#define MT8192_SCP2APMCU_IPC_CLR 0x24084 +#define MT8192_SCP_IPC_INT_BIT BIT(0) +#define MT8192_SCP2SPM_IPC_CLR 0x24094 +#define MT8192_GIPC_IN_SET 0x24098 +#define MT8192_HOST_IPC_INT_BIT BIT(0) + +#define MT8192_CORE0_SW_RSTN_CLR 0x30000 +#define MT8192_CORE0_SW_RSTN_SET 0x30004 +#define MT8192_CORE0_WDT_CFG 0x30034 + #define SCP_FW_VER_LEN 32 #define SCP_SHARE_BUFFER_SIZE 288 @@ -50,6 +67,19 @@ struct scp_ipi_desc { void *priv; }; +struct mtk_scp; + +struct mtk_scp_of_data { + int (*scp_before_load)(struct mtk_scp *scp); + void (*scp_irq_handler)(struct mtk_scp *scp); + void (*scp_reset_assert)(struct mtk_scp *scp); + void (*scp_reset_deassert)(struct mtk_scp *scp); + void (*scp_stop)(struct mtk_scp *scp); + + u32 host_to_scp_reg; + u32 host_to_scp_int_bit; +}; + struct mtk_scp { struct device *dev; struct rproc *rproc; @@ -58,6 +88,8 @@ struct mtk_scp { void __iomem *sram_base; size_t sram_size; + const struct mtk_scp_of_data *data; + struct mtk_share_obj __iomem *recv_buf; struct mtk_share_obj __iomem *send_buf; struct scp_run run; diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c index ac13e7b046a6..577cbd5d421e 100644 --- a/drivers/remoteproc/mtk_scp.c +++ b/drivers/remoteproc/mtk_scp.c @@ -124,9 +124,6 @@ static int scp_ipi_init(struct mtk_scp *scp) size_t send_offset = SCP_FW_END - sizeof(struct mtk_share_obj); size_t recv_offset = send_offset - sizeof(struct mtk_share_obj); - /* Disable SCP to host interrupt */ - writel(MT8183_SCP_IPC_INT_BIT, scp->reg_base + MT8183_SCP_TO_HOST); - /* shared buffer initialization */ scp->recv_buf = (struct mtk_share_obj __iomem *)(scp->sram_base + recv_offset); @@ -138,7 +135,7 @@ static int scp_ipi_init(struct mtk_scp *scp) return 0; } -static void scp_reset_assert(const struct mtk_scp *scp) +static void mt8183_scp_reset_assert(struct mtk_scp *scp) { u32 val; @@ -147,7 +144,7 @@ static void scp_reset_assert(const struct mtk_scp *scp) writel(val, scp->reg_base + MT8183_SW_RSTN); } -static void scp_reset_deassert(const struct mtk_scp *scp) +static void mt8183_scp_reset_deassert(struct mtk_scp *scp) { u32 val; @@ -156,17 +153,19 @@ static void scp_reset_deassert(const struct mtk_scp *scp) writel(val, scp->reg_base + MT8183_SW_RSTN); } -static irqreturn_t scp_irq_handler(int irq, void *priv) +static void mt8192_scp_reset_assert(struct mtk_scp *scp) { - struct mtk_scp *scp = priv; - u32 scp_to_host; - int ret; + writel(1, scp->reg_base + MT8192_CORE0_SW_RSTN_SET); +} - ret = clk_prepare_enable(scp->clk); - if (ret) { - dev_err(scp->dev, "failed to enable clocks\n"); - return IRQ_NONE; - } +static void mt8192_scp_reset_deassert(struct mtk_scp *scp) +{ + writel(1, scp->reg_base + MT8192_CORE0_SW_RSTN_CLR); +} + +static void mt8183_scp_irq_handler(struct mtk_scp *scp) +{ + u32 scp_to_host; scp_to_host = readl(scp->reg_base + MT8183_SCP_TO_HOST); if (scp_to_host & MT8183_SCP_IPC_INT_BIT) @@ -177,6 +176,40 @@ static irqreturn_t scp_irq_handler(int irq, void *priv) /* SCP won't send another interrupt until we set SCP_TO_HOST to 0. */ writel(MT8183_SCP_IPC_INT_BIT | MT8183_SCP_WDT_INT_BIT, scp->reg_base + MT8183_SCP_TO_HOST); +} + +static void mt8192_scp_irq_handler(struct mtk_scp *scp) +{ + u32 scp_to_host; + + scp_to_host = readl(scp->reg_base + MT8192_SCP2APMCU_IPC_SET); + + if (scp_to_host & MT8192_SCP_IPC_INT_BIT) + scp_ipi_handler(scp); + else + scp_wdt_handler(scp, scp_to_host); + + /* + * SCP won't send another interrupt until we clear + * MT8192_SCP2APMCU_IPC. + */ + writel(MT8192_SCP_IPC_INT_BIT, + scp->reg_base + MT8192_SCP2APMCU_IPC_CLR); +} + +static irqreturn_t scp_irq_handler(int irq, void *priv) +{ + struct mtk_scp *scp = priv; + int ret; + + ret = clk_prepare_enable(scp->clk); + if (ret) { + dev_err(scp->dev, "failed to enable clocks\n"); + return IRQ_NONE; + } + + scp->data->scp_irq_handler(scp); + clk_disable_unprepare(scp->clk); return IRQ_HANDLED; @@ -238,20 +271,10 @@ static int scp_elf_load_segments(struct rproc *rproc, const struct firmware *fw) return ret; } -static int scp_load(struct rproc *rproc, const struct firmware *fw) +static int mt8183_scp_before_load(struct mtk_scp *scp) { - const struct mtk_scp *scp = rproc->priv; - struct device *dev = scp->dev; - int ret; - - ret = clk_prepare_enable(scp->clk); - if (ret) { - dev_err(dev, "failed to enable clocks\n"); - return ret; - } - - /* Hold SCP in reset while loading FW. */ - scp_reset_assert(scp); + /* Clear SCP to host interrupt */ + writel(MT8183_SCP_IPC_INT_BIT, scp->reg_base + MT8183_SCP_TO_HOST); /* Reset clocks before loading FW */ writel(0x0, scp->reg_base + MT8183_SCP_CLK_SW_SEL); @@ -272,6 +295,63 @@ static int scp_load(struct rproc *rproc, const struct firmware *fw) scp->reg_base + MT8183_SCP_CACHE_CON); writel(MT8183_SCP_CACHESIZE_8KB, scp->reg_base + MT8183_SCP_DCACHE_CON); + return 0; +} + +static void mt8192_power_on_sram(void *addr) +{ + int i; + + for (i = 31; i >= 0; i--) + writel(GENMASK(i, 0), addr); + writel(0, addr); +} + +static void mt8192_power_off_sram(void *addr) +{ + int i; + + writel(0, addr); + for (i = 0; i < 32; i++) + writel(GENMASK(i, 0), addr); +} + +static int mt8192_scp_before_load(struct mtk_scp *scp) +{ + /* clear SPM interrupt, SCP2SPM_IPC_CLR */ + writel(0xff, scp->reg_base + MT8192_SCP2SPM_IPC_CLR); + + writel(1, scp->reg_base + MT8192_CORE0_SW_RSTN_SET); + + /* enable SRAM clock */ + mt8192_power_on_sram(scp->reg_base + MT8192_L2TCM_SRAM_PD_0); + mt8192_power_on_sram(scp->reg_base + MT8192_L2TCM_SRAM_PD_1); + mt8192_power_on_sram(scp->reg_base + MT8192_L2TCM_SRAM_PD_2); + mt8192_power_on_sram(scp->reg_base + MT8192_L1TCM_SRAM_PDN); + mt8192_power_on_sram(scp->reg_base + MT8192_CPU0_SRAM_PD); + + return 0; +} + +static int scp_load(struct rproc *rproc, const struct firmware *fw) +{ + struct mtk_scp *scp = rproc->priv; + struct device *dev = scp->dev; + int ret; + + ret = clk_prepare_enable(scp->clk); + if (ret) { + dev_err(dev, "failed to enable clocks\n"); + return ret; + } + + /* Hold SCP in reset while loading FW. */ + scp->data->scp_reset_assert(scp); + + ret = scp->data->scp_before_load(scp); + if (ret < 0) + return ret; + ret = scp_elf_load_segments(rproc, fw); clk_disable_unprepare(scp->clk); @@ -293,7 +373,7 @@ static int scp_start(struct rproc *rproc) run->signaled = false; - scp_reset_deassert(scp); + scp->data->scp_reset_deassert(scp); ret = wait_event_interruptible_timeout( run->wq, @@ -309,13 +389,14 @@ static int scp_start(struct rproc *rproc) dev_err(dev, "wait SCP interrupted by a signal!\n"); goto stop; } + clk_disable_unprepare(scp->clk); dev_info(dev, "SCP is ready. FW version %s\n", run->fw_ver); return 0; stop: - scp_reset_assert(scp); + scp->data->scp_reset_assert(scp); clk_disable_unprepare(scp->clk); return ret; } @@ -329,7 +410,7 @@ static void *scp_da_to_va(struct rproc *rproc, u64 da, size_t len) offset = da; if (offset >= 0 && (offset + len) < scp->sram_size) return (void __force *)scp->sram_base + offset; - } else { + } else if (scp->dram_size) { offset = da - scp->dma_addr; if (offset >= 0 && (offset + len) < scp->dram_size) return (void __force *)scp->cpu_addr + offset; @@ -338,6 +419,25 @@ static void *scp_da_to_va(struct rproc *rproc, u64 da, size_t len) return NULL; } +static void mt8183_scp_stop(struct mtk_scp *scp) +{ + /* Disable SCP watchdog */ + writel(0, scp->reg_base + MT8183_WDT_CFG); +} + +static void mt8192_scp_stop(struct mtk_scp *scp) +{ + /* Disable SRAM clock */ + mt8192_power_off_sram(scp->reg_base + MT8192_L2TCM_SRAM_PD_0); + mt8192_power_off_sram(scp->reg_base + MT8192_L2TCM_SRAM_PD_1); + mt8192_power_off_sram(scp->reg_base + MT8192_L2TCM_SRAM_PD_2); + mt8192_power_off_sram(scp->reg_base + MT8192_L1TCM_SRAM_PDN); + mt8192_power_off_sram(scp->reg_base + MT8192_CPU0_SRAM_PD); + + /* Disable SCP watchdog */ + writel(0, scp->reg_base + MT8192_CORE0_WDT_CFG); +} + static int scp_stop(struct rproc *rproc) { struct mtk_scp *scp = (struct mtk_scp *)rproc->priv; @@ -349,9 +449,8 @@ static int scp_stop(struct rproc *rproc) return ret; } - scp_reset_assert(scp); - /* Disable SCP watchdog */ - writel(0, scp->reg_base + MT8183_WDT_CFG); + scp->data->scp_reset_assert(scp); + scp->data->scp_stop(scp); clk_disable_unprepare(scp->clk); return 0; @@ -443,6 +542,13 @@ static int scp_map_memory_region(struct mtk_scp *scp) int ret; ret = of_reserved_mem_device_init(scp->dev); + + /* reserved memory is optional. */ + if (ret == -ENODEV) { + dev_info(scp->dev, "skipping reserved memory initialization."); + return 0; + } + if (ret) { dev_err(scp->dev, "failed to assign memory-region: %d\n", ret); return -ENOMEM; @@ -460,6 +566,9 @@ static int scp_map_memory_region(struct mtk_scp *scp) static void scp_unmap_memory_region(struct mtk_scp *scp) { + if (scp->dram_size == 0) + return; + dma_free_coherent(scp->dev, scp->dram_size, scp->cpu_addr, scp->dma_addr); of_reserved_mem_device_release(scp->dev); @@ -536,6 +645,7 @@ static int scp_probe(struct platform_device *pdev) scp = (struct mtk_scp *)rproc->priv; scp->rproc = rproc; scp->dev = dev; + scp->data = of_device_get_match_data(dev); platform_set_drvdata(pdev, scp); res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sram"); @@ -642,8 +752,29 @@ static int scp_remove(struct platform_device *pdev) return 0; } +static const struct mtk_scp_of_data mt8183_of_data = { + .scp_before_load = mt8183_scp_before_load, + .scp_irq_handler = mt8183_scp_irq_handler, + .scp_reset_assert = mt8183_scp_reset_assert, + .scp_reset_deassert = mt8183_scp_reset_deassert, + .scp_stop = mt8183_scp_stop, + .host_to_scp_reg = MT8183_HOST_TO_SCP, + .host_to_scp_int_bit = MT8183_HOST_IPC_INT_BIT, +}; + +static const struct mtk_scp_of_data mt8192_of_data = { + .scp_before_load = mt8192_scp_before_load, + .scp_irq_handler = mt8192_scp_irq_handler, + .scp_reset_assert = mt8192_scp_reset_assert, + .scp_reset_deassert = mt8192_scp_reset_deassert, + .scp_stop = mt8192_scp_stop, + .host_to_scp_reg = MT8192_GIPC_IN_SET, + .host_to_scp_int_bit = MT8192_HOST_IPC_INT_BIT, +}; + static const struct of_device_id mtk_scp_of_match[] = { - { .compatible = "mediatek,mt8183-scp"}, + { .compatible = "mediatek,mt8183-scp", .data = &mt8183_of_data }, + { .compatible = "mediatek,mt8192-scp", .data = &mt8192_of_data }, {}, }; MODULE_DEVICE_TABLE(of, mtk_scp_of_match); diff --git a/drivers/remoteproc/mtk_scp_ipi.c b/drivers/remoteproc/mtk_scp_ipi.c index 3d3d87210ef2..6dc955ecab80 100644 --- a/drivers/remoteproc/mtk_scp_ipi.c +++ b/drivers/remoteproc/mtk_scp_ipi.c @@ -30,10 +30,8 @@ int scp_ipi_register(struct mtk_scp *scp, scp_ipi_handler_t handler, void *priv) { - if (!scp) { - dev_err(scp->dev, "scp device is not ready\n"); + if (!scp) return -EPROBE_DEFER; - } if (WARN_ON(id >= SCP_IPI_MAX) || WARN_ON(handler == NULL)) return -EINVAL; @@ -182,7 +180,7 @@ int scp_ipi_send(struct mtk_scp *scp, u32 id, void *buf, unsigned int len, ret = -ETIMEDOUT; goto clock_disable; } - } while (readl(scp->reg_base + MT8183_HOST_TO_SCP)); + } while (readl(scp->reg_base + scp->data->host_to_scp_reg)); scp_memcpy_aligned(send_obj->share_buf, buf, len); @@ -191,7 +189,8 @@ int scp_ipi_send(struct mtk_scp *scp, u32 id, void *buf, unsigned int len, scp->ipi_id_ack[id] = false; /* send the command to SCP */ - writel(MT8183_HOST_IPC_INT_BIT, scp->reg_base + MT8183_HOST_TO_SCP); + writel(scp->data->host_to_scp_int_bit, + scp->reg_base + scp->data->host_to_scp_reg); if (wait) { /* wait for SCP's ACK */ diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c index c401bcc263fa..eb3457a6c3b7 100644 --- a/drivers/remoteproc/qcom_q6v5_mss.c +++ b/drivers/remoteproc/qcom_q6v5_mss.c @@ -931,6 +931,17 @@ static int q6v5_mba_load(struct q6v5 *qproc) goto assert_reset; } + /* + * Some versions of the MBA firmware will upon boot wipe the MPSS region as well, so provide + * the Q6 access to this region. + */ + ret = q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm, false, true, + qproc->mpss_phys, qproc->mpss_size); + if (ret) { + dev_err(qproc->dev, "assigning Q6 access to mpss memory failed: %d\n", ret); + goto disable_active_clks; + } + /* Assign MBA image access in DDR to q6 */ ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, false, true, qproc->mba_phys, qproc->mba_size); @@ -1135,10 +1146,9 @@ static int q6v5_mpss_load(struct q6v5 *qproc) max_addr = ALIGN(phdr->p_paddr + phdr->p_memsz, SZ_4K); } - /** + /* * In case of a modem subsystem restart on secure devices, the modem - * memory can be reclaimed only after MBA is loaded. For modem cold - * boot this will be a nop + * memory can be reclaimed only after MBA is loaded. */ q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm, true, false, qproc->mpss_phys, qproc->mpss_size); diff --git a/drivers/remoteproc/remoteproc_coredump.c b/drivers/remoteproc/remoteproc_coredump.c index bb15a29038e8..34530dc20cb4 100644 --- a/drivers/remoteproc/remoteproc_coredump.c +++ b/drivers/remoteproc/remoteproc_coredump.c @@ -257,7 +257,7 @@ void rproc_coredump(struct rproc *rproc) * directly read from device memory. */ data_size += elf_size_of_phdr(class); - if (dump_conf == RPROC_COREDUMP_DEFAULT) + if (dump_conf == RPROC_COREDUMP_ENABLED) data_size += segment->size; phnum++; @@ -297,14 +297,14 @@ void rproc_coredump(struct rproc *rproc) elf_phdr_set_p_flags(class, phdr, PF_R | PF_W | PF_X); elf_phdr_set_p_align(class, phdr, 0); - if (dump_conf == RPROC_COREDUMP_DEFAULT) + if (dump_conf == RPROC_COREDUMP_ENABLED) rproc_copy_segment(rproc, data + offset, segment, 0, segment->size); offset += elf_phdr_get_p_filesz(class, phdr); phdr += elf_size_of_phdr(class); } - if (dump_conf == RPROC_COREDUMP_DEFAULT) { + if (dump_conf == RPROC_COREDUMP_ENABLED) { dev_coredumpv(&rproc->dev, data, data_size, GFP_KERNEL); return; } diff --git a/drivers/remoteproc/remoteproc_debugfs.c b/drivers/remoteproc/remoteproc_debugfs.c index 2e3b3e22e1d0..7e5845376e9f 100644 --- a/drivers/remoteproc/remoteproc_debugfs.c +++ b/drivers/remoteproc/remoteproc_debugfs.c @@ -33,9 +33,9 @@ static struct dentry *rproc_dbg; * enum rproc_coredump_mechanism */ static const char * const rproc_coredump_str[] = { - [RPROC_COREDUMP_DEFAULT] = "default", - [RPROC_COREDUMP_INLINE] = "inline", [RPROC_COREDUMP_DISABLED] = "disabled", + [RPROC_COREDUMP_ENABLED] = "enabled", + [RPROC_COREDUMP_INLINE] = "inline", }; /* Expose the current coredump configuration via debugfs */ @@ -54,20 +54,19 @@ static ssize_t rproc_coredump_read(struct file *filp, char __user *userbuf, /* * By writing to the 'coredump' debugfs entry, we control the behavior of the - * coredump mechanism dynamically. The default value of this entry is "default". + * coredump mechanism dynamically. The default value of this entry is "disabled". * * The 'coredump' debugfs entry supports these commands: * - * default: This is the default coredump mechanism. When the remoteproc - * crashes the entire coredump will be copied to a separate buffer - * and exposed to userspace. + * disabled: By default coredump collection is disabled. Recovery will + * proceed without collecting any dump. + * + * enabled: When the remoteproc crashes the entire coredump will be copied + * to a separate buffer and exposed to userspace. * * inline: The coredump will not be copied to a separate buffer and the * recovery process will have to wait until data is read by * userspace. But this avoid usage of extra memory. - * - * disabled: This will disable coredump. Recovery will proceed without - * collecting any dump. */ static ssize_t rproc_coredump_write(struct file *filp, const char __user *user_buf, size_t count, @@ -94,12 +93,12 @@ static ssize_t rproc_coredump_write(struct file *filp, goto out; } - if (!strncmp(buf, "disable", count)) { + if (!strncmp(buf, "disabled", count)) { rproc->dump_conf = RPROC_COREDUMP_DISABLED; + } else if (!strncmp(buf, "enabled", count)) { + rproc->dump_conf = RPROC_COREDUMP_ENABLED; } else if (!strncmp(buf, "inline", count)) { rproc->dump_conf = RPROC_COREDUMP_INLINE; - } else if (!strncmp(buf, "default", count)) { - rproc->dump_conf = RPROC_COREDUMP_DEFAULT; } else { dev_err(&rproc->dev, "Invalid coredump configuration\n"); err = -EINVAL; diff --git a/drivers/remoteproc/remoteproc_sysfs.c b/drivers/remoteproc/remoteproc_sysfs.c index eea514cec50e..d1cf7bf277c4 100644 --- a/drivers/remoteproc/remoteproc_sysfs.c +++ b/drivers/remoteproc/remoteproc_sysfs.c @@ -10,6 +10,123 @@ #define to_rproc(d) container_of(d, struct rproc, dev) +static ssize_t recovery_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct rproc *rproc = to_rproc(dev); + + return sprintf(buf, "%s", rproc->recovery_disabled ? "disabled\n" : "enabled\n"); +} + +/* + * By writing to the 'recovery' sysfs entry, we control the behavior of the + * recovery mechanism dynamically. The default value of this entry is "enabled". + * + * The 'recovery' sysfs entry supports these commands: + * + * enabled: When enabled, the remote processor will be automatically + * recovered whenever it crashes. Moreover, if the remote + * processor crashes while recovery is disabled, it will + * be automatically recovered too as soon as recovery is enabled. + * + * disabled: When disabled, a remote processor will remain in a crashed + * state if it crashes. This is useful for debugging purposes; + * without it, debugging a crash is substantially harder. + * + * recover: This function will trigger an immediate recovery if the + * remote processor is in a crashed state, without changing + * or checking the recovery state (enabled/disabled). + * This is useful during debugging sessions, when one expects + * additional crashes to happen after enabling recovery. In this + * case, enabling recovery will make it hard to debug subsequent + * crashes, so it's recommended to keep recovery disabled, and + * instead use the "recover" command as needed. + */ +static ssize_t recovery_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct rproc *rproc = to_rproc(dev); + + if (sysfs_streq(buf, "enabled")) { + /* change the flag and begin the recovery process if needed */ + rproc->recovery_disabled = false; + rproc_trigger_recovery(rproc); + } else if (sysfs_streq(buf, "disabled")) { + rproc->recovery_disabled = true; + } else if (sysfs_streq(buf, "recover")) { + /* begin the recovery process without changing the flag */ + rproc_trigger_recovery(rproc); + } else { + return -EINVAL; + } + + return count; +} +static DEVICE_ATTR_RW(recovery); + +/* + * A coredump-configuration-to-string lookup table, for exposing a + * human readable configuration via sysfs. Always keep in sync with + * enum rproc_coredump_mechanism + */ +static const char * const rproc_coredump_str[] = { + [RPROC_COREDUMP_DISABLED] = "disabled", + [RPROC_COREDUMP_ENABLED] = "enabled", + [RPROC_COREDUMP_INLINE] = "inline", +}; + +/* Expose the current coredump configuration via debugfs */ +static ssize_t coredump_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct rproc *rproc = to_rproc(dev); + + return sprintf(buf, "%s\n", rproc_coredump_str[rproc->dump_conf]); +} + +/* + * By writing to the 'coredump' sysfs entry, we control the behavior of the + * coredump mechanism dynamically. The default value of this entry is "default". + * + * The 'coredump' sysfs entry supports these commands: + * + * disabled: This is the default coredump mechanism. Recovery will proceed + * without collecting any dump. + * + * default: When the remoteproc crashes the entire coredump will be + * copied to a separate buffer and exposed to userspace. + * + * inline: The coredump will not be copied to a separate buffer and the + * recovery process will have to wait until data is read by + * userspace. But this avoid usage of extra memory. + */ +static ssize_t coredump_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct rproc *rproc = to_rproc(dev); + + if (rproc->state == RPROC_CRASHED) { + dev_err(&rproc->dev, "can't change coredump configuration\n"); + return -EBUSY; + } + + if (sysfs_streq(buf, "disabled")) { + rproc->dump_conf = RPROC_COREDUMP_DISABLED; + } else if (sysfs_streq(buf, "enabled")) { + rproc->dump_conf = RPROC_COREDUMP_ENABLED; + } else if (sysfs_streq(buf, "inline")) { + rproc->dump_conf = RPROC_COREDUMP_INLINE; + } else { + dev_err(&rproc->dev, "Invalid coredump configuration\n"); + return -EINVAL; + } + + return count; +} +static DEVICE_ATTR_RW(coredump); + /* Expose the loaded / running firmware name via sysfs */ static ssize_t firmware_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -138,6 +255,8 @@ static ssize_t name_show(struct device *dev, struct device_attribute *attr, static DEVICE_ATTR_RO(name); static struct attribute *rproc_attrs[] = { + &dev_attr_coredump.attr, + &dev_attr_recovery.attr, &dev_attr_firmware.attr, &dev_attr_state.attr, &dev_attr_name.attr, diff --git a/drivers/remoteproc/stm32_rproc.c b/drivers/remoteproc/stm32_rproc.c index f4da42fc0eeb..d2414cc1d90d 100644 --- a/drivers/remoteproc/stm32_rproc.c +++ b/drivers/remoteproc/stm32_rproc.c @@ -685,7 +685,7 @@ static int stm32_rproc_get_m4_status(struct stm32_rproc *ddata, * We couldn't get the coprocessor's state, assume * it is not running. */ - state = M4_STATE_OFF; + *state = M4_STATE_OFF; return 0; } diff --git a/drivers/remoteproc/ti_k3_r5_remoteproc.c b/drivers/remoteproc/ti_k3_r5_remoteproc.c new file mode 100644 index 000000000000..d9307935441d --- /dev/null +++ b/drivers/remoteproc/ti_k3_r5_remoteproc.c @@ -0,0 +1,1395 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * TI K3 R5F (MCU) Remote Processor driver + * + * Copyright (C) 2017-2020 Texas Instruments Incorporated - https://www.ti.com/ + * Suman Anna <s-anna@ti.com> + */ + +#include <linux/dma-mapping.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/mailbox_client.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/of_reserved_mem.h> +#include <linux/omap-mailbox.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/remoteproc.h> +#include <linux/reset.h> +#include <linux/slab.h> + +#include "omap_remoteproc.h" +#include "remoteproc_internal.h" +#include "ti_sci_proc.h" + +/* This address can either be for ATCM or BTCM with the other at address 0x0 */ +#define K3_R5_TCM_DEV_ADDR 0x41010000 + +/* R5 TI-SCI Processor Configuration Flags */ +#define PROC_BOOT_CFG_FLAG_R5_DBG_EN 0x00000001 +#define PROC_BOOT_CFG_FLAG_R5_DBG_NIDEN 0x00000002 +#define PROC_BOOT_CFG_FLAG_R5_LOCKSTEP 0x00000100 +#define PROC_BOOT_CFG_FLAG_R5_TEINIT 0x00000200 +#define PROC_BOOT_CFG_FLAG_R5_NMFI_EN 0x00000400 +#define PROC_BOOT_CFG_FLAG_R5_TCM_RSTBASE 0x00000800 +#define PROC_BOOT_CFG_FLAG_R5_BTCM_EN 0x00001000 +#define PROC_BOOT_CFG_FLAG_R5_ATCM_EN 0x00002000 + +/* R5 TI-SCI Processor Control Flags */ +#define PROC_BOOT_CTRL_FLAG_R5_CORE_HALT 0x00000001 + +/* R5 TI-SCI Processor Status Flags */ +#define PROC_BOOT_STATUS_FLAG_R5_WFE 0x00000001 +#define PROC_BOOT_STATUS_FLAG_R5_WFI 0x00000002 +#define PROC_BOOT_STATUS_FLAG_R5_CLK_GATED 0x00000004 +#define PROC_BOOT_STATUS_FLAG_R5_LOCKSTEP_PERMITTED 0x00000100 + +/** + * struct k3_r5_mem - internal memory structure + * @cpu_addr: MPU virtual address of the memory region + * @bus_addr: Bus address used to access the memory region + * @dev_addr: Device address from remoteproc view + * @size: Size of the memory region + */ +struct k3_r5_mem { + void __iomem *cpu_addr; + phys_addr_t bus_addr; + u32 dev_addr; + size_t size; +}; + +enum cluster_mode { + CLUSTER_MODE_SPLIT = 0, + CLUSTER_MODE_LOCKSTEP, +}; + +/** + * struct k3_r5_cluster - K3 R5F Cluster structure + * @dev: cached device pointer + * @mode: Mode to configure the Cluster - Split or LockStep + * @cores: list of R5 cores within the cluster + */ +struct k3_r5_cluster { + struct device *dev; + enum cluster_mode mode; + struct list_head cores; +}; + +/** + * struct k3_r5_core - K3 R5 core structure + * @elem: linked list item + * @dev: cached device pointer + * @rproc: rproc handle representing this core + * @mem: internal memory regions data + * @sram: on-chip SRAM memory regions data + * @num_mems: number of internal memory regions + * @num_sram: number of on-chip SRAM memory regions + * @reset: reset control handle + * @tsp: TI-SCI processor control handle + * @ti_sci: TI-SCI handle + * @ti_sci_id: TI-SCI device identifier + * @atcm_enable: flag to control ATCM enablement + * @btcm_enable: flag to control BTCM enablement + * @loczrama: flag to dictate which TCM is at device address 0x0 + */ +struct k3_r5_core { + struct list_head elem; + struct device *dev; + struct rproc *rproc; + struct k3_r5_mem *mem; + struct k3_r5_mem *sram; + int num_mems; + int num_sram; + struct reset_control *reset; + struct ti_sci_proc *tsp; + const struct ti_sci_handle *ti_sci; + u32 ti_sci_id; + u32 atcm_enable; + u32 btcm_enable; + u32 loczrama; +}; + +/** + * struct k3_r5_rproc - K3 remote processor state + * @dev: cached device pointer + * @cluster: cached pointer to parent cluster structure + * @mbox: mailbox channel handle + * @client: mailbox client to request the mailbox channel + * @rproc: rproc handle + * @core: cached pointer to r5 core structure being used + * @rmem: reserved memory regions data + * @num_rmems: number of reserved memory regions + */ +struct k3_r5_rproc { + struct device *dev; + struct k3_r5_cluster *cluster; + struct mbox_chan *mbox; + struct mbox_client client; + struct rproc *rproc; + struct k3_r5_core *core; + struct k3_r5_mem *rmem; + int num_rmems; +}; + +/** + * k3_r5_rproc_mbox_callback() - inbound mailbox message handler + * @client: mailbox client pointer used for requesting the mailbox channel + * @data: mailbox payload + * + * This handler is invoked by the OMAP mailbox driver whenever a mailbox + * message is received. Usually, the mailbox payload simply contains + * the index of the virtqueue that is kicked by the remote processor, + * and we let remoteproc core handle it. + * + * In addition to virtqueue indices, we also have some out-of-band values + * that indicate different events. Those values are deliberately very + * large so they don't coincide with virtqueue indices. + */ +static void k3_r5_rproc_mbox_callback(struct mbox_client *client, void *data) +{ + struct k3_r5_rproc *kproc = container_of(client, struct k3_r5_rproc, + client); + struct device *dev = kproc->rproc->dev.parent; + const char *name = kproc->rproc->name; + u32 msg = omap_mbox_message(data); + + dev_dbg(dev, "mbox msg: 0x%x\n", msg); + + switch (msg) { + case RP_MBOX_CRASH: + /* + * remoteproc detected an exception, but error recovery is not + * supported. So, just log this for now + */ + dev_err(dev, "K3 R5F rproc %s crashed\n", name); + break; + case RP_MBOX_ECHO_REPLY: + dev_info(dev, "received echo reply from %s\n", name); + break; + default: + /* silently handle all other valid messages */ + if (msg >= RP_MBOX_READY && msg < RP_MBOX_END_MSG) + return; + if (msg > kproc->rproc->max_notifyid) { + dev_dbg(dev, "dropping unknown message 0x%x", msg); + return; + } + /* msg contains the index of the triggered vring */ + if (rproc_vq_interrupt(kproc->rproc, msg) == IRQ_NONE) + dev_dbg(dev, "no message was found in vqid %d\n", msg); + } +} + +/* kick a virtqueue */ +static void k3_r5_rproc_kick(struct rproc *rproc, int vqid) +{ + struct k3_r5_rproc *kproc = rproc->priv; + struct device *dev = rproc->dev.parent; + mbox_msg_t msg = (mbox_msg_t)vqid; + int ret; + + /* send the index of the triggered virtqueue in the mailbox payload */ + ret = mbox_send_message(kproc->mbox, (void *)msg); + if (ret < 0) + dev_err(dev, "failed to send mailbox message, status = %d\n", + ret); +} + +static int k3_r5_split_reset(struct k3_r5_core *core) +{ + int ret; + + ret = reset_control_assert(core->reset); + if (ret) { + dev_err(core->dev, "local-reset assert failed, ret = %d\n", + ret); + return ret; + } + + ret = core->ti_sci->ops.dev_ops.put_device(core->ti_sci, + core->ti_sci_id); + if (ret) { + dev_err(core->dev, "module-reset assert failed, ret = %d\n", + ret); + if (reset_control_deassert(core->reset)) + dev_warn(core->dev, "local-reset deassert back failed\n"); + } + + return ret; +} + +static int k3_r5_split_release(struct k3_r5_core *core) +{ + int ret; + + ret = core->ti_sci->ops.dev_ops.get_device(core->ti_sci, + core->ti_sci_id); + if (ret) { + dev_err(core->dev, "module-reset deassert failed, ret = %d\n", + ret); + return ret; + } + + ret = reset_control_deassert(core->reset); + if (ret) { + dev_err(core->dev, "local-reset deassert failed, ret = %d\n", + ret); + if (core->ti_sci->ops.dev_ops.put_device(core->ti_sci, + core->ti_sci_id)) + dev_warn(core->dev, "module-reset assert back failed\n"); + } + + return ret; +} + +static int k3_r5_lockstep_reset(struct k3_r5_cluster *cluster) +{ + struct k3_r5_core *core; + int ret; + + /* assert local reset on all applicable cores */ + list_for_each_entry(core, &cluster->cores, elem) { + ret = reset_control_assert(core->reset); + if (ret) { + dev_err(core->dev, "local-reset assert failed, ret = %d\n", + ret); + core = list_prev_entry(core, elem); + goto unroll_local_reset; + } + } + + /* disable PSC modules on all applicable cores */ + list_for_each_entry(core, &cluster->cores, elem) { + ret = core->ti_sci->ops.dev_ops.put_device(core->ti_sci, + core->ti_sci_id); + if (ret) { + dev_err(core->dev, "module-reset assert failed, ret = %d\n", + ret); + goto unroll_module_reset; + } + } + + return 0; + +unroll_module_reset: + list_for_each_entry_continue_reverse(core, &cluster->cores, elem) { + if (core->ti_sci->ops.dev_ops.put_device(core->ti_sci, + core->ti_sci_id)) + dev_warn(core->dev, "module-reset assert back failed\n"); + } + core = list_last_entry(&cluster->cores, struct k3_r5_core, elem); +unroll_local_reset: + list_for_each_entry_from_reverse(core, &cluster->cores, elem) { + if (reset_control_deassert(core->reset)) + dev_warn(core->dev, "local-reset deassert back failed\n"); + } + + return ret; +} + +static int k3_r5_lockstep_release(struct k3_r5_cluster *cluster) +{ + struct k3_r5_core *core; + int ret; + + /* enable PSC modules on all applicable cores */ + list_for_each_entry_reverse(core, &cluster->cores, elem) { + ret = core->ti_sci->ops.dev_ops.get_device(core->ti_sci, + core->ti_sci_id); + if (ret) { + dev_err(core->dev, "module-reset deassert failed, ret = %d\n", + ret); + core = list_next_entry(core, elem); + goto unroll_module_reset; + } + } + + /* deassert local reset on all applicable cores */ + list_for_each_entry_reverse(core, &cluster->cores, elem) { + ret = reset_control_deassert(core->reset); + if (ret) { + dev_err(core->dev, "module-reset deassert failed, ret = %d\n", + ret); + goto unroll_local_reset; + } + } + + return 0; + +unroll_local_reset: + list_for_each_entry_continue(core, &cluster->cores, elem) { + if (reset_control_assert(core->reset)) + dev_warn(core->dev, "local-reset assert back failed\n"); + } + core = list_first_entry(&cluster->cores, struct k3_r5_core, elem); +unroll_module_reset: + list_for_each_entry_from(core, &cluster->cores, elem) { + if (core->ti_sci->ops.dev_ops.put_device(core->ti_sci, + core->ti_sci_id)) + dev_warn(core->dev, "module-reset assert back failed\n"); + } + + return ret; +} + +static inline int k3_r5_core_halt(struct k3_r5_core *core) +{ + return ti_sci_proc_set_control(core->tsp, + PROC_BOOT_CTRL_FLAG_R5_CORE_HALT, 0); +} + +static inline int k3_r5_core_run(struct k3_r5_core *core) +{ + return ti_sci_proc_set_control(core->tsp, + 0, PROC_BOOT_CTRL_FLAG_R5_CORE_HALT); +} + +/* + * The R5F cores have controls for both a reset and a halt/run. The code + * execution from DDR requires the initial boot-strapping code to be run + * from the internal TCMs. This function is used to release the resets on + * applicable cores to allow loading into the TCMs. The .prepare() ops is + * invoked by remoteproc core before any firmware loading, and is followed + * by the .start() ops after loading to actually let the R5 cores run. + */ +static int k3_r5_rproc_prepare(struct rproc *rproc) +{ + struct k3_r5_rproc *kproc = rproc->priv; + struct k3_r5_cluster *cluster = kproc->cluster; + struct k3_r5_core *core = kproc->core; + struct device *dev = kproc->dev; + int ret; + + ret = (cluster->mode == CLUSTER_MODE_LOCKSTEP) ? + k3_r5_lockstep_release(cluster) : k3_r5_split_release(core); + if (ret) { + dev_err(dev, "unable to enable cores for TCM loading, ret = %d\n", + ret); + return ret; + } + + /* + * Zero out both TCMs unconditionally (access from v8 Arm core is not + * affected by ATCM & BTCM enable configuration values) so that ECC + * can be effective on all TCM addresses. + */ + dev_dbg(dev, "zeroing out ATCM memory\n"); + memset(core->mem[0].cpu_addr, 0x00, core->mem[0].size); + + dev_dbg(dev, "zeroing out BTCM memory\n"); + memset(core->mem[1].cpu_addr, 0x00, core->mem[1].size); + + return 0; +} + +/* + * This function implements the .unprepare() ops and performs the complimentary + * operations to that of the .prepare() ops. The function is used to assert the + * resets on all applicable cores for the rproc device (depending on LockStep + * or Split mode). This completes the second portion of powering down the R5F + * cores. The cores themselves are only halted in the .stop() ops, and the + * .unprepare() ops is invoked by the remoteproc core after the remoteproc is + * stopped. + */ +static int k3_r5_rproc_unprepare(struct rproc *rproc) +{ + struct k3_r5_rproc *kproc = rproc->priv; + struct k3_r5_cluster *cluster = kproc->cluster; + struct k3_r5_core *core = kproc->core; + struct device *dev = kproc->dev; + int ret; + + ret = (cluster->mode == CLUSTER_MODE_LOCKSTEP) ? + k3_r5_lockstep_reset(cluster) : k3_r5_split_reset(core); + if (ret) + dev_err(dev, "unable to disable cores, ret = %d\n", ret); + + return ret; +} + +/* + * The R5F start sequence includes two different operations + * 1. Configure the boot vector for R5F core(s) + * 2. Unhalt/Run the R5F core(s) + * + * The sequence is different between LockStep and Split modes. The LockStep + * mode requires the boot vector to be configured only for Core0, and then + * unhalt both the cores to start the execution - Core1 needs to be unhalted + * first followed by Core0. The Split-mode requires that Core0 to be maintained + * always in a higher power state that Core1 (implying Core1 needs to be started + * always only after Core0 is started). + */ +static int k3_r5_rproc_start(struct rproc *rproc) +{ + struct k3_r5_rproc *kproc = rproc->priv; + struct k3_r5_cluster *cluster = kproc->cluster; + struct mbox_client *client = &kproc->client; + struct device *dev = kproc->dev; + struct k3_r5_core *core; + u32 boot_addr; + int ret; + + client->dev = dev; + client->tx_done = NULL; + client->rx_callback = k3_r5_rproc_mbox_callback; + client->tx_block = false; + client->knows_txdone = false; + + kproc->mbox = mbox_request_channel(client, 0); + if (IS_ERR(kproc->mbox)) { + ret = -EBUSY; + dev_err(dev, "mbox_request_channel failed: %ld\n", + PTR_ERR(kproc->mbox)); + return ret; + } + + /* + * Ping the remote processor, this is only for sanity-sake for now; + * there is no functional effect whatsoever. + * + * Note that the reply will _not_ arrive immediately: this message + * will wait in the mailbox fifo until the remote processor is booted. + */ + ret = mbox_send_message(kproc->mbox, (void *)RP_MBOX_ECHO_REQUEST); + if (ret < 0) { + dev_err(dev, "mbox_send_message failed: %d\n", ret); + goto put_mbox; + } + + boot_addr = rproc->bootaddr; + /* TODO: add boot_addr sanity checking */ + dev_dbg(dev, "booting R5F core using boot addr = 0x%x\n", boot_addr); + + /* boot vector need not be programmed for Core1 in LockStep mode */ + core = kproc->core; + ret = ti_sci_proc_set_config(core->tsp, boot_addr, 0, 0); + if (ret) + goto put_mbox; + + /* unhalt/run all applicable cores */ + if (cluster->mode == CLUSTER_MODE_LOCKSTEP) { + list_for_each_entry_reverse(core, &cluster->cores, elem) { + ret = k3_r5_core_run(core); + if (ret) + goto unroll_core_run; + } + } else { + ret = k3_r5_core_run(core); + if (ret) + goto put_mbox; + } + + return 0; + +unroll_core_run: + list_for_each_entry_continue(core, &cluster->cores, elem) { + if (k3_r5_core_halt(core)) + dev_warn(core->dev, "core halt back failed\n"); + } +put_mbox: + mbox_free_channel(kproc->mbox); + return ret; +} + +/* + * The R5F stop function includes the following operations + * 1. Halt R5F core(s) + * + * The sequence is different between LockStep and Split modes, and the order + * of cores the operations are performed are also in general reverse to that + * of the start function. The LockStep mode requires each operation to be + * performed first on Core0 followed by Core1. The Split-mode requires that + * Core0 to be maintained always in a higher power state that Core1 (implying + * Core1 needs to be stopped first before Core0). + * + * Note that the R5F halt operation in general is not effective when the R5F + * core is running, but is needed to make sure the core won't run after + * deasserting the reset the subsequent time. The asserting of reset can + * be done here, but is preferred to be done in the .unprepare() ops - this + * maintains the symmetric behavior between the .start(), .stop(), .prepare() + * and .unprepare() ops, and also balances them well between sysfs 'state' + * flow and device bind/unbind or module removal. + */ +static int k3_r5_rproc_stop(struct rproc *rproc) +{ + struct k3_r5_rproc *kproc = rproc->priv; + struct k3_r5_cluster *cluster = kproc->cluster; + struct k3_r5_core *core = kproc->core; + int ret; + + /* halt all applicable cores */ + if (cluster->mode == CLUSTER_MODE_LOCKSTEP) { + list_for_each_entry(core, &cluster->cores, elem) { + ret = k3_r5_core_halt(core); + if (ret) { + core = list_prev_entry(core, elem); + goto unroll_core_halt; + } + } + } else { + ret = k3_r5_core_halt(core); + if (ret) + goto out; + } + + mbox_free_channel(kproc->mbox); + + return 0; + +unroll_core_halt: + list_for_each_entry_from_reverse(core, &cluster->cores, elem) { + if (k3_r5_core_run(core)) + dev_warn(core->dev, "core run back failed\n"); + } +out: + return ret; +} + +/* + * Internal Memory translation helper + * + * Custom function implementing the rproc .da_to_va ops to provide address + * translation (device address to kernel virtual address) for internal RAMs + * present in a DSP or IPU device). The translated addresses can be used + * either by the remoteproc core for loading, or by any rpmsg bus drivers. + */ +static void *k3_r5_rproc_da_to_va(struct rproc *rproc, u64 da, size_t len) +{ + struct k3_r5_rproc *kproc = rproc->priv; + struct k3_r5_core *core = kproc->core; + void __iomem *va = NULL; + phys_addr_t bus_addr; + u32 dev_addr, offset; + size_t size; + int i; + + if (len == 0) + return NULL; + + /* handle both R5 and SoC views of ATCM and BTCM */ + for (i = 0; i < core->num_mems; i++) { + bus_addr = core->mem[i].bus_addr; + dev_addr = core->mem[i].dev_addr; + size = core->mem[i].size; + + /* handle R5-view addresses of TCMs */ + if (da >= dev_addr && ((da + len) <= (dev_addr + size))) { + offset = da - dev_addr; + va = core->mem[i].cpu_addr + offset; + return (__force void *)va; + } + + /* handle SoC-view addresses of TCMs */ + if (da >= bus_addr && ((da + len) <= (bus_addr + size))) { + offset = da - bus_addr; + va = core->mem[i].cpu_addr + offset; + return (__force void *)va; + } + } + + /* handle any SRAM regions using SoC-view addresses */ + for (i = 0; i < core->num_sram; i++) { + dev_addr = core->sram[i].dev_addr; + size = core->sram[i].size; + + if (da >= dev_addr && ((da + len) <= (dev_addr + size))) { + offset = da - dev_addr; + va = core->sram[i].cpu_addr + offset; + return (__force void *)va; + } + } + + /* handle static DDR reserved memory regions */ + for (i = 0; i < kproc->num_rmems; i++) { + dev_addr = kproc->rmem[i].dev_addr; + size = kproc->rmem[i].size; + + if (da >= dev_addr && ((da + len) <= (dev_addr + size))) { + offset = da - dev_addr; + va = kproc->rmem[i].cpu_addr + offset; + return (__force void *)va; + } + } + + return NULL; +} + +static const struct rproc_ops k3_r5_rproc_ops = { + .prepare = k3_r5_rproc_prepare, + .unprepare = k3_r5_rproc_unprepare, + .start = k3_r5_rproc_start, + .stop = k3_r5_rproc_stop, + .kick = k3_r5_rproc_kick, + .da_to_va = k3_r5_rproc_da_to_va, +}; + +/* + * Internal R5F Core configuration + * + * Each R5FSS has a cluster-level setting for configuring the processor + * subsystem either in a safety/fault-tolerant LockStep mode or a performance + * oriented Split mode. Each R5F core has a number of settings to either + * enable/disable each of the TCMs, control which TCM appears at the R5F core's + * address 0x0. These settings need to be configured before the resets for the + * corresponding core are released. These settings are all protected and managed + * by the System Processor. + * + * This function is used to pre-configure these settings for each R5F core, and + * the configuration is all done through various ti_sci_proc functions that + * communicate with the System Processor. The function also ensures that both + * the cores are halted before the .prepare() step. + * + * The function is called from k3_r5_cluster_rproc_init() and is invoked either + * once (in LockStep mode) or twice (in Split mode). Support for LockStep-mode + * is dictated by an eFUSE register bit, and the config settings retrieved from + * DT are adjusted accordingly as per the permitted cluster mode. All cluster + * level settings like Cluster mode and TEINIT (exception handling state + * dictating ARM or Thumb mode) can only be set and retrieved using Core0. + * + * The function behavior is different based on the cluster mode. The R5F cores + * are configured independently as per their individual settings in Split mode. + * They are identically configured in LockStep mode using the primary Core0 + * settings. However, some individual settings cannot be set in LockStep mode. + * This is overcome by switching to Split-mode initially and then programming + * both the cores with the same settings, before reconfiguing again for + * LockStep mode. + */ +static int k3_r5_rproc_configure(struct k3_r5_rproc *kproc) +{ + struct k3_r5_cluster *cluster = kproc->cluster; + struct device *dev = kproc->dev; + struct k3_r5_core *core0, *core, *temp; + u32 ctrl = 0, cfg = 0, stat = 0; + u32 set_cfg = 0, clr_cfg = 0; + u64 boot_vec = 0; + bool lockstep_en; + int ret; + + core0 = list_first_entry(&cluster->cores, struct k3_r5_core, elem); + core = (cluster->mode == CLUSTER_MODE_LOCKSTEP) ? core0 : kproc->core; + + ret = ti_sci_proc_get_status(core->tsp, &boot_vec, &cfg, &ctrl, + &stat); + if (ret < 0) + return ret; + + dev_dbg(dev, "boot_vector = 0x%llx, cfg = 0x%x ctrl = 0x%x stat = 0x%x\n", + boot_vec, cfg, ctrl, stat); + + lockstep_en = !!(stat & PROC_BOOT_STATUS_FLAG_R5_LOCKSTEP_PERMITTED); + if (!lockstep_en && cluster->mode == CLUSTER_MODE_LOCKSTEP) { + dev_err(cluster->dev, "lockstep mode not permitted, force configuring for split-mode\n"); + cluster->mode = CLUSTER_MODE_SPLIT; + } + + /* always enable ARM mode and set boot vector to 0 */ + boot_vec = 0x0; + if (core == core0) { + clr_cfg = PROC_BOOT_CFG_FLAG_R5_TEINIT; + /* + * LockStep configuration bit is Read-only on Split-mode _only_ + * devices and system firmware will NACK any requests with the + * bit configured, so program it only on permitted devices + */ + if (lockstep_en) + clr_cfg |= PROC_BOOT_CFG_FLAG_R5_LOCKSTEP; + } + + if (core->atcm_enable) + set_cfg |= PROC_BOOT_CFG_FLAG_R5_ATCM_EN; + else + clr_cfg |= PROC_BOOT_CFG_FLAG_R5_ATCM_EN; + + if (core->btcm_enable) + set_cfg |= PROC_BOOT_CFG_FLAG_R5_BTCM_EN; + else + clr_cfg |= PROC_BOOT_CFG_FLAG_R5_BTCM_EN; + + if (core->loczrama) + set_cfg |= PROC_BOOT_CFG_FLAG_R5_TCM_RSTBASE; + else + clr_cfg |= PROC_BOOT_CFG_FLAG_R5_TCM_RSTBASE; + + if (cluster->mode == CLUSTER_MODE_LOCKSTEP) { + /* + * work around system firmware limitations to make sure both + * cores are programmed symmetrically in LockStep. LockStep + * and TEINIT config is only allowed with Core0. + */ + list_for_each_entry(temp, &cluster->cores, elem) { + ret = k3_r5_core_halt(temp); + if (ret) + goto out; + + if (temp != core) { + clr_cfg &= ~PROC_BOOT_CFG_FLAG_R5_LOCKSTEP; + clr_cfg &= ~PROC_BOOT_CFG_FLAG_R5_TEINIT; + } + ret = ti_sci_proc_set_config(temp->tsp, boot_vec, + set_cfg, clr_cfg); + if (ret) + goto out; + } + + set_cfg = PROC_BOOT_CFG_FLAG_R5_LOCKSTEP; + clr_cfg = 0; + ret = ti_sci_proc_set_config(core->tsp, boot_vec, + set_cfg, clr_cfg); + } else { + ret = k3_r5_core_halt(core); + if (ret) + goto out; + + ret = ti_sci_proc_set_config(core->tsp, boot_vec, + set_cfg, clr_cfg); + } + +out: + return ret; +} + +static int k3_r5_reserved_mem_init(struct k3_r5_rproc *kproc) +{ + struct device *dev = kproc->dev; + struct device_node *np = dev_of_node(dev); + struct device_node *rmem_np; + struct reserved_mem *rmem; + int num_rmems; + int ret, i; + + num_rmems = of_property_count_elems_of_size(np, "memory-region", + sizeof(phandle)); + if (num_rmems <= 0) { + dev_err(dev, "device does not have reserved memory regions, ret = %d\n", + num_rmems); + return -EINVAL; + } + if (num_rmems < 2) { + dev_err(dev, "device needs atleast two memory regions to be defined, num = %d\n", + num_rmems); + return -EINVAL; + } + + /* use reserved memory region 0 for vring DMA allocations */ + ret = of_reserved_mem_device_init_by_idx(dev, np, 0); + if (ret) { + dev_err(dev, "device cannot initialize DMA pool, ret = %d\n", + ret); + return ret; + } + + num_rmems--; + kproc->rmem = kcalloc(num_rmems, sizeof(*kproc->rmem), GFP_KERNEL); + if (!kproc->rmem) { + ret = -ENOMEM; + goto release_rmem; + } + + /* use remaining reserved memory regions for static carveouts */ + for (i = 0; i < num_rmems; i++) { + rmem_np = of_parse_phandle(np, "memory-region", i + 1); + if (!rmem_np) { + ret = -EINVAL; + goto unmap_rmem; + } + + rmem = of_reserved_mem_lookup(rmem_np); + if (!rmem) { + of_node_put(rmem_np); + ret = -EINVAL; + goto unmap_rmem; + } + of_node_put(rmem_np); + + kproc->rmem[i].bus_addr = rmem->base; + /* + * R5Fs do not have an MMU, but have a Region Address Translator + * (RAT) module that provides a fixed entry translation between + * the 32-bit processor addresses to 64-bit bus addresses. The + * RAT is programmable only by the R5F cores. Support for RAT + * is currently not supported, so 64-bit address regions are not + * supported. The absence of MMUs implies that the R5F device + * addresses/supported memory regions are restricted to 32-bit + * bus addresses, and are identical + */ + kproc->rmem[i].dev_addr = (u32)rmem->base; + kproc->rmem[i].size = rmem->size; + kproc->rmem[i].cpu_addr = ioremap_wc(rmem->base, rmem->size); + if (!kproc->rmem[i].cpu_addr) { + dev_err(dev, "failed to map reserved memory#%d at %pa of size %pa\n", + i + 1, &rmem->base, &rmem->size); + ret = -ENOMEM; + goto unmap_rmem; + } + + dev_dbg(dev, "reserved memory%d: bus addr %pa size 0x%zx va %pK da 0x%x\n", + i + 1, &kproc->rmem[i].bus_addr, + kproc->rmem[i].size, kproc->rmem[i].cpu_addr, + kproc->rmem[i].dev_addr); + } + kproc->num_rmems = num_rmems; + + return 0; + +unmap_rmem: + for (i--; i >= 0; i--) + iounmap(kproc->rmem[i].cpu_addr); + kfree(kproc->rmem); +release_rmem: + of_reserved_mem_device_release(dev); + return ret; +} + +static void k3_r5_reserved_mem_exit(struct k3_r5_rproc *kproc) +{ + int i; + + for (i = 0; i < kproc->num_rmems; i++) + iounmap(kproc->rmem[i].cpu_addr); + kfree(kproc->rmem); + + of_reserved_mem_device_release(kproc->dev); +} + +static int k3_r5_cluster_rproc_init(struct platform_device *pdev) +{ + struct k3_r5_cluster *cluster = platform_get_drvdata(pdev); + struct device *dev = &pdev->dev; + struct k3_r5_rproc *kproc; + struct k3_r5_core *core, *core1; + struct device *cdev; + const char *fw_name; + struct rproc *rproc; + int ret; + + core1 = list_last_entry(&cluster->cores, struct k3_r5_core, elem); + list_for_each_entry(core, &cluster->cores, elem) { + cdev = core->dev; + ret = rproc_of_parse_firmware(cdev, 0, &fw_name); + if (ret) { + dev_err(dev, "failed to parse firmware-name property, ret = %d\n", + ret); + goto out; + } + + rproc = rproc_alloc(cdev, dev_name(cdev), &k3_r5_rproc_ops, + fw_name, sizeof(*kproc)); + if (!rproc) { + ret = -ENOMEM; + goto out; + } + + /* K3 R5s have a Region Address Translator (RAT) but no MMU */ + rproc->has_iommu = false; + /* error recovery is not supported at present */ + rproc->recovery_disabled = true; + + kproc = rproc->priv; + kproc->cluster = cluster; + kproc->core = core; + kproc->dev = cdev; + kproc->rproc = rproc; + core->rproc = rproc; + + ret = k3_r5_rproc_configure(kproc); + if (ret) { + dev_err(dev, "initial configure failed, ret = %d\n", + ret); + goto err_config; + } + + ret = k3_r5_reserved_mem_init(kproc); + if (ret) { + dev_err(dev, "reserved memory init failed, ret = %d\n", + ret); + goto err_config; + } + + ret = rproc_add(rproc); + if (ret) { + dev_err(dev, "rproc_add failed, ret = %d\n", ret); + goto err_add; + } + + /* create only one rproc in lockstep mode */ + if (cluster->mode == CLUSTER_MODE_LOCKSTEP) + break; + } + + return 0; + +err_split: + rproc_del(rproc); +err_add: + k3_r5_reserved_mem_exit(kproc); +err_config: + rproc_free(rproc); + core->rproc = NULL; +out: + /* undo core0 upon any failures on core1 in split-mode */ + if (cluster->mode == CLUSTER_MODE_SPLIT && core == core1) { + core = list_prev_entry(core, elem); + rproc = core->rproc; + kproc = rproc->priv; + goto err_split; + } + return ret; +} + +static int k3_r5_cluster_rproc_exit(struct platform_device *pdev) +{ + struct k3_r5_cluster *cluster = platform_get_drvdata(pdev); + struct k3_r5_rproc *kproc; + struct k3_r5_core *core; + struct rproc *rproc; + + /* + * lockstep mode has only one rproc associated with first core, whereas + * split-mode has two rprocs associated with each core, and requires + * that core1 be powered down first + */ + core = (cluster->mode == CLUSTER_MODE_LOCKSTEP) ? + list_first_entry(&cluster->cores, struct k3_r5_core, elem) : + list_last_entry(&cluster->cores, struct k3_r5_core, elem); + + list_for_each_entry_from_reverse(core, &cluster->cores, elem) { + rproc = core->rproc; + kproc = rproc->priv; + + rproc_del(rproc); + + k3_r5_reserved_mem_exit(kproc); + + rproc_free(rproc); + core->rproc = NULL; + } + + return 0; +} + +static int k3_r5_core_of_get_internal_memories(struct platform_device *pdev, + struct k3_r5_core *core) +{ + static const char * const mem_names[] = {"atcm", "btcm"}; + struct device *dev = &pdev->dev; + struct resource *res; + int num_mems; + int i; + + num_mems = ARRAY_SIZE(mem_names); + core->mem = devm_kcalloc(dev, num_mems, sizeof(*core->mem), GFP_KERNEL); + if (!core->mem) + return -ENOMEM; + + for (i = 0; i < num_mems; i++) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + mem_names[i]); + if (!res) { + dev_err(dev, "found no memory resource for %s\n", + mem_names[i]); + return -EINVAL; + } + if (!devm_request_mem_region(dev, res->start, + resource_size(res), + dev_name(dev))) { + dev_err(dev, "could not request %s region for resource\n", + mem_names[i]); + return -EBUSY; + } + + /* + * TCMs are designed in general to support RAM-like backing + * memories. So, map these as Normal Non-Cached memories. This + * also avoids/fixes any potential alignment faults due to + * unaligned data accesses when using memcpy() or memset() + * functions (normally seen with device type memory). + */ + core->mem[i].cpu_addr = devm_ioremap_wc(dev, res->start, + resource_size(res)); + if (!core->mem[i].cpu_addr) { + dev_err(dev, "failed to map %s memory\n", mem_names[i]); + return -ENOMEM; + } + core->mem[i].bus_addr = res->start; + + /* + * TODO: + * The R5F cores can place ATCM & BTCM anywhere in its address + * based on the corresponding Region Registers in the System + * Control coprocessor. For now, place ATCM and BTCM at + * addresses 0 and 0x41010000 (same as the bus address on AM65x + * SoCs) based on loczrama setting + */ + if (!strcmp(mem_names[i], "atcm")) { + core->mem[i].dev_addr = core->loczrama ? + 0 : K3_R5_TCM_DEV_ADDR; + } else { + core->mem[i].dev_addr = core->loczrama ? + K3_R5_TCM_DEV_ADDR : 0; + } + core->mem[i].size = resource_size(res); + + dev_dbg(dev, "memory %5s: bus addr %pa size 0x%zx va %pK da 0x%x\n", + mem_names[i], &core->mem[i].bus_addr, + core->mem[i].size, core->mem[i].cpu_addr, + core->mem[i].dev_addr); + } + core->num_mems = num_mems; + + return 0; +} + +static int k3_r5_core_of_get_sram_memories(struct platform_device *pdev, + struct k3_r5_core *core) +{ + struct device_node *np = pdev->dev.of_node; + struct device *dev = &pdev->dev; + struct device_node *sram_np; + struct resource res; + int num_sram; + int i, ret; + + num_sram = of_property_count_elems_of_size(np, "sram", sizeof(phandle)); + if (num_sram <= 0) { + dev_dbg(dev, "device does not use reserved on-chip memories, num_sram = %d\n", + num_sram); + return 0; + } + + core->sram = devm_kcalloc(dev, num_sram, sizeof(*core->sram), GFP_KERNEL); + if (!core->sram) + return -ENOMEM; + + for (i = 0; i < num_sram; i++) { + sram_np = of_parse_phandle(np, "sram", i); + if (!sram_np) + return -EINVAL; + + if (!of_device_is_available(sram_np)) { + of_node_put(sram_np); + return -EINVAL; + } + + ret = of_address_to_resource(sram_np, 0, &res); + of_node_put(sram_np); + if (ret) + return -EINVAL; + + core->sram[i].bus_addr = res.start; + core->sram[i].dev_addr = res.start; + core->sram[i].size = resource_size(&res); + core->sram[i].cpu_addr = devm_ioremap_wc(dev, res.start, + resource_size(&res)); + if (!core->sram[i].cpu_addr) { + dev_err(dev, "failed to parse and map sram%d memory at %pad\n", + i, &res.start); + return -ENOMEM; + } + + dev_dbg(dev, "memory sram%d: bus addr %pa size 0x%zx va %pK da 0x%x\n", + i, &core->sram[i].bus_addr, + core->sram[i].size, core->sram[i].cpu_addr, + core->sram[i].dev_addr); + } + core->num_sram = num_sram; + + return 0; +} + +static +struct ti_sci_proc *k3_r5_core_of_get_tsp(struct device *dev, + const struct ti_sci_handle *sci) +{ + struct ti_sci_proc *tsp; + u32 temp[2]; + int ret; + + ret = of_property_read_u32_array(dev_of_node(dev), "ti,sci-proc-ids", + temp, 2); + if (ret < 0) + return ERR_PTR(ret); + + tsp = devm_kzalloc(dev, sizeof(*tsp), GFP_KERNEL); + if (!tsp) + return ERR_PTR(-ENOMEM); + + tsp->dev = dev; + tsp->sci = sci; + tsp->ops = &sci->ops.proc_ops; + tsp->proc_id = temp[0]; + tsp->host_id = temp[1]; + + return tsp; +} + +static int k3_r5_core_of_init(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev_of_node(dev); + struct k3_r5_core *core; + int ret; + + if (!devres_open_group(dev, k3_r5_core_of_init, GFP_KERNEL)) + return -ENOMEM; + + core = devm_kzalloc(dev, sizeof(*core), GFP_KERNEL); + if (!core) { + ret = -ENOMEM; + goto err; + } + + core->dev = dev; + /* + * Use SoC Power-on-Reset values as default if no DT properties are + * used to dictate the TCM configurations + */ + core->atcm_enable = 0; + core->btcm_enable = 1; + core->loczrama = 1; + + ret = of_property_read_u32(np, "ti,atcm-enable", &core->atcm_enable); + if (ret < 0 && ret != -EINVAL) { + dev_err(dev, "invalid format for ti,atcm-enable, ret = %d\n", + ret); + goto err; + } + + ret = of_property_read_u32(np, "ti,btcm-enable", &core->btcm_enable); + if (ret < 0 && ret != -EINVAL) { + dev_err(dev, "invalid format for ti,btcm-enable, ret = %d\n", + ret); + goto err; + } + + ret = of_property_read_u32(np, "ti,loczrama", &core->loczrama); + if (ret < 0 && ret != -EINVAL) { + dev_err(dev, "invalid format for ti,loczrama, ret = %d\n", ret); + goto err; + } + + core->ti_sci = devm_ti_sci_get_by_phandle(dev, "ti,sci"); + if (IS_ERR(core->ti_sci)) { + ret = PTR_ERR(core->ti_sci); + if (ret != -EPROBE_DEFER) { + dev_err(dev, "failed to get ti-sci handle, ret = %d\n", + ret); + } + core->ti_sci = NULL; + goto err; + } + + ret = of_property_read_u32(np, "ti,sci-dev-id", &core->ti_sci_id); + if (ret) { + dev_err(dev, "missing 'ti,sci-dev-id' property\n"); + goto err; + } + + core->reset = devm_reset_control_get_exclusive(dev, NULL); + if (IS_ERR_OR_NULL(core->reset)) { + ret = PTR_ERR_OR_ZERO(core->reset); + if (!ret) + ret = -ENODEV; + if (ret != -EPROBE_DEFER) { + dev_err(dev, "failed to get reset handle, ret = %d\n", + ret); + } + goto err; + } + + core->tsp = k3_r5_core_of_get_tsp(dev, core->ti_sci); + if (IS_ERR(core->tsp)) { + dev_err(dev, "failed to construct ti-sci proc control, ret = %d\n", + ret); + ret = PTR_ERR(core->tsp); + goto err; + } + + ret = k3_r5_core_of_get_internal_memories(pdev, core); + if (ret) { + dev_err(dev, "failed to get internal memories, ret = %d\n", + ret); + goto err; + } + + ret = k3_r5_core_of_get_sram_memories(pdev, core); + if (ret) { + dev_err(dev, "failed to get sram memories, ret = %d\n", ret); + goto err; + } + + ret = ti_sci_proc_request(core->tsp); + if (ret < 0) { + dev_err(dev, "ti_sci_proc_request failed, ret = %d\n", ret); + goto err; + } + + platform_set_drvdata(pdev, core); + devres_close_group(dev, k3_r5_core_of_init); + + return 0; + +err: + devres_release_group(dev, k3_r5_core_of_init); + return ret; +} + +/* + * free the resources explicitly since driver model is not being used + * for the child R5F devices + */ +static void k3_r5_core_of_exit(struct platform_device *pdev) +{ + struct k3_r5_core *core = platform_get_drvdata(pdev); + struct device *dev = &pdev->dev; + int ret; + + ret = ti_sci_proc_release(core->tsp); + if (ret) + dev_err(dev, "failed to release proc, ret = %d\n", ret); + + platform_set_drvdata(pdev, NULL); + devres_release_group(dev, k3_r5_core_of_init); +} + +static void k3_r5_cluster_of_exit(struct platform_device *pdev) +{ + struct k3_r5_cluster *cluster = platform_get_drvdata(pdev); + struct platform_device *cpdev; + struct k3_r5_core *core, *temp; + + list_for_each_entry_safe_reverse(core, temp, &cluster->cores, elem) { + list_del(&core->elem); + cpdev = to_platform_device(core->dev); + k3_r5_core_of_exit(cpdev); + } +} + +static int k3_r5_cluster_of_init(struct platform_device *pdev) +{ + struct k3_r5_cluster *cluster = platform_get_drvdata(pdev); + struct device *dev = &pdev->dev; + struct device_node *np = dev_of_node(dev); + struct platform_device *cpdev; + struct device_node *child; + struct k3_r5_core *core; + int ret; + + for_each_available_child_of_node(np, child) { + cpdev = of_find_device_by_node(child); + if (!cpdev) { + ret = -ENODEV; + dev_err(dev, "could not get R5 core platform device\n"); + goto fail; + } + + ret = k3_r5_core_of_init(cpdev); + if (ret) { + dev_err(dev, "k3_r5_core_of_init failed, ret = %d\n", + ret); + put_device(&cpdev->dev); + goto fail; + } + + core = platform_get_drvdata(cpdev); + put_device(&cpdev->dev); + list_add_tail(&core->elem, &cluster->cores); + } + + return 0; + +fail: + k3_r5_cluster_of_exit(pdev); + return ret; +} + +static int k3_r5_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev_of_node(dev); + struct k3_r5_cluster *cluster; + int ret; + int num_cores; + + cluster = devm_kzalloc(dev, sizeof(*cluster), GFP_KERNEL); + if (!cluster) + return -ENOMEM; + + cluster->dev = dev; + cluster->mode = CLUSTER_MODE_LOCKSTEP; + INIT_LIST_HEAD(&cluster->cores); + + ret = of_property_read_u32(np, "ti,cluster-mode", &cluster->mode); + if (ret < 0 && ret != -EINVAL) { + dev_err(dev, "invalid format for ti,cluster-mode, ret = %d\n", + ret); + return ret; + } + + num_cores = of_get_available_child_count(np); + if (num_cores != 2) { + dev_err(dev, "MCU cluster requires both R5F cores to be enabled, num_cores = %d\n", + num_cores); + return -ENODEV; + } + + platform_set_drvdata(pdev, cluster); + + ret = devm_of_platform_populate(dev); + if (ret) { + dev_err(dev, "devm_of_platform_populate failed, ret = %d\n", + ret); + return ret; + } + + ret = k3_r5_cluster_of_init(pdev); + if (ret) { + dev_err(dev, "k3_r5_cluster_of_init failed, ret = %d\n", ret); + return ret; + } + + ret = devm_add_action_or_reset(dev, + (void(*)(void *))k3_r5_cluster_of_exit, + pdev); + if (ret) + return ret; + + ret = k3_r5_cluster_rproc_init(pdev); + if (ret) { + dev_err(dev, "k3_r5_cluster_rproc_init failed, ret = %d\n", + ret); + return ret; + } + + ret = devm_add_action_or_reset(dev, + (void(*)(void *))k3_r5_cluster_rproc_exit, + pdev); + if (ret) + return ret; + + return 0; +} + +static const struct of_device_id k3_r5_of_match[] = { + { .compatible = "ti,am654-r5fss", }, + { .compatible = "ti,j721e-r5fss", }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, k3_r5_of_match); + +static struct platform_driver k3_r5_rproc_driver = { + .probe = k3_r5_probe, + .driver = { + .name = "k3_r5_rproc", + .of_match_table = k3_r5_of_match, + }, +}; + +module_platform_driver(k3_r5_rproc_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("TI K3 R5F remote processor driver"); +MODULE_AUTHOR("Suman Anna <s-anna@ti.com>"); diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig index 97e848740e13..07d162b179fc 100644 --- a/drivers/reset/Kconfig +++ b/drivers/reset/Kconfig @@ -65,9 +65,10 @@ config RESET_HSDK This enables the reset controller driver for HSDK board. config RESET_IMX7 - bool "i.MX7/8 Reset Driver" if COMPILE_TEST + tristate "i.MX7/8 Reset Driver" depends on HAS_IOMEM - default SOC_IMX7D || (ARM64 && ARCH_MXC) + depends on SOC_IMX7D || (ARM64 && ARCH_MXC) || COMPILE_TEST + default y if SOC_IMX7D select MFD_SYSCON help This enables the reset controller driver for i.MX7 SoCs. diff --git a/drivers/reset/core.c b/drivers/reset/core.c index 01c0c7aa835c..a2df88e90011 100644 --- a/drivers/reset/core.c +++ b/drivers/reset/core.c @@ -32,7 +32,8 @@ static LIST_HEAD(reset_lookup_list); * @refcnt: Number of gets of this reset_control * @acquired: Only one reset_control may be acquired for a given rcdev and id. * @shared: Is this a shared (1), or an exclusive (0) reset_control? - * @deassert_cnt: Number of times this reset line has been deasserted + * @array: Is this an array of reset controls (1)? + * @deassert_count: Number of times this reset line has been deasserted * @triggered_count: Number of times this reset line has been reset. Currently * only used for shared resets, which means that the value * will be either 0 or 1. diff --git a/drivers/reset/reset-imx7.c b/drivers/reset/reset-imx7.c index e8aa8691deb2..185a333df66c 100644 --- a/drivers/reset/reset-imx7.c +++ b/drivers/reset/reset-imx7.c @@ -8,7 +8,7 @@ */ #include <linux/mfd/syscon.h> -#include <linux/mod_devicetable.h> +#include <linux/module.h> #include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/reset-controller.h> @@ -178,6 +178,9 @@ static const struct imx7_src_signal imx8mq_src_signals[IMX8MQ_RESET_NUM] = { [IMX8MQ_RESET_A53_SOC_DBG_RESET] = { SRC_A53RCR0, BIT(20) }, [IMX8MQ_RESET_A53_L2RESET] = { SRC_A53RCR0, BIT(21) }, [IMX8MQ_RESET_SW_NON_SCLR_M4C_RST] = { SRC_M4RCR, BIT(0) }, + [IMX8MQ_RESET_SW_M4C_RST] = { SRC_M4RCR, BIT(1) }, + [IMX8MQ_RESET_SW_M4P_RST] = { SRC_M4RCR, BIT(2) }, + [IMX8MQ_RESET_M4_ENABLE] = { SRC_M4RCR, BIT(3) }, [IMX8MQ_RESET_OTG1_PHY_RESET] = { SRC_USBOPHY1_RCR, BIT(0) }, [IMX8MQ_RESET_OTG2_PHY_RESET] = { SRC_USBOPHY2_RCR, BIT(0) }, [IMX8MQ_RESET_MIPI_DSI_RESET_BYTE_N] = { SRC_MIPIPHY_RCR, BIT(1) }, @@ -238,6 +241,7 @@ static int imx8mq_reset_set(struct reset_controller_dev *rcdev, case IMX8MQ_RESET_MIPI_DSI_DPI_RESET_N: case IMX8MQ_RESET_MIPI_DSI_RESET_N: case IMX8MQ_RESET_MIPI_DSI_RESET_BYTE_N: + case IMX8MQ_RESET_M4_ENABLE: value = assert ? 0 : bit; break; } @@ -386,6 +390,7 @@ static const struct of_device_id imx7_reset_dt_ids[] = { { .compatible = "fsl,imx8mp-src", .data = &variant_imx8mp }, { /* sentinel */ }, }; +MODULE_DEVICE_TABLE(of, imx7_reset_dt_ids); static struct platform_driver imx7_reset_driver = { .probe = imx7_reset_probe, @@ -394,4 +399,8 @@ static struct platform_driver imx7_reset_driver = { .of_match_table = imx7_reset_dt_ids, }, }; -builtin_platform_driver(imx7_reset_driver); +module_platform_driver(imx7_reset_driver); + +MODULE_AUTHOR("Andrey Smirnov <andrew.smirnov@gmail.com>"); +MODULE_DESCRIPTION("NXP i.MX7 reset driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/reset/reset-zynqmp.c b/drivers/reset/reset-zynqmp.c index 373ea8d4f7a1..ebd433fa09dd 100644 --- a/drivers/reset/reset-zynqmp.c +++ b/drivers/reset/reset-zynqmp.c @@ -9,12 +9,20 @@ #include <linux/platform_device.h> #include <linux/reset-controller.h> #include <linux/firmware/xlnx-zynqmp.h> +#include <linux/of_device.h> #define ZYNQMP_NR_RESETS (ZYNQMP_PM_RESET_END - ZYNQMP_PM_RESET_START) #define ZYNQMP_RESET_ID ZYNQMP_PM_RESET_START +#define VERSAL_NR_RESETS 95 + +struct zynqmp_reset_soc_data { + u32 reset_id; + u32 num_resets; +}; struct zynqmp_reset_data { struct reset_controller_dev rcdev; + const struct zynqmp_reset_soc_data *data; }; static inline struct zynqmp_reset_data * @@ -26,23 +34,28 @@ to_zynqmp_reset_data(struct reset_controller_dev *rcdev) static int zynqmp_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) { - return zynqmp_pm_reset_assert(ZYNQMP_RESET_ID + id, + struct zynqmp_reset_data *priv = to_zynqmp_reset_data(rcdev); + + return zynqmp_pm_reset_assert(priv->data->reset_id + id, PM_RESET_ACTION_ASSERT); } static int zynqmp_reset_deassert(struct reset_controller_dev *rcdev, unsigned long id) { - return zynqmp_pm_reset_assert(ZYNQMP_RESET_ID + id, + struct zynqmp_reset_data *priv = to_zynqmp_reset_data(rcdev); + + return zynqmp_pm_reset_assert(priv->data->reset_id + id, PM_RESET_ACTION_RELEASE); } static int zynqmp_reset_status(struct reset_controller_dev *rcdev, unsigned long id) { + struct zynqmp_reset_data *priv = to_zynqmp_reset_data(rcdev); int val, err; - err = zynqmp_pm_reset_get_status(ZYNQMP_RESET_ID + id, &val); + err = zynqmp_pm_reset_get_status(priv->data->reset_id + id, &val); if (err) return err; @@ -52,10 +65,28 @@ static int zynqmp_reset_status(struct reset_controller_dev *rcdev, static int zynqmp_reset_reset(struct reset_controller_dev *rcdev, unsigned long id) { - return zynqmp_pm_reset_assert(ZYNQMP_RESET_ID + id, + struct zynqmp_reset_data *priv = to_zynqmp_reset_data(rcdev); + + return zynqmp_pm_reset_assert(priv->data->reset_id + id, PM_RESET_ACTION_PULSE); } +static int zynqmp_reset_of_xlate(struct reset_controller_dev *rcdev, + const struct of_phandle_args *reset_spec) +{ + return reset_spec->args[0]; +} + +static const struct zynqmp_reset_soc_data zynqmp_reset_data = { + .reset_id = ZYNQMP_RESET_ID, + .num_resets = ZYNQMP_NR_RESETS, +}; + +static const struct zynqmp_reset_soc_data versal_reset_data = { + .reset_id = 0, + .num_resets = VERSAL_NR_RESETS, +}; + static const struct reset_control_ops zynqmp_reset_ops = { .reset = zynqmp_reset_reset, .assert = zynqmp_reset_assert, @@ -71,18 +102,25 @@ static int zynqmp_reset_probe(struct platform_device *pdev) if (!priv) return -ENOMEM; + priv->data = of_device_get_match_data(&pdev->dev); + if (!priv->data) + return -EINVAL; + platform_set_drvdata(pdev, priv); priv->rcdev.ops = &zynqmp_reset_ops; priv->rcdev.owner = THIS_MODULE; priv->rcdev.of_node = pdev->dev.of_node; - priv->rcdev.nr_resets = ZYNQMP_NR_RESETS; + priv->rcdev.nr_resets = priv->data->num_resets; + priv->rcdev.of_reset_n_cells = 1; + priv->rcdev.of_xlate = zynqmp_reset_of_xlate; return devm_reset_controller_register(&pdev->dev, &priv->rcdev); } static const struct of_device_id zynqmp_reset_dt_ids[] = { - { .compatible = "xlnx,zynqmp-reset", }, + { .compatible = "xlnx,zynqmp-reset", .data = &zynqmp_reset_data, }, + { .compatible = "xlnx,versal-reset", .data = &versal_reset_data, }, { /* sentinel */ }, }; diff --git a/drivers/reset/sti/reset-syscfg.c b/drivers/reset/sti/reset-syscfg.c index 91215bb88f62..99b63035fe72 100644 --- a/drivers/reset/sti/reset-syscfg.c +++ b/drivers/reset/sti/reset-syscfg.c @@ -17,7 +17,7 @@ #include "reset-syscfg.h" /** - * Reset channel regmap configuration + * struct syscfg_reset_channel - Reset channel regmap configuration * * @reset: regmap field for the channel's reset bit. * @ack: regmap field for the channel's ack bit (optional). @@ -28,8 +28,9 @@ struct syscfg_reset_channel { }; /** - * A reset controller which groups together a set of related reset bits, which - * may be located in different system configuration registers. + * struct syscfg_reset_controller - A reset controller which groups together + * a set of related reset bits, which may be located in different system + * configuration registers. * * @rst: base reset controller structure. * @active_low: are the resets in this controller active low, i.e. clearing diff --git a/drivers/rpmsg/mtk_rpmsg.c b/drivers/rpmsg/mtk_rpmsg.c index 83f2b8804ee9..96a17ec29140 100644 --- a/drivers/rpmsg/mtk_rpmsg.c +++ b/drivers/rpmsg/mtk_rpmsg.c @@ -200,7 +200,6 @@ static int mtk_rpmsg_register_device(struct mtk_rpmsg_rproc_subdev *mtk_subdev, struct rpmsg_device *rpdev; struct mtk_rpmsg_device *mdev; struct platform_device *pdev = mtk_subdev->pdev; - int ret; mdev = kzalloc(sizeof(*mdev), GFP_KERNEL); if (!mdev) @@ -219,13 +218,7 @@ static int mtk_rpmsg_register_device(struct mtk_rpmsg_rproc_subdev *mtk_subdev, rpdev->dev.parent = &pdev->dev; rpdev->dev.release = mtk_rpmsg_release_device; - ret = rpmsg_register_device(rpdev); - if (ret) { - kfree(mdev); - return ret; - } - - return 0; + return rpmsg_register_device(rpdev); } static void mtk_register_device_work_function(struct work_struct *register_work) diff --git a/drivers/rpmsg/qcom_glink_native.c b/drivers/rpmsg/qcom_glink_native.c index f40312b16da0..27a05167c18c 100644 --- a/drivers/rpmsg/qcom_glink_native.c +++ b/drivers/rpmsg/qcom_glink_native.c @@ -970,7 +970,7 @@ static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid) return -EINVAL; } - complete(&channel->open_ack); + complete_all(&channel->open_ack); return 0; } @@ -1178,7 +1178,7 @@ static int qcom_glink_announce_create(struct rpmsg_device *rpdev) __be32 *val = defaults; int size; - if (glink->intentless) + if (glink->intentless || !completion_done(&channel->open_ack)) return 0; prop = of_find_property(np, "qcom,intents", NULL); @@ -1413,7 +1413,7 @@ static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid, channel->rcid = ret; spin_unlock_irqrestore(&glink->idr_lock, flags); - complete(&channel->open_req); + complete_all(&channel->open_req); if (create_device) { rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); @@ -1574,6 +1574,60 @@ static void qcom_glink_cancel_rx_work(struct qcom_glink *glink) kfree(dcmd); } +static ssize_t rpmsg_name_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int ret = 0; + const char *name; + + ret = of_property_read_string(dev->of_node, "label", &name); + if (ret < 0) + name = dev->of_node->name; + + return snprintf(buf, RPMSG_NAME_SIZE, "%s\n", name); +} +static DEVICE_ATTR_RO(rpmsg_name); + +static struct attribute *qcom_glink_attrs[] = { + &dev_attr_rpmsg_name.attr, + NULL +}; +ATTRIBUTE_GROUPS(qcom_glink); + +static void qcom_glink_device_release(struct device *dev) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + struct glink_channel *channel = to_glink_channel(rpdev->ept); + + /* Release qcom_glink_alloc_channel() reference */ + kref_put(&channel->refcount, qcom_glink_channel_release); + kfree(rpdev); +} + +static int qcom_glink_create_chrdev(struct qcom_glink *glink) +{ + struct rpmsg_device *rpdev; + struct glink_channel *channel; + + rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); + if (!rpdev) + return -ENOMEM; + + channel = qcom_glink_alloc_channel(glink, "rpmsg_chrdev"); + if (IS_ERR(channel)) { + kfree(rpdev); + return PTR_ERR(channel); + } + channel->rpdev = rpdev; + + rpdev->ept = &channel->ept; + rpdev->ops = &glink_device_ops; + rpdev->dev.parent = glink->dev; + rpdev->dev.release = qcom_glink_device_release; + + return rpmsg_chrdev_register_device(rpdev); +} + struct qcom_glink *qcom_glink_native_probe(struct device *dev, unsigned long features, struct qcom_glink_pipe *rx, @@ -1604,6 +1658,12 @@ struct qcom_glink *qcom_glink_native_probe(struct device *dev, idr_init(&glink->lcids); idr_init(&glink->rcids); + glink->dev->groups = qcom_glink_groups; + + ret = device_add_groups(dev, qcom_glink_groups); + if (ret) + dev_err(dev, "failed to add groups\n"); + ret = of_property_read_string(dev->of_node, "label", &glink->name); if (ret < 0) glink->name = dev->of_node->name; @@ -1633,6 +1693,10 @@ struct qcom_glink *qcom_glink_native_probe(struct device *dev, if (ret) return ERR_PTR(ret); + ret = qcom_glink_create_chrdev(glink); + if (ret) + dev_err(glink->dev, "failed to register chrdev\n"); + return glink; } EXPORT_SYMBOL_GPL(qcom_glink_native_probe); diff --git a/drivers/rpmsg/qcom_smd.c b/drivers/rpmsg/qcom_smd.c index 4abbeea782fa..19903de6268d 100644 --- a/drivers/rpmsg/qcom_smd.c +++ b/drivers/rpmsg/qcom_smd.c @@ -1338,7 +1338,7 @@ static int qcom_smd_parse_edge(struct device *dev, ret = of_property_read_u32(node, key, &edge->edge_id); if (ret) { dev_err(dev, "edge missing %s property\n", key); - return -EINVAL; + goto put_node; } edge->remote_pid = QCOM_SMEM_HOST_ANY; @@ -1349,32 +1349,37 @@ static int qcom_smd_parse_edge(struct device *dev, edge->mbox_client.knows_txdone = true; edge->mbox_chan = mbox_request_channel(&edge->mbox_client, 0); if (IS_ERR(edge->mbox_chan)) { - if (PTR_ERR(edge->mbox_chan) != -ENODEV) - return PTR_ERR(edge->mbox_chan); + if (PTR_ERR(edge->mbox_chan) != -ENODEV) { + ret = PTR_ERR(edge->mbox_chan); + goto put_node; + } edge->mbox_chan = NULL; syscon_np = of_parse_phandle(node, "qcom,ipc", 0); if (!syscon_np) { dev_err(dev, "no qcom,ipc node\n"); - return -ENODEV; + ret = -ENODEV; + goto put_node; } edge->ipc_regmap = syscon_node_to_regmap(syscon_np); - if (IS_ERR(edge->ipc_regmap)) - return PTR_ERR(edge->ipc_regmap); + if (IS_ERR(edge->ipc_regmap)) { + ret = PTR_ERR(edge->ipc_regmap); + goto put_node; + } key = "qcom,ipc"; ret = of_property_read_u32_index(node, key, 1, &edge->ipc_offset); if (ret < 0) { dev_err(dev, "no offset in %s\n", key); - return -EINVAL; + goto put_node; } ret = of_property_read_u32_index(node, key, 2, &edge->ipc_bit); if (ret < 0) { dev_err(dev, "no bit in %s\n", key); - return -EINVAL; + goto put_node; } } @@ -1385,7 +1390,8 @@ static int qcom_smd_parse_edge(struct device *dev, irq = irq_of_parse_and_map(node, 0); if (irq < 0) { dev_err(dev, "required smd interrupt missing\n"); - return -EINVAL; + ret = irq; + goto put_node; } ret = devm_request_irq(dev, irq, @@ -1393,12 +1399,18 @@ static int qcom_smd_parse_edge(struct device *dev, node->name, edge); if (ret) { dev_err(dev, "failed to request smd irq\n"); - return ret; + goto put_node; } edge->irq = irq; return 0; + +put_node: + of_node_put(node); + edge->of_node = NULL; + + return ret; } /* diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c index a6361cad608b..91de940896e3 100644 --- a/drivers/rpmsg/rpmsg_core.c +++ b/drivers/rpmsg/rpmsg_core.c @@ -81,7 +81,7 @@ EXPORT_SYMBOL(rpmsg_create_ept); */ void rpmsg_destroy_ept(struct rpmsg_endpoint *ept) { - if (ept) + if (ept && ept->ops) ept->ops->destroy_ept(ept); } EXPORT_SYMBOL(rpmsg_destroy_ept); diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c index 9006fc7f73d0..7d7ed4e5cce7 100644 --- a/drivers/rpmsg/virtio_rpmsg_bus.c +++ b/drivers/rpmsg/virtio_rpmsg_bus.c @@ -123,7 +123,12 @@ enum rpmsg_ns_flags { }; /** - * @vrp: the remote processor this channel belongs to + * struct virtio_rpmsg_channel - rpmsg channel descriptor + * @rpdev: the rpmsg channel device + * @vrp: the virtio remote processor device this channel belongs to + * + * This structure stores the channel that links the rpmsg device to the virtio + * remote processor device. */ struct virtio_rpmsg_channel { struct rpmsg_device rpdev; diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index 48c536acd777..65ad9d0b47ab 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -669,6 +669,16 @@ config RTC_DRV_RV3028 This driver can also be built as a module. If so, the module will be called rtc-rv3028. +config RTC_DRV_RV3032 + tristate "Micro Crystal RV3032" + select REGMAP_I2C + help + If you say yes here you get support for the Micro Crystal + RV3032. + + This driver can also be built as a module. If so, the module + will be called rtc-rv3032. + config RTC_DRV_RV8803 tristate "Micro Crystal RV8803, Epson RX8900" help diff --git a/drivers/rtc/Makefile b/drivers/rtc/Makefile index 880e08a409c3..bfb57464118d 100644 --- a/drivers/rtc/Makefile +++ b/drivers/rtc/Makefile @@ -141,6 +141,7 @@ obj-$(CONFIG_RTC_DRV_RS5C372) += rtc-rs5c372.o obj-$(CONFIG_RTC_DRV_RTD119X) += rtc-rtd119x.o obj-$(CONFIG_RTC_DRV_RV3028) += rtc-rv3028.o obj-$(CONFIG_RTC_DRV_RV3029C2) += rtc-rv3029c2.o +obj-$(CONFIG_RTC_DRV_RV3032) += rtc-rv3032.o obj-$(CONFIG_RTC_DRV_RV8803) += rtc-rv8803.o obj-$(CONFIG_RTC_DRV_RX4581) += rtc-rx4581.o obj-$(CONFIG_RTC_DRV_RX6110) += rtc-rx6110.o diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c index bcc96ab7793f..c633319cdb91 100644 --- a/drivers/rtc/rtc-cmos.c +++ b/drivers/rtc/rtc-cmos.c @@ -1006,6 +1006,7 @@ static int cmos_suspend(struct device *dev) enable_irq_wake(cmos->irq); } + memset(&cmos->saved_wkalrm, 0, sizeof(struct rtc_wkalrm)); cmos_read_alarm(dev, &cmos->saved_wkalrm); dev_dbg(dev, "suspend%s, ctrl %02x\n", @@ -1054,6 +1055,7 @@ static void cmos_check_wkalrm(struct device *dev) return; } + memset(¤t_alarm, 0, sizeof(struct rtc_wkalrm)); cmos_read_alarm(dev, ¤t_alarm); t_current_expires = rtc_tm_to_time64(¤t_alarm.time); t_saved_expires = rtc_tm_to_time64(&cmos->saved_wkalrm.time); diff --git a/drivers/rtc/rtc-ds1307.c b/drivers/rtc/rtc-ds1307.c index 54c85cdd019d..9f5f54ca039d 100644 --- a/drivers/rtc/rtc-ds1307.c +++ b/drivers/rtc/rtc-ds1307.c @@ -122,6 +122,9 @@ enum ds_type { #define RX8130_REG_FLAG_AF BIT(3) #define RX8130_REG_CONTROL0 0x1e #define RX8130_REG_CONTROL0_AIE BIT(3) +#define RX8130_REG_CONTROL1 0x1f +#define RX8130_REG_CONTROL1_INIEN BIT(4) +#define RX8130_REG_CONTROL1_CHGEN BIT(5) #define MCP794XX_REG_CONTROL 0x07 # define MCP794XX_BIT_ALM0_EN 0x10 @@ -153,6 +156,7 @@ enum ds_type { #define DS1388_REG_CONTROL 0x0c # define DS1388_BIT_RST BIT(0) # define DS1388_BIT_WDE BIT(1) +# define DS1388_BIT_nEOSC BIT(7) /* negative offset step is -2.034ppm */ #define M41TXX_NEG_OFFSET_STEP_PPB 2034 @@ -190,6 +194,15 @@ struct chip_desc { u16 trickle_charger_reg; u8 (*do_trickle_setup)(struct ds1307 *, u32, bool); + /* Does the RTC require trickle-resistor-ohms to select the value of + * the resistor between Vcc and Vbackup? + */ + bool requires_trickle_resistor; + /* Some RTC's batteries and supercaps were charged by default, others + * allow charging but were not configured previously to do so. + * Remember this behavior to stay backwards compatible. + */ + bool charge_default; }; static const struct chip_desc chips[last_ds_type]; @@ -352,6 +365,10 @@ static int ds1307_set_time(struct device *dev, struct rtc_time *t) regmap_update_bits(ds1307->regmap, DS1340_REG_FLAG, DS1340_BIT_OSF, 0); break; + case ds_1388: + regmap_update_bits(ds1307->regmap, DS1388_REG_FLAG, + DS1388_BIT_OSF, 0); + break; case mcp794xx: /* * these bits were cleared when preparing the date/time @@ -507,6 +524,8 @@ static u8 do_trickle_setup_ds1339(struct ds1307 *ds1307, u32 ohms, bool diode) u8 setup = (diode) ? DS1307_TRICKLE_CHARGER_DIODE : DS1307_TRICKLE_CHARGER_NO_DIODE; + setup |= DS13XX_TRICKLE_CHARGER_MAGIC; + switch (ohms) { case 250: setup |= DS1307_TRICKLE_CHARGER_250_OHM; @@ -525,6 +544,16 @@ static u8 do_trickle_setup_ds1339(struct ds1307 *ds1307, u32 ohms, bool diode) return setup; } +static u8 do_trickle_setup_rx8130(struct ds1307 *ds1307, u32 ohms, bool diode) +{ + /* make sure that the backup battery is enabled */ + u8 setup = RX8130_REG_CONTROL1_INIEN; + if (diode) + setup |= RX8130_REG_CONTROL1_CHGEN; + + return setup; +} + static irqreturn_t rx8130_irq(int irq, void *dev_id) { struct ds1307 *ds1307 = dev_id; @@ -979,6 +1008,8 @@ static const struct chip_desc chips[last_ds_type] = { .bbsqi_bit = DS1339_BIT_BBSQI, .trickle_charger_reg = 0x10, .do_trickle_setup = &do_trickle_setup_ds1339, + .requires_trickle_resistor = true, + .charge_default = true, }, [ds_1340] = { .century_reg = DS1307_REG_HOUR, @@ -986,6 +1017,8 @@ static const struct chip_desc chips[last_ds_type] = { .century_bit = DS1340_BIT_CENTURY, .do_trickle_setup = &do_trickle_setup_ds1339, .trickle_charger_reg = 0x08, + .requires_trickle_resistor = true, + .charge_default = true, }, [ds_1341] = { .century_reg = DS1307_REG_MONTH, @@ -1009,6 +1042,8 @@ static const struct chip_desc chips[last_ds_type] = { .offset = 0x10, .irq_handler = rx8130_irq, .rtc_ops = &rx8130_rtc_ops, + .trickle_charger_reg = RX8130_REG_CONTROL1, + .do_trickle_setup = &do_trickle_setup_rx8130, }, [m41t0] = { .rtc_ops = &m41txx_rtc_ops, @@ -1293,18 +1328,37 @@ static int ds1307_nvram_write(void *priv, unsigned int offset, void *val, static u8 ds1307_trickle_init(struct ds1307 *ds1307, const struct chip_desc *chip) { - u32 ohms; - bool diode = true; + u32 ohms, chargeable; + bool diode = chip->charge_default; if (!chip->do_trickle_setup) return 0; if (device_property_read_u32(ds1307->dev, "trickle-resistor-ohms", - &ohms)) + &ohms) && chip->requires_trickle_resistor) return 0; - if (device_property_read_bool(ds1307->dev, "trickle-diode-disable")) + /* aux-voltage-chargeable takes precedence over the deprecated + * trickle-diode-disable + */ + if (!device_property_read_u32(ds1307->dev, "aux-voltage-chargeable", + &chargeable)) { + switch (chargeable) { + case 0: + diode = false; + break; + case 1: + diode = true; + break; + default: + dev_warn(ds1307->dev, + "unsupported aux-voltage-chargeable value\n"); + break; + } + } else if (device_property_read_bool(ds1307->dev, + "trickle-diode-disable")) { diode = false; + } return chip->do_trickle_setup(ds1307, ohms, diode); } @@ -1758,7 +1812,6 @@ static int ds1307_probe(struct i2c_client *client, trickle_charger_setup = pdata->trickle_charger_setup; if (trickle_charger_setup && chip->trickle_charger_reg) { - trickle_charger_setup |= DS13XX_TRICKLE_CHARGER_MAGIC; dev_dbg(ds1307->dev, "writing trickle charger info 0x%x to 0x%x\n", trickle_charger_setup, chip->trickle_charger_reg); @@ -1881,6 +1934,19 @@ static int ds1307_probe(struct i2c_client *client, DS1307_REG_HOUR << 4 | 0x08, hour); } break; + case ds_1388: + err = regmap_read(ds1307->regmap, DS1388_REG_CONTROL, &tmp); + if (err) { + dev_dbg(ds1307->dev, "read error %d\n", err); + goto exit; + } + + /* oscillator off? turn it on, so clock can tick. */ + if (tmp & DS1388_BIT_nEOSC) { + tmp &= ~DS1388_BIT_nEOSC; + regmap_write(ds1307->regmap, DS1388_REG_CONTROL, tmp); + } + break; default: break; } diff --git a/drivers/rtc/rtc-ds1685.c b/drivers/rtc/rtc-ds1685.c index 56c670af2e50..dfbd7b88b2b9 100644 --- a/drivers/rtc/rtc-ds1685.c +++ b/drivers/rtc/rtc-ds1685.c @@ -193,12 +193,12 @@ ds1685_rtc_begin_data_access(struct ds1685_priv *rtc) rtc->write(rtc, RTC_CTRL_B, (rtc->read(rtc, RTC_CTRL_B) | RTC_CTRL_B_SET)); + /* Switch to Bank 1 */ + ds1685_rtc_switch_to_bank1(rtc); + /* Read Ext Ctrl 4A and check the INCR bit to avoid a lockout. */ while (rtc->read(rtc, RTC_EXT_CTRL_4A) & RTC_CTRL_4A_INCR) cpu_relax(); - - /* Switch to Bank 1 */ - ds1685_rtc_switch_to_bank1(rtc); } /** @@ -213,7 +213,7 @@ static inline void ds1685_rtc_end_data_access(struct ds1685_priv *rtc) { /* Switch back to Bank 0 */ - ds1685_rtc_switch_to_bank1(rtc); + ds1685_rtc_switch_to_bank0(rtc); /* Clear the SET bit in Ctrl B */ rtc->write(rtc, RTC_CTRL_B, diff --git a/drivers/rtc/rtc-fsl-ftm-alarm.c b/drivers/rtc/rtc-fsl-ftm-alarm.c index 68f0a1801a2e..48d3b38ea348 100644 --- a/drivers/rtc/rtc-fsl-ftm-alarm.c +++ b/drivers/rtc/rtc-fsl-ftm-alarm.c @@ -3,7 +3,7 @@ * Freescale FlexTimer Module (FTM) alarm device driver. * * Copyright 2014 Freescale Semiconductor, Inc. - * Copyright 2019 NXP + * Copyright 2019-2020 NXP * */ @@ -312,7 +312,7 @@ static const struct of_device_id ftm_rtc_match[] = { }; static const struct acpi_device_id ftm_imx_acpi_ids[] = { - {"NXP0011",}, + {"NXP0014",}, { } }; MODULE_DEVICE_TABLE(acpi, ftm_imx_acpi_ids); diff --git a/drivers/rtc/rtc-meson-vrtc.c b/drivers/rtc/rtc-meson-vrtc.c index 89e5ba0dae69..e6bd0808a092 100644 --- a/drivers/rtc/rtc-meson-vrtc.c +++ b/drivers/rtc/rtc-meson-vrtc.c @@ -65,7 +65,6 @@ static const struct rtc_class_ops meson_vrtc_ops = { static int meson_vrtc_probe(struct platform_device *pdev) { struct meson_vrtc_data *vrtc; - int ret; vrtc = devm_kzalloc(&pdev->dev, sizeof(*vrtc), GFP_KERNEL); if (!vrtc) @@ -84,11 +83,7 @@ static int meson_vrtc_probe(struct platform_device *pdev) return PTR_ERR(vrtc->rtc); vrtc->rtc->ops = &meson_vrtc_ops; - ret = rtc_register_device(vrtc->rtc); - if (ret) - return ret; - - return 0; + return rtc_register_device(vrtc->rtc); } static int __maybe_unused meson_vrtc_suspend(struct device *dev) diff --git a/drivers/rtc/rtc-mt6397.c b/drivers/rtc/rtc-mt6397.c index f8b1353777ba..1894aded4c85 100644 --- a/drivers/rtc/rtc-mt6397.c +++ b/drivers/rtc/rtc-mt6397.c @@ -31,7 +31,8 @@ static int mtk_rtc_write_trigger(struct mt6397_rtc *rtc) MTK_RTC_POLL_DELAY_US, MTK_RTC_POLL_TIMEOUT); if (ret < 0) - dev_err(rtc->dev, "failed to write WRTGE: %d\n", ret); + dev_err(rtc->rtc_dev->dev.parent, + "failed to write WRTGR: %d\n", ret); return ret; } diff --git a/drivers/rtc/rtc-pcf2127.c b/drivers/rtc/rtc-pcf2127.c index ed6316992cbb..07a5630ec841 100644 --- a/drivers/rtc/rtc-pcf2127.c +++ b/drivers/rtc/rtc-pcf2127.c @@ -559,7 +559,7 @@ static int pcf2127_probe(struct device *dev, struct regmap *regmap, pcf2127->rtc->set_start_time = true; /* Sets actual start to 1970 */ pcf2127->rtc->uie_unsupported = 1; - if (alarm_irq >= 0) { + if (alarm_irq > 0) { ret = devm_request_threaded_irq(dev, alarm_irq, NULL, pcf2127_rtc_irq, IRQF_TRIGGER_LOW | IRQF_ONESHOT, @@ -570,7 +570,7 @@ static int pcf2127_probe(struct device *dev, struct regmap *regmap, } } - if (alarm_irq >= 0 || device_property_read_bool(dev, "wakeup-source")) { + if (alarm_irq > 0 || device_property_read_bool(dev, "wakeup-source")) { device_init_wakeup(dev, true); pcf2127->rtc->ops = &pcf2127_rtc_alrm_ops; } diff --git a/drivers/rtc/rtc-r9701.c b/drivers/rtc/rtc-r9701.c index 84f0d25259ae..7ceb968f0e44 100644 --- a/drivers/rtc/rtc-r9701.c +++ b/drivers/rtc/rtc-r9701.c @@ -75,8 +75,6 @@ static int r9701_get_datetime(struct device *dev, struct rtc_time *dt) if (ret) return ret; - memset(dt, 0, sizeof(*dt)); - dt->tm_sec = bcd2bin(buf[0]); /* RSECCNT */ dt->tm_min = bcd2bin(buf[1]); /* RMINCNT */ dt->tm_hour = bcd2bin(buf[2]); /* RHRCNT */ @@ -85,20 +83,12 @@ static int r9701_get_datetime(struct device *dev, struct rtc_time *dt) dt->tm_mon = bcd2bin(buf[4]) - 1; /* RMONCNT */ dt->tm_year = bcd2bin(buf[5]) + 100; /* RYRCNT */ - /* the rtc device may contain illegal values on power up - * according to the data sheet. make sure they are valid. - */ - return 0; } static int r9701_set_datetime(struct device *dev, struct rtc_time *dt) { - int ret, year; - - year = dt->tm_year + 1900; - if (year >= 2100 || year < 2000) - return -EINVAL; + int ret; ret = write_reg(dev, RHRCNT, bin2bcd(dt->tm_hour)); ret = ret ? ret : write_reg(dev, RMINCNT, bin2bcd(dt->tm_min)); @@ -106,7 +96,6 @@ static int r9701_set_datetime(struct device *dev, struct rtc_time *dt) ret = ret ? ret : write_reg(dev, RDAYCNT, bin2bcd(dt->tm_mday)); ret = ret ? ret : write_reg(dev, RMONCNT, bin2bcd(dt->tm_mon + 1)); ret = ret ? ret : write_reg(dev, RYRCNT, bin2bcd(dt->tm_year - 100)); - ret = ret ? ret : write_reg(dev, RWKCNT, 1 << dt->tm_wday); return ret; } @@ -119,7 +108,6 @@ static const struct rtc_class_ops r9701_rtc_ops = { static int r9701_probe(struct spi_device *spi) { struct rtc_device *rtc; - struct rtc_time dt; unsigned char tmp; int res; @@ -130,35 +118,16 @@ static int r9701_probe(struct spi_device *spi) return -ENODEV; } - /* - * The device seems to be present. Now check if the registers - * contain invalid values. If so, try to write a default date: - * 2000/1/1 00:00:00 - */ - if (r9701_get_datetime(&spi->dev, &dt)) { - dev_info(&spi->dev, "trying to repair invalid date/time\n"); - dt.tm_sec = 0; - dt.tm_min = 0; - dt.tm_hour = 0; - dt.tm_mday = 1; - dt.tm_mon = 0; - dt.tm_year = 100; - - if (r9701_set_datetime(&spi->dev, &dt) || - r9701_get_datetime(&spi->dev, &dt)) { - dev_err(&spi->dev, "cannot repair RTC register\n"); - return -ENODEV; - } - } - - rtc = devm_rtc_device_register(&spi->dev, "r9701", - &r9701_rtc_ops, THIS_MODULE); + rtc = devm_rtc_allocate_device(&spi->dev); if (IS_ERR(rtc)) return PTR_ERR(rtc); spi_set_drvdata(spi, rtc); + rtc->ops = &r9701_rtc_ops; + rtc->range_min = RTC_TIMESTAMP_BEGIN_2000; + rtc->range_max = RTC_TIMESTAMP_END_2099; - return 0; + return rtc_register_device(rtc); } static struct spi_driver r9701_driver = { diff --git a/drivers/rtc/rtc-rs5c313.c b/drivers/rtc/rtc-rs5c313.c index 89f38e3e917d..e98f85f34206 100644 --- a/drivers/rtc/rtc-rs5c313.c +++ b/drivers/rtc/rtc-rs5c313.c @@ -366,15 +366,15 @@ static const struct rtc_class_ops rs5c313_rtc_ops = { static int rs5c313_rtc_probe(struct platform_device *pdev) { - struct rtc_device *rtc = devm_rtc_device_register(&pdev->dev, "rs5c313", - &rs5c313_rtc_ops, THIS_MODULE); + struct rtc_device *rtc; - if (IS_ERR(rtc)) - return PTR_ERR(rtc); + rs5c313_init_port(); + rs5c313_check_xstp_bit(); - platform_set_drvdata(pdev, rtc); + rtc = devm_rtc_device_register(&pdev->dev, "rs5c313", &rs5c313_rtc_ops, + THIS_MODULE); - return 0; + return PTR_ERR_OR_ZERO(rtc); } static struct platform_driver rs5c313_rtc_platform_driver = { @@ -384,27 +384,7 @@ static struct platform_driver rs5c313_rtc_platform_driver = { .probe = rs5c313_rtc_probe, }; -static int __init rs5c313_rtc_init(void) -{ - int err; - - err = platform_driver_register(&rs5c313_rtc_platform_driver); - if (err) - return err; - - rs5c313_init_port(); - rs5c313_check_xstp_bit(); - - return 0; -} - -static void __exit rs5c313_rtc_exit(void) -{ - platform_driver_unregister(&rs5c313_rtc_platform_driver); -} - -module_init(rs5c313_rtc_init); -module_exit(rs5c313_rtc_exit); +module_platform_driver(rs5c313_rtc_platform_driver); MODULE_AUTHOR("kogiidena , Nobuhiro Iwamatsu <iwamatsu@nigauri.org>"); MODULE_DESCRIPTION("Ricoh RS5C313 RTC device driver"); diff --git a/drivers/rtc/rtc-rv3028.c b/drivers/rtc/rtc-rv3028.c index ec84db0b3d7a..fa226f0fe67d 100644 --- a/drivers/rtc/rtc-rv3028.c +++ b/drivers/rtc/rtc-rv3028.c @@ -71,6 +71,7 @@ #define RV3028_EVT_CTRL_TSR BIT(2) +#define RV3028_EEPROM_CMD_UPDATE 0x11 #define RV3028_EEPROM_CMD_WRITE 0x21 #define RV3028_EEPROM_CMD_READ 0x22 @@ -95,7 +96,7 @@ struct rv3028_data { #endif }; -static u16 rv3028_trickle_resistors[] = {1000, 3000, 6000, 11000}; +static u16 rv3028_trickle_resistors[] = {3000, 5000, 9000, 15000}; static ssize_t timestamp0_store(struct device *dev, struct device_attribute *attr, @@ -171,6 +172,88 @@ static const struct attribute_group rv3028_attr_group = { .attrs = rv3028_attrs, }; +static int rv3028_exit_eerd(struct rv3028_data *rv3028, u32 eerd) +{ + if (eerd) + return 0; + + return regmap_update_bits(rv3028->regmap, RV3028_CTRL1, RV3028_CTRL1_EERD, 0); +} + +static int rv3028_enter_eerd(struct rv3028_data *rv3028, u32 *eerd) +{ + u32 ctrl1, status; + int ret; + + ret = regmap_read(rv3028->regmap, RV3028_CTRL1, &ctrl1); + if (ret) + return ret; + + *eerd = ctrl1 & RV3028_CTRL1_EERD; + if (*eerd) + return 0; + + ret = regmap_update_bits(rv3028->regmap, RV3028_CTRL1, + RV3028_CTRL1_EERD, RV3028_CTRL1_EERD); + if (ret) + return ret; + + ret = regmap_read_poll_timeout(rv3028->regmap, RV3028_STATUS, status, + !(status & RV3028_STATUS_EEBUSY), + RV3028_EEBUSY_POLL, RV3028_EEBUSY_TIMEOUT); + if (ret) { + rv3028_exit_eerd(rv3028, *eerd); + + return ret; + } + + return 0; +} + +static int rv3028_update_eeprom(struct rv3028_data *rv3028, u32 eerd) +{ + u32 status; + int ret; + + ret = regmap_write(rv3028->regmap, RV3028_EEPROM_CMD, 0x0); + if (ret) + goto exit_eerd; + + ret = regmap_write(rv3028->regmap, RV3028_EEPROM_CMD, RV3028_EEPROM_CMD_UPDATE); + if (ret) + goto exit_eerd; + + usleep_range(63000, RV3028_EEBUSY_TIMEOUT); + + ret = regmap_read_poll_timeout(rv3028->regmap, RV3028_STATUS, status, + !(status & RV3028_STATUS_EEBUSY), + RV3028_EEBUSY_POLL, RV3028_EEBUSY_TIMEOUT); + +exit_eerd: + rv3028_exit_eerd(rv3028, eerd); + + return ret; +} + +static int rv3028_update_cfg(struct rv3028_data *rv3028, unsigned int reg, + unsigned int mask, unsigned int val) +{ + u32 eerd; + int ret; + + ret = rv3028_enter_eerd(rv3028, &eerd); + if (ret) + return ret; + + ret = regmap_update_bits(rv3028->regmap, reg, mask, val); + if (ret) { + rv3028_exit_eerd(rv3028, eerd); + return ret; + } + + return rv3028_update_eeprom(rv3028, eerd); +} + static irqreturn_t rv3028_handle_irq(int irq, void *dev_id) { struct rv3028_data *rv3028 = dev_id; @@ -404,17 +487,32 @@ static int rv3028_read_offset(struct device *dev, long *offset) static int rv3028_set_offset(struct device *dev, long offset) { struct rv3028_data *rv3028 = dev_get_drvdata(dev); + u32 eerd; int ret; offset = clamp(offset, -244141L, 243187L) * 1000; offset = DIV_ROUND_CLOSEST(offset, OFFSET_STEP_PPT); + ret = rv3028_enter_eerd(rv3028, &eerd); + if (ret) + return ret; + ret = regmap_write(rv3028->regmap, RV3028_OFFSET, offset >> 1); if (ret < 0) - return ret; + goto exit_eerd; + + ret = regmap_update_bits(rv3028->regmap, RV3028_BACKUP, BIT(7), + offset << 7); + if (ret < 0) + goto exit_eerd; + + return rv3028_update_eeprom(rv3028, eerd); + +exit_eerd: + rv3028_exit_eerd(rv3028, eerd); + + return ret; - return regmap_update_bits(rv3028->regmap, RV3028_BACKUP, BIT(7), - offset << 7); } static int rv3028_ioctl(struct device *dev, unsigned int cmd, unsigned long arg) @@ -451,49 +549,36 @@ static int rv3028_nvram_read(void *priv, unsigned int offset, void *val, static int rv3028_eeprom_write(void *priv, unsigned int offset, void *val, size_t bytes) { - u32 status, ctrl1; - int i, ret, err; + struct rv3028_data *rv3028 = priv; + u32 status, eerd; + int i, ret; u8 *buf = val; - ret = regmap_read(priv, RV3028_CTRL1, &ctrl1); + ret = rv3028_enter_eerd(rv3028, &eerd); if (ret) return ret; - if (!(ctrl1 & RV3028_CTRL1_EERD)) { - ret = regmap_update_bits(priv, RV3028_CTRL1, - RV3028_CTRL1_EERD, RV3028_CTRL1_EERD); - if (ret) - return ret; - - ret = regmap_read_poll_timeout(priv, RV3028_STATUS, status, - !(status & RV3028_STATUS_EEBUSY), - RV3028_EEBUSY_POLL, - RV3028_EEBUSY_TIMEOUT); - if (ret) - goto restore_eerd; - } - for (i = 0; i < bytes; i++) { - ret = regmap_write(priv, RV3028_EEPROM_ADDR, offset + i); + ret = regmap_write(rv3028->regmap, RV3028_EEPROM_ADDR, offset + i); if (ret) goto restore_eerd; - ret = regmap_write(priv, RV3028_EEPROM_DATA, buf[i]); + ret = regmap_write(rv3028->regmap, RV3028_EEPROM_DATA, buf[i]); if (ret) goto restore_eerd; - ret = regmap_write(priv, RV3028_EEPROM_CMD, 0x0); + ret = regmap_write(rv3028->regmap, RV3028_EEPROM_CMD, 0x0); if (ret) goto restore_eerd; - ret = regmap_write(priv, RV3028_EEPROM_CMD, + ret = regmap_write(rv3028->regmap, RV3028_EEPROM_CMD, RV3028_EEPROM_CMD_WRITE); if (ret) goto restore_eerd; usleep_range(RV3028_EEBUSY_POLL, RV3028_EEBUSY_TIMEOUT); - ret = regmap_read_poll_timeout(priv, RV3028_STATUS, status, + ret = regmap_read_poll_timeout(rv3028->regmap, RV3028_STATUS, status, !(status & RV3028_STATUS_EEBUSY), RV3028_EEBUSY_POLL, RV3028_EEBUSY_TIMEOUT); @@ -502,13 +587,7 @@ static int rv3028_eeprom_write(void *priv, unsigned int offset, void *val, } restore_eerd: - if (!(ctrl1 & RV3028_CTRL1_EERD)) - { - err = regmap_update_bits(priv, RV3028_CTRL1, RV3028_CTRL1_EERD, - 0); - if (err && !ret) - ret = err; - } + rv3028_exit_eerd(rv3028, eerd); return ret; } @@ -516,63 +595,44 @@ restore_eerd: static int rv3028_eeprom_read(void *priv, unsigned int offset, void *val, size_t bytes) { - u32 status, ctrl1, data; - int i, ret, err; + struct rv3028_data *rv3028 = priv; + u32 status, eerd, data; + int i, ret; u8 *buf = val; - ret = regmap_read(priv, RV3028_CTRL1, &ctrl1); + ret = rv3028_enter_eerd(rv3028, &eerd); if (ret) return ret; - if (!(ctrl1 & RV3028_CTRL1_EERD)) { - ret = regmap_update_bits(priv, RV3028_CTRL1, - RV3028_CTRL1_EERD, RV3028_CTRL1_EERD); - if (ret) - return ret; - - ret = regmap_read_poll_timeout(priv, RV3028_STATUS, status, - !(status & RV3028_STATUS_EEBUSY), - RV3028_EEBUSY_POLL, - RV3028_EEBUSY_TIMEOUT); - if (ret) - goto restore_eerd; - } - for (i = 0; i < bytes; i++) { - ret = regmap_write(priv, RV3028_EEPROM_ADDR, offset + i); + ret = regmap_write(rv3028->regmap, RV3028_EEPROM_ADDR, offset + i); if (ret) goto restore_eerd; - ret = regmap_write(priv, RV3028_EEPROM_CMD, 0x0); + ret = regmap_write(rv3028->regmap, RV3028_EEPROM_CMD, 0x0); if (ret) goto restore_eerd; - ret = regmap_write(priv, RV3028_EEPROM_CMD, + ret = regmap_write(rv3028->regmap, RV3028_EEPROM_CMD, RV3028_EEPROM_CMD_READ); if (ret) goto restore_eerd; - ret = regmap_read_poll_timeout(priv, RV3028_STATUS, status, + ret = regmap_read_poll_timeout(rv3028->regmap, RV3028_STATUS, status, !(status & RV3028_STATUS_EEBUSY), RV3028_EEBUSY_POLL, RV3028_EEBUSY_TIMEOUT); if (ret) goto restore_eerd; - ret = regmap_read(priv, RV3028_EEPROM_DATA, &data); + ret = regmap_read(rv3028->regmap, RV3028_EEPROM_DATA, &data); if (ret) goto restore_eerd; buf[i] = data; } restore_eerd: - if (!(ctrl1 & RV3028_CTRL1_EERD)) - { - err = regmap_update_bits(priv, RV3028_CTRL1, RV3028_CTRL1_EERD, - 0); - if (err && !ret) - ret = err; - } + rv3028_exit_eerd(rv3028, eerd); return ret; } @@ -619,24 +679,23 @@ static int rv3028_clkout_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { int i, ret; + u32 enabled; struct rv3028_data *rv3028 = clkout_hw_to_rv3028(hw); + ret = regmap_read(rv3028->regmap, RV3028_CLKOUT, &enabled); + if (ret < 0) + return ret; + ret = regmap_write(rv3028->regmap, RV3028_CLKOUT, 0x0); if (ret < 0) return ret; - for (i = 0; i < ARRAY_SIZE(clkout_rates); i++) { - if (clkout_rates[i] == rate) { - ret = regmap_update_bits(rv3028->regmap, - RV3028_CLKOUT, - RV3028_CLKOUT_FD_MASK, i); - if (ret < 0) - return ret; + enabled &= RV3028_CLKOUT_CLKOE; - return regmap_write(rv3028->regmap, RV3028_CLKOUT, - RV3028_CLKOUT_CLKSY | RV3028_CLKOUT_CLKOE); - } - } + for (i = 0; i < ARRAY_SIZE(clkout_rates); i++) + if (clkout_rates[i] == rate) + return rv3028_update_cfg(rv3028, RV3028_CLKOUT, 0xff, + RV3028_CLKOUT_CLKSY | enabled | i); return -EINVAL; } @@ -811,10 +870,8 @@ static int rv3028_probe(struct i2c_client *client) break; if (i < ARRAY_SIZE(rv3028_trickle_resistors)) { - ret = regmap_update_bits(rv3028->regmap, RV3028_BACKUP, - RV3028_BACKUP_TCE | - RV3028_BACKUP_TCR_MASK, - RV3028_BACKUP_TCE | i); + ret = rv3028_update_cfg(rv3028, RV3028_BACKUP, RV3028_BACKUP_TCE | + RV3028_BACKUP_TCR_MASK, RV3028_BACKUP_TCE | i); if (ret) return ret; } else { @@ -835,7 +892,7 @@ static int rv3028_probe(struct i2c_client *client) nvmem_cfg.priv = rv3028->regmap; rtc_nvmem_register(rv3028->rtc, &nvmem_cfg); - eeprom_cfg.priv = rv3028->regmap; + eeprom_cfg.priv = rv3028; rtc_nvmem_register(rv3028->rtc, &eeprom_cfg); rv3028->rtc->max_user_freq = 1; diff --git a/drivers/rtc/rtc-rv3032.c b/drivers/rtc/rtc-rv3032.c new file mode 100644 index 000000000000..3e67f71f4261 --- /dev/null +++ b/drivers/rtc/rtc-rv3032.c @@ -0,0 +1,925 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * RTC driver for the Micro Crystal RV3032 + * + * Copyright (C) 2020 Micro Crystal SA + * + * Alexandre Belloni <alexandre.belloni@bootlin.com> + * + */ + +#include <linux/clk.h> +#include <linux/clk-provider.h> +#include <linux/bcd.h> +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/hwmon.h> +#include <linux/i2c.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/log2.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/regmap.h> +#include <linux/rtc.h> + +#define RV3032_SEC 0x01 +#define RV3032_MIN 0x02 +#define RV3032_HOUR 0x03 +#define RV3032_WDAY 0x04 +#define RV3032_DAY 0x05 +#define RV3032_MONTH 0x06 +#define RV3032_YEAR 0x07 +#define RV3032_ALARM_MIN 0x08 +#define RV3032_ALARM_HOUR 0x09 +#define RV3032_ALARM_DAY 0x0A +#define RV3032_STATUS 0x0D +#define RV3032_TLSB 0x0E +#define RV3032_TMSB 0x0F +#define RV3032_CTRL1 0x10 +#define RV3032_CTRL2 0x11 +#define RV3032_CTRL3 0x12 +#define RV3032_TS_CTRL 0x13 +#define RV3032_CLK_IRQ 0x14 +#define RV3032_EEPROM_ADDR 0x3D +#define RV3032_EEPROM_DATA 0x3E +#define RV3032_EEPROM_CMD 0x3F +#define RV3032_RAM1 0x40 +#define RV3032_PMU 0xC0 +#define RV3032_OFFSET 0xC1 +#define RV3032_CLKOUT1 0xC2 +#define RV3032_CLKOUT2 0xC3 +#define RV3032_TREF0 0xC4 +#define RV3032_TREF1 0xC5 + +#define RV3032_STATUS_VLF BIT(0) +#define RV3032_STATUS_PORF BIT(1) +#define RV3032_STATUS_EVF BIT(2) +#define RV3032_STATUS_AF BIT(3) +#define RV3032_STATUS_TF BIT(4) +#define RV3032_STATUS_UF BIT(5) +#define RV3032_STATUS_TLF BIT(6) +#define RV3032_STATUS_THF BIT(7) + +#define RV3032_TLSB_CLKF BIT(1) +#define RV3032_TLSB_EEBUSY BIT(2) +#define RV3032_TLSB_TEMP GENMASK(7, 4) + +#define RV3032_CLKOUT2_HFD_MSK GENMASK(4, 0) +#define RV3032_CLKOUT2_FD_MSK GENMASK(6, 5) +#define RV3032_CLKOUT2_OS BIT(7) + +#define RV3032_CTRL1_EERD BIT(3) +#define RV3032_CTRL1_WADA BIT(5) + +#define RV3032_CTRL2_STOP BIT(0) +#define RV3032_CTRL2_EIE BIT(2) +#define RV3032_CTRL2_AIE BIT(3) +#define RV3032_CTRL2_TIE BIT(4) +#define RV3032_CTRL2_UIE BIT(5) +#define RV3032_CTRL2_CLKIE BIT(6) +#define RV3032_CTRL2_TSE BIT(7) + +#define RV3032_PMU_TCM GENMASK(1, 0) +#define RV3032_PMU_TCR GENMASK(3, 2) +#define RV3032_PMU_BSM GENMASK(5, 4) +#define RV3032_PMU_NCLKE BIT(6) + +#define RV3032_PMU_BSM_DSM 1 +#define RV3032_PMU_BSM_LSM 2 + +#define RV3032_OFFSET_MSK GENMASK(5, 0) + +#define RV3032_EVT_CTRL_TSR BIT(2) + +#define RV3032_EEPROM_CMD_UPDATE 0x11 +#define RV3032_EEPROM_CMD_WRITE 0x21 +#define RV3032_EEPROM_CMD_READ 0x22 + +#define RV3032_EEPROM_USER 0xCB + +#define RV3032_EEBUSY_POLL 10000 +#define RV3032_EEBUSY_TIMEOUT 100000 + +#define OFFSET_STEP_PPT 238419 + +struct rv3032_data { + struct regmap *regmap; + struct rtc_device *rtc; +#ifdef CONFIG_COMMON_CLK + struct clk_hw clkout_hw; +#endif +}; + +static u16 rv3032_trickle_resistors[] = {1000, 2000, 7000, 11000}; +static u16 rv3032_trickle_voltages[] = {0, 1750, 3000, 4400}; + +static int rv3032_exit_eerd(struct rv3032_data *rv3032, u32 eerd) +{ + if (eerd) + return 0; + + return regmap_update_bits(rv3032->regmap, RV3032_CTRL1, RV3032_CTRL1_EERD, 0); +} + +static int rv3032_enter_eerd(struct rv3032_data *rv3032, u32 *eerd) +{ + u32 ctrl1, status; + int ret; + + ret = regmap_read(rv3032->regmap, RV3032_CTRL1, &ctrl1); + if (ret) + return ret; + + *eerd = ctrl1 & RV3032_CTRL1_EERD; + if (*eerd) + return 0; + + ret = regmap_update_bits(rv3032->regmap, RV3032_CTRL1, + RV3032_CTRL1_EERD, RV3032_CTRL1_EERD); + if (ret) + return ret; + + ret = regmap_read_poll_timeout(rv3032->regmap, RV3032_TLSB, status, + !(status & RV3032_TLSB_EEBUSY), + RV3032_EEBUSY_POLL, RV3032_EEBUSY_TIMEOUT); + if (ret) { + rv3032_exit_eerd(rv3032, *eerd); + + return ret; + } + + return 0; +} + +static int rv3032_update_cfg(struct rv3032_data *rv3032, unsigned int reg, + unsigned int mask, unsigned int val) +{ + u32 status, eerd; + int ret; + + ret = rv3032_enter_eerd(rv3032, &eerd); + if (ret) + return ret; + + ret = regmap_update_bits(rv3032->regmap, reg, mask, val); + if (ret) + goto exit_eerd; + + ret = regmap_write(rv3032->regmap, RV3032_EEPROM_CMD, RV3032_EEPROM_CMD_UPDATE); + if (ret) + goto exit_eerd; + + usleep_range(46000, RV3032_EEBUSY_TIMEOUT); + + ret = regmap_read_poll_timeout(rv3032->regmap, RV3032_TLSB, status, + !(status & RV3032_TLSB_EEBUSY), + RV3032_EEBUSY_POLL, RV3032_EEBUSY_TIMEOUT); + +exit_eerd: + rv3032_exit_eerd(rv3032, eerd); + + return ret; +} + +static irqreturn_t rv3032_handle_irq(int irq, void *dev_id) +{ + struct rv3032_data *rv3032 = dev_id; + unsigned long events = 0; + u32 status = 0, ctrl = 0; + + if (regmap_read(rv3032->regmap, RV3032_STATUS, &status) < 0 || + status == 0) { + return IRQ_NONE; + } + + if (status & RV3032_STATUS_TF) { + status |= RV3032_STATUS_TF; + ctrl |= RV3032_CTRL2_TIE; + events |= RTC_PF; + } + + if (status & RV3032_STATUS_AF) { + status |= RV3032_STATUS_AF; + ctrl |= RV3032_CTRL2_AIE; + events |= RTC_AF; + } + + if (status & RV3032_STATUS_UF) { + status |= RV3032_STATUS_UF; + ctrl |= RV3032_CTRL2_UIE; + events |= RTC_UF; + } + + if (events) { + rtc_update_irq(rv3032->rtc, 1, events); + regmap_update_bits(rv3032->regmap, RV3032_STATUS, status, 0); + regmap_update_bits(rv3032->regmap, RV3032_CTRL2, ctrl, 0); + } + + return IRQ_HANDLED; +} + +static int rv3032_get_time(struct device *dev, struct rtc_time *tm) +{ + struct rv3032_data *rv3032 = dev_get_drvdata(dev); + u8 date[7]; + int ret, status; + + ret = regmap_read(rv3032->regmap, RV3032_STATUS, &status); + if (ret < 0) + return ret; + + if (status & (RV3032_STATUS_PORF | RV3032_STATUS_VLF)) + return -EINVAL; + + ret = regmap_bulk_read(rv3032->regmap, RV3032_SEC, date, sizeof(date)); + if (ret) + return ret; + + tm->tm_sec = bcd2bin(date[0] & 0x7f); + tm->tm_min = bcd2bin(date[1] & 0x7f); + tm->tm_hour = bcd2bin(date[2] & 0x3f); + tm->tm_wday = date[3] & 0x7; + tm->tm_mday = bcd2bin(date[4] & 0x3f); + tm->tm_mon = bcd2bin(date[5] & 0x1f) - 1; + tm->tm_year = bcd2bin(date[6]) + 100; + + return 0; +} + +static int rv3032_set_time(struct device *dev, struct rtc_time *tm) +{ + struct rv3032_data *rv3032 = dev_get_drvdata(dev); + u8 date[7]; + int ret; + + date[0] = bin2bcd(tm->tm_sec); + date[1] = bin2bcd(tm->tm_min); + date[2] = bin2bcd(tm->tm_hour); + date[3] = tm->tm_wday; + date[4] = bin2bcd(tm->tm_mday); + date[5] = bin2bcd(tm->tm_mon + 1); + date[6] = bin2bcd(tm->tm_year - 100); + + ret = regmap_bulk_write(rv3032->regmap, RV3032_SEC, date, + sizeof(date)); + if (ret) + return ret; + + ret = regmap_update_bits(rv3032->regmap, RV3032_STATUS, + RV3032_STATUS_PORF | RV3032_STATUS_VLF, 0); + + return ret; +} + +static int rv3032_get_alarm(struct device *dev, struct rtc_wkalrm *alrm) +{ + struct rv3032_data *rv3032 = dev_get_drvdata(dev); + u8 alarmvals[3]; + int status, ctrl, ret; + + ret = regmap_bulk_read(rv3032->regmap, RV3032_ALARM_MIN, alarmvals, + sizeof(alarmvals)); + if (ret) + return ret; + + ret = regmap_read(rv3032->regmap, RV3032_STATUS, &status); + if (ret < 0) + return ret; + + ret = regmap_read(rv3032->regmap, RV3032_CTRL2, &ctrl); + if (ret < 0) + return ret; + + alrm->time.tm_sec = 0; + alrm->time.tm_min = bcd2bin(alarmvals[0] & 0x7f); + alrm->time.tm_hour = bcd2bin(alarmvals[1] & 0x3f); + alrm->time.tm_mday = bcd2bin(alarmvals[2] & 0x3f); + + alrm->enabled = !!(ctrl & RV3032_CTRL2_AIE); + alrm->pending = (status & RV3032_STATUS_AF) && alrm->enabled; + + return 0; +} + +static int rv3032_set_alarm(struct device *dev, struct rtc_wkalrm *alrm) +{ + struct rv3032_data *rv3032 = dev_get_drvdata(dev); + u8 alarmvals[3]; + u8 ctrl = 0; + int ret; + + /* The alarm has no seconds, round up to nearest minute */ + if (alrm->time.tm_sec) { + time64_t alarm_time = rtc_tm_to_time64(&alrm->time); + + alarm_time += 60 - alrm->time.tm_sec; + rtc_time64_to_tm(alarm_time, &alrm->time); + } + + ret = regmap_update_bits(rv3032->regmap, RV3032_CTRL2, + RV3032_CTRL2_AIE | RV3032_CTRL2_UIE, 0); + if (ret) + return ret; + + alarmvals[0] = bin2bcd(alrm->time.tm_min); + alarmvals[1] = bin2bcd(alrm->time.tm_hour); + alarmvals[2] = bin2bcd(alrm->time.tm_mday); + + ret = regmap_update_bits(rv3032->regmap, RV3032_STATUS, + RV3032_STATUS_AF, 0); + if (ret) + return ret; + + ret = regmap_bulk_write(rv3032->regmap, RV3032_ALARM_MIN, alarmvals, + sizeof(alarmvals)); + if (ret) + return ret; + + if (alrm->enabled) { + if (rv3032->rtc->uie_rtctimer.enabled) + ctrl |= RV3032_CTRL2_UIE; + if (rv3032->rtc->aie_timer.enabled) + ctrl |= RV3032_CTRL2_AIE; + } + + ret = regmap_update_bits(rv3032->regmap, RV3032_CTRL2, + RV3032_CTRL2_UIE | RV3032_CTRL2_AIE, ctrl); + + return ret; +} + +static int rv3032_alarm_irq_enable(struct device *dev, unsigned int enabled) +{ + struct rv3032_data *rv3032 = dev_get_drvdata(dev); + int ctrl = 0, ret; + + if (enabled) { + if (rv3032->rtc->uie_rtctimer.enabled) + ctrl |= RV3032_CTRL2_UIE; + if (rv3032->rtc->aie_timer.enabled) + ctrl |= RV3032_CTRL2_AIE; + } + + ret = regmap_update_bits(rv3032->regmap, RV3032_STATUS, + RV3032_STATUS_AF | RV3032_STATUS_UF, 0); + if (ret) + return ret; + + ret = regmap_update_bits(rv3032->regmap, RV3032_CTRL2, + RV3032_CTRL2_UIE | RV3032_CTRL2_AIE, ctrl); + if (ret) + return ret; + + return 0; +} + +static int rv3032_read_offset(struct device *dev, long *offset) +{ + struct rv3032_data *rv3032 = dev_get_drvdata(dev); + int ret, value, steps; + + ret = regmap_read(rv3032->regmap, RV3032_OFFSET, &value); + if (ret < 0) + return ret; + + steps = sign_extend32(FIELD_GET(RV3032_OFFSET_MSK, value), 5); + + *offset = DIV_ROUND_CLOSEST(steps * OFFSET_STEP_PPT, 1000); + + return 0; +} + +static int rv3032_set_offset(struct device *dev, long offset) +{ + struct rv3032_data *rv3032 = dev_get_drvdata(dev); + + offset = clamp(offset, -7629L, 7391L) * 1000; + offset = DIV_ROUND_CLOSEST(offset, OFFSET_STEP_PPT); + + return rv3032_update_cfg(rv3032, RV3032_OFFSET, RV3032_OFFSET_MSK, + FIELD_PREP(RV3032_OFFSET_MSK, offset)); +} + +static int rv3032_ioctl(struct device *dev, unsigned int cmd, unsigned long arg) +{ + struct rv3032_data *rv3032 = dev_get_drvdata(dev); + int status, val = 0, ret = 0; + + switch (cmd) { + case RTC_VL_READ: + ret = regmap_read(rv3032->regmap, RV3032_STATUS, &status); + if (ret < 0) + return ret; + + if (status & (RV3032_STATUS_PORF | RV3032_STATUS_VLF)) + val = RTC_VL_DATA_INVALID; + return put_user(val, (unsigned int __user *)arg); + + default: + return -ENOIOCTLCMD; + } +} + +static int rv3032_nvram_write(void *priv, unsigned int offset, void *val, size_t bytes) +{ + return regmap_bulk_write(priv, RV3032_RAM1 + offset, val, bytes); +} + +static int rv3032_nvram_read(void *priv, unsigned int offset, void *val, size_t bytes) +{ + return regmap_bulk_read(priv, RV3032_RAM1 + offset, val, bytes); +} + +static int rv3032_eeprom_write(void *priv, unsigned int offset, void *val, size_t bytes) +{ + struct rv3032_data *rv3032 = priv; + u32 status, eerd; + int i, ret; + u8 *buf = val; + + ret = rv3032_enter_eerd(rv3032, &eerd); + if (ret) + return ret; + + for (i = 0; i < bytes; i++) { + ret = regmap_write(rv3032->regmap, RV3032_EEPROM_ADDR, + RV3032_EEPROM_USER + offset + i); + if (ret) + goto exit_eerd; + + ret = regmap_write(rv3032->regmap, RV3032_EEPROM_DATA, buf[i]); + if (ret) + goto exit_eerd; + + ret = regmap_write(rv3032->regmap, RV3032_EEPROM_CMD, + RV3032_EEPROM_CMD_WRITE); + if (ret) + goto exit_eerd; + + usleep_range(RV3032_EEBUSY_POLL, RV3032_EEBUSY_TIMEOUT); + + ret = regmap_read_poll_timeout(rv3032->regmap, RV3032_TLSB, status, + !(status & RV3032_TLSB_EEBUSY), + RV3032_EEBUSY_POLL, RV3032_EEBUSY_TIMEOUT); + if (ret) + goto exit_eerd; + } + +exit_eerd: + rv3032_exit_eerd(rv3032, eerd); + + return ret; +} + +static int rv3032_eeprom_read(void *priv, unsigned int offset, void *val, size_t bytes) +{ + struct rv3032_data *rv3032 = priv; + u32 status, eerd, data; + int i, ret; + u8 *buf = val; + + ret = rv3032_enter_eerd(rv3032, &eerd); + if (ret) + return ret; + + for (i = 0; i < bytes; i++) { + ret = regmap_write(rv3032->regmap, RV3032_EEPROM_ADDR, + RV3032_EEPROM_USER + offset + i); + if (ret) + goto exit_eerd; + + ret = regmap_write(rv3032->regmap, RV3032_EEPROM_CMD, + RV3032_EEPROM_CMD_READ); + if (ret) + goto exit_eerd; + + ret = regmap_read_poll_timeout(rv3032->regmap, RV3032_TLSB, status, + !(status & RV3032_TLSB_EEBUSY), + RV3032_EEBUSY_POLL, RV3032_EEBUSY_TIMEOUT); + if (ret) + goto exit_eerd; + + ret = regmap_read(rv3032->regmap, RV3032_EEPROM_DATA, &data); + if (ret) + goto exit_eerd; + buf[i] = data; + } + +exit_eerd: + rv3032_exit_eerd(rv3032, eerd); + + return ret; +} + +static int rv3032_trickle_charger_setup(struct device *dev, struct rv3032_data *rv3032) +{ + u32 val, ohms, voltage; + int i; + + val = FIELD_PREP(RV3032_PMU_TCM, 1) | FIELD_PREP(RV3032_PMU_BSM, RV3032_PMU_BSM_DSM); + if (!device_property_read_u32(dev, "trickle-voltage-millivolt", &voltage)) { + for (i = 0; i < ARRAY_SIZE(rv3032_trickle_voltages); i++) + if (voltage == rv3032_trickle_voltages[i]) + break; + if (i < ARRAY_SIZE(rv3032_trickle_voltages)) + val = FIELD_PREP(RV3032_PMU_TCM, i) | + FIELD_PREP(RV3032_PMU_BSM, RV3032_PMU_BSM_LSM); + } + + if (device_property_read_u32(dev, "trickle-resistor-ohms", &ohms)) + return 0; + + for (i = 0; i < ARRAY_SIZE(rv3032_trickle_resistors); i++) + if (ohms == rv3032_trickle_resistors[i]) + break; + + if (i >= ARRAY_SIZE(rv3032_trickle_resistors)) { + dev_warn(dev, "invalid trickle resistor value\n"); + + return 0; + } + + return rv3032_update_cfg(rv3032, RV3032_PMU, + RV3032_PMU_TCR | RV3032_PMU_TCM | RV3032_PMU_BSM, + val | FIELD_PREP(RV3032_PMU_TCR, i)); +} + +#ifdef CONFIG_COMMON_CLK +#define clkout_hw_to_rv3032(hw) container_of(hw, struct rv3032_data, clkout_hw) + +static int clkout_xtal_rates[] = { + 32768, + 1024, + 64, + 1, +}; + +#define RV3032_HFD_STEP 8192 + +static unsigned long rv3032_clkout_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + int clkout, ret; + struct rv3032_data *rv3032 = clkout_hw_to_rv3032(hw); + + ret = regmap_read(rv3032->regmap, RV3032_CLKOUT2, &clkout); + if (ret < 0) + return 0; + + if (clkout & RV3032_CLKOUT2_OS) { + unsigned long rate = FIELD_GET(RV3032_CLKOUT2_HFD_MSK, clkout) << 8; + + ret = regmap_read(rv3032->regmap, RV3032_CLKOUT1, &clkout); + if (ret < 0) + return 0; + + rate += clkout + 1; + + return rate * RV3032_HFD_STEP; + } + + return clkout_xtal_rates[FIELD_GET(RV3032_CLKOUT2_FD_MSK, clkout)]; +} + +static long rv3032_clkout_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *prate) +{ + int i, hfd; + + if (rate < RV3032_HFD_STEP) + for (i = 0; i < ARRAY_SIZE(clkout_xtal_rates); i++) + if (clkout_xtal_rates[i] <= rate) + return clkout_xtal_rates[i]; + + hfd = DIV_ROUND_CLOSEST(rate, RV3032_HFD_STEP); + + return RV3032_HFD_STEP * clamp(hfd, 0, 8192); +} + +static int rv3032_clkout_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct rv3032_data *rv3032 = clkout_hw_to_rv3032(hw); + u32 status, eerd; + int i, hfd, ret; + + for (i = 0; i < ARRAY_SIZE(clkout_xtal_rates); i++) { + if (clkout_xtal_rates[i] == rate) { + return rv3032_update_cfg(rv3032, RV3032_CLKOUT2, 0xff, + FIELD_PREP(RV3032_CLKOUT2_FD_MSK, i)); + } + } + + hfd = DIV_ROUND_CLOSEST(rate, RV3032_HFD_STEP); + hfd = clamp(hfd, 1, 8192) - 1; + + ret = rv3032_enter_eerd(rv3032, &eerd); + if (ret) + goto exit_eerd; + + ret = regmap_write(rv3032->regmap, RV3032_CLKOUT1, hfd & 0xff); + if (ret) + return ret; + + ret = regmap_write(rv3032->regmap, RV3032_CLKOUT2, RV3032_CLKOUT2_OS | + FIELD_PREP(RV3032_CLKOUT2_HFD_MSK, hfd >> 8)); + if (ret) + goto exit_eerd; + + ret = regmap_write(rv3032->regmap, RV3032_EEPROM_CMD, RV3032_EEPROM_CMD_UPDATE); + if (ret) + goto exit_eerd; + + usleep_range(46000, RV3032_EEBUSY_TIMEOUT); + + ret = regmap_read_poll_timeout(rv3032->regmap, RV3032_TLSB, status, + !(status & RV3032_TLSB_EEBUSY), + RV3032_EEBUSY_POLL, RV3032_EEBUSY_TIMEOUT); + +exit_eerd: + rv3032_exit_eerd(rv3032, eerd); + + return ret; +} + +static int rv3032_clkout_prepare(struct clk_hw *hw) +{ + struct rv3032_data *rv3032 = clkout_hw_to_rv3032(hw); + + return rv3032_update_cfg(rv3032, RV3032_PMU, RV3032_PMU_NCLKE, 0); +} + +static void rv3032_clkout_unprepare(struct clk_hw *hw) +{ + struct rv3032_data *rv3032 = clkout_hw_to_rv3032(hw); + + rv3032_update_cfg(rv3032, RV3032_PMU, RV3032_PMU_NCLKE, RV3032_PMU_NCLKE); +} + +static int rv3032_clkout_is_prepared(struct clk_hw *hw) +{ + int val, ret; + struct rv3032_data *rv3032 = clkout_hw_to_rv3032(hw); + + ret = regmap_read(rv3032->regmap, RV3032_PMU, &val); + if (ret < 0) + return ret; + + return !(val & RV3032_PMU_NCLKE); +} + +static const struct clk_ops rv3032_clkout_ops = { + .prepare = rv3032_clkout_prepare, + .unprepare = rv3032_clkout_unprepare, + .is_prepared = rv3032_clkout_is_prepared, + .recalc_rate = rv3032_clkout_recalc_rate, + .round_rate = rv3032_clkout_round_rate, + .set_rate = rv3032_clkout_set_rate, +}; + +static int rv3032_clkout_register_clk(struct rv3032_data *rv3032, + struct i2c_client *client) +{ + int ret; + struct clk *clk; + struct clk_init_data init; + struct device_node *node = client->dev.of_node; + + ret = regmap_update_bits(rv3032->regmap, RV3032_TLSB, RV3032_TLSB_CLKF, 0); + if (ret < 0) + return ret; + + ret = regmap_update_bits(rv3032->regmap, RV3032_CTRL2, RV3032_CTRL2_CLKIE, 0); + if (ret < 0) + return ret; + + ret = regmap_write(rv3032->regmap, RV3032_CLK_IRQ, 0); + if (ret < 0) + return ret; + + init.name = "rv3032-clkout"; + init.ops = &rv3032_clkout_ops; + init.flags = 0; + init.parent_names = NULL; + init.num_parents = 0; + rv3032->clkout_hw.init = &init; + + of_property_read_string(node, "clock-output-names", &init.name); + + clk = devm_clk_register(&client->dev, &rv3032->clkout_hw); + if (!IS_ERR(clk)) + of_clk_add_provider(node, of_clk_src_simple_get, clk); + + return 0; +} +#endif + +static int rv3032_hwmon_read_temp(struct device *dev, long *mC) +{ + struct rv3032_data *rv3032 = dev_get_drvdata(dev); + u8 buf[2]; + int temp, prev = 0; + int ret; + + ret = regmap_bulk_read(rv3032->regmap, RV3032_TLSB, buf, sizeof(buf)); + if (ret) + return ret; + + temp = sign_extend32(buf[1], 7) << 4; + temp |= FIELD_GET(RV3032_TLSB_TEMP, buf[0]); + + /* No blocking or shadowing on RV3032_TLSB and RV3032_TMSB */ + do { + prev = temp; + + ret = regmap_bulk_read(rv3032->regmap, RV3032_TLSB, buf, sizeof(buf)); + if (ret) + return ret; + + temp = sign_extend32(buf[1], 7) << 4; + temp |= FIELD_GET(RV3032_TLSB_TEMP, buf[0]); + } while (temp != prev); + + *mC = (temp * 1000) / 16; + + return 0; +} + +static umode_t rv3032_hwmon_is_visible(const void *data, enum hwmon_sensor_types type, + u32 attr, int channel) +{ + if (type != hwmon_temp) + return 0; + + switch (attr) { + case hwmon_temp_input: + return 0444; + default: + return 0; + } +} + +static int rv3032_hwmon_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *temp) +{ + int err; + + switch (attr) { + case hwmon_temp_input: + err = rv3032_hwmon_read_temp(dev, temp); + break; + default: + err = -EOPNOTSUPP; + break; + } + + return err; +} + +static const struct hwmon_channel_info *rv3032_hwmon_info[] = { + HWMON_CHANNEL_INFO(chip, HWMON_C_REGISTER_TZ), + HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_MAX_HYST), + NULL +}; + +static const struct hwmon_ops rv3032_hwmon_hwmon_ops = { + .is_visible = rv3032_hwmon_is_visible, + .read = rv3032_hwmon_read, +}; + +static const struct hwmon_chip_info rv3032_hwmon_chip_info = { + .ops = &rv3032_hwmon_hwmon_ops, + .info = rv3032_hwmon_info, +}; + +static void rv3032_hwmon_register(struct device *dev) +{ + struct rv3032_data *rv3032 = dev_get_drvdata(dev); + + if (!IS_REACHABLE(CONFIG_HWMON)) + return; + + devm_hwmon_device_register_with_info(dev, "rv3032", rv3032, &rv3032_hwmon_chip_info, NULL); +} + +static struct rtc_class_ops rv3032_rtc_ops = { + .read_time = rv3032_get_time, + .set_time = rv3032_set_time, + .read_offset = rv3032_read_offset, + .set_offset = rv3032_set_offset, + .ioctl = rv3032_ioctl, +}; + +static const struct regmap_config regmap_config = { + .reg_bits = 8, + .val_bits = 8, + .max_register = 0xCA, +}; + +static int rv3032_probe(struct i2c_client *client) +{ + struct rv3032_data *rv3032; + int ret, status; + struct nvmem_config nvmem_cfg = { + .name = "rv3032_nvram", + .word_size = 1, + .stride = 1, + .size = 16, + .type = NVMEM_TYPE_BATTERY_BACKED, + .reg_read = rv3032_nvram_read, + .reg_write = rv3032_nvram_write, + }; + struct nvmem_config eeprom_cfg = { + .name = "rv3032_eeprom", + .word_size = 1, + .stride = 1, + .size = 32, + .type = NVMEM_TYPE_EEPROM, + .reg_read = rv3032_eeprom_read, + .reg_write = rv3032_eeprom_write, + }; + + rv3032 = devm_kzalloc(&client->dev, sizeof(struct rv3032_data), + GFP_KERNEL); + if (!rv3032) + return -ENOMEM; + + rv3032->regmap = devm_regmap_init_i2c(client, ®map_config); + if (IS_ERR(rv3032->regmap)) + return PTR_ERR(rv3032->regmap); + + i2c_set_clientdata(client, rv3032); + + ret = regmap_read(rv3032->regmap, RV3032_STATUS, &status); + if (ret < 0) + return ret; + + rv3032->rtc = devm_rtc_allocate_device(&client->dev); + if (IS_ERR(rv3032->rtc)) + return PTR_ERR(rv3032->rtc); + + if (client->irq > 0) { + ret = devm_request_threaded_irq(&client->dev, client->irq, + NULL, rv3032_handle_irq, + IRQF_TRIGGER_LOW | IRQF_ONESHOT, + "rv3032", rv3032); + if (ret) { + dev_warn(&client->dev, "unable to request IRQ, alarms disabled\n"); + client->irq = 0; + } else { + rv3032_rtc_ops.read_alarm = rv3032_get_alarm; + rv3032_rtc_ops.set_alarm = rv3032_set_alarm; + rv3032_rtc_ops.alarm_irq_enable = rv3032_alarm_irq_enable; + } + } + + ret = regmap_update_bits(rv3032->regmap, RV3032_CTRL1, + RV3032_CTRL1_WADA, RV3032_CTRL1_WADA); + if (ret) + return ret; + + rv3032_trickle_charger_setup(&client->dev, rv3032); + + rv3032->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000; + rv3032->rtc->range_max = RTC_TIMESTAMP_END_2099; + rv3032->rtc->ops = &rv3032_rtc_ops; + ret = rtc_register_device(rv3032->rtc); + if (ret) + return ret; + + nvmem_cfg.priv = rv3032; + rtc_nvmem_register(rv3032->rtc, &nvmem_cfg); + eeprom_cfg.priv = rv3032; + rtc_nvmem_register(rv3032->rtc, &eeprom_cfg); + + rv3032->rtc->max_user_freq = 1; + +#ifdef CONFIG_COMMON_CLK + rv3032_clkout_register_clk(rv3032, client); +#endif + + rv3032_hwmon_register(&client->dev); + + return 0; +} + +static const struct of_device_id rv3032_of_match[] = { + { .compatible = "microcrystal,rv3032", }, + { } +}; +MODULE_DEVICE_TABLE(of, rv3032_of_match); + +static struct i2c_driver rv3032_driver = { + .driver = { + .name = "rtc-rv3032", + .of_match_table = of_match_ptr(rv3032_of_match), + }, + .probe_new = rv3032_probe, +}; +module_i2c_driver(rv3032_driver); + +MODULE_AUTHOR("Alexandre Belloni <alexandre.belloni@bootlin.com>"); +MODULE_DESCRIPTION("Micro Crystal RV3032 RTC driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/rtc/rtc-rv8803.c b/drivers/rtc/rtc-rv8803.c index 93c3a6b627bd..c6d8e3425688 100644 --- a/drivers/rtc/rtc-rv8803.c +++ b/drivers/rtc/rtc-rv8803.c @@ -454,13 +454,7 @@ static int rv8803_ioctl(struct device *dev, unsigned int cmd, unsigned long arg) static int rv8803_nvram_write(void *priv, unsigned int offset, void *val, size_t bytes) { - int ret; - - ret = rv8803_write_reg(priv, RV8803_RAM, *(u8 *)val); - if (ret) - return ret; - - return 0; + return rv8803_write_reg(priv, RV8803_RAM, *(u8 *)val); } static int rv8803_nvram_read(void *priv, unsigned int offset, diff --git a/drivers/rtc/rtc-rx8010.c b/drivers/rtc/rtc-rx8010.c index fe010151ec8f..dca41a2a39b2 100644 --- a/drivers/rtc/rtc-rx8010.c +++ b/drivers/rtc/rtc-rx8010.c @@ -11,42 +11,43 @@ #include <linux/i2c.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/regmap.h> #include <linux/rtc.h> -#define RX8010_SEC 0x10 -#define RX8010_MIN 0x11 -#define RX8010_HOUR 0x12 -#define RX8010_WDAY 0x13 -#define RX8010_MDAY 0x14 -#define RX8010_MONTH 0x15 -#define RX8010_YEAR 0x16 -#define RX8010_RESV17 0x17 -#define RX8010_ALMIN 0x18 -#define RX8010_ALHOUR 0x19 -#define RX8010_ALWDAY 0x1A -#define RX8010_TCOUNT0 0x1B -#define RX8010_TCOUNT1 0x1C -#define RX8010_EXT 0x1D -#define RX8010_FLAG 0x1E -#define RX8010_CTRL 0x1F +#define RX8010_SEC 0x10 +#define RX8010_MIN 0x11 +#define RX8010_HOUR 0x12 +#define RX8010_WDAY 0x13 +#define RX8010_MDAY 0x14 +#define RX8010_MONTH 0x15 +#define RX8010_YEAR 0x16 +#define RX8010_RESV17 0x17 +#define RX8010_ALMIN 0x18 +#define RX8010_ALHOUR 0x19 +#define RX8010_ALWDAY 0x1A +#define RX8010_TCOUNT0 0x1B +#define RX8010_TCOUNT1 0x1C +#define RX8010_EXT 0x1D +#define RX8010_FLAG 0x1E +#define RX8010_CTRL 0x1F /* 0x20 to 0x2F are user registers */ -#define RX8010_RESV30 0x30 -#define RX8010_RESV31 0x31 -#define RX8010_IRQ 0x32 +#define RX8010_RESV30 0x30 +#define RX8010_RESV31 0x31 +#define RX8010_IRQ 0x32 -#define RX8010_EXT_WADA BIT(3) +#define RX8010_EXT_WADA BIT(3) -#define RX8010_FLAG_VLF BIT(1) -#define RX8010_FLAG_AF BIT(3) -#define RX8010_FLAG_TF BIT(4) -#define RX8010_FLAG_UF BIT(5) +#define RX8010_FLAG_VLF BIT(1) +#define RX8010_FLAG_AF BIT(3) +#define RX8010_FLAG_TF BIT(4) +#define RX8010_FLAG_UF BIT(5) -#define RX8010_CTRL_AIE BIT(3) -#define RX8010_CTRL_UIE BIT(5) -#define RX8010_CTRL_STOP BIT(6) -#define RX8010_CTRL_TEST BIT(7) +#define RX8010_CTRL_AIE BIT(3) +#define RX8010_CTRL_UIE BIT(5) +#define RX8010_CTRL_STOP BIT(6) +#define RX8010_CTRL_TEST BIT(7) -#define RX8010_ALARM_AE BIT(7) +#define RX8010_ALARM_AE BIT(7) static const struct i2c_device_id rx8010_id[] = { { "rx8010", 0 }, @@ -61,7 +62,7 @@ static const struct of_device_id rx8010_of_match[] = { MODULE_DEVICE_TABLE(of, rx8010_of_match); struct rx8010_data { - struct i2c_client *client; + struct regmap *regs; struct rtc_device *rtc; u8 ctrlreg; }; @@ -70,13 +71,12 @@ static irqreturn_t rx8010_irq_1_handler(int irq, void *dev_id) { struct i2c_client *client = dev_id; struct rx8010_data *rx8010 = i2c_get_clientdata(client); - int flagreg; + int flagreg, err; mutex_lock(&rx8010->rtc->ops_lock); - flagreg = i2c_smbus_read_byte_data(client, RX8010_FLAG); - - if (flagreg <= 0) { + err = regmap_read(rx8010->regs, RX8010_FLAG, &flagreg); + if (err) { mutex_unlock(&rx8010->rtc->ops_lock); return IRQ_NONE; } @@ -99,32 +99,29 @@ static irqreturn_t rx8010_irq_1_handler(int irq, void *dev_id) rtc_update_irq(rx8010->rtc, 1, RTC_UF | RTC_IRQF); } - i2c_smbus_write_byte_data(client, RX8010_FLAG, flagreg); - + err = regmap_write(rx8010->regs, RX8010_FLAG, flagreg); mutex_unlock(&rx8010->rtc->ops_lock); - return IRQ_HANDLED; + return err ? IRQ_NONE : IRQ_HANDLED; } static int rx8010_get_time(struct device *dev, struct rtc_time *dt) { struct rx8010_data *rx8010 = dev_get_drvdata(dev); - u8 date[7]; - int flagreg; - int err; + u8 date[RX8010_YEAR - RX8010_SEC + 1]; + int flagreg, err; - flagreg = i2c_smbus_read_byte_data(rx8010->client, RX8010_FLAG); - if (flagreg < 0) - return flagreg; + err = regmap_read(rx8010->regs, RX8010_FLAG, &flagreg); + if (err) + return err; if (flagreg & RX8010_FLAG_VLF) { dev_warn(dev, "Frequency stop detected\n"); return -EINVAL; } - err = i2c_smbus_read_i2c_block_data(rx8010->client, RX8010_SEC, - 7, date); - if (err != 7) - return err < 0 ? err : -EIO; + err = regmap_bulk_read(rx8010->regs, RX8010_SEC, date, sizeof(date)); + if (err) + return err; dt->tm_sec = bcd2bin(date[RX8010_SEC - RX8010_SEC] & 0x7f); dt->tm_min = bcd2bin(date[RX8010_MIN - RX8010_SEC] & 0x7f); @@ -140,22 +137,13 @@ static int rx8010_get_time(struct device *dev, struct rtc_time *dt) static int rx8010_set_time(struct device *dev, struct rtc_time *dt) { struct rx8010_data *rx8010 = dev_get_drvdata(dev); - u8 date[7]; - int ctrl, flagreg; - int ret; - - if ((dt->tm_year < 100) || (dt->tm_year > 199)) - return -EINVAL; + u8 date[RX8010_YEAR - RX8010_SEC + 1]; + int err; /* set STOP bit before changing clock/calendar */ - ctrl = i2c_smbus_read_byte_data(rx8010->client, RX8010_CTRL); - if (ctrl < 0) - return ctrl; - rx8010->ctrlreg = ctrl | RX8010_CTRL_STOP; - ret = i2c_smbus_write_byte_data(rx8010->client, RX8010_CTRL, - rx8010->ctrlreg); - if (ret < 0) - return ret; + err = regmap_set_bits(rx8010->regs, RX8010_CTRL, RX8010_CTRL_STOP); + if (err) + return err; date[RX8010_SEC - RX8010_SEC] = bin2bcd(dt->tm_sec); date[RX8010_MIN - RX8010_SEC] = bin2bcd(dt->tm_min); @@ -165,66 +153,54 @@ static int rx8010_set_time(struct device *dev, struct rtc_time *dt) date[RX8010_YEAR - RX8010_SEC] = bin2bcd(dt->tm_year - 100); date[RX8010_WDAY - RX8010_SEC] = bin2bcd(1 << dt->tm_wday); - ret = i2c_smbus_write_i2c_block_data(rx8010->client, - RX8010_SEC, 7, date); - if (ret < 0) - return ret; + err = regmap_bulk_write(rx8010->regs, RX8010_SEC, date, sizeof(date)); + if (err) + return err; /* clear STOP bit after changing clock/calendar */ - ctrl = i2c_smbus_read_byte_data(rx8010->client, RX8010_CTRL); - if (ctrl < 0) - return ctrl; - rx8010->ctrlreg = ctrl & ~RX8010_CTRL_STOP; - ret = i2c_smbus_write_byte_data(rx8010->client, RX8010_CTRL, - rx8010->ctrlreg); - if (ret < 0) - return ret; - - flagreg = i2c_smbus_read_byte_data(rx8010->client, RX8010_FLAG); - if (flagreg < 0) { - return flagreg; - } + err = regmap_clear_bits(rx8010->regs, RX8010_CTRL, RX8010_CTRL_STOP); + if (err) + return err; - if (flagreg & RX8010_FLAG_VLF) - ret = i2c_smbus_write_byte_data(rx8010->client, RX8010_FLAG, - flagreg & ~RX8010_FLAG_VLF); + err = regmap_clear_bits(rx8010->regs, RX8010_FLAG, RX8010_FLAG_VLF); + if (err) + return err; return 0; } -static int rx8010_init_client(struct i2c_client *client) +static int rx8010_init(struct device *dev) { - struct rx8010_data *rx8010 = i2c_get_clientdata(client); + struct rx8010_data *rx8010 = dev_get_drvdata(dev); u8 ctrl[2]; - int need_clear = 0, err = 0; + int need_clear = 0, err; /* Initialize reserved registers as specified in datasheet */ - err = i2c_smbus_write_byte_data(client, RX8010_RESV17, 0xD8); - if (err < 0) + err = regmap_write(rx8010->regs, RX8010_RESV17, 0xD8); + if (err) return err; - err = i2c_smbus_write_byte_data(client, RX8010_RESV30, 0x00); - if (err < 0) + err = regmap_write(rx8010->regs, RX8010_RESV30, 0x00); + if (err) return err; - err = i2c_smbus_write_byte_data(client, RX8010_RESV31, 0x08); - if (err < 0) + err = regmap_write(rx8010->regs, RX8010_RESV31, 0x08); + if (err) return err; - err = i2c_smbus_write_byte_data(client, RX8010_IRQ, 0x00); - if (err < 0) + err = regmap_write(rx8010->regs, RX8010_IRQ, 0x00); + if (err) return err; - err = i2c_smbus_read_i2c_block_data(rx8010->client, RX8010_FLAG, - 2, ctrl); - if (err != 2) - return err < 0 ? err : -EIO; + err = regmap_bulk_read(rx8010->regs, RX8010_FLAG, ctrl, 2); + if (err) + return err; if (ctrl[0] & RX8010_FLAG_VLF) - dev_warn(&client->dev, "Frequency stop was detected\n"); + dev_warn(dev, "Frequency stop was detected\n"); if (ctrl[0] & RX8010_FLAG_AF) { - dev_warn(&client->dev, "Alarm was detected\n"); + dev_warn(dev, "Alarm was detected\n"); need_clear = 1; } @@ -236,8 +212,8 @@ static int rx8010_init_client(struct i2c_client *client) if (need_clear) { ctrl[0] &= ~(RX8010_FLAG_AF | RX8010_FLAG_TF | RX8010_FLAG_UF); - err = i2c_smbus_write_byte_data(client, RX8010_FLAG, ctrl[0]); - if (err < 0) + err = regmap_write(rx8010->regs, RX8010_FLAG, ctrl[0]); + if (err) return err; } @@ -249,18 +225,16 @@ static int rx8010_init_client(struct i2c_client *client) static int rx8010_read_alarm(struct device *dev, struct rtc_wkalrm *t) { struct rx8010_data *rx8010 = dev_get_drvdata(dev); - struct i2c_client *client = rx8010->client; u8 alarmvals[3]; - int flagreg; - int err; + int flagreg, err; - err = i2c_smbus_read_i2c_block_data(client, RX8010_ALMIN, 3, alarmvals); - if (err != 3) - return err < 0 ? err : -EIO; + err = regmap_bulk_read(rx8010->regs, RX8010_ALMIN, alarmvals, 3); + if (err) + return err; - flagreg = i2c_smbus_read_byte_data(client, RX8010_FLAG); - if (flagreg < 0) - return flagreg; + err = regmap_read(rx8010->regs, RX8010_FLAG, &flagreg); + if (err) + return err; t->time.tm_sec = 0; t->time.tm_min = bcd2bin(alarmvals[0] & 0x7f); @@ -277,55 +251,38 @@ static int rx8010_read_alarm(struct device *dev, struct rtc_wkalrm *t) static int rx8010_set_alarm(struct device *dev, struct rtc_wkalrm *t) { - struct i2c_client *client = to_i2c_client(dev); struct rx8010_data *rx8010 = dev_get_drvdata(dev); u8 alarmvals[3]; - int extreg, flagreg; int err; - flagreg = i2c_smbus_read_byte_data(client, RX8010_FLAG); - if (flagreg < 0) { - return flagreg; - } - if (rx8010->ctrlreg & (RX8010_CTRL_AIE | RX8010_CTRL_UIE)) { rx8010->ctrlreg &= ~(RX8010_CTRL_AIE | RX8010_CTRL_UIE); - err = i2c_smbus_write_byte_data(rx8010->client, RX8010_CTRL, - rx8010->ctrlreg); - if (err < 0) { + err = regmap_write(rx8010->regs, RX8010_CTRL, rx8010->ctrlreg); + if (err) return err; - } } - flagreg &= ~RX8010_FLAG_AF; - err = i2c_smbus_write_byte_data(rx8010->client, RX8010_FLAG, flagreg); - if (err < 0) + err = regmap_clear_bits(rx8010->regs, RX8010_FLAG, RX8010_FLAG_AF); + if (err) return err; alarmvals[0] = bin2bcd(t->time.tm_min); alarmvals[1] = bin2bcd(t->time.tm_hour); alarmvals[2] = bin2bcd(t->time.tm_mday); - err = i2c_smbus_write_i2c_block_data(rx8010->client, RX8010_ALMIN, - 2, alarmvals); - if (err < 0) + err = regmap_bulk_write(rx8010->regs, RX8010_ALMIN, alarmvals, 2); + if (err) return err; - extreg = i2c_smbus_read_byte_data(client, RX8010_EXT); - if (extreg < 0) - return extreg; - - extreg |= RX8010_EXT_WADA; - err = i2c_smbus_write_byte_data(rx8010->client, RX8010_EXT, extreg); - if (err < 0) + err = regmap_clear_bits(rx8010->regs, RX8010_EXT, RX8010_EXT_WADA); + if (err) return err; if (alarmvals[2] == 0) alarmvals[2] |= RX8010_ALARM_AE; - err = i2c_smbus_write_byte_data(rx8010->client, RX8010_ALWDAY, - alarmvals[2]); - if (err < 0) + err = regmap_write(rx8010->regs, RX8010_ALWDAY, alarmvals[2]); + if (err) return err; if (t->enabled) { @@ -335,9 +292,8 @@ static int rx8010_set_alarm(struct device *dev, struct rtc_wkalrm *t) rx8010->ctrlreg |= (RX8010_CTRL_AIE | RX8010_CTRL_UIE); - err = i2c_smbus_write_byte_data(rx8010->client, RX8010_CTRL, - rx8010->ctrlreg); - if (err < 0) + err = regmap_write(rx8010->regs, RX8010_CTRL, rx8010->ctrlreg); + if (err) return err; } @@ -347,11 +303,9 @@ static int rx8010_set_alarm(struct device *dev, struct rtc_wkalrm *t) static int rx8010_alarm_irq_enable(struct device *dev, unsigned int enabled) { - struct i2c_client *client = to_i2c_client(dev); struct rx8010_data *rx8010 = dev_get_drvdata(dev); - int flagreg; - u8 ctrl; int err; + u8 ctrl; ctrl = rx8010->ctrlreg; @@ -367,20 +321,14 @@ static int rx8010_alarm_irq_enable(struct device *dev, ctrl &= ~RX8010_CTRL_AIE; } - flagreg = i2c_smbus_read_byte_data(client, RX8010_FLAG); - if (flagreg < 0) - return flagreg; - - flagreg &= ~RX8010_FLAG_AF; - err = i2c_smbus_write_byte_data(rx8010->client, RX8010_FLAG, flagreg); - if (err < 0) + err = regmap_clear_bits(rx8010->regs, RX8010_FLAG, RX8010_FLAG_AF); + if (err) return err; if (ctrl != rx8010->ctrlreg) { rx8010->ctrlreg = ctrl; - err = i2c_smbus_write_byte_data(rx8010->client, RX8010_CTRL, - rx8010->ctrlreg); - if (err < 0) + err = regmap_write(rx8010->regs, RX8010_CTRL, rx8010->ctrlreg); + if (err) return err; } @@ -390,14 +338,13 @@ static int rx8010_alarm_irq_enable(struct device *dev, static int rx8010_ioctl(struct device *dev, unsigned int cmd, unsigned long arg) { struct rx8010_data *rx8010 = dev_get_drvdata(dev); - int tmp; - int flagreg; + int tmp, flagreg, err; switch (cmd) { case RTC_VL_READ: - flagreg = i2c_smbus_read_byte_data(rx8010->client, RX8010_FLAG); - if (flagreg < 0) - return flagreg; + err = regmap_read(rx8010->regs, RX8010_FLAG, &flagreg); + if (err) + return err; tmp = flagreg & RX8010_FLAG_VLF ? RTC_VL_DATA_INVALID : 0; return put_user(tmp, (unsigned int __user *)arg); @@ -407,65 +354,72 @@ static int rx8010_ioctl(struct device *dev, unsigned int cmd, unsigned long arg) } } -static struct rtc_class_ops rx8010_rtc_ops = { +static const struct rtc_class_ops rx8010_rtc_ops_default = { + .read_time = rx8010_get_time, + .set_time = rx8010_set_time, + .ioctl = rx8010_ioctl, +}; + +static const struct rtc_class_ops rx8010_rtc_ops_alarm = { .read_time = rx8010_get_time, .set_time = rx8010_set_time, .ioctl = rx8010_ioctl, + .read_alarm = rx8010_read_alarm, + .set_alarm = rx8010_set_alarm, + .alarm_irq_enable = rx8010_alarm_irq_enable, +}; + +static const struct regmap_config rx8010_regmap_config = { + .name = "rx8010-rtc", + .reg_bits = 8, + .val_bits = 8, }; -static int rx8010_probe(struct i2c_client *client, - const struct i2c_device_id *id) +static int rx8010_probe(struct i2c_client *client) { - struct i2c_adapter *adapter = client->adapter; + struct device *dev = &client->dev; struct rx8010_data *rx8010; int err = 0; - if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE_DATA - | I2C_FUNC_SMBUS_I2C_BLOCK)) { - dev_err(&adapter->dev, "doesn't support required functionality\n"); - return -EIO; - } - - rx8010 = devm_kzalloc(&client->dev, sizeof(struct rx8010_data), - GFP_KERNEL); + rx8010 = devm_kzalloc(dev, sizeof(*rx8010), GFP_KERNEL); if (!rx8010) return -ENOMEM; - rx8010->client = client; i2c_set_clientdata(client, rx8010); - err = rx8010_init_client(client); + rx8010->regs = devm_regmap_init_i2c(client, &rx8010_regmap_config); + if (IS_ERR(rx8010->regs)) + return PTR_ERR(rx8010->regs); + + err = rx8010_init(dev); if (err) return err; + rx8010->rtc = devm_rtc_allocate_device(dev); + if (IS_ERR(rx8010->rtc)) + return PTR_ERR(rx8010->rtc); + if (client->irq > 0) { - dev_info(&client->dev, "IRQ %d supplied\n", client->irq); - err = devm_request_threaded_irq(&client->dev, client->irq, NULL, + dev_info(dev, "IRQ %d supplied\n", client->irq); + err = devm_request_threaded_irq(dev, client->irq, NULL, rx8010_irq_1_handler, IRQF_TRIGGER_LOW | IRQF_ONESHOT, "rx8010", client); - if (err) { - dev_err(&client->dev, "unable to request IRQ\n"); - client->irq = 0; - } else { - rx8010_rtc_ops.read_alarm = rx8010_read_alarm; - rx8010_rtc_ops.set_alarm = rx8010_set_alarm; - rx8010_rtc_ops.alarm_irq_enable = rx8010_alarm_irq_enable; + dev_err(dev, "unable to request IRQ\n"); + return err; } - } - rx8010->rtc = devm_rtc_device_register(&client->dev, client->name, - &rx8010_rtc_ops, THIS_MODULE); - - if (IS_ERR(rx8010->rtc)) { - dev_err(&client->dev, "unable to register the class device\n"); - return PTR_ERR(rx8010->rtc); + rx8010->rtc->ops = &rx8010_rtc_ops_alarm; + } else { + rx8010->rtc->ops = &rx8010_rtc_ops_default; } rx8010->rtc->max_user_freq = 1; + rx8010->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000; + rx8010->rtc->range_max = RTC_TIMESTAMP_END_2099; - return 0; + return rtc_register_device(rx8010->rtc); } static struct i2c_driver rx8010_driver = { @@ -473,7 +427,7 @@ static struct i2c_driver rx8010_driver = { .name = "rtc-rx8010", .of_match_table = of_match_ptr(rx8010_of_match), }, - .probe = rx8010_probe, + .probe_new = rx8010_probe, .id_table = rx8010_id, }; diff --git a/drivers/rtc/rtc-s3c.c b/drivers/rtc/rtc-s3c.c index e1b50e682fc4..24a41909f049 100644 --- a/drivers/rtc/rtc-s3c.c +++ b/drivers/rtc/rtc-s3c.c @@ -494,13 +494,8 @@ static int s3c_rtc_probe(struct platform_device *pdev) if (info->data->needs_src_clk) { info->rtc_src_clk = devm_clk_get(&pdev->dev, "rtc_src"); if (IS_ERR(info->rtc_src_clk)) { - ret = PTR_ERR(info->rtc_src_clk); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, - "failed to find rtc source clock\n"); - else - dev_dbg(&pdev->dev, - "probe deferred due to missing rtc src clk\n"); + ret = dev_err_probe(&pdev->dev, PTR_ERR(info->rtc_src_clk), + "failed to find rtc source clock\n"); goto err_src_clk; } ret = clk_prepare_enable(info->rtc_src_clk); diff --git a/drivers/rtc/rtc-st-lpc.c b/drivers/rtc/rtc-st-lpc.c index 51041dc08af4..0c65448b85ee 100644 --- a/drivers/rtc/rtc-st-lpc.c +++ b/drivers/rtc/rtc-st-lpc.c @@ -173,7 +173,7 @@ static int st_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *t) return 0; } -static struct rtc_class_ops st_rtc_ops = { +static const struct rtc_class_ops st_rtc_ops = { .read_time = st_rtc_read_time, .set_time = st_rtc_set_time, .read_alarm = st_rtc_read_alarm, diff --git a/drivers/s390/char/Makefile b/drivers/s390/char/Makefile index 845e12ac5954..c6fdb81a068a 100644 --- a/drivers/s390/char/Makefile +++ b/drivers/s390/char/Makefile @@ -34,6 +34,8 @@ obj-$(CONFIG_SCLP_VT220_TTY) += sclp_vt220.o obj-$(CONFIG_PCI) += sclp_pci.o +obj-$(subst m,y,$(CONFIG_ZCRYPT)) += sclp_ap.o + obj-$(CONFIG_VMLOGRDR) += vmlogrdr.o obj-$(CONFIG_VMCP) += vmcp.o diff --git a/drivers/s390/char/con3215.c b/drivers/s390/char/con3215.c index 92757f9bd010..d8acabbb1ed3 100644 --- a/drivers/s390/char/con3215.c +++ b/drivers/s390/char/con3215.c @@ -978,7 +978,6 @@ static int tty3215_install(struct tty_driver *driver, struct tty_struct *tty) static int tty3215_open(struct tty_struct *tty, struct file * filp) { struct raw3215_info *raw = tty->driver_data; - int retval; tty_port_tty_set(&raw->port, tty); @@ -986,11 +985,7 @@ static int tty3215_open(struct tty_struct *tty, struct file * filp) /* * Start up 3215 device */ - retval = raw3215_startup(raw); - if (retval) - return retval; - - return 0; + return raw3215_startup(raw); } /* diff --git a/drivers/s390/char/raw3270.h b/drivers/s390/char/raw3270.h index 08f36e973b43..8d979e0ee605 100644 --- a/drivers/s390/char/raw3270.h +++ b/drivers/s390/char/raw3270.h @@ -110,7 +110,6 @@ struct raw3270_request { }; struct raw3270_request *raw3270_request_alloc(size_t size); -struct raw3270_request *raw3270_request_alloc_bootmem(size_t size); void raw3270_request_free(struct raw3270_request *); void raw3270_request_reset(struct raw3270_request *); void raw3270_request_set_cmd(struct raw3270_request *, u8 cmd); diff --git a/drivers/s390/char/sclp.h b/drivers/s390/char/sclp.h index 196333013e54..69d9cde9ff5a 100644 --- a/drivers/s390/char/sclp.h +++ b/drivers/s390/char/sclp.h @@ -229,7 +229,7 @@ static inline void sclp_fill_core_info(struct sclp_core_info *info, #define SCLP_HAS_CPU_INFO (sclp.facilities & 0x0800000000000000ULL) #define SCLP_HAS_CPU_RECONFIG (sclp.facilities & 0x0400000000000000ULL) #define SCLP_HAS_PCI_RECONFIG (sclp.facilities & 0x0000000040000000ULL) - +#define SCLP_HAS_AP_RECONFIG (sclp.facilities & 0x0000000100000000ULL) struct gds_subvector { u8 length; @@ -305,9 +305,7 @@ int sclp_deactivate(void); int sclp_reactivate(void); int sclp_sync_request(sclp_cmdw_t command, void *sccb); int sclp_sync_request_timeout(sclp_cmdw_t command, void *sccb, int timeout); - int sclp_sdias_init(void); -void sclp_sdias_exit(void); enum { sclp_init_state_uninitialized, diff --git a/drivers/s390/char/sclp_ap.c b/drivers/s390/char/sclp_ap.c new file mode 100644 index 000000000000..0dd1ca712795 --- /dev/null +++ b/drivers/s390/char/sclp_ap.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * s390 crypto adapter related sclp functions. + * + * Copyright IBM Corp. 2020 + */ +#define KMSG_COMPONENT "sclp_cmd" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +#include <linux/export.h> +#include <linux/slab.h> +#include <asm/sclp.h> +#include "sclp.h" + +#define SCLP_CMDW_CONFIGURE_AP 0x001f0001 +#define SCLP_CMDW_DECONFIGURE_AP 0x001e0001 + +struct ap_cfg_sccb { + struct sccb_header header; +} __packed; + +static int do_ap_configure(sclp_cmdw_t cmd, u32 apid) +{ + struct ap_cfg_sccb *sccb; + int rc; + + if (!SCLP_HAS_AP_RECONFIG) + return -EOPNOTSUPP; + + sccb = (struct ap_cfg_sccb *) get_zeroed_page(GFP_KERNEL | GFP_DMA); + if (!sccb) + return -ENOMEM; + + sccb->header.length = PAGE_SIZE; + cmd |= (apid & 0xFF) << 8; + rc = sclp_sync_request(cmd, sccb); + if (rc) + goto out; + switch (sccb->header.response_code) { + case 0x0020: case 0x0120: case 0x0440: case 0x0450: + break; + default: + pr_warn("configure AP adapter %u failed: cmd=0x%08x response=0x%04x\n", + apid, cmd, sccb->header.response_code); + rc = -EIO; + break; + } +out: + free_page((unsigned long) sccb); + return rc; +} + +int sclp_ap_configure(u32 apid) +{ + return do_ap_configure(SCLP_CMDW_CONFIGURE_AP, apid); +} +EXPORT_SYMBOL(sclp_ap_configure); + +int sclp_ap_deconfigure(u32 apid) +{ + return do_ap_configure(SCLP_CMDW_DECONFIGURE_AP, apid); +} +EXPORT_SYMBOL(sclp_ap_deconfigure); diff --git a/drivers/s390/char/sclp_cmd.c b/drivers/s390/char/sclp_cmd.c index a864b21af602..f6e97f0830f6 100644 --- a/drivers/s390/char/sclp_cmd.c +++ b/drivers/s390/char/sclp_cmd.c @@ -406,7 +406,7 @@ static void __init add_memory_merged(u16 rn) if (!size) goto skip_add; for (addr = start; addr < start + size; addr += block_size) - add_memory(0, addr, block_size); + add_memory(0, addr, block_size, MHP_NONE); skip_add: first_rn = rn; num = 1; diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index 7737470f8498..a960afa974bf 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -17,12 +17,12 @@ static struct read_info_sccb __bootdata(sclp_info_sccb); static int __bootdata(sclp_info_sccb_valid); char *sclp_early_sccb = (char *) EARLY_SCCB_OFFSET; -int sclp_init_state __section(.data) = sclp_init_state_uninitialized; +int sclp_init_state = sclp_init_state_uninitialized; /* * Used to keep track of the size of the event masks. Qemu until version 2.11 * only supports 4 and needs a workaround. */ -bool sclp_mask_compat_mode __section(.data); +bool sclp_mask_compat_mode; void sclp_early_wait_irq(void) { @@ -214,11 +214,11 @@ static int sclp_early_setup(int disable, int *have_linemode, int *have_vt220) * Output one or more lines of text on the SCLP console (VT220 and / * or line-mode). */ -void __sclp_early_printk(const char *str, unsigned int len, unsigned int force) +void __sclp_early_printk(const char *str, unsigned int len) { int have_linemode, have_vt220; - if (!force && sclp_init_state != sclp_init_state_uninitialized) + if (sclp_init_state != sclp_init_state_uninitialized) return; if (sclp_early_setup(0, &have_linemode, &have_vt220) != 0) return; @@ -231,12 +231,7 @@ void __sclp_early_printk(const char *str, unsigned int len, unsigned int force) void sclp_early_printk(const char *str) { - __sclp_early_printk(str, strlen(str), 0); -} - -void sclp_early_printk_force(const char *str) -{ - __sclp_early_printk(str, strlen(str), 1); + __sclp_early_printk(str, strlen(str)); } int __init sclp_early_read_info(void) diff --git a/drivers/s390/char/sclp_rw.c b/drivers/s390/char/sclp_rw.c index 44594a492553..d6c84e354df5 100644 --- a/drivers/s390/char/sclp_rw.c +++ b/drivers/s390/char/sclp_rw.c @@ -337,24 +337,6 @@ sclp_chars_in_buffer(struct sclp_buffer *buffer) } /* - * sets or provides some values that influence the drivers behaviour - */ -void -sclp_set_columns(struct sclp_buffer *buffer, unsigned short columns) -{ - buffer->columns = columns; - if (buffer->current_line != NULL && - buffer->current_length > buffer->columns) - sclp_finalize_mto(buffer); -} - -void -sclp_set_htab(struct sclp_buffer *buffer, unsigned short htab) -{ - buffer->htab = htab; -} - -/* * called by sclp_console_init and/or sclp_tty_init */ int diff --git a/drivers/s390/char/sclp_rw.h b/drivers/s390/char/sclp_rw.h index a2eb22f67393..93d706e4935c 100644 --- a/drivers/s390/char/sclp_rw.h +++ b/drivers/s390/char/sclp_rw.h @@ -86,8 +86,6 @@ void *sclp_unmake_buffer(struct sclp_buffer *); int sclp_buffer_space(struct sclp_buffer *); int sclp_write(struct sclp_buffer *buffer, const unsigned char *, int); int sclp_emit_buffer(struct sclp_buffer *,void (*)(struct sclp_buffer *,int)); -void sclp_set_columns(struct sclp_buffer *, unsigned short); -void sclp_set_htab(struct sclp_buffer *, unsigned short); int sclp_chars_in_buffer(struct sclp_buffer *); #ifdef CONFIG_SCLP_CONSOLE diff --git a/drivers/s390/char/sclp_sdias.c b/drivers/s390/char/sclp_sdias.c index 644b61013679..215d4b4a5ff5 100644 --- a/drivers/s390/char/sclp_sdias.c +++ b/drivers/s390/char/sclp_sdias.c @@ -257,7 +257,7 @@ static int __init sclp_sdias_init_async(void) int __init sclp_sdias_init(void) { - if (ipl_info.type != IPL_TYPE_FCP_DUMP) + if (!is_ipl_type_dump()) return 0; sclp_sdias_sccb = (void *) __get_free_page(GFP_KERNEL | GFP_DMA); BUG_ON(!sclp_sdias_sccb); @@ -275,9 +275,3 @@ out: TRACE("init done\n"); return 0; } - -void __exit sclp_sdias_exit(void) -{ - debug_unregister(sdias_dbf); - sclp_unregister(&sclp_sdias_register); -} diff --git a/drivers/s390/char/tape.h b/drivers/s390/char/tape.h index 8bec5f9ea92c..e2c60475dfa8 100644 --- a/drivers/s390/char/tape.h +++ b/drivers/s390/char/tape.h @@ -238,7 +238,6 @@ extern int tape_do_io(struct tape_device *, struct tape_request *); extern int tape_do_io_async(struct tape_device *, struct tape_request *); extern int tape_do_io_interruptible(struct tape_device *, struct tape_request *); extern int tape_cancel_io(struct tape_device *, struct tape_request *); -void tape_hotplug_event(struct tape_device *, int major, int action); static inline int tape_do_io_free(struct tape_device *device, struct tape_request *request) @@ -258,8 +257,6 @@ tape_do_io_async_free(struct tape_device *device, struct tape_request *request) tape_do_io_async(device, request); } -extern int tape_oper_handler(int irq, int status); -extern void tape_noper_handler(int irq, int status); extern int tape_open(struct tape_device *); extern int tape_release(struct tape_device *); extern int tape_mtop(struct tape_device *, int, int); diff --git a/drivers/s390/char/tape_std.h b/drivers/s390/char/tape_std.h index 53ec8e2870d4..dcc63ff587f9 100644 --- a/drivers/s390/char/tape_std.h +++ b/drivers/s390/char/tape_std.h @@ -101,7 +101,6 @@ struct tape_request *tape_std_read_block(struct tape_device *, size_t); void tape_std_read_backward(struct tape_device *device, struct tape_request *request); struct tape_request *tape_std_write_block(struct tape_device *, size_t); -void tape_std_check_locate(struct tape_device *, struct tape_request *); /* Some non-mtop commands. */ int tape_std_assign(struct tape_device *); @@ -131,19 +130,8 @@ int tape_std_mtunload(struct tape_device *, int); int tape_std_mtweof(struct tape_device *, int); /* Event handlers */ -void tape_std_default_handler(struct tape_device *); -void tape_std_unexpect_uchk_handler(struct tape_device *); -void tape_std_irq(struct tape_device *); void tape_std_process_eov(struct tape_device *); -// the error recovery stuff: -void tape_std_error_recovery(struct tape_device *); -void tape_std_error_recovery_has_failed(struct tape_device *,int error_id); -void tape_std_error_recovery_succeded(struct tape_device *); -void tape_std_error_recovery_do_retry(struct tape_device *); -void tape_std_error_recovery_read_opposite(struct tape_device *); -void tape_std_error_recovery_HWBUG(struct tape_device *, int condno); - /* S390 tape types */ enum s390_tape_type { tape_3480, diff --git a/drivers/s390/char/zcore.c b/drivers/s390/char/zcore.c index d29f1b71618e..1515fdc3c1ab 100644 --- a/drivers/s390/char/zcore.c +++ b/drivers/s390/char/zcore.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-1.0+ /* * zcore module to export memory content and register sets for creating system - * dumps on SCSI disks (zfcpdump). + * dumps on SCSI/NVMe disks (zfcp/nvme dump). * * For more information please refer to Documentation/s390/zfcpdump.rst * @@ -243,7 +243,7 @@ static int __init zcore_init(void) unsigned char arch; int rc; - if (ipl_info.type != IPL_TYPE_FCP_DUMP) + if (!is_ipl_type_dump()) return -ENODATA; if (OLDMEM_BASE) return -ENODATA; @@ -252,9 +252,16 @@ static int __init zcore_init(void) debug_register_view(zcore_dbf, &debug_sprintf_view); debug_set_level(zcore_dbf, 6); - TRACE("devno: %x\n", ipl_info.data.fcp.dev_id.devno); - TRACE("wwpn: %llx\n", (unsigned long long) ipl_info.data.fcp.wwpn); - TRACE("lun: %llx\n", (unsigned long long) ipl_info.data.fcp.lun); + if (ipl_info.type == IPL_TYPE_FCP_DUMP) { + TRACE("type: fcp\n"); + TRACE("devno: %x\n", ipl_info.data.fcp.dev_id.devno); + TRACE("wwpn: %llx\n", (unsigned long long) ipl_info.data.fcp.wwpn); + TRACE("lun: %llx\n", (unsigned long long) ipl_info.data.fcp.lun); + } else if (ipl_info.type == IPL_TYPE_NVME_DUMP) { + TRACE("type: nvme\n"); + TRACE("fid: %x\n", ipl_info.data.nvme.fid); + TRACE("nsid: %x\n", ipl_info.data.nvme.nsid); + } rc = sclp_sdias_init(); if (rc) diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c index 38017c4a31e9..fc06a4002168 100644 --- a/drivers/s390/cio/chsc.c +++ b/drivers/s390/cio/chsc.c @@ -1265,6 +1265,27 @@ int chsc_sstpi(void *page, void *result, size_t size) return (rr->response.code == 0x0001) ? 0 : -EIO; } +int chsc_stzi(void *page, void *result, size_t size) +{ + struct { + struct chsc_header request; + unsigned int rsvd0[3]; + struct chsc_header response; + char data[]; + } *rr; + int rc; + + memset(page, 0, PAGE_SIZE); + rr = page; + rr->request.length = 0x0010; + rr->request.code = 0x003e; + rc = chsc(rr); + if (rc) + return -EIO; + memcpy(result, &rr->data, size); + return (rr->response.code == 0x0001) ? 0 : -EIO; +} + int chsc_siosl(struct subchannel_id schid) { struct { diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c index 1981eb62d329..cca1a7c4bb33 100644 --- a/drivers/s390/cio/css.c +++ b/drivers/s390/cio/css.c @@ -1355,20 +1355,6 @@ static int __init channel_subsystem_init_sync(void) } subsys_initcall_sync(channel_subsystem_init_sync); -void channel_subsystem_reinit(void) -{ - struct channel_path *chp; - struct chp_id chpid; - - chsc_enable_facility(CHSC_SDA_OC_MSS); - chp_id_for_each(&chpid) { - chp = chpid_to_chp(chpid); - if (chp) - chp_update_desc(chp); - } - cmf_reactivate(); -} - #ifdef CONFIG_PROC_FS static ssize_t cio_settle_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) diff --git a/drivers/s390/cio/device.h b/drivers/s390/cio/device.h index f5c427ec24b1..853b6a8ca095 100644 --- a/drivers/s390/cio/device.h +++ b/drivers/s390/cio/device.h @@ -96,7 +96,6 @@ int ccw_device_online(struct ccw_device *); int ccw_device_offline(struct ccw_device *); void ccw_device_update_sense_data(struct ccw_device *); int ccw_device_test_sense_data(struct ccw_device *); -void ccw_device_schedule_sch_unregister(struct ccw_device *); int ccw_purge_blacklisted(void); void ccw_device_sched_todo(struct ccw_device *cdev, enum cdev_todo todo); struct ccw_device *get_ccwdev_by_dev_id(struct ccw_dev_id *dev_id); diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c index 4fab8bba2cdd..f9a31c7819ae 100644 --- a/drivers/s390/cio/qdio_main.c +++ b/drivers/s390/cio/qdio_main.c @@ -531,26 +531,6 @@ static inline int qdio_inbound_q_done(struct qdio_q *q, unsigned int start) return 1; } -static inline void qdio_handle_aobs(struct qdio_q *q, int start, int count) -{ - unsigned char state = 0; - int j, b = start; - - for (j = 0; j < count; ++j) { - get_buf_state(q, b, &state, 0); - if (state == SLSB_P_OUTPUT_PENDING) { - struct qaob *aob = q->u.out.aobs[b]; - if (aob == NULL) - continue; - - q->u.out.sbal_state[b].flags |= - QDIO_OUTBUF_STATE_FLAG_PENDING; - q->u.out.aobs[b] = NULL; - } - b = next_buf(b); - } -} - static inline unsigned long qdio_aob_for_buffer(struct qdio_output_q *q, int bufnr) { @@ -640,6 +620,19 @@ void qdio_inbound_processing(unsigned long data) __qdio_inbound_processing(q); } +static void qdio_check_pending(struct qdio_q *q, unsigned int index) +{ + unsigned char state; + + if (get_buf_state(q, index, &state, 0) > 0 && + state == SLSB_P_OUTPUT_PENDING && + q->u.out.aobs[index]) { + q->u.out.sbal_state[index].flags |= + QDIO_OUTBUF_STATE_FLAG_PENDING; + q->u.out.aobs[index] = NULL; + } +} + static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start) { unsigned char state = 0; @@ -712,8 +705,13 @@ static inline int qdio_outbound_q_moved(struct qdio_q *q, unsigned int start) if (count) { DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out moved:%1d", q->nr); - if (q->u.out.use_cq) - qdio_handle_aobs(q, start, count); + + if (q->u.out.use_cq) { + unsigned int i; + + for (i = 0; i < count; i++) + qdio_check_pending(q, QDIO_BUFNR(start + i)); + } } return count; @@ -1221,7 +1219,6 @@ static void qdio_trace_init_data(struct qdio_irq *irq, struct qdio_initialize *data) { DBF_DEV_EVENT(DBF_ERR, irq, "qfmt:%1u", data->q_format); - DBF_DEV_HEX(irq, data->adapter_name, 8, DBF_ERR); DBF_DEV_EVENT(DBF_ERR, irq, "qpff%4x", data->qib_param_field_format); DBF_DEV_HEX(irq, &data->qib_param_field, sizeof(void *), DBF_ERR); DBF_DEV_HEX(irq, &data->input_slib_elements, sizeof(void *), DBF_ERR); diff --git a/drivers/s390/cio/qdio_setup.c b/drivers/s390/cio/qdio_setup.c index 2c5cc6ec668e..a5b2e16b7aa8 100644 --- a/drivers/s390/cio/qdio_setup.c +++ b/drivers/s390/cio/qdio_setup.c @@ -9,6 +9,8 @@ #include <linux/slab.h> #include <linux/export.h> #include <linux/io.h> + +#include <asm/ebcdic.h> #include <asm/qdio.h> #include "cio.h" @@ -403,28 +405,22 @@ void qdio_free_async_data(struct qdio_irq *irq_ptr) } } -static void __qdio_allocate_fill_qdr(struct qdio_irq *irq_ptr, - struct qdio_q **irq_ptr_qs, - int i, int nr) +static void qdio_fill_qdr_desc(struct qdesfmt0 *desc, struct qdio_q *queue) { - irq_ptr->qdr->qdf0[i + nr].sliba = - (unsigned long)irq_ptr_qs[i]->slib; - - irq_ptr->qdr->qdf0[i + nr].sla = - (unsigned long)irq_ptr_qs[i]->sl; - - irq_ptr->qdr->qdf0[i + nr].slsba = - (unsigned long)&irq_ptr_qs[i]->slsb.val[0]; - - irq_ptr->qdr->qdf0[i + nr].akey = PAGE_DEFAULT_KEY >> 4; - irq_ptr->qdr->qdf0[i + nr].bkey = PAGE_DEFAULT_KEY >> 4; - irq_ptr->qdr->qdf0[i + nr].ckey = PAGE_DEFAULT_KEY >> 4; - irq_ptr->qdr->qdf0[i + nr].dkey = PAGE_DEFAULT_KEY >> 4; + desc->sliba = virt_to_phys(queue->slib); + desc->sla = virt_to_phys(queue->sl); + desc->slsba = virt_to_phys(&queue->slsb); + + desc->akey = PAGE_DEFAULT_KEY >> 4; + desc->bkey = PAGE_DEFAULT_KEY >> 4; + desc->ckey = PAGE_DEFAULT_KEY >> 4; + desc->dkey = PAGE_DEFAULT_KEY >> 4; } static void setup_qdr(struct qdio_irq *irq_ptr, struct qdio_initialize *qdio_init) { + struct qdesfmt0 *desc = &irq_ptr->qdr->qdf0[0]; int i; irq_ptr->qdr->qfmt = qdio_init->q_format; @@ -433,15 +429,14 @@ static void setup_qdr(struct qdio_irq *irq_ptr, irq_ptr->qdr->oqdcnt = qdio_init->no_output_qs; irq_ptr->qdr->iqdsz = sizeof(struct qdesfmt0) / 4; /* size in words */ irq_ptr->qdr->oqdsz = sizeof(struct qdesfmt0) / 4; - irq_ptr->qdr->qiba = (unsigned long)&irq_ptr->qib; + irq_ptr->qdr->qiba = virt_to_phys(&irq_ptr->qib); irq_ptr->qdr->qkey = PAGE_DEFAULT_KEY >> 4; for (i = 0; i < qdio_init->no_input_qs; i++) - __qdio_allocate_fill_qdr(irq_ptr, irq_ptr->input_qs, i, 0); + qdio_fill_qdr_desc(desc++, irq_ptr->input_qs[i]); for (i = 0; i < qdio_init->no_output_qs; i++) - __qdio_allocate_fill_qdr(irq_ptr, irq_ptr->output_qs, i, - qdio_init->no_input_qs); + qdio_fill_qdr_desc(desc++, irq_ptr->output_qs[i]); } static void setup_qib(struct qdio_irq *irq_ptr, @@ -459,7 +454,8 @@ static void setup_qib(struct qdio_irq *irq_ptr, if (init_data->no_output_qs) irq_ptr->qib.osliba = (unsigned long)(irq_ptr->output_qs[0]->slib); - memcpy(irq_ptr->qib.ebcnam, init_data->adapter_name, 8); + memcpy(irq_ptr->qib.ebcnam, dev_name(&irq_ptr->cdev->dev), 8); + ASCEBC(irq_ptr->qib.ebcnam, 8); } int qdio_setup_irq(struct qdio_irq *irq_ptr, struct qdio_initialize *init_data) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 24a1940b829e..485cbfcbf06e 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -214,7 +214,7 @@ static inline int ap_fetch_qci_info(struct ap_config_info *info) static void __init ap_init_qci_info(void) { if (!ap_qci_available()) { - AP_DBF(DBF_INFO, "%s QCI not supported\n", __func__); + AP_DBF_INFO("%s QCI not supported\n", __func__); return; } @@ -226,18 +226,18 @@ static void __init ap_init_qci_info(void) ap_qci_info = NULL; return; } - AP_DBF(DBF_INFO, "%s successful fetched initial qci info\n", __func__); + AP_DBF_INFO("%s successful fetched initial qci info\n", __func__); if (ap_qci_info->apxa) { if (ap_qci_info->Na) { ap_max_adapter_id = ap_qci_info->Na; - AP_DBF(DBF_INFO, "%s new ap_max_adapter_id is %d\n", - __func__, ap_max_adapter_id); + AP_DBF_INFO("%s new ap_max_adapter_id is %d\n", + __func__, ap_max_adapter_id); } if (ap_qci_info->Nd) { ap_max_domain_id = ap_qci_info->Nd; - AP_DBF(DBF_INFO, "%s new ap_max_domain_id is %d\n", - __func__, ap_max_domain_id); + AP_DBF_INFO("%s new ap_max_domain_id is %d\n", + __func__, ap_max_domain_id); } } } @@ -307,7 +307,7 @@ EXPORT_SYMBOL(ap_test_config_ctrl_domain); * false otherwise. */ static bool ap_queue_info(ap_qid_t qid, int *q_type, - unsigned int *q_fac, int *q_depth) + unsigned int *q_fac, int *q_depth, bool *q_decfg) { struct ap_queue_status status; unsigned long info = 0; @@ -322,6 +322,9 @@ static bool ap_queue_info(ap_qid_t qid, int *q_type, switch (status.response_code) { case AP_RESPONSE_NORMAL: case AP_RESPONSE_RESET_IN_PROGRESS: + case AP_RESPONSE_DECONFIGURED: + case AP_RESPONSE_CHECKSTOPPED: + case AP_RESPONSE_BUSY: /* * According to the architecture in all these cases the * info should be filled. All bits 0 is not possible as @@ -332,6 +335,7 @@ static bool ap_queue_info(ap_qid_t qid, int *q_type, *q_type = (int)((info >> 24) & 0xff); *q_fac = (unsigned int)(info >> 32); *q_depth = (int)(info & 0xff); + *q_decfg = status.response_code == AP_RESPONSE_DECONFIGURED; switch (*q_type) { /* For CEX2 and CEX3 the available functions * are not reflected by the facilities bits. @@ -618,8 +622,8 @@ static int __ap_revise_reserved(struct device *dev, void *dummy) drvres = to_ap_drv(dev->driver)->flags & AP_DRIVER_FLAG_DEFAULT; if (!!devres != !!drvres) { - AP_DBF(DBF_DEBUG, "reprobing queue=%02x.%04x\n", - card, queue); + AP_DBF_DBG("reprobing queue=%02x.%04x\n", + card, queue); rc = device_reprobe(dev); } } @@ -796,7 +800,7 @@ EXPORT_SYMBOL(ap_bus_force_rescan); */ void ap_bus_cfg_chg(void) { - AP_DBF(DBF_INFO, "%s config change, forcing bus rescan\n", __func__); + AP_DBF_DBG("%s config change, forcing bus rescan\n", __func__); ap_bus_force_rescan(); } @@ -947,7 +951,7 @@ static ssize_t ap_domain_store(struct bus_type *bus, ap_domain_index = domain; spin_unlock_bh(&ap_domain_lock); - AP_DBF(DBF_INFO, "stored new default domain=%d\n", domain); + AP_DBF_INFO("stored new default domain=%d\n", domain); return count; } @@ -1208,8 +1212,8 @@ static void ap_select_domain(void) } if (dom <= ap_max_domain_id) { ap_domain_index = dom; - AP_DBF(DBF_DEBUG, "%s new default domain is %d\n", - __func__, ap_domain_index); + AP_DBF_INFO("%s new default domain is %d\n", + __func__, ap_domain_index); } out: spin_unlock_bh(&ap_domain_lock); @@ -1225,8 +1229,11 @@ static int ap_get_compatible_type(ap_qid_t qid, int rawtype, unsigned int func) int comp_type = 0; /* < CEX2A is not supported */ - if (rawtype < AP_DEVICE_TYPE_CEX2A) + if (rawtype < AP_DEVICE_TYPE_CEX2A) { + AP_DBF_WARN("get_comp_type queue=%02x.%04x unsupported type %d\n", + AP_QID_CARD(qid), AP_QID_QUEUE(qid), rawtype); return 0; + } /* up to CEX7 known and fully supported */ if (rawtype <= AP_DEVICE_TYPE_CEX7) return rawtype; @@ -1248,11 +1255,12 @@ static int ap_get_compatible_type(ap_qid_t qid, int rawtype, unsigned int func) comp_type = apinfo.cat; } if (!comp_type) - AP_DBF(DBF_WARN, "queue=%02x.%04x unable to map type %d\n", - AP_QID_CARD(qid), AP_QID_QUEUE(qid), rawtype); + AP_DBF_WARN("get_comp_type queue=%02x.%04x unable to map type %d\n", + AP_QID_CARD(qid), AP_QID_QUEUE(qid), rawtype); else if (comp_type != rawtype) - AP_DBF(DBF_INFO, "queue=%02x.%04x map type %d to %d\n", - AP_QID_CARD(qid), AP_QID_QUEUE(qid), rawtype, comp_type); + AP_DBF_INFO("get_comp_type queue=%02x.%04x map type %d to %d\n", + AP_QID_CARD(qid), AP_QID_QUEUE(qid), + rawtype, comp_type); return comp_type; } @@ -1286,155 +1294,278 @@ static int __match_queue_device_with_queue_id(struct device *dev, const void *da /* * Helper function for ap_scan_bus(). - * Does the scan bus job for the given adapter id. + * Remove card device and associated queue devices. */ -static void _ap_scan_bus_adapter(int id) +static inline void ap_scan_rm_card_dev_and_queue_devs(struct ap_card *ac) { - bool broken; + bus_for_each_dev(&ap_bus_type, NULL, + (void *)(long) ac->id, + __ap_queue_devices_with_id_unregister); + device_unregister(&ac->ap_dev.device); +} + +/* + * Helper function for ap_scan_bus(). + * Does the scan bus job for all the domains within + * a valid adapter given by an ap_card ptr. + */ +static inline void ap_scan_domains(struct ap_card *ac) +{ + bool decfg; ap_qid_t qid; unsigned int func; - struct ap_card *ac; struct device *dev; struct ap_queue *aq; + int rc, dom, depth, type; + + /* + * Go through the configuration for the domains and compare them + * to the existing queue devices. Also take care of the config + * and error state for the queue devices. + */ + + for (dom = 0; dom <= ap_max_domain_id; dom++) { + qid = AP_MKQID(ac->id, dom); + dev = bus_find_device(&ap_bus_type, NULL, + (void *)(long) qid, + __match_queue_device_with_qid); + aq = dev ? to_ap_queue(dev) : NULL; + if (!ap_test_config_usage_domain(dom)) { + if (dev) { + AP_DBF_INFO("%s(%d,%d) not in config any more, rm queue device\n", + __func__, ac->id, dom); + device_unregister(dev); + put_device(dev); + } + continue; + } + /* domain is valid, get info from this APQN */ + if (!ap_queue_info(qid, &type, &func, &depth, &decfg)) { + if (aq) { + AP_DBF_INFO( + "%s(%d,%d) ap_queue_info() not successful, rm queue device\n", + __func__, ac->id, dom); + device_unregister(dev); + put_device(dev); + } + continue; + } + /* if no queue device exists, create a new one */ + if (!aq) { + aq = ap_queue_create(qid, ac->ap_dev.device_type); + if (!aq) { + AP_DBF_WARN("%s(%d,%d) ap_queue_create() failed\n", + __func__, ac->id, dom); + continue; + } + aq->card = ac; + aq->config = !decfg; + dev = &aq->ap_dev.device; + dev->bus = &ap_bus_type; + dev->parent = &ac->ap_dev.device; + dev_set_name(dev, "%02x.%04x", ac->id, dom); + /* register queue device */ + rc = device_register(dev); + if (rc) { + AP_DBF_WARN("%s(%d,%d) device_register() failed\n", + __func__, ac->id, dom); + goto put_dev_and_continue; + } + if (decfg) + AP_DBF_INFO("%s(%d,%d) new (decfg) queue device created\n", + __func__, ac->id, dom); + else + AP_DBF_INFO("%s(%d,%d) new queue device created\n", + __func__, ac->id, dom); + goto put_dev_and_continue; + } + /* Check config state on the already existing queue device */ + spin_lock_bh(&aq->lock); + if (decfg && aq->config) { + /* config off this queue device */ + aq->config = false; + if (aq->dev_state > AP_DEV_STATE_UNINITIATED) { + aq->dev_state = AP_DEV_STATE_ERROR; + aq->last_err_rc = AP_RESPONSE_DECONFIGURED; + } + spin_unlock_bh(&aq->lock); + AP_DBF_INFO("%s(%d,%d) queue device config off\n", + __func__, ac->id, dom); + /* 'receive' pending messages with -EAGAIN */ + ap_flush_queue(aq); + goto put_dev_and_continue; + } + if (!decfg && !aq->config) { + /* config on this queue device */ + aq->config = true; + if (aq->dev_state > AP_DEV_STATE_UNINITIATED) { + aq->dev_state = AP_DEV_STATE_OPERATING; + aq->sm_state = AP_SM_STATE_RESET_START; + } + spin_unlock_bh(&aq->lock); + AP_DBF_INFO("%s(%d,%d) queue device config on\n", + __func__, ac->id, dom); + goto put_dev_and_continue; + } + /* handle other error states */ + if (!decfg && aq->dev_state == AP_DEV_STATE_ERROR) { + spin_unlock_bh(&aq->lock); + /* 'receive' pending messages with -EAGAIN */ + ap_flush_queue(aq); + /* re-init (with reset) the queue device */ + ap_queue_init_state(aq); + AP_DBF_INFO("%s(%d,%d) queue device reinit enforced\n", + __func__, ac->id, dom); + goto put_dev_and_continue; + } + spin_unlock_bh(&aq->lock); +put_dev_and_continue: + put_device(dev); + } +} + +/* + * Helper function for ap_scan_bus(). + * Does the scan bus job for the given adapter id. + */ +static inline void ap_scan_adapter(int ap) +{ + bool decfg; + ap_qid_t qid; + unsigned int func; + struct device *dev; + struct ap_card *ac; int rc, dom, depth, type, comp_type; - /* check if there is a card device registered with this id */ + /* Is there currently a card device for this adapter ? */ dev = bus_find_device(&ap_bus_type, NULL, - (void *)(long) id, + (void *)(long) ap, __match_card_device_with_id); ac = dev ? to_ap_card(dev) : NULL; - if (!ap_test_config_card_id(id)) { - if (dev) { - /* Card device has been removed from configuration */ - bus_for_each_dev(&ap_bus_type, NULL, - (void *)(long) id, - __ap_queue_devices_with_id_unregister); - device_unregister(dev); + + /* Adapter not in configuration ? */ + if (!ap_test_config_card_id(ap)) { + if (ac) { + AP_DBF_INFO("%s(%d) ap not in config any more, rm card and queue devices\n", + __func__, ap); + ap_scan_rm_card_dev_and_queue_devs(ac); put_device(dev); } return; } /* - * This card id is enabled in the configuration. If we already have - * a card device with this id, check if type and functions are still - * the very same. Also verify that at least one queue is available. + * Adapter ap is valid in the current configuration. So do some checks: + * If no card device exists, build one. If a card device exists, check + * for type and functions changed. For all this we need to find a valid + * APQN first. */ - if (ac) { - /* find the first valid queue */ - for (dom = 0; dom < AP_DOMAINS; dom++) { - qid = AP_MKQID(id, dom); - if (ap_queue_info(qid, &type, &func, &depth)) + + for (dom = 0; dom <= ap_max_domain_id; dom++) + if (ap_test_config_usage_domain(dom)) { + qid = AP_MKQID(ap, dom); + if (ap_queue_info(qid, &type, &func, &depth, &decfg)) break; } - broken = false; - if (dom >= AP_DOMAINS) { - /* no accessible queue on this card */ - broken = true; - } else if (ac->raw_hwtype != type) { - /* card type has changed */ - AP_DBF(DBF_INFO, "card=%02x type changed.\n", id); - broken = true; - } else if (ac->functions != func) { - /* card functions have changed */ - AP_DBF(DBF_INFO, "card=%02x functions changed.\n", id); - broken = true; + if (dom > ap_max_domain_id) { + /* Could not find a valid APQN for this adapter */ + if (ac) { + AP_DBF_INFO( + "%s(%d) no type info (no APQN found), rm card and queue devices\n", + __func__, ap); + ap_scan_rm_card_dev_and_queue_devs(ac); + put_device(dev); + } else { + AP_DBF_DBG("%s(%d) no type info (no APQN found), ignored\n", + __func__, ap); } - if (broken) { - /* unregister card device and associated queues */ - bus_for_each_dev(&ap_bus_type, NULL, - (void *)(long) id, - __ap_queue_devices_with_id_unregister); - device_unregister(dev); + return; + } + if (!type) { + /* No apdater type info available, an unusable adapter */ + if (ac) { + AP_DBF_INFO("%s(%d) no valid type (0) info, rm card and queue devices\n", + __func__, ap); + ap_scan_rm_card_dev_and_queue_devs(ac); put_device(dev); - /* go back if there is no valid queue on this card */ - if (dom >= AP_DOMAINS) - return; - ac = NULL; + } else { + AP_DBF_DBG("%s(%d) no valid type (0) info, ignored\n", + __func__, ap); } + return; } - /* - * Go through all possible queue ids. Check and maybe create or release - * queue devices for this card. If there exists no card device yet, - * create a card device also. - */ - for (dom = 0; dom < AP_DOMAINS; dom++) { - qid = AP_MKQID(id, dom); - dev = bus_find_device(&ap_bus_type, NULL, - (void *)(long) qid, - __match_queue_device_with_qid); - aq = dev ? to_ap_queue(dev) : NULL; - if (!ap_test_config_usage_domain(dom)) { - if (dev) { - /* Queue device exists but has been - * removed from configuration. - */ - device_unregister(dev); - put_device(dev); - } - continue; - } - /* try to fetch infos about this queue */ - broken = !ap_queue_info(qid, &type, &func, &depth); - if (dev) { - if (!broken) { - spin_lock_bh(&aq->lock); - broken = aq->sm_state == AP_SM_STATE_BORKED; - spin_unlock_bh(&aq->lock); + if (ac) { + /* Check APQN against existing card device for changes */ + if (ac->raw_hwtype != type) { + AP_DBF_INFO("%s(%d) hwtype %d changed, rm card and queue devices\n", + __func__, ap, type); + ap_scan_rm_card_dev_and_queue_devs(ac); + put_device(dev); + ac = NULL; + } else if (ac->functions != func) { + AP_DBF_INFO("%s(%d) functions 0x%08x changed, rm card and queue devices\n", + __func__, ap, type); + ap_scan_rm_card_dev_and_queue_devs(ac); + put_device(dev); + ac = NULL; + } else { + if (decfg && ac->config) { + ac->config = false; + AP_DBF_INFO("%s(%d) card device config off\n", + __func__, ap); + } - if (broken) { - /* Remove broken device */ - AP_DBF(DBF_DEBUG, - "removing broken queue=%02x.%04x\n", - id, dom); - device_unregister(dev); + if (!decfg && !ac->config) { + ac->config = true; + AP_DBF_INFO("%s(%d) card device config on\n", + __func__, ap); } - put_device(dev); - continue; } - if (broken) - continue; - /* a new queue device is needed, check out comp type */ + } + + if (!ac) { + /* Build a new card device */ comp_type = ap_get_compatible_type(qid, type, func); - if (!comp_type) - continue; - /* maybe a card device needs to be created first */ + if (!comp_type) { + AP_DBF_WARN("%s(%d) type %d, can't get compatibility type\n", + __func__, ap, type); + return; + } + ac = ap_card_create(ap, depth, type, comp_type, func); if (!ac) { - ac = ap_card_create(id, depth, type, comp_type, func); - if (!ac) - continue; - ac->ap_dev.device.bus = &ap_bus_type; - ac->ap_dev.device.parent = ap_root_device; - dev_set_name(&ac->ap_dev.device, "card%02x", id); - /* Register card device with AP bus */ - rc = device_register(&ac->ap_dev.device); - if (rc) { - put_device(&ac->ap_dev.device); - ac = NULL; - break; - } - /* get it and thus adjust reference counter */ - get_device(&ac->ap_dev.device); + AP_DBF_WARN("%s(%d) ap_card_create() failed\n", + __func__, ap); + return; } - /* now create the new queue device */ - aq = ap_queue_create(qid, comp_type); - if (!aq) - continue; - aq->card = ac; - aq->ap_dev.device.bus = &ap_bus_type; - aq->ap_dev.device.parent = &ac->ap_dev.device; - dev_set_name(&aq->ap_dev.device, "%02x.%04x", id, dom); - /* Register queue device */ - rc = device_register(&aq->ap_dev.device); + ac->config = !decfg; + dev = &ac->ap_dev.device; + dev->bus = &ap_bus_type; + dev->parent = ap_root_device; + dev_set_name(dev, "card%02x", ap); + /* Register the new card device with AP bus */ + rc = device_register(dev); if (rc) { - put_device(&aq->ap_dev.device); - continue; + AP_DBF_WARN("%s(%d) device_register() failed\n", + __func__, ap); + put_device(dev); + return; } - } /* end domain loop */ + /* get it and thus adjust reference counter */ + get_device(dev); + if (decfg) + AP_DBF_INFO("%s(%d) new (decfg) card device type=%d func=0x%08x created\n", + __func__, ap, type, func); + else + AP_DBF_INFO("%s(%d) new card device type=%d func=0x%08x created\n", + __func__, ap, type, func); + } + + /* Verify the domains and the queue devices for this card */ + ap_scan_domains(ac); - if (ac) - put_device(&ac->ap_dev.device); + /* release the card device */ + put_device(&ac->ap_dev.device); } /** @@ -1443,16 +1574,16 @@ static void _ap_scan_bus_adapter(int id) */ static void ap_scan_bus(struct work_struct *unused) { - int id; + int ap; ap_fetch_qci_info(ap_qci_info); ap_select_domain(); - AP_DBF(DBF_DEBUG, "%s running\n", __func__); + AP_DBF_DBG("%s running\n", __func__); /* loop over all possible adapters */ - for (id = 0; id < AP_DEVICES; id++) - _ap_scan_bus_adapter(id); + for (ap = 0; ap <= ap_max_adapter_id; ap++) + ap_scan_adapter(ap); /* check if there is at least one queue available with default domain */ if (ap_domain_index >= 0) { @@ -1463,9 +1594,8 @@ static void ap_scan_bus(struct work_struct *unused) if (dev) put_device(dev); else - AP_DBF(DBF_INFO, - "no queue device with default domain %d available\n", - ap_domain_index); + AP_DBF_INFO("no queue device with default domain %d available\n", + ap_domain_index); } mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ); @@ -1575,7 +1705,6 @@ static int __init ap_module_init(void) */ if (MACHINE_IS_VM) poll_timeout = 1500000; - spin_lock_init(&ap_poll_timer_lock); hrtimer_init(&ap_poll_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); ap_poll_timer.function = ap_poll_timeout; diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index 1ea046324e8f..5029b80132aa 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -50,6 +50,7 @@ static inline int ap_test_bit(unsigned int *ptr, unsigned int nr) #define AP_RESPONSE_NO_FIRST_PART 0x13 #define AP_RESPONSE_MESSAGE_TOO_BIG 0x15 #define AP_RESPONSE_REQ_FAC_NOT_INST 0x16 +#define AP_RESPONSE_INVALID_DOMAIN 0x42 /* * Known device types @@ -86,15 +87,12 @@ static inline int ap_test_bit(unsigned int *ptr, unsigned int nr) * AP queue state machine states */ enum ap_sm_state { - AP_SM_STATE_RESET_START, + AP_SM_STATE_RESET_START = 0, AP_SM_STATE_RESET_WAIT, AP_SM_STATE_SETIRQ_WAIT, AP_SM_STATE_IDLE, AP_SM_STATE_WORKING, AP_SM_STATE_QUEUE_FULL, - AP_SM_STATE_REMOVE, /* about to be removed from driver */ - AP_SM_STATE_UNBOUND, /* momentary not bound to a driver */ - AP_SM_STATE_BORKED, /* broken */ NR_AP_SM_STATES }; @@ -118,6 +116,17 @@ enum ap_sm_wait { NR_AP_SM_WAIT }; +/* + * AP queue device states + */ +enum ap_dev_state { + AP_DEV_STATE_UNINITIATED = 0, /* fresh and virgin, not touched */ + AP_DEV_STATE_OPERATING, /* queue dev is working normal */ + AP_DEV_STATE_SHUTDOWN, /* remove/unbind/shutdown in progress */ + AP_DEV_STATE_ERROR, /* device is in error state */ + NR_AP_DEV_STATES +}; + struct ap_device; struct ap_message; @@ -158,6 +167,7 @@ struct ap_card { unsigned int functions; /* AP device function bitfield. */ int queue_depth; /* AP queue depth.*/ int id; /* AP card number. */ + bool config; /* configured state */ atomic64_t total_request_count; /* # requests ever for this AP device.*/ }; @@ -169,10 +179,11 @@ struct ap_queue { struct ap_card *card; /* Ptr to assoc. AP card. */ spinlock_t lock; /* Per device lock. */ void *private; /* ap driver private pointer. */ + enum ap_dev_state dev_state; /* queue device state */ + bool config; /* configured state */ ap_qid_t qid; /* AP queue id. */ int interrupt; /* indicate if interrupts are enabled */ int queue_count; /* # messages currently on AP queue. */ - enum ap_sm_state sm_state; /* ap queue state machine state */ int pendingq_count; /* # requests on pendingq list. */ int requestq_count; /* # requests on requestq list. */ u64 total_request_count; /* # requests ever for this AP device.*/ @@ -181,18 +192,45 @@ struct ap_queue { struct list_head pendingq; /* List of message sent to AP queue. */ struct list_head requestq; /* List of message yet to be sent. */ struct ap_message *reply; /* Per device reply message. */ + enum ap_sm_state sm_state; /* ap queue state machine state */ + int last_err_rc; /* last error state response code */ }; #define to_ap_queue(x) container_of((x), struct ap_queue, ap_dev.device) typedef enum ap_sm_wait (ap_func_t)(struct ap_queue *queue); +/* failure injection cmd struct */ +struct ap_fi { + union { + u16 cmd; /* fi flags + action */ + struct { + u8 flags; /* fi flags only */ + u8 action; /* fi action only */ + }; + }; +}; + +/* all currently known fi actions */ +enum ap_fi_actions { + AP_FI_ACTION_CCA_AGENT_FF = 0x01, + AP_FI_ACTION_CCA_DOM_INVAL = 0x02, + AP_FI_ACTION_NQAP_QID_INVAL = 0x03, +}; + +/* all currently known fi flags */ +enum ap_fi_flags { + AP_FI_FLAG_NO_RETRY = 0x01, + AP_FI_FLAG_TOGGLE_SPECIAL = 0x02, +}; + struct ap_message { struct list_head list; /* Request queueing. */ unsigned long long psmid; /* Message id. */ void *msg; /* Pointer to message buffer. */ unsigned int len; /* Message length. */ - u32 flags; /* Flags, see AP_MSG_FLAG_xxx */ + u16 flags; /* Flags, see AP_MSG_FLAG_xxx */ + struct ap_fi fi; /* Failure Injection cmd */ int rc; /* Return code for this message */ void *private; /* ap driver private pointer. */ /* receive is called from tasklet context */ @@ -200,7 +238,7 @@ struct ap_message { struct ap_message *); }; -#define AP_MSG_FLAG_SPECIAL (1 << 16) /* flag msg as 'special' with NQAP */ +#define AP_MSG_FLAG_SPECIAL 1 /* flag msg as 'special' with NQAP */ /** * ap_init_message() - Initialize ap_message. @@ -234,7 +272,7 @@ int ap_recv(ap_qid_t, unsigned long long *, void *, size_t); enum ap_sm_wait ap_sm_event(struct ap_queue *aq, enum ap_sm_event event); enum ap_sm_wait ap_sm_event_loop(struct ap_queue *aq, enum ap_sm_event event); -void ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg); +int ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg); void ap_cancel_message(struct ap_queue *aq, struct ap_message *ap_msg); void ap_flush_queue(struct ap_queue *aq); diff --git a/drivers/s390/crypto/ap_card.c b/drivers/s390/crypto/ap_card.c index 6588713319ba..d98bdd28d23e 100644 --- a/drivers/s390/crypto/ap_card.c +++ b/drivers/s390/crypto/ap_card.c @@ -12,6 +12,7 @@ #include <linux/init.h> #include <linux/slab.h> #include <asm/facility.h> +#include <asm/sclp.h> #include "ap_bus.h" @@ -139,6 +140,38 @@ static ssize_t modalias_show(struct device *dev, static DEVICE_ATTR_RO(modalias); +static ssize_t config_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ap_card *ac = to_ap_card(dev); + + return scnprintf(buf, PAGE_SIZE, "%d\n", ac->config ? 1 : 0); +} + +static ssize_t config_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + int rc = 0, cfg; + struct ap_card *ac = to_ap_card(dev); + + if (sscanf(buf, "%d\n", &cfg) != 1 || cfg < 0 || cfg > 1) + return -EINVAL; + + if (cfg && !ac->config) + rc = sclp_ap_configure(ac->id); + else if (!cfg && ac->config) + rc = sclp_ap_deconfigure(ac->id); + if (rc) + return rc; + + ac->config = cfg ? true : false; + + return count; +} + +static DEVICE_ATTR_RW(config); + static struct attribute *ap_card_dev_attrs[] = { &dev_attr_hwtype.attr, &dev_attr_raw_hwtype.attr, @@ -148,6 +181,7 @@ static struct attribute *ap_card_dev_attrs[] = { &dev_attr_requestq_count.attr, &dev_attr_pendingq_count.attr, &dev_attr_modalias.attr, + &dev_attr_config.attr, NULL }; diff --git a/drivers/s390/crypto/ap_debug.h b/drivers/s390/crypto/ap_debug.h index dc675eb5aef6..34b0350d0b1a 100644 --- a/drivers/s390/crypto/ap_debug.h +++ b/drivers/s390/crypto/ap_debug.h @@ -20,6 +20,14 @@ #define AP_DBF(...) \ debug_sprintf_event(ap_dbf_info, ##__VA_ARGS__) +#define AP_DBF_ERR(...) \ + debug_sprintf_event(ap_dbf_info, DBF_ERR, ##__VA_ARGS__) +#define AP_DBF_WARN(...) \ + debug_sprintf_event(ap_dbf_info, DBF_WARN, ##__VA_ARGS__) +#define AP_DBF_INFO(...) \ + debug_sprintf_event(ap_dbf_info, DBF_INFO, ##__VA_ARGS__) +#define AP_DBF_DBG(...) \ + debug_sprintf_event(ap_dbf_info, DBF_DEBUG, ##__VA_ARGS__) extern debug_info_t *ap_dbf_info; diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c index 688ebebbf98c..ecefc25eff0c 100644 --- a/drivers/s390/crypto/ap_queue.c +++ b/drivers/s390/crypto/ap_queue.c @@ -195,7 +195,11 @@ static enum ap_sm_wait ap_sm_read(struct ap_queue *aq) aq->sm_state = AP_SM_STATE_IDLE; return AP_SM_WAIT_NONE; default: - aq->sm_state = AP_SM_STATE_BORKED; + aq->dev_state = AP_DEV_STATE_ERROR; + aq->last_err_rc = status.response_code; + AP_DBF_WARN("%s RC 0x%02x on 0x%02x.%04x -> AP_DEV_STATE_ERROR\n", + __func__, status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return AP_SM_WAIT_NONE; } } @@ -210,12 +214,20 @@ static enum ap_sm_wait ap_sm_write(struct ap_queue *aq) { struct ap_queue_status status; struct ap_message *ap_msg; + ap_qid_t qid = aq->qid; if (aq->requestq_count <= 0) return AP_SM_WAIT_NONE; /* Start the next request on the queue. */ ap_msg = list_entry(aq->requestq.next, struct ap_message, list); - status = __ap_send(aq->qid, ap_msg->psmid, +#ifdef CONFIG_ZCRYPT_DEBUG + if (ap_msg->fi.action == AP_FI_ACTION_NQAP_QID_INVAL) { + AP_DBF_WARN("%s fi cmd 0x%04x: forcing invalid qid 0xFF00\n", + __func__, ap_msg->fi.cmd); + qid = 0xFF00; + } +#endif + status = __ap_send(qid, ap_msg->psmid, ap_msg->msg, ap_msg->len, ap_msg->flags & AP_MSG_FLAG_SPECIAL); switch (status.response_code) { @@ -237,6 +249,9 @@ static enum ap_sm_wait ap_sm_write(struct ap_queue *aq) case AP_RESPONSE_RESET_IN_PROGRESS: aq->sm_state = AP_SM_STATE_RESET_WAIT; return AP_SM_WAIT_TIMEOUT; + case AP_RESPONSE_INVALID_DOMAIN: + AP_DBF(DBF_WARN, "AP_RESPONSE_INVALID_DOMAIN on NQAP\n"); + fallthrough; case AP_RESPONSE_MESSAGE_TOO_BIG: case AP_RESPONSE_REQ_FAC_NOT_INST: list_del_init(&ap_msg->list); @@ -245,7 +260,11 @@ static enum ap_sm_wait ap_sm_write(struct ap_queue *aq) ap_msg->receive(aq, ap_msg, NULL); return AP_SM_WAIT_AGAIN; default: - aq->sm_state = AP_SM_STATE_BORKED; + aq->dev_state = AP_DEV_STATE_ERROR; + aq->last_err_rc = status.response_code; + AP_DBF_WARN("%s RC 0x%02x on 0x%02x.%04x -> AP_DEV_STATE_ERROR\n", + __func__, status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return AP_SM_WAIT_NONE; } } @@ -278,13 +297,12 @@ static enum ap_sm_wait ap_sm_reset(struct ap_queue *aq) aq->sm_state = AP_SM_STATE_RESET_WAIT; aq->interrupt = AP_INTR_DISABLED; return AP_SM_WAIT_TIMEOUT; - case AP_RESPONSE_BUSY: - return AP_SM_WAIT_TIMEOUT; - case AP_RESPONSE_Q_NOT_AVAIL: - case AP_RESPONSE_DECONFIGURED: - case AP_RESPONSE_CHECKSTOPPED: default: - aq->sm_state = AP_SM_STATE_BORKED; + aq->dev_state = AP_DEV_STATE_ERROR; + aq->last_err_rc = status.response_code; + AP_DBF_WARN("%s RC 0x%02x on 0x%02x.%04x -> AP_DEV_STATE_ERROR\n", + __func__, status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return AP_SM_WAIT_NONE; } } @@ -323,7 +341,11 @@ static enum ap_sm_wait ap_sm_reset_wait(struct ap_queue *aq) case AP_RESPONSE_DECONFIGURED: case AP_RESPONSE_CHECKSTOPPED: default: - aq->sm_state = AP_SM_STATE_BORKED; + aq->dev_state = AP_DEV_STATE_ERROR; + aq->last_err_rc = status.response_code; + AP_DBF_WARN("%s RC 0x%02x on 0x%02x.%04x -> AP_DEV_STATE_ERROR\n", + __func__, status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return AP_SM_WAIT_NONE; } } @@ -360,7 +382,11 @@ static enum ap_sm_wait ap_sm_setirq_wait(struct ap_queue *aq) case AP_RESPONSE_NO_PENDING_REPLY: return AP_SM_WAIT_TIMEOUT; default: - aq->sm_state = AP_SM_STATE_BORKED; + aq->dev_state = AP_DEV_STATE_ERROR; + aq->last_err_rc = status.response_code; + AP_DBF_WARN("%s RC 0x%02x on 0x%02x.%04x -> AP_DEV_STATE_ERROR\n", + __func__, status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return AP_SM_WAIT_NONE; } } @@ -393,23 +419,14 @@ static ap_func_t *ap_jumptable[NR_AP_SM_STATES][NR_AP_SM_EVENTS] = { [AP_SM_EVENT_POLL] = ap_sm_read, [AP_SM_EVENT_TIMEOUT] = ap_sm_reset, }, - [AP_SM_STATE_REMOVE] = { - [AP_SM_EVENT_POLL] = ap_sm_nop, - [AP_SM_EVENT_TIMEOUT] = ap_sm_nop, - }, - [AP_SM_STATE_UNBOUND] = { - [AP_SM_EVENT_POLL] = ap_sm_nop, - [AP_SM_EVENT_TIMEOUT] = ap_sm_nop, - }, - [AP_SM_STATE_BORKED] = { - [AP_SM_EVENT_POLL] = ap_sm_nop, - [AP_SM_EVENT_TIMEOUT] = ap_sm_nop, - }, }; enum ap_sm_wait ap_sm_event(struct ap_queue *aq, enum ap_sm_event event) { - return ap_jumptable[aq->sm_state][event](aq); + if (aq->dev_state > AP_DEV_STATE_UNINITIATED) + return ap_jumptable[aq->sm_state][event](aq); + else + return AP_SM_WAIT_NONE; } enum ap_sm_wait ap_sm_event_loop(struct ap_queue *aq, enum ap_sm_event event) @@ -429,12 +446,20 @@ static ssize_t request_count_show(struct device *dev, char *buf) { struct ap_queue *aq = to_ap_queue(dev); + bool valid = false; u64 req_cnt; spin_lock_bh(&aq->lock); - req_cnt = aq->total_request_count; + if (aq->dev_state > AP_DEV_STATE_UNINITIATED) { + req_cnt = aq->total_request_count; + valid = true; + } spin_unlock_bh(&aq->lock); - return scnprintf(buf, PAGE_SIZE, "%llu\n", req_cnt); + + if (valid) + return scnprintf(buf, PAGE_SIZE, "%llu\n", req_cnt); + else + return scnprintf(buf, PAGE_SIZE, "-\n"); } static ssize_t request_count_store(struct device *dev, @@ -459,7 +484,8 @@ static ssize_t requestq_count_show(struct device *dev, unsigned int reqq_cnt = 0; spin_lock_bh(&aq->lock); - reqq_cnt = aq->requestq_count; + if (aq->dev_state > AP_DEV_STATE_UNINITIATED) + reqq_cnt = aq->requestq_count; spin_unlock_bh(&aq->lock); return scnprintf(buf, PAGE_SIZE, "%d\n", reqq_cnt); } @@ -473,7 +499,8 @@ static ssize_t pendingq_count_show(struct device *dev, unsigned int penq_cnt = 0; spin_lock_bh(&aq->lock); - penq_cnt = aq->pendingq_count; + if (aq->dev_state > AP_DEV_STATE_UNINITIATED) + penq_cnt = aq->pendingq_count; spin_unlock_bh(&aq->lock); return scnprintf(buf, PAGE_SIZE, "%d\n", penq_cnt); } @@ -542,12 +569,138 @@ static ssize_t interrupt_show(struct device *dev, static DEVICE_ATTR_RO(interrupt); +static ssize_t config_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ap_queue *aq = to_ap_queue(dev); + int rc; + + spin_lock_bh(&aq->lock); + rc = scnprintf(buf, PAGE_SIZE, "%d\n", aq->config ? 1 : 0); + spin_unlock_bh(&aq->lock); + return rc; +} + +static DEVICE_ATTR_RO(config); + +#ifdef CONFIG_ZCRYPT_DEBUG +static ssize_t states_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ap_queue *aq = to_ap_queue(dev); + int rc = 0; + + spin_lock_bh(&aq->lock); + /* queue device state */ + switch (aq->dev_state) { + case AP_DEV_STATE_UNINITIATED: + rc = scnprintf(buf, PAGE_SIZE, "UNINITIATED\n"); + break; + case AP_DEV_STATE_OPERATING: + rc = scnprintf(buf, PAGE_SIZE, "OPERATING"); + break; + case AP_DEV_STATE_SHUTDOWN: + rc = scnprintf(buf, PAGE_SIZE, "SHUTDOWN"); + break; + case AP_DEV_STATE_ERROR: + rc = scnprintf(buf, PAGE_SIZE, "ERROR"); + break; + default: + rc = scnprintf(buf, PAGE_SIZE, "UNKNOWN"); + } + /* state machine state */ + if (aq->dev_state) { + switch (aq->sm_state) { + case AP_SM_STATE_RESET_START: + rc += scnprintf(buf + rc, PAGE_SIZE - rc, + " [RESET_START]\n"); + break; + case AP_SM_STATE_RESET_WAIT: + rc += scnprintf(buf + rc, PAGE_SIZE - rc, + " [RESET_WAIT]\n"); + break; + case AP_SM_STATE_SETIRQ_WAIT: + rc += scnprintf(buf + rc, PAGE_SIZE - rc, + " [SETIRQ_WAIT]\n"); + break; + case AP_SM_STATE_IDLE: + rc += scnprintf(buf + rc, PAGE_SIZE - rc, + " [IDLE]\n"); + break; + case AP_SM_STATE_WORKING: + rc += scnprintf(buf + rc, PAGE_SIZE - rc, + " [WORKING]\n"); + break; + case AP_SM_STATE_QUEUE_FULL: + rc += scnprintf(buf + rc, PAGE_SIZE - rc, + " [FULL]\n"); + break; + default: + rc += scnprintf(buf + rc, PAGE_SIZE - rc, + " [UNKNOWN]\n"); + } + } + spin_unlock_bh(&aq->lock); + + return rc; +} +static DEVICE_ATTR_RO(states); + +static ssize_t last_err_rc_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ap_queue *aq = to_ap_queue(dev); + int rc; + + spin_lock_bh(&aq->lock); + rc = aq->last_err_rc; + spin_unlock_bh(&aq->lock); + + switch (rc) { + case AP_RESPONSE_NORMAL: + return scnprintf(buf, PAGE_SIZE, "NORMAL\n"); + case AP_RESPONSE_Q_NOT_AVAIL: + return scnprintf(buf, PAGE_SIZE, "Q_NOT_AVAIL\n"); + case AP_RESPONSE_RESET_IN_PROGRESS: + return scnprintf(buf, PAGE_SIZE, "RESET_IN_PROGRESS\n"); + case AP_RESPONSE_DECONFIGURED: + return scnprintf(buf, PAGE_SIZE, "DECONFIGURED\n"); + case AP_RESPONSE_CHECKSTOPPED: + return scnprintf(buf, PAGE_SIZE, "CHECKSTOPPED\n"); + case AP_RESPONSE_BUSY: + return scnprintf(buf, PAGE_SIZE, "BUSY\n"); + case AP_RESPONSE_INVALID_ADDRESS: + return scnprintf(buf, PAGE_SIZE, "INVALID_ADDRESS\n"); + case AP_RESPONSE_OTHERWISE_CHANGED: + return scnprintf(buf, PAGE_SIZE, "OTHERWISE_CHANGED\n"); + case AP_RESPONSE_Q_FULL: + return scnprintf(buf, PAGE_SIZE, "Q_FULL/NO_PENDING_REPLY\n"); + case AP_RESPONSE_INDEX_TOO_BIG: + return scnprintf(buf, PAGE_SIZE, "INDEX_TOO_BIG\n"); + case AP_RESPONSE_NO_FIRST_PART: + return scnprintf(buf, PAGE_SIZE, "NO_FIRST_PART\n"); + case AP_RESPONSE_MESSAGE_TOO_BIG: + return scnprintf(buf, PAGE_SIZE, "MESSAGE_TOO_BIG\n"); + case AP_RESPONSE_REQ_FAC_NOT_INST: + return scnprintf(buf, PAGE_SIZE, "REQ_FAC_NOT_INST\n"); + default: + return scnprintf(buf, PAGE_SIZE, "response code %d\n", rc); + } +} +static DEVICE_ATTR_RO(last_err_rc); +#endif + static struct attribute *ap_queue_dev_attrs[] = { &dev_attr_request_count.attr, &dev_attr_requestq_count.attr, &dev_attr_pendingq_count.attr, &dev_attr_reset.attr, &dev_attr_interrupt.attr, + &dev_attr_config.attr, +#ifdef CONFIG_ZCRYPT_DEBUG + &dev_attr_states.attr, + &dev_attr_last_err_rc.attr, +#endif NULL }; @@ -587,7 +740,6 @@ struct ap_queue *ap_queue_create(ap_qid_t qid, int device_type) aq->ap_dev.device.type = &ap_queue_type; aq->ap_dev.device_type = device_type; aq->qid = qid; - aq->sm_state = AP_SM_STATE_UNBOUND; aq->interrupt = AP_INTR_DISABLED; spin_lock_init(&aq->lock); INIT_LIST_HEAD(&aq->pendingq); @@ -612,22 +764,30 @@ EXPORT_SYMBOL(ap_queue_init_reply); * @aq: The AP device to queue the message to * @ap_msg: The message that is to be added */ -void ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg) +int ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg) { - /* For asynchronous message handling a valid receive-callback - * is required. - */ + int rc = 0; + + /* msg needs to have a valid receive-callback */ BUG_ON(!ap_msg->receive); spin_lock_bh(&aq->lock); - /* Queue the message. */ - list_add_tail(&ap_msg->list, &aq->requestq); - aq->requestq_count++; - aq->total_request_count++; - atomic64_inc(&aq->card->total_request_count); + + /* only allow to queue new messages if device state is ok */ + if (aq->dev_state == AP_DEV_STATE_OPERATING) { + list_add_tail(&ap_msg->list, &aq->requestq); + aq->requestq_count++; + aq->total_request_count++; + atomic64_inc(&aq->card->total_request_count); + } else + rc = -ENODEV; + /* Send/receive as many request from the queue as possible. */ ap_wait(ap_sm_event_loop(aq, AP_SM_EVENT_POLL)); + spin_unlock_bh(&aq->lock); + + return rc; } EXPORT_SYMBOL(ap_queue_message); @@ -698,8 +858,8 @@ void ap_queue_prepare_remove(struct ap_queue *aq) spin_lock_bh(&aq->lock); /* flush queue */ __ap_flush_queue(aq); - /* set REMOVE state to prevent new messages are queued in */ - aq->sm_state = AP_SM_STATE_REMOVE; + /* move queue device state to SHUTDOWN in progress */ + aq->dev_state = AP_DEV_STATE_SHUTDOWN; spin_unlock_bh(&aq->lock); del_timer_sync(&aq->timeout); } @@ -707,21 +867,21 @@ void ap_queue_prepare_remove(struct ap_queue *aq) void ap_queue_remove(struct ap_queue *aq) { /* - * all messages have been flushed and the state is - * AP_SM_STATE_REMOVE. Now reset with zero which also - * clears the irq registration and move the state - * to AP_SM_STATE_UNBOUND to signal that this queue - * is not used by any driver currently. + * all messages have been flushed and the device state + * is SHUTDOWN. Now reset with zero which also clears + * the irq registration and move the device state + * to the initial value AP_DEV_STATE_UNINITIATED. */ spin_lock_bh(&aq->lock); ap_zapq(aq->qid); - aq->sm_state = AP_SM_STATE_UNBOUND; + aq->dev_state = AP_DEV_STATE_UNINITIATED; spin_unlock_bh(&aq->lock); } void ap_queue_init_state(struct ap_queue *aq) { spin_lock_bh(&aq->lock); + aq->dev_state = AP_DEV_STATE_OPERATING; aq->sm_state = AP_SM_STATE_RESET_START; ap_wait(ap_sm_event(aq, AP_SM_EVENT_POLL)); spin_unlock_bh(&aq->lock); diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index 5896e5282a4e..99cb60ea663d 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -31,8 +31,9 @@ MODULE_LICENSE("GPL"); MODULE_AUTHOR("IBM Corporation"); MODULE_DESCRIPTION("s390 protected key interface"); -#define KEYBLOBBUFSIZE 8192 /* key buffer size used for internal processing */ -#define MAXAPQNSINLIST 64 /* max 64 apqns within a apqn list */ +#define KEYBLOBBUFSIZE 8192 /* key buffer size used for internal processing */ +#define PROTKEYBLOBBUFSIZE 256 /* protected key buffer size used internal */ +#define MAXAPQNSINLIST 64 /* max 64 apqns within a apqn list */ /* mask of available pckmo subfunctions, fetched once at module init */ static cpacf_mask_t pckmo_functions; @@ -237,8 +238,9 @@ static int pkey_ep11key2pkey(const u8 *key, struct pkey_protkey *pkey) for (rc = -ENODEV, i = 0; i < nr_apqns; i++) { card = apqns[i] >> 16; dom = apqns[i] & 0xFFFF; - rc = ep11_key2protkey(card, dom, key, kb->head.len, - pkey->protkey, &pkey->len, &pkey->type); + pkey->len = sizeof(pkey->protkey); + rc = ep11_kblob2protkey(card, dom, key, kb->head.len, + pkey->protkey, &pkey->len, &pkey->type); if (rc == 0) break; } @@ -449,15 +451,21 @@ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, break; } case TOKVER_EP11_AES: { - if (keylen < MINEP11AESKEYBLOBSIZE) - goto out; /* check ep11 key for exportable as protected key */ - rc = ep11_check_aeskeyblob(debug_info, 3, key, 0, 1); + rc = ep11_check_aes_key(debug_info, 3, key, keylen, 1); if (rc) goto out; rc = pkey_ep11key2pkey(key, protkey); break; } + case TOKVER_EP11_AES_WITH_HEADER: + /* check ep11 key with header for exportable as protected key */ + rc = ep11_check_aes_key_with_hdr(debug_info, 3, key, keylen, 1); + if (rc) + goto out; + rc = pkey_ep11key2pkey(key + sizeof(struct ep11kblob_header), + protkey); + break; default: DEBUG_ERR("%s unknown/unsupported non-CCA token version %d\n", __func__, hdr->version); @@ -661,13 +669,14 @@ static int pkey_verifykey2(const u8 *key, size_t keylen, *ksize = (enum pkey_key_size) t->bitsize; rc = cca_findcard2(&_apqns, &_nr_apqns, *cardnr, *domain, - ZCRYPT_CEX3C, t->mkvp, 0, 1); + ZCRYPT_CEX3C, AES_MK_SET, t->mkvp, 0, 1); if (rc == 0 && flags) *flags = PKEY_FLAGS_MATCH_CUR_MKVP; if (rc == -ENODEV) { rc = cca_findcard2(&_apqns, &_nr_apqns, *cardnr, *domain, - ZCRYPT_CEX3C, 0, t->mkvp, 1); + ZCRYPT_CEX3C, AES_MK_SET, + 0, t->mkvp, 1); if (rc == 0 && flags) *flags = PKEY_FLAGS_MATCH_ALT_MKVP; } @@ -697,13 +706,14 @@ static int pkey_verifykey2(const u8 *key, size_t keylen, } rc = cca_findcard2(&_apqns, &_nr_apqns, *cardnr, *domain, - ZCRYPT_CEX6, t->mkvp0, 0, 1); + ZCRYPT_CEX6, AES_MK_SET, t->mkvp0, 0, 1); if (rc == 0 && flags) *flags = PKEY_FLAGS_MATCH_CUR_MKVP; if (rc == -ENODEV) { rc = cca_findcard2(&_apqns, &_nr_apqns, *cardnr, *domain, - ZCRYPT_CEX6, 0, t->mkvp0, 1); + ZCRYPT_CEX6, AES_MK_SET, + 0, t->mkvp0, 1); if (rc == 0 && flags) *flags = PKEY_FLAGS_MATCH_ALT_MKVP; } @@ -717,7 +727,7 @@ static int pkey_verifykey2(const u8 *key, size_t keylen, && hdr->version == TOKVER_EP11_AES) { struct ep11keyblob *kb = (struct ep11keyblob *)key; - rc = ep11_check_aeskeyblob(debug_info, 3, key, 0, 1); + rc = ep11_check_aes_key(debug_info, 3, key, keylen, 1); if (rc) goto out; if (ktype) @@ -778,7 +788,7 @@ static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, if (hdr->version == TOKVER_EP11_AES) { if (keylen < sizeof(struct ep11keyblob)) return -EINVAL; - if (ep11_check_aeskeyblob(debug_info, 3, key, 0, 1)) + if (ep11_check_aes_key(debug_info, 3, key, keylen, 1)) return -EINVAL; } else { return pkey_nonccatok2pkey(key, keylen, pkey); @@ -804,9 +814,10 @@ static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, else { /* EP11 AES secure key blob */ struct ep11keyblob *kb = (struct ep11keyblob *) key; - rc = ep11_key2protkey(card, dom, key, kb->head.len, - pkey->protkey, &pkey->len, - &pkey->type); + pkey->len = sizeof(pkey->protkey); + rc = ep11_kblob2protkey(card, dom, key, kb->head.len, + pkey->protkey, &pkey->len, + &pkey->type); } if (rc == 0) break; @@ -825,7 +836,27 @@ static int pkey_apqns4key(const u8 *key, size_t keylen, u32 flags, if (keylen < sizeof(struct keytoken_header) || flags == 0) return -EINVAL; - if (hdr->type == TOKTYPE_NON_CCA && hdr->version == TOKVER_EP11_AES) { + if (hdr->type == TOKTYPE_NON_CCA + && (hdr->version == TOKVER_EP11_AES_WITH_HEADER + || hdr->version == TOKVER_EP11_ECC_WITH_HEADER) + && is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { + int minhwtype = 0, api = 0; + struct ep11keyblob *kb = (struct ep11keyblob *) + (key + sizeof(struct ep11kblob_header)); + + if (flags != PKEY_FLAGS_MATCH_CUR_MKVP) + return -EINVAL; + if (kb->attr & EP11_BLOB_PKEY_EXTRACTABLE) { + minhwtype = ZCRYPT_CEX7; + api = EP11_API_V; + } + rc = ep11_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + minhwtype, api, kb->wkvp); + if (rc) + goto out; + } else if (hdr->type == TOKTYPE_NON_CCA + && hdr->version == TOKVER_EP11_AES + && is_ep11_keyblob(key)) { int minhwtype = 0, api = 0; struct ep11keyblob *kb = (struct ep11keyblob *) key; @@ -863,7 +894,26 @@ static int pkey_apqns4key(const u8 *key, size_t keylen, u32 flags, return -EINVAL; } rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - minhwtype, cur_mkvp, old_mkvp, 1); + minhwtype, AES_MK_SET, + cur_mkvp, old_mkvp, 1); + if (rc) + goto out; + } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { + u64 cur_mkvp = 0, old_mkvp = 0; + struct eccprivkeytoken *t = (struct eccprivkeytoken *)key; + + if (t->secid == 0x20) { + if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) + cur_mkvp = t->mkvp; + if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) + old_mkvp = t->mkvp; + } else { + /* unknown cca internal 2 token type */ + return -EINVAL; + } + rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX7, APKA_MK_SET, + cur_mkvp, old_mkvp, 1); if (rc) goto out; } else @@ -900,10 +950,26 @@ static int pkey_apqns4keytype(enum pkey_key_type ktype, if (ktype == PKEY_TYPE_CCA_CIPHER) minhwtype = ZCRYPT_CEX6; rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - minhwtype, cur_mkvp, old_mkvp, 1); + minhwtype, AES_MK_SET, + cur_mkvp, old_mkvp, 1); if (rc) goto out; - } else if (ktype == PKEY_TYPE_EP11) { + } else if (ktype == PKEY_TYPE_CCA_ECC) { + u64 cur_mkvp = 0, old_mkvp = 0; + + if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) + cur_mkvp = *((u64 *) cur_mkvp); + if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) + old_mkvp = *((u64 *) alt_mkvp); + rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX7, APKA_MK_SET, + cur_mkvp, old_mkvp, 1); + if (rc) + goto out; + + } else if (ktype == PKEY_TYPE_EP11 || + ktype == PKEY_TYPE_EP11_AES || + ktype == PKEY_TYPE_EP11_ECC) { u8 *wkvp = NULL; if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) @@ -929,6 +995,111 @@ out: return rc; } +static int pkey_keyblob2pkey3(const struct pkey_apqn *apqns, size_t nr_apqns, + const u8 *key, size_t keylen, u32 *protkeytype, + u8 *protkey, u32 *protkeylen) +{ + int i, card, dom, rc; + struct keytoken_header *hdr = (struct keytoken_header *)key; + + /* check for at least one apqn given */ + if (!apqns || !nr_apqns) + return -EINVAL; + + if (keylen < sizeof(struct keytoken_header)) + return -EINVAL; + + if (hdr->type == TOKTYPE_NON_CCA + && hdr->version == TOKVER_EP11_AES_WITH_HEADER + && is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { + /* EP11 AES key blob with header */ + if (ep11_check_aes_key_with_hdr(debug_info, 3, key, keylen, 1)) + return -EINVAL; + } else if (hdr->type == TOKTYPE_NON_CCA + && hdr->version == TOKVER_EP11_ECC_WITH_HEADER + && is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { + /* EP11 ECC key blob with header */ + if (ep11_check_ecc_key_with_hdr(debug_info, 3, key, keylen, 1)) + return -EINVAL; + } else if (hdr->type == TOKTYPE_NON_CCA + && hdr->version == TOKVER_EP11_AES + && is_ep11_keyblob(key)) { + /* EP11 AES key blob with header in session field */ + if (ep11_check_aes_key(debug_info, 3, key, keylen, 1)) + return -EINVAL; + } else if (hdr->type == TOKTYPE_CCA_INTERNAL) { + if (hdr->version == TOKVER_CCA_AES) { + /* CCA AES data key */ + if (keylen != sizeof(struct secaeskeytoken)) + return -EINVAL; + if (cca_check_secaeskeytoken(debug_info, 3, key, 0)) + return -EINVAL; + } else if (hdr->version == TOKVER_CCA_VLSC) { + /* CCA AES cipher key */ + if (keylen < hdr->len || keylen > MAXCCAVLSCTOKENSIZE) + return -EINVAL; + if (cca_check_secaescipherkey(debug_info, 3, key, 0, 1)) + return -EINVAL; + } else { + DEBUG_ERR("%s unknown CCA internal token version %d\n", + __func__, hdr->version); + return -EINVAL; + } + } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { + /* CCA ECC (private) key */ + if (keylen < sizeof(struct eccprivkeytoken)) + return -EINVAL; + if (cca_check_sececckeytoken(debug_info, 3, key, keylen, 1)) + return -EINVAL; + } else if (hdr->type == TOKTYPE_NON_CCA) { + struct pkey_protkey pkey; + + rc = pkey_nonccatok2pkey(key, keylen, &pkey); + if (rc) + return rc; + memcpy(protkey, pkey.protkey, pkey.len); + *protkeylen = pkey.len; + *protkeytype = pkey.type; + return 0; + } else { + DEBUG_ERR("%s unknown/unsupported blob type %d\n", + __func__, hdr->type); + return -EINVAL; + } + + /* simple try all apqns from the list */ + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + card = apqns[i].card; + dom = apqns[i].domain; + if (hdr->type == TOKTYPE_NON_CCA + && (hdr->version == TOKVER_EP11_AES_WITH_HEADER + || hdr->version == TOKVER_EP11_ECC_WITH_HEADER) + && is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) + rc = ep11_kblob2protkey(card, dom, key, hdr->len, + protkey, protkeylen, protkeytype); + else if (hdr->type == TOKTYPE_NON_CCA + && hdr->version == TOKVER_EP11_AES + && is_ep11_keyblob(key)) + rc = ep11_kblob2protkey(card, dom, key, hdr->len, + protkey, protkeylen, protkeytype); + else if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_AES) + rc = cca_sec2protkey(card, dom, key, protkey, + protkeylen, protkeytype); + else if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_VLSC) + rc = cca_cipher2protkey(card, dom, key, protkey, + protkeylen, protkeytype); + else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) + rc = cca_ecc2protkey(card, dom, key, protkey, + protkeylen, protkeytype); + else + return -EINVAL; + } + + return rc; +} + /* * File io functions */ @@ -1329,6 +1500,55 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, kfree(apqns); break; } + case PKEY_KBLOB2PROTK3: { + struct pkey_kblob2pkey3 __user *utp = (void __user *) arg; + struct pkey_kblob2pkey3 ktp; + struct pkey_apqn *apqns = NULL; + u32 protkeylen = PROTKEYBLOBBUFSIZE; + u8 *kkey, *protkey; + + if (copy_from_user(&ktp, utp, sizeof(ktp))) + return -EFAULT; + apqns = _copy_apqns_from_user(ktp.apqns, ktp.apqn_entries); + if (IS_ERR(apqns)) + return PTR_ERR(apqns); + kkey = _copy_key_from_user(ktp.key, ktp.keylen); + if (IS_ERR(kkey)) { + kfree(apqns); + return PTR_ERR(kkey); + } + protkey = kmalloc(protkeylen, GFP_KERNEL); + if (!protkey) { + kfree(apqns); + kfree(kkey); + return -ENOMEM; + } + rc = pkey_keyblob2pkey3(apqns, ktp.apqn_entries, kkey, + ktp.keylen, &ktp.pkeytype, + protkey, &protkeylen); + DEBUG_DBG("%s pkey_keyblob2pkey3()=%d\n", __func__, rc); + kfree(apqns); + kfree(kkey); + if (rc) { + kfree(protkey); + break; + } + if (ktp.pkey && ktp.pkeylen) { + if (protkeylen > ktp.pkeylen) { + kfree(protkey); + return -EINVAL; + } + if (copy_to_user(ktp.pkey, protkey, protkeylen)) { + kfree(protkey); + return -EFAULT; + } + } + kfree(protkey); + ktp.pkeylen = protkeylen; + if (copy_to_user(utp, &ktp, sizeof(ktp))) + return -EFAULT; + break; + } default: /* unknown/unsupported ioctl cmd */ return -ENOTTY; @@ -1589,7 +1809,7 @@ static ssize_t pkey_ccacipher_aes_attr_read(enum pkey_key_size keybits, /* build a list of apqns able to generate an cipher key */ rc = cca_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX6, 0, 0, 0); + ZCRYPT_CEX6, 0, 0, 0, 0); if (rc) return rc; diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index f314936b5462..f60f9fb25214 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -25,6 +25,7 @@ #include <linux/debugfs.h> #include <linux/cdev.h> #include <linux/ctype.h> +#include <linux/capability.h> #include <asm/debug.h> #define CREATE_TRACE_POINTS @@ -602,13 +603,13 @@ static inline bool zcrypt_card_compare(struct zcrypt_card *zc, unsigned int pref_weight) { if (!pref_zc) - return false; + return true; weight += atomic_read(&zc->load); pref_weight += atomic_read(&pref_zc->load); if (weight == pref_weight) - return atomic64_read(&zc->card->total_request_count) > + return atomic64_read(&zc->card->total_request_count) < atomic64_read(&pref_zc->card->total_request_count); - return weight > pref_weight; + return weight < pref_weight; } static inline bool zcrypt_queue_compare(struct zcrypt_queue *zq, @@ -617,30 +618,39 @@ static inline bool zcrypt_queue_compare(struct zcrypt_queue *zq, unsigned int pref_weight) { if (!pref_zq) - return false; + return true; weight += atomic_read(&zq->load); pref_weight += atomic_read(&pref_zq->load); if (weight == pref_weight) - return zq->queue->total_request_count > + return zq->queue->total_request_count < pref_zq->queue->total_request_count; - return weight > pref_weight; + return weight < pref_weight; } /* * zcrypt ioctls. */ static long zcrypt_rsa_modexpo(struct ap_perms *perms, + struct zcrypt_track *tr, struct ica_rsa_modexpo *mex) { struct zcrypt_card *zc, *pref_zc; struct zcrypt_queue *zq, *pref_zq; - unsigned int weight = 0, pref_weight = 0; + struct ap_message ap_msg; + unsigned int wgt = 0, pref_wgt = 0; unsigned int func_code; - int qid = 0, rc = -ENODEV; + int cpen, qpen, qid = 0, rc = -ENODEV; struct module *mod; trace_s390_zcrypt_req(mex, TP_ICARSAMODEXPO); + ap_init_message(&ap_msg); + +#ifdef CONFIG_ZCRYPT_DEBUG + if (tr && tr->fi.cmd) + ap_msg.fi.cmd = tr->fi.cmd; +#endif + if (mex->outputdatalength < mex->inputdatalength) { func_code = 0; rc = -EINVAL; @@ -662,8 +672,9 @@ static long zcrypt_rsa_modexpo(struct ap_perms *perms, pref_zq = NULL; spin_lock(&zcrypt_list_lock); for_each_zcrypt_card(zc) { - /* Check for online accelarator and CCA cards */ - if (!zc->online || !(zc->card->functions & 0x18000000)) + /* Check for useable accelarator or CCA card */ + if (!zc->online || !zc->card->config || + !(zc->card->functions & 0x18000000)) continue; /* Check for size limits */ if (zc->min_mod_size > mex->inputdatalength || @@ -673,26 +684,35 @@ static long zcrypt_rsa_modexpo(struct ap_perms *perms, if (!zcrypt_check_card(perms, zc->card->id)) continue; /* get weight index of the card device */ - weight = zc->speed_rating[func_code]; - if (zcrypt_card_compare(zc, pref_zc, weight, pref_weight)) + wgt = zc->speed_rating[func_code]; + /* penalty if this msg was previously sent via this card */ + cpen = (tr && tr->again_counter && tr->last_qid && + AP_QID_CARD(tr->last_qid) == zc->card->id) ? + TRACK_AGAIN_CARD_WEIGHT_PENALTY : 0; + if (!zcrypt_card_compare(zc, pref_zc, wgt + cpen, pref_wgt)) continue; for_each_zcrypt_queue(zq, zc) { - /* check if device is online and eligible */ - if (!zq->online || !zq->ops->rsa_modexpo) + /* check if device is useable and eligible */ + if (!zq->online || !zq->ops->rsa_modexpo || + !zq->queue->config) continue; /* check if device node has admission for this queue */ if (!zcrypt_check_queue(perms, AP_QID_QUEUE(zq->queue->qid))) continue; - if (zcrypt_queue_compare(zq, pref_zq, - weight, pref_weight)) + /* penalty if the msg was previously sent at this qid */ + qpen = (tr && tr->again_counter && tr->last_qid && + tr->last_qid == zq->queue->qid) ? + TRACK_AGAIN_QUEUE_WEIGHT_PENALTY : 0; + if (!zcrypt_queue_compare(zq, pref_zq, + wgt + cpen + qpen, pref_wgt)) continue; pref_zc = zc; pref_zq = zq; - pref_weight = weight; + pref_wgt = wgt + cpen + qpen; } } - pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, weight); + pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, wgt); spin_unlock(&zcrypt_list_lock); if (!pref_zq) { @@ -701,30 +721,44 @@ static long zcrypt_rsa_modexpo(struct ap_perms *perms, } qid = pref_zq->queue->qid; - rc = pref_zq->ops->rsa_modexpo(pref_zq, mex); + rc = pref_zq->ops->rsa_modexpo(pref_zq, mex, &ap_msg); spin_lock(&zcrypt_list_lock); - zcrypt_drop_queue(pref_zc, pref_zq, mod, weight); + zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt); spin_unlock(&zcrypt_list_lock); out: + ap_release_message(&ap_msg); + if (tr) { + tr->last_rc = rc; + tr->last_qid = qid; + } trace_s390_zcrypt_rep(mex, func_code, rc, AP_QID_CARD(qid), AP_QID_QUEUE(qid)); return rc; } static long zcrypt_rsa_crt(struct ap_perms *perms, + struct zcrypt_track *tr, struct ica_rsa_modexpo_crt *crt) { struct zcrypt_card *zc, *pref_zc; struct zcrypt_queue *zq, *pref_zq; - unsigned int weight = 0, pref_weight = 0; + struct ap_message ap_msg; + unsigned int wgt = 0, pref_wgt = 0; unsigned int func_code; - int qid = 0, rc = -ENODEV; + int cpen, qpen, qid = 0, rc = -ENODEV; struct module *mod; trace_s390_zcrypt_req(crt, TP_ICARSACRT); + ap_init_message(&ap_msg); + +#ifdef CONFIG_ZCRYPT_DEBUG + if (tr && tr->fi.cmd) + ap_msg.fi.cmd = tr->fi.cmd; +#endif + if (crt->outputdatalength < crt->inputdatalength) { func_code = 0; rc = -EINVAL; @@ -746,8 +780,9 @@ static long zcrypt_rsa_crt(struct ap_perms *perms, pref_zq = NULL; spin_lock(&zcrypt_list_lock); for_each_zcrypt_card(zc) { - /* Check for online accelarator and CCA cards */ - if (!zc->online || !(zc->card->functions & 0x18000000)) + /* Check for useable accelarator or CCA card */ + if (!zc->online || !zc->card->config || + !(zc->card->functions & 0x18000000)) continue; /* Check for size limits */ if (zc->min_mod_size > crt->inputdatalength || @@ -757,26 +792,35 @@ static long zcrypt_rsa_crt(struct ap_perms *perms, if (!zcrypt_check_card(perms, zc->card->id)) continue; /* get weight index of the card device */ - weight = zc->speed_rating[func_code]; - if (zcrypt_card_compare(zc, pref_zc, weight, pref_weight)) + wgt = zc->speed_rating[func_code]; + /* penalty if this msg was previously sent via this card */ + cpen = (tr && tr->again_counter && tr->last_qid && + AP_QID_CARD(tr->last_qid) == zc->card->id) ? + TRACK_AGAIN_CARD_WEIGHT_PENALTY : 0; + if (!zcrypt_card_compare(zc, pref_zc, wgt + cpen, pref_wgt)) continue; for_each_zcrypt_queue(zq, zc) { - /* check if device is online and eligible */ - if (!zq->online || !zq->ops->rsa_modexpo_crt) + /* check if device is useable and eligible */ + if (!zq->online || !zq->ops->rsa_modexpo_crt || + !zq->queue->config) continue; /* check if device node has admission for this queue */ if (!zcrypt_check_queue(perms, AP_QID_QUEUE(zq->queue->qid))) continue; - if (zcrypt_queue_compare(zq, pref_zq, - weight, pref_weight)) + /* penalty if the msg was previously sent at this qid */ + qpen = (tr && tr->again_counter && tr->last_qid && + tr->last_qid == zq->queue->qid) ? + TRACK_AGAIN_QUEUE_WEIGHT_PENALTY : 0; + if (!zcrypt_queue_compare(zq, pref_zq, + wgt + cpen + qpen, pref_wgt)) continue; pref_zc = zc; pref_zq = zq; - pref_weight = weight; + pref_wgt = wgt + cpen + qpen; } } - pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, weight); + pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, wgt); spin_unlock(&zcrypt_list_lock); if (!pref_zq) { @@ -785,35 +829,52 @@ static long zcrypt_rsa_crt(struct ap_perms *perms, } qid = pref_zq->queue->qid; - rc = pref_zq->ops->rsa_modexpo_crt(pref_zq, crt); + rc = pref_zq->ops->rsa_modexpo_crt(pref_zq, crt, &ap_msg); spin_lock(&zcrypt_list_lock); - zcrypt_drop_queue(pref_zc, pref_zq, mod, weight); + zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt); spin_unlock(&zcrypt_list_lock); out: + ap_release_message(&ap_msg); + if (tr) { + tr->last_rc = rc; + tr->last_qid = qid; + } trace_s390_zcrypt_rep(crt, func_code, rc, AP_QID_CARD(qid), AP_QID_QUEUE(qid)); return rc; } -static long _zcrypt_send_cprb(struct ap_perms *perms, +static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, + struct zcrypt_track *tr, struct ica_xcRB *xcRB) { struct zcrypt_card *zc, *pref_zc; struct zcrypt_queue *zq, *pref_zq; struct ap_message ap_msg; - unsigned int weight = 0, pref_weight = 0; + unsigned int wgt = 0, pref_wgt = 0; unsigned int func_code; unsigned short *domain, tdom; - int qid = 0, rc = -ENODEV; + int cpen, qpen, qid = 0, rc = -ENODEV; struct module *mod; trace_s390_zcrypt_req(xcRB, TB_ZSECSENDCPRB); xcRB->status = 0; ap_init_message(&ap_msg); - rc = get_cprb_fc(xcRB, &ap_msg, &func_code, &domain); + +#ifdef CONFIG_ZCRYPT_DEBUG + if (tr && tr->fi.cmd) + ap_msg.fi.cmd = tr->fi.cmd; + if (tr && tr->fi.action == AP_FI_ACTION_CCA_AGENT_FF) { + ZCRYPT_DBF_WARN("%s fi cmd 0x%04x: forcing invalid agent_ID 'FF'\n", + __func__, tr->fi.cmd); + xcRB->agent_ID = 0x4646; + } +#endif + + rc = get_cprb_fc(userspace, xcRB, &ap_msg, &func_code, &domain); if (rc) goto out; @@ -832,8 +893,9 @@ static long _zcrypt_send_cprb(struct ap_perms *perms, pref_zq = NULL; spin_lock(&zcrypt_list_lock); for_each_zcrypt_card(zc) { - /* Check for online CCA cards */ - if (!zc->online || !(zc->card->functions & 0x10000000)) + /* Check for useable CCA card */ + if (!zc->online || !zc->card->config || + !(zc->card->functions & 0x10000000)) continue; /* Check for user selected CCA card */ if (xcRB->user_defined != AUTOSELECT && @@ -843,13 +905,18 @@ static long _zcrypt_send_cprb(struct ap_perms *perms, if (!zcrypt_check_card(perms, zc->card->id)) continue; /* get weight index of the card device */ - weight = speed_idx_cca(func_code) * zc->speed_rating[SECKEY]; - if (zcrypt_card_compare(zc, pref_zc, weight, pref_weight)) + wgt = speed_idx_cca(func_code) * zc->speed_rating[SECKEY]; + /* penalty if this msg was previously sent via this card */ + cpen = (tr && tr->again_counter && tr->last_qid && + AP_QID_CARD(tr->last_qid) == zc->card->id) ? + TRACK_AGAIN_CARD_WEIGHT_PENALTY : 0; + if (!zcrypt_card_compare(zc, pref_zc, wgt + cpen, pref_wgt)) continue; for_each_zcrypt_queue(zq, zc) { - /* check if device is online and eligible */ + /* check for device useable and eligible */ if (!zq->online || !zq->ops->send_cprb || + !zq->queue->config || (tdom != AUTOSEL_DOM && tdom != AP_QID_QUEUE(zq->queue->qid))) continue; @@ -857,15 +924,19 @@ static long _zcrypt_send_cprb(struct ap_perms *perms, if (!zcrypt_check_queue(perms, AP_QID_QUEUE(zq->queue->qid))) continue; - if (zcrypt_queue_compare(zq, pref_zq, - weight, pref_weight)) + /* penalty if the msg was previously sent at this qid */ + qpen = (tr && tr->again_counter && tr->last_qid && + tr->last_qid == zq->queue->qid) ? + TRACK_AGAIN_QUEUE_WEIGHT_PENALTY : 0; + if (!zcrypt_queue_compare(zq, pref_zq, + wgt + cpen + qpen, pref_wgt)) continue; pref_zc = zc; pref_zq = zq; - pref_weight = weight; + pref_wgt = wgt + cpen + qpen; } } - pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, weight); + pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, wgt); spin_unlock(&zcrypt_list_lock); if (!pref_zq) { @@ -878,14 +949,26 @@ static long _zcrypt_send_cprb(struct ap_perms *perms, if (*domain == AUTOSEL_DOM) *domain = AP_QID_QUEUE(qid); - rc = pref_zq->ops->send_cprb(pref_zq, xcRB, &ap_msg); +#ifdef CONFIG_ZCRYPT_DEBUG + if (tr && tr->fi.action == AP_FI_ACTION_CCA_DOM_INVAL) { + ZCRYPT_DBF_WARN("%s fi cmd 0x%04x: forcing invalid domain\n", + __func__, tr->fi.cmd); + *domain = 99; + } +#endif + + rc = pref_zq->ops->send_cprb(userspace, pref_zq, xcRB, &ap_msg); spin_lock(&zcrypt_list_lock); - zcrypt_drop_queue(pref_zc, pref_zq, mod, weight); + zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt); spin_unlock(&zcrypt_list_lock); out: ap_release_message(&ap_msg); + if (tr) { + tr->last_rc = rc; + tr->last_qid = qid; + } trace_s390_zcrypt_rep(xcRB, func_code, rc, AP_QID_CARD(qid), AP_QID_QUEUE(qid)); return rc; @@ -893,7 +976,7 @@ out: long zcrypt_send_cprb(struct ica_xcRB *xcRB) { - return _zcrypt_send_cprb(&ap_perms, xcRB); + return _zcrypt_send_cprb(false, &ap_perms, NULL, xcRB); } EXPORT_SYMBOL(zcrypt_send_cprb); @@ -924,23 +1007,29 @@ static bool is_desired_ep11_queue(unsigned int dev_qid, return false; } -static long _zcrypt_send_ep11_cprb(struct ap_perms *perms, +static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, + struct zcrypt_track *tr, struct ep11_urb *xcrb) { struct zcrypt_card *zc, *pref_zc; struct zcrypt_queue *zq, *pref_zq; struct ep11_target_dev *targets; unsigned short target_num; - unsigned int weight = 0, pref_weight = 0; + unsigned int wgt = 0, pref_wgt = 0; unsigned int func_code; struct ap_message ap_msg; - int qid = 0, rc = -ENODEV; + int cpen, qpen, qid = 0, rc = -ENODEV; struct module *mod; trace_s390_zcrypt_req(xcrb, TP_ZSENDEP11CPRB); ap_init_message(&ap_msg); +#ifdef CONFIG_ZCRYPT_DEBUG + if (tr && tr->fi.cmd) + ap_msg.fi.cmd = tr->fi.cmd; +#endif + target_num = (unsigned short) xcrb->targets_num; /* empty list indicates autoselect (all available targets) */ @@ -956,7 +1045,7 @@ static long _zcrypt_send_ep11_cprb(struct ap_perms *perms, } uptr = (struct ep11_target_dev __force __user *) xcrb->targets; - if (copy_from_user(targets, uptr, + if (z_copy_from_user(userspace, targets, uptr, target_num * sizeof(*targets))) { func_code = 0; rc = -EFAULT; @@ -964,7 +1053,7 @@ static long _zcrypt_send_ep11_cprb(struct ap_perms *perms, } } - rc = get_ep11cprb_fc(xcrb, &ap_msg, &func_code); + rc = get_ep11cprb_fc(userspace, xcrb, &ap_msg, &func_code); if (rc) goto out_free; @@ -972,8 +1061,9 @@ static long _zcrypt_send_ep11_cprb(struct ap_perms *perms, pref_zq = NULL; spin_lock(&zcrypt_list_lock); for_each_zcrypt_card(zc) { - /* Check for online EP11 cards */ - if (!zc->online || !(zc->card->functions & 0x04000000)) + /* Check for useable EP11 card */ + if (!zc->online || !zc->card->config || + !(zc->card->functions & 0x04000000)) continue; /* Check for user selected EP11 card */ if (targets && @@ -983,13 +1073,18 @@ static long _zcrypt_send_ep11_cprb(struct ap_perms *perms, if (!zcrypt_check_card(perms, zc->card->id)) continue; /* get weight index of the card device */ - weight = speed_idx_ep11(func_code) * zc->speed_rating[SECKEY]; - if (zcrypt_card_compare(zc, pref_zc, weight, pref_weight)) + wgt = speed_idx_ep11(func_code) * zc->speed_rating[SECKEY]; + /* penalty if this msg was previously sent via this card */ + cpen = (tr && tr->again_counter && tr->last_qid && + AP_QID_CARD(tr->last_qid) == zc->card->id) ? + TRACK_AGAIN_CARD_WEIGHT_PENALTY : 0; + if (!zcrypt_card_compare(zc, pref_zc, wgt + cpen, pref_wgt)) continue; for_each_zcrypt_queue(zq, zc) { - /* check if device is online and eligible */ + /* check if device is useable and eligible */ if (!zq->online || !zq->ops->send_ep11_cprb || + !zq->queue->config || (targets && !is_desired_ep11_queue(zq->queue->qid, target_num, targets))) @@ -998,15 +1093,19 @@ static long _zcrypt_send_ep11_cprb(struct ap_perms *perms, if (!zcrypt_check_queue(perms, AP_QID_QUEUE(zq->queue->qid))) continue; - if (zcrypt_queue_compare(zq, pref_zq, - weight, pref_weight)) + /* penalty if the msg was previously sent at this qid */ + qpen = (tr && tr->again_counter && tr->last_qid && + tr->last_qid == zq->queue->qid) ? + TRACK_AGAIN_QUEUE_WEIGHT_PENALTY : 0; + if (!zcrypt_queue_compare(zq, pref_zq, + wgt + cpen + qpen, pref_wgt)) continue; pref_zc = zc; pref_zq = zq; - pref_weight = weight; + pref_wgt = wgt + cpen + qpen; } } - pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, weight); + pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, wgt); spin_unlock(&zcrypt_list_lock); if (!pref_zq) { @@ -1015,16 +1114,20 @@ static long _zcrypt_send_ep11_cprb(struct ap_perms *perms, } qid = pref_zq->queue->qid; - rc = pref_zq->ops->send_ep11_cprb(pref_zq, xcrb, &ap_msg); + rc = pref_zq->ops->send_ep11_cprb(userspace, pref_zq, xcrb, &ap_msg); spin_lock(&zcrypt_list_lock); - zcrypt_drop_queue(pref_zc, pref_zq, mod, weight); + zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt); spin_unlock(&zcrypt_list_lock); out_free: kfree(targets); out: ap_release_message(&ap_msg); + if (tr) { + tr->last_rc = rc; + tr->last_qid = qid; + } trace_s390_zcrypt_rep(xcrb, func_code, rc, AP_QID_CARD(qid), AP_QID_QUEUE(qid)); return rc; @@ -1032,7 +1135,7 @@ out: long zcrypt_send_ep11_cprb(struct ep11_urb *xcrb) { - return _zcrypt_send_ep11_cprb(&ap_perms, xcrb); + return _zcrypt_send_ep11_cprb(false, &ap_perms, NULL, xcrb); } EXPORT_SYMBOL(zcrypt_send_ep11_cprb); @@ -1040,7 +1143,7 @@ static long zcrypt_rng(char *buffer) { struct zcrypt_card *zc, *pref_zc; struct zcrypt_queue *zq, *pref_zq; - unsigned int weight = 0, pref_weight = 0; + unsigned int wgt = 0, pref_wgt = 0; unsigned int func_code; struct ap_message ap_msg; unsigned int domain; @@ -1058,26 +1161,27 @@ static long zcrypt_rng(char *buffer) pref_zq = NULL; spin_lock(&zcrypt_list_lock); for_each_zcrypt_card(zc) { - /* Check for online CCA cards */ - if (!zc->online || !(zc->card->functions & 0x10000000)) + /* Check for useable CCA card */ + if (!zc->online || !zc->card->config || + !(zc->card->functions & 0x10000000)) continue; /* get weight index of the card device */ - weight = zc->speed_rating[func_code]; - if (zcrypt_card_compare(zc, pref_zc, weight, pref_weight)) + wgt = zc->speed_rating[func_code]; + if (!zcrypt_card_compare(zc, pref_zc, wgt, pref_wgt)) continue; for_each_zcrypt_queue(zq, zc) { - /* check if device is online and eligible */ - if (!zq->online || !zq->ops->rng) + /* check if device is useable and eligible */ + if (!zq->online || !zq->ops->rng || + !zq->queue->config) continue; - if (zcrypt_queue_compare(zq, pref_zq, - weight, pref_weight)) + if (!zcrypt_queue_compare(zq, pref_zq, wgt, pref_wgt)) continue; pref_zc = zc; pref_zq = zq; - pref_weight = weight; + pref_wgt = wgt; } } - pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, weight); + pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, wgt); spin_unlock(&zcrypt_list_lock); if (!pref_zq) { @@ -1089,7 +1193,7 @@ static long zcrypt_rng(char *buffer) rc = pref_zq->ops->rng(pref_zq, buffer, &ap_msg); spin_lock(&zcrypt_list_lock); - zcrypt_drop_queue(pref_zc, pref_zq, mod, weight); + zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt); spin_unlock(&zcrypt_list_lock); out: @@ -1301,19 +1405,39 @@ static int zcrypt_requestq_count(void) static int icarsamodexpo_ioctl(struct ap_perms *perms, unsigned long arg) { int rc; + struct zcrypt_track tr; struct ica_rsa_modexpo mex; struct ica_rsa_modexpo __user *umex = (void __user *) arg; + memset(&tr, 0, sizeof(tr)); if (copy_from_user(&mex, umex, sizeof(mex))) return -EFAULT; + +#ifdef CONFIG_ZCRYPT_DEBUG + if (mex.inputdatalength & (1U << 31)) { + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + tr.fi.cmd = (u16)(mex.inputdatalength >> 16); + } + mex.inputdatalength &= 0x0000FFFF; +#endif + do { - rc = zcrypt_rsa_modexpo(perms, &mex); - } while (rc == -EAGAIN); + rc = zcrypt_rsa_modexpo(perms, &tr, &mex); + if (rc == -EAGAIN) + tr.again_counter++; +#ifdef CONFIG_ZCRYPT_DEBUG + if (rc == -EAGAIN && (tr.fi.flags & AP_FI_FLAG_NO_RETRY)) + break; +#endif + } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); /* on failure: retry once again after a requested rescan */ if ((rc == -ENODEV) && (zcrypt_process_rescan())) do { - rc = zcrypt_rsa_modexpo(perms, &mex); - } while (rc == -EAGAIN); + rc = zcrypt_rsa_modexpo(perms, &tr, &mex); + if (rc == -EAGAIN) + tr.again_counter++; + } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); if (rc) { ZCRYPT_DBF(DBF_DEBUG, "ioctl ICARSAMODEXPO rc=%d\n", rc); return rc; @@ -1324,19 +1448,39 @@ static int icarsamodexpo_ioctl(struct ap_perms *perms, unsigned long arg) static int icarsacrt_ioctl(struct ap_perms *perms, unsigned long arg) { int rc; + struct zcrypt_track tr; struct ica_rsa_modexpo_crt crt; struct ica_rsa_modexpo_crt __user *ucrt = (void __user *) arg; + memset(&tr, 0, sizeof(tr)); if (copy_from_user(&crt, ucrt, sizeof(crt))) return -EFAULT; + +#ifdef CONFIG_ZCRYPT_DEBUG + if (crt.inputdatalength & (1U << 31)) { + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + tr.fi.cmd = (u16)(crt.inputdatalength >> 16); + } + crt.inputdatalength &= 0x0000FFFF; +#endif + do { - rc = zcrypt_rsa_crt(perms, &crt); - } while (rc == -EAGAIN); + rc = zcrypt_rsa_crt(perms, &tr, &crt); + if (rc == -EAGAIN) + tr.again_counter++; +#ifdef CONFIG_ZCRYPT_DEBUG + if (rc == -EAGAIN && (tr.fi.flags & AP_FI_FLAG_NO_RETRY)) + break; +#endif + } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); /* on failure: retry once again after a requested rescan */ if ((rc == -ENODEV) && (zcrypt_process_rescan())) do { - rc = zcrypt_rsa_crt(perms, &crt); - } while (rc == -EAGAIN); + rc = zcrypt_rsa_crt(perms, &tr, &crt); + if (rc == -EAGAIN) + tr.again_counter++; + } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); if (rc) { ZCRYPT_DBF(DBF_DEBUG, "ioctl ICARSACRT rc=%d\n", rc); return rc; @@ -1348,18 +1492,38 @@ static int zsecsendcprb_ioctl(struct ap_perms *perms, unsigned long arg) { int rc; struct ica_xcRB xcRB; + struct zcrypt_track tr; struct ica_xcRB __user *uxcRB = (void __user *) arg; + memset(&tr, 0, sizeof(tr)); if (copy_from_user(&xcRB, uxcRB, sizeof(xcRB))) return -EFAULT; + +#ifdef CONFIG_ZCRYPT_DEBUG + if (xcRB.status & (1U << 31)) { + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + tr.fi.cmd = (u16)(xcRB.status >> 16); + } + xcRB.status &= 0x0000FFFF; +#endif + do { - rc = _zcrypt_send_cprb(perms, &xcRB); - } while (rc == -EAGAIN); + rc = _zcrypt_send_cprb(true, perms, &tr, &xcRB); + if (rc == -EAGAIN) + tr.again_counter++; +#ifdef CONFIG_ZCRYPT_DEBUG + if (rc == -EAGAIN && (tr.fi.flags & AP_FI_FLAG_NO_RETRY)) + break; +#endif + } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); /* on failure: retry once again after a requested rescan */ if ((rc == -ENODEV) && (zcrypt_process_rescan())) do { - rc = _zcrypt_send_cprb(perms, &xcRB); - } while (rc == -EAGAIN); + rc = _zcrypt_send_cprb(true, perms, &tr, &xcRB); + if (rc == -EAGAIN) + tr.again_counter++; + } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); if (rc) ZCRYPT_DBF(DBF_DEBUG, "ioctl ZSENDCPRB rc=%d status=0x%x\n", rc, xcRB.status); @@ -1372,18 +1536,38 @@ static int zsendep11cprb_ioctl(struct ap_perms *perms, unsigned long arg) { int rc; struct ep11_urb xcrb; + struct zcrypt_track tr; struct ep11_urb __user *uxcrb = (void __user *)arg; + memset(&tr, 0, sizeof(tr)); if (copy_from_user(&xcrb, uxcrb, sizeof(xcrb))) return -EFAULT; + +#ifdef CONFIG_ZCRYPT_DEBUG + if (xcrb.req_len & (1ULL << 63)) { + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + tr.fi.cmd = (u16)(xcrb.req_len >> 48); + } + xcrb.req_len &= 0x0000FFFFFFFFFFFFULL; +#endif + do { - rc = _zcrypt_send_ep11_cprb(perms, &xcrb); - } while (rc == -EAGAIN); + rc = _zcrypt_send_ep11_cprb(true, perms, &tr, &xcrb); + if (rc == -EAGAIN) + tr.again_counter++; +#ifdef CONFIG_ZCRYPT_DEBUG + if (rc == -EAGAIN && (tr.fi.flags & AP_FI_FLAG_NO_RETRY)) + break; +#endif + } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); /* on failure: retry once again after a requested rescan */ if ((rc == -ENODEV) && (zcrypt_process_rescan())) do { - rc = _zcrypt_send_ep11_cprb(perms, &xcrb); - } while (rc == -EAGAIN); + rc = _zcrypt_send_ep11_cprb(true, perms, &tr, &xcrb); + if (rc == -EAGAIN) + tr.again_counter++; + } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); if (rc) ZCRYPT_DBF(DBF_DEBUG, "ioctl ZSENDEP11CPRB rc=%d\n", rc); if (copy_to_user(uxcrb, &xcrb, sizeof(xcrb))) @@ -1536,8 +1720,10 @@ static long trans_modexpo32(struct ap_perms *perms, struct file *filp, struct compat_ica_rsa_modexpo __user *umex32 = compat_ptr(arg); struct compat_ica_rsa_modexpo mex32; struct ica_rsa_modexpo mex64; + struct zcrypt_track tr; long rc; + memset(&tr, 0, sizeof(tr)); if (copy_from_user(&mex32, umex32, sizeof(mex32))) return -EFAULT; mex64.inputdata = compat_ptr(mex32.inputdata); @@ -1547,13 +1733,17 @@ static long trans_modexpo32(struct ap_perms *perms, struct file *filp, mex64.b_key = compat_ptr(mex32.b_key); mex64.n_modulus = compat_ptr(mex32.n_modulus); do { - rc = zcrypt_rsa_modexpo(perms, &mex64); - } while (rc == -EAGAIN); + rc = zcrypt_rsa_modexpo(perms, &tr, &mex64); + if (rc == -EAGAIN) + tr.again_counter++; + } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); /* on failure: retry once again after a requested rescan */ if ((rc == -ENODEV) && (zcrypt_process_rescan())) do { - rc = zcrypt_rsa_modexpo(perms, &mex64); - } while (rc == -EAGAIN); + rc = zcrypt_rsa_modexpo(perms, &tr, &mex64); + if (rc == -EAGAIN) + tr.again_counter++; + } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); if (rc) return rc; return put_user(mex64.outputdatalength, @@ -1578,8 +1768,10 @@ static long trans_modexpo_crt32(struct ap_perms *perms, struct file *filp, struct compat_ica_rsa_modexpo_crt __user *ucrt32 = compat_ptr(arg); struct compat_ica_rsa_modexpo_crt crt32; struct ica_rsa_modexpo_crt crt64; + struct zcrypt_track tr; long rc; + memset(&tr, 0, sizeof(tr)); if (copy_from_user(&crt32, ucrt32, sizeof(crt32))) return -EFAULT; crt64.inputdata = compat_ptr(crt32.inputdata); @@ -1592,13 +1784,17 @@ static long trans_modexpo_crt32(struct ap_perms *perms, struct file *filp, crt64.nq_prime = compat_ptr(crt32.nq_prime); crt64.u_mult_inv = compat_ptr(crt32.u_mult_inv); do { - rc = zcrypt_rsa_crt(perms, &crt64); - } while (rc == -EAGAIN); + rc = zcrypt_rsa_crt(perms, &tr, &crt64); + if (rc == -EAGAIN) + tr.again_counter++; + } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); /* on failure: retry once again after a requested rescan */ if ((rc == -ENODEV) && (zcrypt_process_rescan())) do { - rc = zcrypt_rsa_crt(perms, &crt64); - } while (rc == -EAGAIN); + rc = zcrypt_rsa_crt(perms, &tr, &crt64); + if (rc == -EAGAIN) + tr.again_counter++; + } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); if (rc) return rc; return put_user(crt64.outputdatalength, @@ -1630,9 +1826,11 @@ static long trans_xcRB32(struct ap_perms *perms, struct file *filp, { struct compat_ica_xcRB __user *uxcRB32 = compat_ptr(arg); struct compat_ica_xcRB xcRB32; + struct zcrypt_track tr; struct ica_xcRB xcRB64; long rc; + memset(&tr, 0, sizeof(tr)); if (copy_from_user(&xcRB32, uxcRB32, sizeof(xcRB32))) return -EFAULT; xcRB64.agent_ID = xcRB32.agent_ID; @@ -1656,13 +1854,17 @@ static long trans_xcRB32(struct ap_perms *perms, struct file *filp, xcRB64.priority_window = xcRB32.priority_window; xcRB64.status = xcRB32.status; do { - rc = _zcrypt_send_cprb(perms, &xcRB64); - } while (rc == -EAGAIN); + rc = _zcrypt_send_cprb(true, perms, &tr, &xcRB64); + if (rc == -EAGAIN) + tr.again_counter++; + } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); /* on failure: retry once again after a requested rescan */ if ((rc == -ENODEV) && (zcrypt_process_rescan())) do { - rc = _zcrypt_send_cprb(perms, &xcRB64); - } while (rc == -EAGAIN); + rc = _zcrypt_send_cprb(true, perms, &tr, &xcRB64); + if (rc == -EAGAIN) + tr.again_counter++; + } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); xcRB32.reply_control_blk_length = xcRB64.reply_control_blk_length; xcRB32.reply_data_length = xcRB64.reply_data_length; xcRB32.status = xcRB64.status; diff --git a/drivers/s390/crypto/zcrypt_api.h b/drivers/s390/crypto/zcrypt_api.h index 599e68bf53f7..51c0b8bdef50 100644 --- a/drivers/s390/crypto/zcrypt_api.h +++ b/drivers/s390/crypto/zcrypt_api.h @@ -55,13 +55,30 @@ enum crypto_ops { struct zcrypt_queue; +/* struct to hold tracking information for a userspace request/response */ +struct zcrypt_track { + int again_counter; /* retry attempts counter */ + int last_qid; /* last qid used */ + int last_rc; /* last return code */ +#ifdef CONFIG_ZCRYPT_DEBUG + struct ap_fi fi; /* failure injection cmd */ +#endif +}; + +/* defines related to message tracking */ +#define TRACK_AGAIN_MAX 10 +#define TRACK_AGAIN_CARD_WEIGHT_PENALTY 1000 +#define TRACK_AGAIN_QUEUE_WEIGHT_PENALTY 10000 + struct zcrypt_ops { - long (*rsa_modexpo)(struct zcrypt_queue *, struct ica_rsa_modexpo *); + long (*rsa_modexpo)(struct zcrypt_queue *, struct ica_rsa_modexpo *, + struct ap_message *); long (*rsa_modexpo_crt)(struct zcrypt_queue *, - struct ica_rsa_modexpo_crt *); - long (*send_cprb)(struct zcrypt_queue *, struct ica_xcRB *, + struct ica_rsa_modexpo_crt *, + struct ap_message *); + long (*send_cprb)(bool userspace, struct zcrypt_queue *, struct ica_xcRB *, struct ap_message *); - long (*send_ep11_cprb)(struct zcrypt_queue *, struct ep11_urb *, + long (*send_ep11_cprb)(bool userspace, struct zcrypt_queue *, struct ep11_urb *, struct ap_message *); long (*rng)(struct zcrypt_queue *, char *, struct ap_message *); struct list_head list; /* zcrypt ops list. */ @@ -82,7 +99,7 @@ struct zcrypt_card { int min_mod_size; /* Min number of bits. */ int max_mod_size; /* Max number of bits. */ int max_exp_bit_length; - int speed_rating[NUM_OPS]; /* Speed idx of crypto ops. */ + const int *speed_rating; /* Speed idx of crypto ops. */ atomic_t load; /* Utilization of the crypto device */ int request_count; /* # current requests. */ @@ -145,4 +162,26 @@ void zcrypt_device_status_mask_ext(struct zcrypt_device_status_ext *devstatus); int zcrypt_device_status_ext(int card, int queue, struct zcrypt_device_status_ext *devstatus); +static inline unsigned long z_copy_from_user(bool userspace, + void *to, + const void __user *from, + unsigned long n) +{ + if (likely(userspace)) + return copy_from_user(to, from, n); + memcpy(to, (void __force *) from, n); + return 0; +} + +static inline unsigned long z_copy_to_user(bool userspace, + void __user *to, + const void *from, + unsigned long n) +{ + if (likely(userspace)) + return copy_to_user(to, from, n); + memcpy((void __force *) to, from, n); + return 0; +} + #endif /* _ZCRYPT_API_H_ */ diff --git a/drivers/s390/crypto/zcrypt_card.c b/drivers/s390/crypto/zcrypt_card.c index c53cab4b0c9e..e342eb86acd1 100644 --- a/drivers/s390/crypto/zcrypt_card.c +++ b/drivers/s390/crypto/zcrypt_card.c @@ -50,22 +50,28 @@ static ssize_t online_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct zcrypt_card *zc = to_ap_card(dev)->private; + struct ap_card *ac = to_ap_card(dev); + struct zcrypt_card *zc = ac->private; + int online = ac->config && zc->online ? 1 : 0; - return scnprintf(buf, PAGE_SIZE, "%d\n", zc->online); + return scnprintf(buf, PAGE_SIZE, "%d\n", online); } static ssize_t online_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct zcrypt_card *zc = to_ap_card(dev)->private; + struct ap_card *ac = to_ap_card(dev); + struct zcrypt_card *zc = ac->private; struct zcrypt_queue *zq; int online, id; if (sscanf(buf, "%d\n", &online) != 1 || online < 0 || online > 1) return -EINVAL; + if (online && !ac->config) + return -ENODEV; + zc->online = online; id = zc->card->id; diff --git a/drivers/s390/crypto/zcrypt_ccamisc.c b/drivers/s390/crypto/zcrypt_ccamisc.c index c793dcabd551..b1046811450f 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.c +++ b/drivers/s390/crypto/zcrypt_ccamisc.c @@ -173,6 +173,49 @@ int cca_check_secaescipherkey(debug_info_t *dbg, int dbflvl, EXPORT_SYMBOL(cca_check_secaescipherkey); /* + * Simple check if the token is a valid CCA secure ECC private + * key token. Returns 0 on success or errno value on failure. + */ +int cca_check_sececckeytoken(debug_info_t *dbg, int dbflvl, + const u8 *token, size_t keysize, + int checkcpacfexport) +{ + struct eccprivkeytoken *t = (struct eccprivkeytoken *) token; + +#define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__) + + if (t->type != TOKTYPE_CCA_INTERNAL_PKA) { + if (dbg) + DBF("%s token check failed, type 0x%02x != 0x%02x\n", + __func__, (int) t->type, TOKTYPE_CCA_INTERNAL_PKA); + return -EINVAL; + } + if (t->len > keysize) { + if (dbg) + DBF("%s token check failed, len %d > keysize %zu\n", + __func__, (int) t->len, keysize); + return -EINVAL; + } + if (t->secid != 0x20) { + if (dbg) + DBF("%s token check failed, secid 0x%02x != 0x20\n", + __func__, (int) t->secid); + return -EINVAL; + } + if (checkcpacfexport && !(t->kutc & 0x01)) { + if (dbg) + DBF("%s token check failed, XPRTCPAC bit is 0\n", + __func__); + return -EINVAL; + } + +#undef DBF + + return 0; +} +EXPORT_SYMBOL(cca_check_sececckeytoken); + +/* * Allocate consecutive memory for request CPRB, request param * block, reply CPRB and reply param block and fill in values * for the common fields. Returns 0 on success or errno value @@ -249,24 +292,6 @@ static inline void prep_xcrb(struct ica_xcRB *pxcrb, } /* - * Helper function which calls zcrypt_send_cprb with - * memory management segment adjusted to kernel space - * so that the copy_from_user called within this - * function do in fact copy from kernel space. - */ -static inline int _zcrypt_send_cprb(struct ica_xcRB *xcrb) -{ - int rc; - mm_segment_t old_fs = get_fs(); - - set_fs(KERNEL_DS); - rc = zcrypt_send_cprb(xcrb); - set_fs(old_fs); - - return rc; -} - -/* * Generate (random) CCA AES DATA secure key. */ int cca_genseckey(u16 cardnr, u16 domain, @@ -359,7 +384,7 @@ int cca_genseckey(u16 cardnr, u16 domain, prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ - rc = _zcrypt_send_cprb(&xcrb); + rc = zcrypt_send_cprb(&xcrb); if (rc) { DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, errno %d\n", __func__, (int) cardnr, (int) domain, rc); @@ -497,7 +522,7 @@ int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize, prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ - rc = _zcrypt_send_cprb(&xcrb); + rc = zcrypt_send_cprb(&xcrb); if (rc) { DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", __func__, (int) cardnr, (int) domain, rc); @@ -624,7 +649,7 @@ int cca_sec2protkey(u16 cardnr, u16 domain, prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ - rc = _zcrypt_send_cprb(&xcrb); + rc = zcrypt_send_cprb(&xcrb); if (rc) { DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", __func__, (int) cardnr, (int) domain, rc); @@ -850,7 +875,7 @@ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ - rc = _zcrypt_send_cprb(&xcrb); + rc = zcrypt_send_cprb(&xcrb); if (rc) { DEBUG_ERR( "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", @@ -1018,7 +1043,7 @@ static int _ip_cprb_helper(u16 cardnr, u16 domain, prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ - rc = _zcrypt_send_cprb(&xcrb); + rc = zcrypt_send_cprb(&xcrb); if (rc) { DEBUG_ERR( "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", @@ -1235,7 +1260,7 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ - rc = _zcrypt_send_cprb(&xcrb); + rc = zcrypt_send_cprb(&xcrb); if (rc) { DEBUG_ERR( "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", @@ -1316,6 +1341,156 @@ out: EXPORT_SYMBOL(cca_cipher2protkey); /* + * Derive protected key from CCA ECC secure private key. + */ +int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + int rc; + u8 *mem, *ptr; + struct CPRBX *preqcblk, *prepcblk; + struct ica_xcRB xcrb; + struct aureqparm { + u8 subfunc_code[2]; + u16 rule_array_len; + u8 rule_array[8]; + struct { + u16 len; + u16 tk_blob_len; + u16 tk_blob_tag; + u8 tk_blob[66]; + } vud; + struct { + u16 len; + u16 cca_key_token_len; + u16 cca_key_token_flags; + u8 cca_key_token[0]; + } kb; + } __packed * preqparm; + struct aurepparm { + u8 subfunc_code[2]; + u16 rule_array_len; + struct { + u16 len; + u16 sublen; + u16 tag; + struct cpacfkeyblock { + u8 version; /* version of this struct */ + u8 flags[2]; + u8 algo; + u8 form; + u8 pad1[3]; + u16 keylen; + u8 key[0]; /* the key (keylen bytes) */ + u16 keyattrlen; + u8 keyattr[32]; + u8 pad2[1]; + u8 vptype; + u8 vp[32]; /* verification pattern */ + } ckb; + } vud; + struct { + u16 len; + } kb; + } __packed * prepparm; + int keylen = ((struct eccprivkeytoken *)key)->len; + + /* get already prepared memory for 2 cprbs with param block each */ + rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, &preqcblk, &prepcblk); + if (rc) + return rc; + + /* fill request cprb struct */ + preqcblk->domain = domain; + + /* fill request cprb param block with AU request */ + preqparm = (struct aureqparm __force *) preqcblk->req_parmb; + memcpy(preqparm->subfunc_code, "AU", 2); + preqparm->rule_array_len = + sizeof(preqparm->rule_array_len) + + sizeof(preqparm->rule_array); + memcpy(preqparm->rule_array, "EXPT-SK ", 8); + /* vud, tk blob */ + preqparm->vud.len = sizeof(preqparm->vud); + preqparm->vud.tk_blob_len = sizeof(preqparm->vud.tk_blob) + + 2 * sizeof(uint16_t); + preqparm->vud.tk_blob_tag = 0x00C2; + /* kb, cca token */ + preqparm->kb.len = keylen + 3 * sizeof(uint16_t); + preqparm->kb.cca_key_token_len = keylen + 2 * sizeof(uint16_t); + memcpy(preqparm->kb.cca_key_token, key, keylen); + /* now fill length of param block into cprb */ + preqcblk->req_parml = sizeof(struct aureqparm) + keylen; + + /* fill xcrb struct */ + prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); + + /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ + rc = zcrypt_send_cprb(&xcrb); + if (rc) { + DEBUG_ERR( + "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int) cardnr, (int) domain, rc); + goto out; + } + + /* check response returncode and reasoncode */ + if (prepcblk->ccp_rtcode != 0) { + DEBUG_ERR( + "%s unwrap secure key failure, card response %d/%d\n", + __func__, + (int) prepcblk->ccp_rtcode, + (int) prepcblk->ccp_rscode); + rc = -EIO; + goto out; + } + if (prepcblk->ccp_rscode != 0) { + DEBUG_WARN( + "%s unwrap secure key warning, card response %d/%d\n", + __func__, + (int) prepcblk->ccp_rtcode, + (int) prepcblk->ccp_rscode); + } + + /* process response cprb param block */ + ptr = ((u8 *) prepcblk) + sizeof(struct CPRBX); + prepcblk->rpl_parmb = (u8 __user *) ptr; + prepparm = (struct aurepparm *) ptr; + + /* check the returned keyblock */ + if (prepparm->vud.ckb.version != 0x02) { + DEBUG_ERR("%s reply param keyblock version mismatch 0x%02x != 0x02\n", + __func__, (int) prepparm->vud.ckb.version); + rc = -EIO; + goto out; + } + if (prepparm->vud.ckb.algo != 0x81) { + DEBUG_ERR( + "%s reply param keyblock algo mismatch 0x%02x != 0x81\n", + __func__, (int) prepparm->vud.ckb.algo); + rc = -EIO; + goto out; + } + + /* copy the translated protected key */ + if (prepparm->vud.ckb.keylen > *protkeylen) { + DEBUG_ERR("%s prot keylen mismatch %d > buffersize %u\n", + __func__, prepparm->vud.ckb.keylen, *protkeylen); + rc = -EIO; + goto out; + } + memcpy(protkey, prepparm->vud.ckb.key, prepparm->vud.ckb.keylen); + *protkeylen = prepparm->vud.ckb.keylen; + if (protkeytype) + *protkeytype = PKEY_KEYTYPE_ECC; + +out: + free_cprbmem(mem, PARMBSIZE, 0); + return rc; +} +EXPORT_SYMBOL(cca_ecc2protkey); + +/* * query cryptographic facility from CCA adapter */ int cca_query_crypto_facility(u16 cardnr, u16 domain, @@ -1366,7 +1541,7 @@ int cca_query_crypto_facility(u16 cardnr, u16 domain, prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk); /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ - rc = _zcrypt_send_cprb(&xcrb); + rc = zcrypt_send_cprb(&xcrb); if (rc) { DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", __func__, (int) cardnr, (int) domain, rc); @@ -1524,21 +1699,38 @@ static int fetch_cca_info(u16 cardnr, u16 domain, struct cca_info *ci) rarray, &rlen, varray, &vlen); if (rc == 0 && rlen >= 10*8 && vlen >= 204) { memcpy(ci->serial, rarray, 8); - ci->new_mk_state = (char) rarray[7*8]; - ci->cur_mk_state = (char) rarray[8*8]; - ci->old_mk_state = (char) rarray[9*8]; - if (ci->old_mk_state == '2') - memcpy(&ci->old_mkvp, varray + 172, 8); - if (ci->cur_mk_state == '2') - memcpy(&ci->cur_mkvp, varray + 184, 8); - if (ci->new_mk_state == '3') - memcpy(&ci->new_mkvp, varray + 196, 8); - found = 1; + ci->new_aes_mk_state = (char) rarray[7*8]; + ci->cur_aes_mk_state = (char) rarray[8*8]; + ci->old_aes_mk_state = (char) rarray[9*8]; + if (ci->old_aes_mk_state == '2') + memcpy(&ci->old_aes_mkvp, varray + 172, 8); + if (ci->cur_aes_mk_state == '2') + memcpy(&ci->cur_aes_mkvp, varray + 184, 8); + if (ci->new_aes_mk_state == '3') + memcpy(&ci->new_aes_mkvp, varray + 196, 8); + found++; + } + if (!found) + goto out; + rlen = vlen = PAGE_SIZE/2; + rc = cca_query_crypto_facility(cardnr, domain, "STATICSB", + rarray, &rlen, varray, &vlen); + if (rc == 0 && rlen >= 10*8 && vlen >= 240) { + ci->new_apka_mk_state = (char) rarray[7*8]; + ci->cur_apka_mk_state = (char) rarray[8*8]; + ci->old_apka_mk_state = (char) rarray[9*8]; + if (ci->old_apka_mk_state == '2') + memcpy(&ci->old_apka_mkvp, varray + 208, 8); + if (ci->cur_apka_mk_state == '2') + memcpy(&ci->cur_apka_mkvp, varray + 220, 8); + if (ci->new_apka_mk_state == '3') + memcpy(&ci->new_apka_mkvp, varray + 232, 8); + found++; } +out: free_page((unsigned long) pg); - - return found ? 0 : -ENOENT; + return found == 2 ? 0 : -ENOENT; } /* @@ -1592,16 +1784,16 @@ static int findcard(u64 mkvp, u16 *pcardnr, u16 *pdomain, /* enabled CCA card, check current mkvp from cache */ if (cca_info_cache_fetch(card, dom, &ci) == 0 && ci.hwtype >= minhwtype && - ci.cur_mk_state == '2' && - ci.cur_mkvp == mkvp) { + ci.cur_aes_mk_state == '2' && + ci.cur_aes_mkvp == mkvp) { if (!verify) break; /* verify: refresh card info */ if (fetch_cca_info(card, dom, &ci) == 0) { cca_info_cache_update(card, dom, &ci); if (ci.hwtype >= minhwtype && - ci.cur_mk_state == '2' && - ci.cur_mkvp == mkvp) + ci.cur_aes_mk_state == '2' && + ci.cur_aes_mkvp == mkvp) break; } } @@ -1623,12 +1815,12 @@ static int findcard(u64 mkvp, u16 *pcardnr, u16 *pdomain, if (fetch_cca_info(card, dom, &ci) == 0) { cca_info_cache_update(card, dom, &ci); if (ci.hwtype >= minhwtype && - ci.cur_mk_state == '2' && - ci.cur_mkvp == mkvp) + ci.cur_aes_mk_state == '2' && + ci.cur_aes_mkvp == mkvp) break; if (ci.hwtype >= minhwtype && - ci.old_mk_state == '2' && - ci.old_mkvp == mkvp && + ci.old_aes_mk_state == '2' && + ci.old_aes_mkvp == mkvp && oi < 0) oi = i; } @@ -1682,15 +1874,14 @@ int cca_findcard(const u8 *key, u16 *pcardnr, u16 *pdomain, int verify) EXPORT_SYMBOL(cca_findcard); int cca_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, - int minhwtype, u64 cur_mkvp, u64 old_mkvp, int verify) + int minhwtype, int mktype, u64 cur_mkvp, u64 old_mkvp, + int verify) { struct zcrypt_device_status_ext *device_status; - int i, n, card, dom, curmatch, oldmatch, rc = 0; + u32 *_apqns = NULL, _nr_apqns = 0; + int i, card, dom, curmatch, oldmatch, rc = 0; struct cca_info ci; - *apqns = NULL; - *nr_apqns = 0; - /* fetch status of all crypto cards */ device_status = kvmalloc_array(MAX_ZDEV_ENTRIES_EXT, sizeof(struct zcrypt_device_status_ext), @@ -1699,67 +1890,73 @@ int cca_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, return -ENOMEM; zcrypt_device_status_mask_ext(device_status); - /* loop two times: first gather eligible apqns, then store them */ - while (1) { - n = 0; - /* walk through all the crypto cards */ - for (i = 0; i < MAX_ZDEV_ENTRIES_EXT; i++) { - card = AP_QID_CARD(device_status[i].qid); - dom = AP_QID_QUEUE(device_status[i].qid); - /* check online state */ - if (!device_status[i].online) - continue; - /* check for cca functions */ - if (!(device_status[i].functions & 0x04)) - continue; - /* check cardnr */ - if (cardnr != 0xFFFF && card != cardnr) - continue; - /* check domain */ - if (domain != 0xFFFF && dom != domain) - continue; - /* get cca info on this apqn */ - if (cca_get_info(card, dom, &ci, verify)) - continue; - /* current master key needs to be valid */ - if (ci.cur_mk_state != '2') - continue; - /* check min hardware type */ - if (minhwtype > 0 && minhwtype > ci.hwtype) - continue; - if (cur_mkvp || old_mkvp) { - /* check mkvps */ - curmatch = oldmatch = 0; - if (cur_mkvp && cur_mkvp == ci.cur_mkvp) + /* allocate 1k space for up to 256 apqns */ + _apqns = kmalloc_array(256, sizeof(u32), GFP_KERNEL); + if (!_apqns) { + kvfree(device_status); + return -ENOMEM; + } + + /* walk through all the crypto apqnss */ + for (i = 0; i < MAX_ZDEV_ENTRIES_EXT; i++) { + card = AP_QID_CARD(device_status[i].qid); + dom = AP_QID_QUEUE(device_status[i].qid); + /* check online state */ + if (!device_status[i].online) + continue; + /* check for cca functions */ + if (!(device_status[i].functions & 0x04)) + continue; + /* check cardnr */ + if (cardnr != 0xFFFF && card != cardnr) + continue; + /* check domain */ + if (domain != 0xFFFF && dom != domain) + continue; + /* get cca info on this apqn */ + if (cca_get_info(card, dom, &ci, verify)) + continue; + /* current master key needs to be valid */ + if (mktype == AES_MK_SET && ci.cur_aes_mk_state != '2') + continue; + if (mktype == APKA_MK_SET && ci.cur_apka_mk_state != '2') + continue; + /* check min hardware type */ + if (minhwtype > 0 && minhwtype > ci.hwtype) + continue; + if (cur_mkvp || old_mkvp) { + /* check mkvps */ + curmatch = oldmatch = 0; + if (mktype == AES_MK_SET) { + if (cur_mkvp && cur_mkvp == ci.cur_aes_mkvp) + curmatch = 1; + if (old_mkvp && ci.old_aes_mk_state == '2' && + old_mkvp == ci.old_aes_mkvp) + oldmatch = 1; + } else { + if (cur_mkvp && cur_mkvp == ci.cur_apka_mkvp) curmatch = 1; - if (old_mkvp && ci.old_mk_state == '2' && - old_mkvp == ci.old_mkvp) + if (old_mkvp && ci.old_apka_mk_state == '2' && + old_mkvp == ci.old_apka_mkvp) oldmatch = 1; - if ((cur_mkvp || old_mkvp) && - (curmatch + oldmatch < 1)) - continue; } - /* apqn passed all filtering criterons */ - if (*apqns && n < *nr_apqns) - (*apqns)[n] = (((u16)card) << 16) | ((u16) dom); - n++; - } - /* loop 2nd time: array has been filled */ - if (*apqns) - break; - /* loop 1st time: have # of eligible apqns in n */ - if (!n) { - rc = -ENODEV; /* no eligible apqns found */ - break; - } - *nr_apqns = n; - /* allocate array to store n apqns into */ - *apqns = kmalloc_array(n, sizeof(u32), GFP_KERNEL); - if (!*apqns) { - rc = -ENOMEM; - break; + if (curmatch + oldmatch < 1) + continue; } - verify = 0; + /* apqn passed all filtering criterons, add to the array */ + if (_nr_apqns < 256) + _apqns[_nr_apqns++] = (((u16)card) << 16) | ((u16) dom); + } + + /* nothing found ? */ + if (!_nr_apqns) { + kfree(_apqns); + rc = -ENODEV; + } else { + /* no re-allocation, simple return the _apqns array */ + *apqns = _apqns; + *nr_apqns = _nr_apqns; + rc = 0; } kvfree(device_status); diff --git a/drivers/s390/crypto/zcrypt_ccamisc.h b/drivers/s390/crypto/zcrypt_ccamisc.h index 8b7a641671c9..e7105443d5cb 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.h +++ b/drivers/s390/crypto/zcrypt_ccamisc.h @@ -14,8 +14,9 @@ #include <asm/pkey.h> /* Key token types */ -#define TOKTYPE_NON_CCA 0x00 /* Non-CCA key token */ -#define TOKTYPE_CCA_INTERNAL 0x01 /* CCA internal key token */ +#define TOKTYPE_NON_CCA 0x00 /* Non-CCA key token */ +#define TOKTYPE_CCA_INTERNAL 0x01 /* CCA internal sym key token */ +#define TOKTYPE_CCA_INTERNAL_PKA 0x1f /* CCA internal asym key token */ /* For TOKTYPE_NON_CCA: */ #define TOKVER_PROTECTED_KEY 0x01 /* Protected key token */ @@ -93,6 +94,31 @@ struct cipherkeytoken { u8 vdata[]; /* variable part data follows */ } __packed; +/* inside view of an CCA secure ECC private key */ +struct eccprivkeytoken { + u8 type; /* 0x1f for internal asym key token */ + u8 version; /* should be 0x00 */ + u16 len; /* total key token length in bytes */ + u8 res1[4]; + u8 secid; /* 0x20 for ECC priv key section marker */ + u8 secver; /* section version */ + u16 seclen; /* section length */ + u8 wtype; /* wrapping method, 0x00 clear, 0x01 AES */ + u8 htype; /* hash method, 0x02 for SHA-256 */ + u8 res2[2]; + u8 kutc; /* key usage and translation control */ + u8 ctype; /* curve type */ + u8 kfs; /* key format and security */ + u8 ksrc; /* key source */ + u16 pbitlen; /* length of prime p in bits */ + u16 ibmadlen; /* IBM associated data length in bytes */ + u64 mkvp; /* master key verification pattern */ + u8 opk[48]; /* encrypted object protection key data */ + u16 adatalen; /* associated data length in bytes */ + u16 fseclen; /* formated section length in bytes */ + u8 more_data[]; /* more data follows */ +} __packed; + /* Some defines for the CCA AES cipherkeytoken kmf1 field */ #define KMF1_XPRT_SYM 0x8000 #define KMF1_XPRT_UASY 0x4000 @@ -123,6 +149,14 @@ int cca_check_secaescipherkey(debug_info_t *dbg, int dbflvl, int checkcpacfexport); /* + * Simple check if the token is a valid CCA secure ECC private + * key token. Returns 0 on success or errno value on failure. + */ +int cca_check_sececckeytoken(debug_info_t *dbg, int dbflvl, + const u8 *token, size_t keysize, + int checkcpacfexport); + +/* * Generate (random) CCA AES DATA secure key. */ int cca_genseckey(u16 cardnr, u16 domain, u32 keybitsize, u8 *seckey); @@ -159,6 +193,12 @@ int cca_clr2cipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, const u8 *clrkey, u8 *keybuf, size_t *keybufsize); /* + * Derive proteced key from CCA ECC secure private key. + */ +int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, + u8 *protkey, u32 *protkeylen, u32 *protkeytype); + +/* * Query cryptographic facility from CCA adapter */ int cca_query_crypto_facility(u16 cardnr, u16 domain, @@ -186,6 +226,8 @@ int cca_findcard(const u8 *key, u16 *pcardnr, u16 *pdomain, int verify); * - if verify is enabled and a cur_mkvp and/or old_mkvp * value is given, then refetch the cca_info and make sure the current * cur_mkvp or old_mkvp values of the apqn are used. + * The mktype determines which set of master keys to use: + * 0 = AES_MK_SET - AES MK set, 1 = APKA MK_SET - APKA MK set * The array of apqn entries is allocated with kmalloc and returned in *apqns; * the number of apqns stored into the list is returned in *nr_apqns. One apqn * entry is simple a 32 bit value with 16 bit cardnr and 16 bit domain nr and @@ -194,18 +236,28 @@ int cca_findcard(const u8 *key, u16 *pcardnr, u16 *pdomain, int verify); * -ENODEV is returned. */ int cca_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, - int minhwtype, u64 cur_mkvp, u64 old_mkvp, int verify); + int minhwtype, int mktype, u64 cur_mkvp, u64 old_mkvp, + int verify); + +#define AES_MK_SET 0 +#define APKA_MK_SET 1 /* struct to hold info for each CCA queue */ struct cca_info { - int hwtype; /* one of the defined AP_DEVICE_TYPE_* */ - char new_mk_state; /* '1' empty, '2' partially full, '3' full */ - char cur_mk_state; /* '1' invalid, '2' valid */ - char old_mk_state; /* '1' invalid, '2' valid */ - u64 new_mkvp; /* truncated sha256 hash of new master key */ - u64 cur_mkvp; /* truncated sha256 hash of current master key */ - u64 old_mkvp; /* truncated sha256 hash of old master key */ - char serial[9]; /* serial number string (8 ascii numbers + 0x00) */ + int hwtype; /* one of the defined AP_DEVICE_TYPE_* */ + char new_aes_mk_state; /* '1' empty, '2' partially full, '3' full */ + char cur_aes_mk_state; /* '1' invalid, '2' valid */ + char old_aes_mk_state; /* '1' invalid, '2' valid */ + char new_apka_mk_state; /* '1' empty, '2' partially full, '3' full */ + char cur_apka_mk_state; /* '1' invalid, '2' valid */ + char old_apka_mk_state; /* '1' invalid, '2' valid */ + u64 new_aes_mkvp; /* truncated sha256 of new aes master key */ + u64 cur_aes_mkvp; /* truncated sha256 of current aes master key */ + u64 old_aes_mkvp; /* truncated sha256 of old aes master key */ + u64 new_apka_mkvp; /* truncated sha256 of new apka master key */ + u64 cur_apka_mkvp; /* truncated sha256 of current apka mk */ + u64 old_apka_mkvp; /* truncated sha256 of old apka mk */ + char serial[9]; /* serial number (8 ascii numbers + 0x00) */ }; /* diff --git a/drivers/s390/crypto/zcrypt_cex2a.c b/drivers/s390/crypto/zcrypt_cex2a.c index b447f3e9e4a2..226a5612e855 100644 --- a/drivers/s390/crypto/zcrypt_cex2a.c +++ b/drivers/s390/crypto/zcrypt_cex2a.c @@ -94,8 +94,7 @@ static int zcrypt_cex2a_card_probe(struct ap_device *ap_dev) if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX2A) { zc->min_mod_size = CEX2A_MIN_MOD_SIZE; zc->max_mod_size = CEX2A_MAX_MOD_SIZE; - memcpy(zc->speed_rating, CEX2A_SPEED_IDX, - sizeof(CEX2A_SPEED_IDX)); + zc->speed_rating = CEX2A_SPEED_IDX; zc->max_exp_bit_length = CEX2A_MAX_MOD_SIZE; zc->type_string = "CEX2A"; zc->user_space_type = ZCRYPT_CEX2A; @@ -108,8 +107,7 @@ static int zcrypt_cex2a_card_probe(struct ap_device *ap_dev) zc->max_mod_size = CEX3A_MAX_MOD_SIZE; zc->max_exp_bit_length = CEX3A_MAX_MOD_SIZE; } - memcpy(zc->speed_rating, CEX3A_SPEED_IDX, - sizeof(CEX3A_SPEED_IDX)); + zc->speed_rating = CEX3A_SPEED_IDX; zc->type_string = "CEX3A"; zc->user_space_type = ZCRYPT_CEX3A; } else { diff --git a/drivers/s390/crypto/zcrypt_cex2c.c b/drivers/s390/crypto/zcrypt_cex2c.c index f00127a78bab..7a8cbdbe4408 100644 --- a/drivers/s390/crypto/zcrypt_cex2c.c +++ b/drivers/s390/crypto/zcrypt_cex2c.c @@ -109,26 +109,53 @@ static ssize_t cca_mkvps_show(struct device *dev, AP_QID_QUEUE(zq->queue->qid), &ci, zq->online); - if (ci.new_mk_state >= '1' && ci.new_mk_state <= '3') + if (ci.new_aes_mk_state >= '1' && ci.new_aes_mk_state <= '3') n = scnprintf(buf, PAGE_SIZE, "AES NEW: %s 0x%016llx\n", - new_state[ci.new_mk_state - '1'], ci.new_mkvp); + new_state[ci.new_aes_mk_state - '1'], + ci.new_aes_mkvp); else n = scnprintf(buf, PAGE_SIZE, "AES NEW: - -\n"); - if (ci.cur_mk_state >= '1' && ci.cur_mk_state <= '2') + if (ci.cur_aes_mk_state >= '1' && ci.cur_aes_mk_state <= '2') n += scnprintf(buf + n, PAGE_SIZE - n, "AES CUR: %s 0x%016llx\n", - cao_state[ci.cur_mk_state - '1'], ci.cur_mkvp); + cao_state[ci.cur_aes_mk_state - '1'], + ci.cur_aes_mkvp); else n += scnprintf(buf + n, PAGE_SIZE - n, "AES CUR: - -\n"); - if (ci.old_mk_state >= '1' && ci.old_mk_state <= '2') + if (ci.old_aes_mk_state >= '1' && ci.old_aes_mk_state <= '2') n += scnprintf(buf + n, PAGE_SIZE - n, "AES OLD: %s 0x%016llx\n", - cao_state[ci.old_mk_state - '1'], ci.old_mkvp); + cao_state[ci.old_aes_mk_state - '1'], + ci.old_aes_mkvp); else n += scnprintf(buf + n, PAGE_SIZE - n, "AES OLD: - -\n"); + if (ci.new_apka_mk_state >= '1' && ci.new_apka_mk_state <= '3') + n += scnprintf(buf + n, PAGE_SIZE - n, + "APKA NEW: %s 0x%016llx\n", + new_state[ci.new_apka_mk_state - '1'], + ci.new_apka_mkvp); + else + n += scnprintf(buf + n, PAGE_SIZE - n, "APKA NEW: - -\n"); + + if (ci.cur_apka_mk_state >= '1' && ci.cur_apka_mk_state <= '2') + n += scnprintf(buf + n, PAGE_SIZE - n, + "APKA CUR: %s 0x%016llx\n", + cao_state[ci.cur_apka_mk_state - '1'], + ci.cur_apka_mkvp); + else + n += scnprintf(buf + n, PAGE_SIZE - n, "APKA CUR: - -\n"); + + if (ci.old_apka_mk_state >= '1' && ci.old_apka_mk_state <= '2') + n += scnprintf(buf + n, PAGE_SIZE - n, + "APKA OLD: %s 0x%016llx\n", + cao_state[ci.old_apka_mk_state - '1'], + ci.old_apka_mkvp); + else + n += scnprintf(buf + n, PAGE_SIZE - n, "APKA OLD: - -\n"); + return n; } @@ -239,8 +266,7 @@ static int zcrypt_cex2c_card_probe(struct ap_device *ap_dev) case AP_DEVICE_TYPE_CEX2C: zc->user_space_type = ZCRYPT_CEX2C; zc->type_string = "CEX2C"; - memcpy(zc->speed_rating, CEX2C_SPEED_IDX, - sizeof(CEX2C_SPEED_IDX)); + zc->speed_rating = CEX2C_SPEED_IDX; zc->min_mod_size = CEX2C_MIN_MOD_SIZE; zc->max_mod_size = CEX2C_MAX_MOD_SIZE; zc->max_exp_bit_length = CEX2C_MAX_MOD_SIZE; @@ -248,8 +274,7 @@ static int zcrypt_cex2c_card_probe(struct ap_device *ap_dev) case AP_DEVICE_TYPE_CEX3C: zc->user_space_type = ZCRYPT_CEX3C; zc->type_string = "CEX3C"; - memcpy(zc->speed_rating, CEX3C_SPEED_IDX, - sizeof(CEX3C_SPEED_IDX)); + zc->speed_rating = CEX3C_SPEED_IDX; zc->min_mod_size = CEX3C_MIN_MOD_SIZE; zc->max_mod_size = CEX3C_MAX_MOD_SIZE; zc->max_exp_bit_length = CEX3C_MAX_MOD_SIZE; diff --git a/drivers/s390/crypto/zcrypt_cex4.c b/drivers/s390/crypto/zcrypt_cex4.c index dc20d983e468..f5195bca1d85 100644 --- a/drivers/s390/crypto/zcrypt_cex4.c +++ b/drivers/s390/crypto/zcrypt_cex4.c @@ -121,26 +121,53 @@ static ssize_t cca_mkvps_show(struct device *dev, AP_QID_QUEUE(zq->queue->qid), &ci, zq->online); - if (ci.new_mk_state >= '1' && ci.new_mk_state <= '3') + if (ci.new_aes_mk_state >= '1' && ci.new_aes_mk_state <= '3') n = scnprintf(buf, PAGE_SIZE, "AES NEW: %s 0x%016llx\n", - new_state[ci.new_mk_state - '1'], ci.new_mkvp); + new_state[ci.new_aes_mk_state - '1'], + ci.new_aes_mkvp); else n = scnprintf(buf, PAGE_SIZE, "AES NEW: - -\n"); - if (ci.cur_mk_state >= '1' && ci.cur_mk_state <= '2') + if (ci.cur_aes_mk_state >= '1' && ci.cur_aes_mk_state <= '2') n += scnprintf(buf + n, PAGE_SIZE - n, "AES CUR: %s 0x%016llx\n", - cao_state[ci.cur_mk_state - '1'], ci.cur_mkvp); + cao_state[ci.cur_aes_mk_state - '1'], + ci.cur_aes_mkvp); else n += scnprintf(buf + n, PAGE_SIZE - n, "AES CUR: - -\n"); - if (ci.old_mk_state >= '1' && ci.old_mk_state <= '2') + if (ci.old_aes_mk_state >= '1' && ci.old_aes_mk_state <= '2') n += scnprintf(buf + n, PAGE_SIZE - n, "AES OLD: %s 0x%016llx\n", - cao_state[ci.old_mk_state - '1'], ci.old_mkvp); + cao_state[ci.old_aes_mk_state - '1'], + ci.old_aes_mkvp); else n += scnprintf(buf + n, PAGE_SIZE - n, "AES OLD: - -\n"); + if (ci.new_apka_mk_state >= '1' && ci.new_apka_mk_state <= '3') + n += scnprintf(buf + n, PAGE_SIZE - n, + "APKA NEW: %s 0x%016llx\n", + new_state[ci.new_apka_mk_state - '1'], + ci.new_apka_mkvp); + else + n += scnprintf(buf + n, PAGE_SIZE - n, "APKA NEW: - -\n"); + + if (ci.cur_apka_mk_state >= '1' && ci.cur_apka_mk_state <= '2') + n += scnprintf(buf + n, PAGE_SIZE - n, + "APKA CUR: %s 0x%016llx\n", + cao_state[ci.cur_apka_mk_state - '1'], + ci.cur_apka_mkvp); + else + n += scnprintf(buf + n, PAGE_SIZE - n, "APKA CUR: - -\n"); + + if (ci.old_apka_mk_state >= '1' && ci.old_apka_mk_state <= '2') + n += scnprintf(buf + n, PAGE_SIZE - n, + "APKA OLD: %s 0x%016llx\n", + cao_state[ci.old_apka_mk_state - '1'], + ci.old_apka_mkvp); + else + n += scnprintf(buf + n, PAGE_SIZE - n, "APKA OLD: - -\n"); + return n; } @@ -382,31 +409,31 @@ static int zcrypt_cex4_card_probe(struct ap_device *ap_dev) * Normalized speed ratings per crypto adapter * MEX_1k, MEX_2k, MEX_4k, CRT_1k, CRT_2k, CRT_4k, RNG, SECKEY */ - static const int CEX4A_SPEED_IDX[] = { + static const int CEX4A_SPEED_IDX[NUM_OPS] = { 14, 19, 249, 42, 228, 1458, 0, 0}; - static const int CEX5A_SPEED_IDX[] = { + static const int CEX5A_SPEED_IDX[NUM_OPS] = { 8, 9, 20, 18, 66, 458, 0, 0}; - static const int CEX6A_SPEED_IDX[] = { + static const int CEX6A_SPEED_IDX[NUM_OPS] = { 6, 9, 20, 17, 65, 438, 0, 0}; - static const int CEX7A_SPEED_IDX[] = { + static const int CEX7A_SPEED_IDX[NUM_OPS] = { 6, 8, 17, 15, 54, 362, 0, 0}; - static const int CEX4C_SPEED_IDX[] = { + static const int CEX4C_SPEED_IDX[NUM_OPS] = { 59, 69, 308, 83, 278, 2204, 209, 40}; static const int CEX5C_SPEED_IDX[] = { 24, 31, 50, 37, 90, 479, 27, 10}; - static const int CEX6C_SPEED_IDX[] = { + static const int CEX6C_SPEED_IDX[NUM_OPS] = { 16, 20, 32, 27, 77, 455, 24, 9}; - static const int CEX7C_SPEED_IDX[] = { + static const int CEX7C_SPEED_IDX[NUM_OPS] = { 14, 16, 26, 23, 64, 376, 23, 8}; - static const int CEX4P_SPEED_IDX[] = { + static const int CEX4P_SPEED_IDX[NUM_OPS] = { 0, 0, 0, 0, 0, 0, 0, 50}; - static const int CEX5P_SPEED_IDX[] = { + static const int CEX5P_SPEED_IDX[NUM_OPS] = { 0, 0, 0, 0, 0, 0, 0, 10}; - static const int CEX6P_SPEED_IDX[] = { + static const int CEX6P_SPEED_IDX[NUM_OPS] = { 0, 0, 0, 0, 0, 0, 0, 9}; - static const int CEX7P_SPEED_IDX[] = { + static const int CEX7P_SPEED_IDX[NUM_OPS] = { 0, 0, 0, 0, 0, 0, 0, 8}; struct ap_card *ac = to_ap_card(&ap_dev->device); @@ -422,26 +449,22 @@ static int zcrypt_cex4_card_probe(struct ap_device *ap_dev) if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX4) { zc->type_string = "CEX4A"; zc->user_space_type = ZCRYPT_CEX4; - memcpy(zc->speed_rating, CEX4A_SPEED_IDX, - sizeof(CEX4A_SPEED_IDX)); + zc->speed_rating = CEX4A_SPEED_IDX; } else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX5) { zc->type_string = "CEX5A"; zc->user_space_type = ZCRYPT_CEX5; - memcpy(zc->speed_rating, CEX5A_SPEED_IDX, - sizeof(CEX5A_SPEED_IDX)); + zc->speed_rating = CEX5A_SPEED_IDX; } else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX6) { zc->type_string = "CEX6A"; zc->user_space_type = ZCRYPT_CEX6; - memcpy(zc->speed_rating, CEX6A_SPEED_IDX, - sizeof(CEX6A_SPEED_IDX)); + zc->speed_rating = CEX6A_SPEED_IDX; } else { zc->type_string = "CEX7A"; /* wrong user space type, just for compatibility * with the ZCRYPT_STATUS_MASK ioctl. */ zc->user_space_type = ZCRYPT_CEX6; - memcpy(zc->speed_rating, CEX7A_SPEED_IDX, - sizeof(CEX7A_SPEED_IDX)); + zc->speed_rating = CEX7A_SPEED_IDX; } zc->min_mod_size = CEX4A_MIN_MOD_SIZE; if (ap_test_bit(&ac->functions, AP_FUNC_MEX4K) && @@ -461,32 +484,28 @@ static int zcrypt_cex4_card_probe(struct ap_device *ap_dev) * just keep it for cca compatibility */ zc->user_space_type = ZCRYPT_CEX3C; - memcpy(zc->speed_rating, CEX4C_SPEED_IDX, - sizeof(CEX4C_SPEED_IDX)); + zc->speed_rating = CEX4C_SPEED_IDX; } else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX5) { zc->type_string = "CEX5C"; /* wrong user space type, must be CEX5 * just keep it for cca compatibility */ zc->user_space_type = ZCRYPT_CEX3C; - memcpy(zc->speed_rating, CEX5C_SPEED_IDX, - sizeof(CEX5C_SPEED_IDX)); + zc->speed_rating = CEX5C_SPEED_IDX; } else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX6) { zc->type_string = "CEX6C"; /* wrong user space type, must be CEX6 * just keep it for cca compatibility */ zc->user_space_type = ZCRYPT_CEX3C; - memcpy(zc->speed_rating, CEX6C_SPEED_IDX, - sizeof(CEX6C_SPEED_IDX)); + zc->speed_rating = CEX6C_SPEED_IDX; } else { zc->type_string = "CEX7C"; /* wrong user space type, must be CEX7 * just keep it for cca compatibility */ zc->user_space_type = ZCRYPT_CEX3C; - memcpy(zc->speed_rating, CEX7C_SPEED_IDX, - sizeof(CEX7C_SPEED_IDX)); + zc->speed_rating = CEX7C_SPEED_IDX; } zc->min_mod_size = CEX4C_MIN_MOD_SIZE; zc->max_mod_size = CEX4C_MAX_MOD_SIZE; @@ -495,26 +514,22 @@ static int zcrypt_cex4_card_probe(struct ap_device *ap_dev) if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX4) { zc->type_string = "CEX4P"; zc->user_space_type = ZCRYPT_CEX4; - memcpy(zc->speed_rating, CEX4P_SPEED_IDX, - sizeof(CEX4P_SPEED_IDX)); + zc->speed_rating = CEX4P_SPEED_IDX; } else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX5) { zc->type_string = "CEX5P"; zc->user_space_type = ZCRYPT_CEX5; - memcpy(zc->speed_rating, CEX5P_SPEED_IDX, - sizeof(CEX5P_SPEED_IDX)); + zc->speed_rating = CEX5P_SPEED_IDX; } else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX6) { zc->type_string = "CEX6P"; zc->user_space_type = ZCRYPT_CEX6; - memcpy(zc->speed_rating, CEX6P_SPEED_IDX, - sizeof(CEX6P_SPEED_IDX)); + zc->speed_rating = CEX6P_SPEED_IDX; } else { zc->type_string = "CEX7P"; /* wrong user space type, just for compatibility * with the ZCRYPT_STATUS_MASK ioctl. */ zc->user_space_type = ZCRYPT_CEX6; - memcpy(zc->speed_rating, CEX7P_SPEED_IDX, - sizeof(CEX7P_SPEED_IDX)); + zc->speed_rating = CEX7P_SPEED_IDX; } zc->min_mod_size = CEX4C_MIN_MOD_SIZE; zc->max_mod_size = CEX4C_MAX_MOD_SIZE; diff --git a/drivers/s390/crypto/zcrypt_debug.h b/drivers/s390/crypto/zcrypt_debug.h index 241dbb5f75bf..3225489a1c41 100644 --- a/drivers/s390/crypto/zcrypt_debug.h +++ b/drivers/s390/crypto/zcrypt_debug.h @@ -21,6 +21,14 @@ #define ZCRYPT_DBF(...) \ debug_sprintf_event(zcrypt_dbf_info, ##__VA_ARGS__) +#define ZCRYPT_DBF_ERR(...) \ + debug_sprintf_event(zcrypt_dbf_info, DBF_ERR, ##__VA_ARGS__) +#define ZCRYPT_DBF_WARN(...) \ + debug_sprintf_event(zcrypt_dbf_info, DBF_WARN, ##__VA_ARGS__) +#define ZCRYPT_DBF_INFO(...) \ + debug_sprintf_event(zcrypt_dbf_info, DBF_INFO, ##__VA_ARGS__) +#define ZCRYPT_DBF_DBG(...) \ + debug_sprintf_event(zcrypt_dbf_info, DBF_DEBUG, ##__VA_ARGS__) extern debug_info_t *zcrypt_dbf_info; diff --git a/drivers/s390/crypto/zcrypt_ep11misc.c b/drivers/s390/crypto/zcrypt_ep11misc.c index 3c3d403abe92..9ce5a71da69b 100644 --- a/drivers/s390/crypto/zcrypt_ep11misc.c +++ b/drivers/s390/crypto/zcrypt_ep11misc.c @@ -15,6 +15,7 @@ #include <linux/random.h> #include <asm/zcrypt.h> #include <asm/pkey.h> +#include <crypto/aes.h> #include "ap_bus.h" #include "zcrypt_api.h" @@ -113,79 +114,199 @@ static void __exit card_cache_free(void) } /* - * Simple check if the key blob is a valid EP11 secure AES key. + * Simple check if the key blob is a valid EP11 AES key blob with header. */ -int ep11_check_aeskeyblob(debug_info_t *dbg, int dbflvl, - const u8 *key, int keybitsize, - int checkcpacfexport) +int ep11_check_aes_key_with_hdr(debug_info_t *dbg, int dbflvl, + const u8 *key, size_t keylen, int checkcpacfexp) { - struct ep11keyblob *kb = (struct ep11keyblob *) key; + struct ep11kblob_header *hdr = (struct ep11kblob_header *) key; + struct ep11keyblob *kb = (struct ep11keyblob *) (key + sizeof(*hdr)); #define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__) - if (kb->head.type != TOKTYPE_NON_CCA) { + if (keylen < sizeof(*hdr) + sizeof(*kb)) { + DBF("%s key check failed, keylen %zu < %zu\n", + __func__, keylen, sizeof(*hdr) + sizeof(*kb)); + return -EINVAL; + } + + if (hdr->type != TOKTYPE_NON_CCA) { if (dbg) DBF("%s key check failed, type 0x%02x != 0x%02x\n", - __func__, (int) kb->head.type, TOKTYPE_NON_CCA); + __func__, (int) hdr->type, TOKTYPE_NON_CCA); return -EINVAL; } - if (kb->head.version != TOKVER_EP11_AES) { + if (hdr->hver != 0x00) { + if (dbg) + DBF("%s key check failed, header version 0x%02x != 0x00\n", + __func__, (int) hdr->hver); + return -EINVAL; + } + if (hdr->version != TOKVER_EP11_AES_WITH_HEADER) { if (dbg) DBF("%s key check failed, version 0x%02x != 0x%02x\n", - __func__, (int) kb->head.version, TOKVER_EP11_AES); + __func__, (int) hdr->version, TOKVER_EP11_AES_WITH_HEADER); + return -EINVAL; + } + if (hdr->len > keylen) { + if (dbg) + DBF("%s key check failed, header len %d keylen %zu mismatch\n", + __func__, (int) hdr->len, keylen); + return -EINVAL; + } + if (hdr->len < sizeof(*hdr) + sizeof(*kb)) { + if (dbg) + DBF("%s key check failed, header len %d < %zu\n", + __func__, (int) hdr->len, sizeof(*hdr) + sizeof(*kb)); return -EINVAL; } + if (kb->version != EP11_STRUCT_MAGIC) { if (dbg) - DBF("%s key check failed, magic 0x%04x != 0x%04x\n", + DBF("%s key check failed, blob magic 0x%04x != 0x%04x\n", __func__, (int) kb->version, EP11_STRUCT_MAGIC); return -EINVAL; } - switch (kb->head.keybitlen) { - case 128: - case 192: - case 256: - break; - default: + if (checkcpacfexp && !(kb->attr & EP11_BLOB_PKEY_EXTRACTABLE)) { if (dbg) - DBF("%s key check failed, keybitlen %d invalid\n", - __func__, (int) kb->head.keybitlen); + DBF("%s key check failed, PKEY_EXTRACTABLE is off\n", + __func__); return -EINVAL; } - if (keybitsize > 0 && keybitsize != (int) kb->head.keybitlen) { - DBF("%s key check failed, keybitsize %d\n", - __func__, keybitsize); + +#undef DBF + + return 0; +} +EXPORT_SYMBOL(ep11_check_aes_key_with_hdr); + +/* + * Simple check if the key blob is a valid EP11 ECC key blob with header. + */ +int ep11_check_ecc_key_with_hdr(debug_info_t *dbg, int dbflvl, + const u8 *key, size_t keylen, int checkcpacfexp) +{ + struct ep11kblob_header *hdr = (struct ep11kblob_header *) key; + struct ep11keyblob *kb = (struct ep11keyblob *) (key + sizeof(*hdr)); + +#define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__) + + if (keylen < sizeof(*hdr) + sizeof(*kb)) { + DBF("%s key check failed, keylen %zu < %zu\n", + __func__, keylen, sizeof(*hdr) + sizeof(*kb)); + return -EINVAL; + } + + if (hdr->type != TOKTYPE_NON_CCA) { + if (dbg) + DBF("%s key check failed, type 0x%02x != 0x%02x\n", + __func__, (int) hdr->type, TOKTYPE_NON_CCA); + return -EINVAL; + } + if (hdr->hver != 0x00) { + if (dbg) + DBF("%s key check failed, header version 0x%02x != 0x00\n", + __func__, (int) hdr->hver); + return -EINVAL; + } + if (hdr->version != TOKVER_EP11_ECC_WITH_HEADER) { + if (dbg) + DBF("%s key check failed, version 0x%02x != 0x%02x\n", + __func__, (int) hdr->version, TOKVER_EP11_ECC_WITH_HEADER); return -EINVAL; } - if (checkcpacfexport && !(kb->attr & EP11_BLOB_PKEY_EXTRACTABLE)) { + if (hdr->len > keylen) { if (dbg) - DBF("%s key check failed, PKEY_EXTRACTABLE is 0\n", + DBF("%s key check failed, header len %d keylen %zu mismatch\n", + __func__, (int) hdr->len, keylen); + return -EINVAL; + } + if (hdr->len < sizeof(*hdr) + sizeof(*kb)) { + if (dbg) + DBF("%s key check failed, header len %d < %zu\n", + __func__, (int) hdr->len, sizeof(*hdr) + sizeof(*kb)); + return -EINVAL; + } + + if (kb->version != EP11_STRUCT_MAGIC) { + if (dbg) + DBF("%s key check failed, blob magic 0x%04x != 0x%04x\n", + __func__, (int) kb->version, EP11_STRUCT_MAGIC); + return -EINVAL; + } + if (checkcpacfexp && !(kb->attr & EP11_BLOB_PKEY_EXTRACTABLE)) { + if (dbg) + DBF("%s key check failed, PKEY_EXTRACTABLE is off\n", __func__); return -EINVAL; } + #undef DBF return 0; } -EXPORT_SYMBOL(ep11_check_aeskeyblob); +EXPORT_SYMBOL(ep11_check_ecc_key_with_hdr); /* - * Helper function which calls zcrypt_send_ep11_cprb with - * memory management segment adjusted to kernel space - * so that the copy_from_user called within this - * function do in fact copy from kernel space. + * Simple check if the key blob is a valid EP11 AES key blob with + * the header in the session field (old style EP11 AES key). */ -static inline int _zcrypt_send_ep11_cprb(struct ep11_urb *urb) +int ep11_check_aes_key(debug_info_t *dbg, int dbflvl, + const u8 *key, size_t keylen, int checkcpacfexp) { - int rc; - mm_segment_t old_fs = get_fs(); + struct ep11keyblob *kb = (struct ep11keyblob *) key; - set_fs(KERNEL_DS); - rc = zcrypt_send_ep11_cprb(urb); - set_fs(old_fs); +#define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__) - return rc; + if (keylen < sizeof(*kb)) { + DBF("%s key check failed, keylen %zu < %zu\n", + __func__, keylen, sizeof(*kb)); + return -EINVAL; + } + + if (kb->head.type != TOKTYPE_NON_CCA) { + if (dbg) + DBF("%s key check failed, type 0x%02x != 0x%02x\n", + __func__, (int) kb->head.type, TOKTYPE_NON_CCA); + return -EINVAL; + } + if (kb->head.version != TOKVER_EP11_AES) { + if (dbg) + DBF("%s key check failed, version 0x%02x != 0x%02x\n", + __func__, (int) kb->head.version, TOKVER_EP11_AES); + return -EINVAL; + } + if (kb->head.len > keylen) { + if (dbg) + DBF("%s key check failed, header len %d keylen %zu mismatch\n", + __func__, (int) kb->head.len, keylen); + return -EINVAL; + } + if (kb->head.len < sizeof(*kb)) { + if (dbg) + DBF("%s key check failed, header len %d < %zu\n", + __func__, (int) kb->head.len, sizeof(*kb)); + return -EINVAL; + } + + if (kb->version != EP11_STRUCT_MAGIC) { + if (dbg) + DBF("%s key check failed, blob magic 0x%04x != 0x%04x\n", + __func__, (int) kb->version, EP11_STRUCT_MAGIC); + return -EINVAL; + } + if (checkcpacfexp && !(kb->attr & EP11_BLOB_PKEY_EXTRACTABLE)) { + if (dbg) + DBF("%s key check failed, PKEY_EXTRACTABLE is off\n", + __func__); + return -EINVAL; + } + +#undef DBF + + return 0; } +EXPORT_SYMBOL(ep11_check_aes_key); /* * Allocate and prepare ep11 cprb plus additional payload. @@ -399,7 +520,7 @@ static int ep11_query_info(u16 cardnr, u16 domain, u32 query_type, req, sizeof(*req) + sizeof(*req_pl), rep, sizeof(*rep) + sizeof(*rep_pl) + buflen); - rc = _zcrypt_send_ep11_cprb(urb); + rc = zcrypt_send_ep11_cprb(urb); if (rc) { DEBUG_ERR( "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", @@ -637,7 +758,7 @@ int ep11_genaeskey(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, req, sizeof(*req) + sizeof(*req_pl), rep, sizeof(*rep) + sizeof(*rep_pl)); - rc = _zcrypt_send_ep11_cprb(urb); + rc = zcrypt_send_ep11_cprb(urb); if (rc) { DEBUG_ERR( "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", @@ -757,7 +878,7 @@ static int ep11_cryptsingle(u16 card, u16 domain, req, sizeof(*req) + req_pl_size, rep, sizeof(*rep) + rep_pl_size); - rc = _zcrypt_send_ep11_cprb(urb); + rc = zcrypt_send_ep11_cprb(urb); if (rc) { DEBUG_ERR( "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", @@ -905,7 +1026,7 @@ static int ep11_unwrapkey(u16 card, u16 domain, req, sizeof(*req) + req_pl_size, rep, sizeof(*rep) + sizeof(*rep_pl)); - rc = _zcrypt_send_ep11_cprb(urb); + rc = zcrypt_send_ep11_cprb(urb); if (rc) { DEBUG_ERR( "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", @@ -972,7 +1093,7 @@ static int ep11_wrapkey(u16 card, u16 domain, u8 data_tag; u8 data_lenfmt; u16 data_len; - u8 data[512]; + u8 data[1024]; } __packed * rep_pl; struct ep11_cprb *req = NULL, *rep = NULL; struct ep11_target_dev target; @@ -980,8 +1101,17 @@ static int ep11_wrapkey(u16 card, u16 domain, struct ep11keyblob *kb; size_t req_pl_size; int api, rc = -ENOMEM; + bool has_header = false; u8 *p; + /* maybe the session field holds a header with key info */ + kb = (struct ep11keyblob *) key; + if (kb->head.type == TOKTYPE_NON_CCA && + kb->head.version == TOKVER_EP11_AES) { + has_header = true; + keysize = kb->head.len < keysize ? kb->head.len : keysize; + } + /* request cprb and payload */ req_pl_size = sizeof(struct wk_req_pl) + (iv ? 16 : 0) + ASN1TAGLEN(keysize) + 4; @@ -1007,9 +1137,10 @@ static int ep11_wrapkey(u16 card, u16 domain, /* key blob */ p += asn1tag_write(p, 0x04, key, keysize); /* maybe the key argument needs the head data cleaned out */ - kb = (struct ep11keyblob *)(p - keysize); - if (kb->head.version == TOKVER_EP11_AES) + if (has_header) { + kb = (struct ep11keyblob *)(p - keysize); memset(&kb->head, 0, sizeof(kb->head)); + } /* empty kek tag */ *p++ = 0x04; *p++ = 0; @@ -1033,7 +1164,7 @@ static int ep11_wrapkey(u16 card, u16 domain, req, sizeof(*req) + req_pl_size, rep, sizeof(*rep) + sizeof(*rep_pl)); - rc = _zcrypt_send_ep11_cprb(urb); + rc = zcrypt_send_ep11_cprb(urb); if (rc) { DEBUG_ERR( "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", @@ -1132,12 +1263,12 @@ out: } EXPORT_SYMBOL(ep11_clr2keyblob); -int ep11_key2protkey(u16 card, u16 dom, const u8 *key, size_t keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) +int ep11_kblob2protkey(u16 card, u16 dom, const u8 *keyblob, size_t keybloblen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) { int rc = -EIO; u8 *wkbuf = NULL; - size_t wkbuflen = 256; + size_t wkbuflen, keylen; struct wk_info { u16 version; u8 res1[16]; @@ -1147,8 +1278,33 @@ int ep11_key2protkey(u16 card, u16 dom, const u8 *key, size_t keylen, u8 res2[8]; u8 pkey[0]; } __packed * wki; + const u8 *key; + struct ep11kblob_header *hdr; + + /* key with or without header ? */ + hdr = (struct ep11kblob_header *) keyblob; + if (hdr->type == TOKTYPE_NON_CCA + && (hdr->version == TOKVER_EP11_AES_WITH_HEADER + || hdr->version == TOKVER_EP11_ECC_WITH_HEADER) + && is_ep11_keyblob(keyblob + sizeof(struct ep11kblob_header))) { + /* EP11 AES or ECC key with header */ + key = keyblob + sizeof(struct ep11kblob_header); + keylen = hdr->len - sizeof(struct ep11kblob_header); + } else if (hdr->type == TOKTYPE_NON_CCA + && hdr->version == TOKVER_EP11_AES + && is_ep11_keyblob(keyblob)) { + /* EP11 AES key (old style) */ + key = keyblob; + keylen = hdr->len; + } else if (is_ep11_keyblob(keyblob)) { + /* raw EP11 key blob */ + key = keyblob; + keylen = keybloblen; + } else + return -EINVAL; /* alloc temp working buffer */ + wkbuflen = (keylen + AES_BLOCK_SIZE) & (~(AES_BLOCK_SIZE - 1)); wkbuf = kmalloc(wkbuflen, GFP_ATOMIC); if (!wkbuf) return -ENOMEM; @@ -1165,46 +1321,68 @@ int ep11_key2protkey(u16 card, u16 dom, const u8 *key, size_t keylen, wki = (struct wk_info *) wkbuf; /* check struct version and pkey type */ - if (wki->version != 1 || wki->pkeytype != 1) { + if (wki->version != 1 || wki->pkeytype < 1 || wki->pkeytype > 5) { DEBUG_ERR("%s wk info version %d or pkeytype %d mismatch.\n", __func__, (int) wki->version, (int) wki->pkeytype); rc = -EIO; goto out; } - /* copy the tanslated protected key */ - switch (wki->pkeysize) { - case 16+32: - /* AES 128 protected key */ - if (protkeytype) - *protkeytype = PKEY_KEYTYPE_AES_128; - break; - case 24+32: - /* AES 192 protected key */ - if (protkeytype) - *protkeytype = PKEY_KEYTYPE_AES_192; + /* check protected key type field */ + switch (wki->pkeytype) { + case 1: /* AES */ + switch (wki->pkeysize) { + case 16+32: + /* AES 128 protected key */ + if (protkeytype) + *protkeytype = PKEY_KEYTYPE_AES_128; + break; + case 24+32: + /* AES 192 protected key */ + if (protkeytype) + *protkeytype = PKEY_KEYTYPE_AES_192; + break; + case 32+32: + /* AES 256 protected key */ + if (protkeytype) + *protkeytype = PKEY_KEYTYPE_AES_256; + break; + default: + DEBUG_ERR("%s unknown/unsupported AES pkeysize %d\n", + __func__, (int) wki->pkeysize); + rc = -EIO; + goto out; + } break; - case 32+32: - /* AES 256 protected key */ + case 3: /* EC-P */ + case 4: /* EC-ED */ + case 5: /* EC-BP */ if (protkeytype) - *protkeytype = PKEY_KEYTYPE_AES_256; + *protkeytype = PKEY_KEYTYPE_ECC; break; + case 2: /* TDES */ default: - DEBUG_ERR("%s unknown/unsupported pkeysize %d\n", - __func__, (int) wki->pkeysize); + DEBUG_ERR("%s unknown/unsupported key type %d\n", + __func__, (int) wki->pkeytype); rc = -EIO; goto out; } + + /* copy the tanslated protected key */ + if (wki->pkeysize > *protkeylen) { + DEBUG_ERR("%s wk info pkeysize %llu > protkeysize %u\n", + __func__, wki->pkeysize, *protkeylen); + rc = -EINVAL; + goto out; + } memcpy(protkey, wki->pkey, wki->pkeysize); - if (protkeylen) - *protkeylen = (u32) wki->pkeysize; - rc = 0; + *protkeylen = wki->pkeysize; out: kfree(wkbuf); return rc; } -EXPORT_SYMBOL(ep11_key2protkey); +EXPORT_SYMBOL(ep11_kblob2protkey); int ep11_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, int minhwtype, int minapi, const u8 *wkvp) diff --git a/drivers/s390/crypto/zcrypt_ep11misc.h b/drivers/s390/crypto/zcrypt_ep11misc.h index e3ed5ed1de86..1e02b197c003 100644 --- a/drivers/s390/crypto/zcrypt_ep11misc.h +++ b/drivers/s390/crypto/zcrypt_ep11misc.h @@ -12,22 +12,28 @@ #include <asm/zcrypt.h> #include <asm/pkey.h> -#define TOKVER_EP11_AES 0x03 /* EP11 AES key blob */ - #define EP11_API_V 4 /* highest known and supported EP11 API version */ - #define EP11_STRUCT_MAGIC 0x1234 -#define EP11_BLOB_PKEY_EXTRACTABLE 0x200000 +#define EP11_BLOB_PKEY_EXTRACTABLE 0x00200000 + +/* + * Internal used values for the version field of the key header. + * Should match to the enum pkey_key_type in pkey.h. + */ +#define TOKVER_EP11_AES 0x03 /* EP11 AES key blob (old style) */ +#define TOKVER_EP11_AES_WITH_HEADER 0x06 /* EP11 AES key blob with header */ +#define TOKVER_EP11_ECC_WITH_HEADER 0x07 /* EP11 ECC key blob with header */ /* inside view of an EP11 secure key blob */ struct ep11keyblob { union { u8 session[32]; + /* only used for PKEY_TYPE_EP11: */ struct { u8 type; /* 0x00 (TOKTYPE_NON_CCA) */ u8 res0; /* unused */ u16 len; /* total length in bytes of this blob */ - u8 version; /* 0x06 (TOKVER_EP11_AES) */ + u8 version; /* 0x03 (TOKVER_EP11_AES) */ u8 res1; /* unused */ u16 keybitlen; /* clear key bit len, 0 for unknown */ } head; @@ -41,16 +47,41 @@ struct ep11keyblob { u8 mac[32]; } __packed; +/* check ep11 key magic to find out if this is an ep11 key blob */ +static inline bool is_ep11_keyblob(const u8 *key) +{ + struct ep11keyblob *kb = (struct ep11keyblob *) key; + + return (kb->version == EP11_STRUCT_MAGIC); +} + +/* + * Simple check if the key blob is a valid EP11 AES key blob with header. + * If checkcpacfexport is enabled, the key is also checked for the + * attributes needed to export this key for CPACF use. + * Returns 0 on success or errno value on failure. + */ +int ep11_check_aes_key_with_hdr(debug_info_t *dbg, int dbflvl, + const u8 *key, size_t keylen, int checkcpacfexp); + /* - * Simple check if the key blob is a valid EP11 secure AES key. - * If keybitsize is given, the bitsize of the key is also checked. + * Simple check if the key blob is a valid EP11 ECC key blob with header. * If checkcpacfexport is enabled, the key is also checked for the * attributes needed to export this key for CPACF use. * Returns 0 on success or errno value on failure. */ -int ep11_check_aeskeyblob(debug_info_t *dbg, int dbflvl, - const u8 *key, int keybitsize, - int checkcpacfexport); +int ep11_check_ecc_key_with_hdr(debug_info_t *dbg, int dbflvl, + const u8 *key, size_t keylen, int checkcpacfexp); + +/* + * Simple check if the key blob is a valid EP11 AES key blob with + * the header in the session field (old style EP11 AES key). + * If checkcpacfexport is enabled, the key is also checked for the + * attributes needed to export this key for CPACF use. + * Returns 0 on success or errno value on failure. + */ +int ep11_check_aes_key(debug_info_t *dbg, int dbflvl, + const u8 *key, size_t keylen, int checkcpacfexp); /* EP11 card info struct */ struct ep11_card_info { @@ -92,12 +123,6 @@ int ep11_clr2keyblob(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, const u8 *clrkey, u8 *keybuf, size_t *keybufsize); /* - * Derive proteced key from EP11 AES secure key blob. - */ -int ep11_key2protkey(u16 cardnr, u16 domain, const u8 *key, size_t keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype); - -/* * Build a list of ep11 apqns meeting the following constrains: * - apqn is online and is in fact an EP11 apqn * - if cardnr is not FFFF only apqns with this cardnr @@ -119,6 +144,12 @@ int ep11_key2protkey(u16 cardnr, u16 domain, const u8 *key, size_t keylen, int ep11_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, int minhwtype, int minapi, const u8 *wkvp); +/* + * Derive proteced key from EP11 key blob (AES and ECC keys). + */ +int ep11_kblob2protkey(u16 card, u16 dom, const u8 *key, size_t keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype); + void zcrypt_ep11misc_exit(void); #endif /* _ZCRYPT_EP11MISC_H_ */ diff --git a/drivers/s390/crypto/zcrypt_error.h b/drivers/s390/crypto/zcrypt_error.h index 54a04f8c38ef..39e626e3a379 100644 --- a/drivers/s390/crypto/zcrypt_error.h +++ b/drivers/s390/crypto/zcrypt_error.h @@ -52,7 +52,6 @@ struct error_hdr { #define REP82_ERROR_INVALID_COMMAND 0x30 #define REP82_ERROR_MALFORMED_MSG 0x40 #define REP82_ERROR_INVALID_SPECIAL_CMD 0x41 -#define REP82_ERROR_INVALID_DOMAIN_PRECHECK 0x42 #define REP82_ERROR_RESERVED_FIELDO 0x50 /* old value */ #define REP82_ERROR_WORD_ALIGNMENT 0x60 #define REP82_ERROR_MESSAGE_LENGTH 0x80 @@ -67,7 +66,6 @@ struct error_hdr { #define REP82_ERROR_ZERO_BUFFER_LEN 0xB0 #define REP88_ERROR_MODULE_FAILURE 0x10 - #define REP88_ERROR_MESSAGE_TYPE 0x20 #define REP88_ERROR_MESSAGE_MALFORMD 0x22 #define REP88_ERROR_MESSAGE_LENGTH 0x23 @@ -85,78 +83,56 @@ static inline int convert_error(struct zcrypt_queue *zq, int queue = AP_QID_QUEUE(zq->queue->qid); switch (ehdr->reply_code) { - case REP82_ERROR_OPERAND_INVALID: - case REP82_ERROR_OPERAND_SIZE: - case REP82_ERROR_EVEN_MOD_IN_OPND: - case REP88_ERROR_MESSAGE_MALFORMD: - case REP82_ERROR_INVALID_DOMAIN_PRECHECK: - case REP82_ERROR_INVALID_DOMAIN_PENDING: - case REP82_ERROR_INVALID_SPECIAL_CMD: - case REP82_ERROR_FILTERED_BY_HYPERVISOR: - // REP88_ERROR_INVALID_KEY // '82' CEX2A - // REP88_ERROR_OPERAND // '84' CEX2A - // REP88_ERROR_OPERAND_EVEN_MOD // '85' CEX2A - /* Invalid input data. */ + case REP82_ERROR_INVALID_MSG_LEN: /* 0x23 */ + case REP82_ERROR_RESERVD_FIELD: /* 0x24 */ + case REP82_ERROR_FORMAT_FIELD: /* 0x29 */ + case REP82_ERROR_MALFORMED_MSG: /* 0x40 */ + case REP82_ERROR_INVALID_SPECIAL_CMD: /* 0x41 */ + case REP82_ERROR_MESSAGE_LENGTH: /* 0x80 */ + case REP82_ERROR_OPERAND_INVALID: /* 0x82 */ + case REP82_ERROR_OPERAND_SIZE: /* 0x84 */ + case REP82_ERROR_EVEN_MOD_IN_OPND: /* 0x85 */ + case REP82_ERROR_INVALID_DOMAIN_PENDING: /* 0x8A */ + case REP82_ERROR_FILTERED_BY_HYPERVISOR: /* 0x8B */ + case REP82_ERROR_PACKET_TRUNCATED: /* 0xA0 */ + case REP88_ERROR_MESSAGE_MALFORMD: /* 0x22 */ + case REP88_ERROR_KEY_TYPE: /* 0x34 */ + /* RY indicates malformed request */ ZCRYPT_DBF(DBF_WARN, - "device=%02x.%04x reply=0x%02x => rc=EINVAL\n", + "dev=%02x.%04x RY=0x%02x => rc=EINVAL\n", card, queue, ehdr->reply_code); return -EINVAL; - case REP82_ERROR_MESSAGE_TYPE: - // REP88_ERROR_MESSAGE_TYPE // '20' CEX2A + case REP82_ERROR_MACHINE_FAILURE: /* 0x10 */ + case REP82_ERROR_MESSAGE_TYPE: /* 0x20 */ + case REP82_ERROR_TRANSPORT_FAIL: /* 0x90 */ /* - * To sent a message of the wrong type is a bug in the - * device driver. Send error msg, disable the device - * and then repeat the request. + * Msg to wrong type or card/infrastructure failure. + * Trigger rescan of the ap bus, trigger retry request. */ atomic_set(&zcrypt_rescan_req, 1); - zq->online = 0; - pr_err("Cryptographic device %02x.%04x failed and was set offline\n", - card, queue); - ZCRYPT_DBF(DBF_ERR, - "device=%02x.%04x reply=0x%02x => online=0 rc=EAGAIN\n", - card, queue, ehdr->reply_code); - return -EAGAIN; - case REP82_ERROR_TRANSPORT_FAIL: - /* Card or infrastructure failure, disable card */ - atomic_set(&zcrypt_rescan_req, 1); - zq->online = 0; - pr_err("Cryptographic device %02x.%04x failed and was set offline\n", - card, queue); /* For type 86 response show the apfs value (failure reason) */ - if (ehdr->type == TYPE86_RSP_CODE) { + if (ehdr->reply_code == REP82_ERROR_TRANSPORT_FAIL && + ehdr->type == TYPE86_RSP_CODE) { struct { struct type86_hdr hdr; struct type86_fmt2_ext fmt2; } __packed * head = reply->msg; unsigned int apfs = *((u32 *)head->fmt2.apfs); - ZCRYPT_DBF(DBF_ERR, - "device=%02x.%04x reply=0x%02x apfs=0x%x => online=0 rc=EAGAIN\n", - card, queue, apfs, ehdr->reply_code); + ZCRYPT_DBF(DBF_WARN, + "dev=%02x.%04x RY=0x%02x apfs=0x%x => bus rescan, rc=EAGAIN\n", + card, queue, ehdr->reply_code, apfs); } else - ZCRYPT_DBF(DBF_ERR, - "device=%02x.%04x reply=0x%02x => online=0 rc=EAGAIN\n", + ZCRYPT_DBF(DBF_WARN, + "dev=%02x.%04x RY=0x%02x => bus rescan, rc=EAGAIN\n", card, queue, ehdr->reply_code); return -EAGAIN; - case REP82_ERROR_MACHINE_FAILURE: - // REP88_ERROR_MODULE_FAILURE // '10' CEX2A - /* If a card fails disable it and repeat the request. */ - atomic_set(&zcrypt_rescan_req, 1); - zq->online = 0; - pr_err("Cryptographic device %02x.%04x failed and was set offline\n", - card, queue); - ZCRYPT_DBF(DBF_ERR, - "device=%02x.%04x reply=0x%02x => online=0 rc=EAGAIN\n", - card, queue, ehdr->reply_code); - return -EAGAIN; default: - zq->online = 0; - pr_err("Cryptographic device %02x.%04x failed and was set offline\n", - card, queue); - ZCRYPT_DBF(DBF_ERR, - "device=%02x.%04x reply=0x%02x => online=0 rc=EAGAIN\n", + /* Assume request is valid and a retry will be worth it */ + ZCRYPT_DBF(DBF_WARN, + "dev=%02x.%04x RY=0x%02x => rc=EAGAIN\n", card, queue, ehdr->reply_code); - return -EAGAIN; /* repeat the request on a different device. */ + return -EAGAIN; } } diff --git a/drivers/s390/crypto/zcrypt_msgtype50.c b/drivers/s390/crypto/zcrypt_msgtype50.c index 7aedc338b445..bf14ee445f89 100644 --- a/drivers/s390/crypto/zcrypt_msgtype50.c +++ b/drivers/s390/crypto/zcrypt_msgtype50.c @@ -246,6 +246,12 @@ static int ICAMEX_msg_to_type50MEX_msg(struct zcrypt_queue *zq, copy_from_user(exp, mex->b_key, mod_len) || copy_from_user(inp, mex->inputdata, mod_len)) return -EFAULT; + +#ifdef CONFIG_ZCRYPT_DEBUG + if (ap_msg->fi.flags & AP_FI_FLAG_TOGGLE_SPECIAL) + ap_msg->flags ^= AP_MSG_FLAG_SPECIAL; +#endif + return 0; } @@ -332,6 +338,11 @@ static int ICACRT_msg_to_type50CRT_msg(struct zcrypt_queue *zq, copy_from_user(inp, crt->inputdata, mod_len)) return -EFAULT; +#ifdef CONFIG_ZCRYPT_DEBUG + if (ap_msg->fi.flags & AP_FI_FLAG_TOGGLE_SPECIAL) + ap_msg->flags ^= AP_MSG_FLAG_SPECIAL; +#endif + return 0; } @@ -356,15 +367,15 @@ static int convert_type80(struct zcrypt_queue *zq, if (t80h->len < sizeof(*t80h) + outputdatalength) { /* The result is too short, the CEXxA card may not do that.. */ zq->online = 0; - pr_err("Cryptographic device %02x.%04x failed and was set offline\n", + pr_err("Crypto dev=%02x.%04x code=0x%02x => online=0 rc=EAGAIN\n", AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid)); - ZCRYPT_DBF(DBF_ERR, - "device=%02x.%04x code=0x%02x => online=0 rc=EAGAIN\n", - AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), - t80h->code); - return -EAGAIN; /* repeat the request on a different device. */ + AP_QID_QUEUE(zq->queue->qid), + t80h->code); + ZCRYPT_DBF_ERR("dev=%02x.%04x code=0x%02x => online=0 rc=EAGAIN\n", + AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), + t80h->code); + return -EAGAIN; } if (zq->zcard->user_space_type == ZCRYPT_CEX2A) BUG_ON(t80h->len > CEX2A_MAX_RESPONSE_SIZE); @@ -376,10 +387,10 @@ static int convert_type80(struct zcrypt_queue *zq, return 0; } -static int convert_response(struct zcrypt_queue *zq, - struct ap_message *reply, - char __user *outputdata, - unsigned int outputdatalength) +static int convert_response_cex2a(struct zcrypt_queue *zq, + struct ap_message *reply, + char __user *outputdata, + unsigned int outputdatalength) { /* Response type byte is the second byte in the response. */ unsigned char rtype = ((unsigned char *) reply->msg)[1]; @@ -393,15 +404,15 @@ static int convert_response(struct zcrypt_queue *zq, outputdata, outputdatalength); default: /* Unknown response type, this should NEVER EVER happen */ zq->online = 0; - pr_err("Cryptographic device %02x.%04x failed and was set offline\n", + pr_err("Crypto dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n", AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid)); - ZCRYPT_DBF(DBF_ERR, - "device=%02x.%04x rtype=0x%02x => online=0 rc=EAGAIN\n", - AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), - (unsigned int) rtype); - return -EAGAIN; /* repeat the request on a different device. */ + AP_QID_QUEUE(zq->queue->qid), + (int) rtype); + ZCRYPT_DBF_ERR("dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n", + AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), + (int) rtype); + return -EAGAIN; } } @@ -450,39 +461,41 @@ static atomic_t zcrypt_step = ATOMIC_INIT(0); * @mex: pointer to the modexpo request buffer */ static long zcrypt_cex2a_modexpo(struct zcrypt_queue *zq, - struct ica_rsa_modexpo *mex) + struct ica_rsa_modexpo *mex, + struct ap_message *ap_msg) { - struct ap_message ap_msg; struct completion work; int rc; - ap_init_message(&ap_msg); if (zq->zcard->user_space_type == ZCRYPT_CEX2A) - ap_msg.msg = kmalloc(MSGTYPE50_CRB2_MAX_MSG_SIZE, GFP_KERNEL); + ap_msg->msg = kmalloc(MSGTYPE50_CRB2_MAX_MSG_SIZE, GFP_KERNEL); else - ap_msg.msg = kmalloc(MSGTYPE50_CRB3_MAX_MSG_SIZE, GFP_KERNEL); - if (!ap_msg.msg) + ap_msg->msg = kmalloc(MSGTYPE50_CRB3_MAX_MSG_SIZE, GFP_KERNEL); + if (!ap_msg->msg) return -ENOMEM; - ap_msg.receive = zcrypt_cex2a_receive; - ap_msg.psmid = (((unsigned long long) current->pid) << 32) + - atomic_inc_return(&zcrypt_step); - ap_msg.private = &work; - rc = ICAMEX_msg_to_type50MEX_msg(zq, &ap_msg, mex); + ap_msg->receive = zcrypt_cex2a_receive; + ap_msg->psmid = (((unsigned long long) current->pid) << 32) + + atomic_inc_return(&zcrypt_step); + ap_msg->private = &work; + rc = ICAMEX_msg_to_type50MEX_msg(zq, ap_msg, mex); if (rc) - goto out_free; + goto out; init_completion(&work); - ap_queue_message(zq->queue, &ap_msg); + rc = ap_queue_message(zq->queue, ap_msg); + if (rc) + goto out; rc = wait_for_completion_interruptible(&work); if (rc == 0) { - rc = ap_msg.rc; + rc = ap_msg->rc; if (rc == 0) - rc = convert_response(zq, &ap_msg, mex->outputdata, - mex->outputdatalength); + rc = convert_response_cex2a(zq, ap_msg, + mex->outputdata, + mex->outputdatalength); } else /* Signal pending. */ - ap_cancel_message(zq->queue, &ap_msg); -out_free: - kfree(ap_msg.msg); + ap_cancel_message(zq->queue, ap_msg); +out: + ap_msg->private = NULL; return rc; } @@ -494,39 +507,41 @@ out_free: * @crt: pointer to the modexpoc_crt request buffer */ static long zcrypt_cex2a_modexpo_crt(struct zcrypt_queue *zq, - struct ica_rsa_modexpo_crt *crt) + struct ica_rsa_modexpo_crt *crt, + struct ap_message *ap_msg) { - struct ap_message ap_msg; struct completion work; int rc; - ap_init_message(&ap_msg); if (zq->zcard->user_space_type == ZCRYPT_CEX2A) - ap_msg.msg = kmalloc(MSGTYPE50_CRB2_MAX_MSG_SIZE, GFP_KERNEL); + ap_msg->msg = kmalloc(MSGTYPE50_CRB2_MAX_MSG_SIZE, GFP_KERNEL); else - ap_msg.msg = kmalloc(MSGTYPE50_CRB3_MAX_MSG_SIZE, GFP_KERNEL); - if (!ap_msg.msg) + ap_msg->msg = kmalloc(MSGTYPE50_CRB3_MAX_MSG_SIZE, GFP_KERNEL); + if (!ap_msg->msg) return -ENOMEM; - ap_msg.receive = zcrypt_cex2a_receive; - ap_msg.psmid = (((unsigned long long) current->pid) << 32) + - atomic_inc_return(&zcrypt_step); - ap_msg.private = &work; - rc = ICACRT_msg_to_type50CRT_msg(zq, &ap_msg, crt); + ap_msg->receive = zcrypt_cex2a_receive; + ap_msg->psmid = (((unsigned long long) current->pid) << 32) + + atomic_inc_return(&zcrypt_step); + ap_msg->private = &work; + rc = ICACRT_msg_to_type50CRT_msg(zq, ap_msg, crt); if (rc) - goto out_free; + goto out; init_completion(&work); - ap_queue_message(zq->queue, &ap_msg); + rc = ap_queue_message(zq->queue, ap_msg); + if (rc) + goto out; rc = wait_for_completion_interruptible(&work); if (rc == 0) { - rc = ap_msg.rc; + rc = ap_msg->rc; if (rc == 0) - rc = convert_response(zq, &ap_msg, crt->outputdata, - crt->outputdatalength); + rc = convert_response_cex2a(zq, ap_msg, + crt->outputdata, + crt->outputdatalength); } else /* Signal pending. */ - ap_cancel_message(zq->queue, &ap_msg); -out_free: - kfree(ap_msg.msg); + ap_cancel_message(zq->queue, ap_msg); +out: + ap_msg->private = NULL; return rc; } diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c index d77991c74c25..307f90657d1d 100644 --- a/drivers/s390/crypto/zcrypt_msgtype6.c +++ b/drivers/s390/crypto/zcrypt_msgtype6.c @@ -388,7 +388,7 @@ struct type86_fmt2_msg { struct type86_fmt2_ext fmt2; } __packed; -static int XCRB_msg_to_type6CPRB_msgX(struct ap_message *ap_msg, +static int XCRB_msg_to_type6CPRB_msgX(bool userspace, struct ap_message *ap_msg, struct ica_xcRB *xcRB, unsigned int *fcode, unsigned short **dom) @@ -465,8 +465,8 @@ static int XCRB_msg_to_type6CPRB_msgX(struct ap_message *ap_msg, msg->hdr.FromCardLen2 = xcRB->reply_data_length; /* prepare CPRB */ - if (copy_from_user(&(msg->cprbx), xcRB->request_control_blk_addr, - xcRB->request_control_blk_length)) + if (z_copy_from_user(userspace, &(msg->cprbx), xcRB->request_control_blk_addr, + xcRB->request_control_blk_length)) return -EFAULT; if (msg->cprbx.cprb_len + sizeof(msg->hdr.function_code) > xcRB->request_control_blk_length) @@ -482,18 +482,23 @@ static int XCRB_msg_to_type6CPRB_msgX(struct ap_message *ap_msg, || memcmp(function_code, "AU", 2) == 0) ap_msg->flags |= AP_MSG_FLAG_SPECIAL; +#ifdef CONFIG_ZCRYPT_DEBUG + if (ap_msg->fi.flags & AP_FI_FLAG_TOGGLE_SPECIAL) + ap_msg->flags ^= AP_MSG_FLAG_SPECIAL; +#endif + /* copy data block */ if (xcRB->request_data_length && - copy_from_user(req_data, xcRB->request_data_address, - xcRB->request_data_length)) + z_copy_from_user(userspace, req_data, xcRB->request_data_address, + xcRB->request_data_length)) return -EFAULT; return 0; } -static int xcrb_msg_to_type6_ep11cprb_msgx(struct ap_message *ap_msg, - struct ep11_urb *xcRB, - unsigned int *fcode) +static int xcrb_msg_to_type6_ep11cprb_msgx(bool userspace, struct ap_message *ap_msg, + struct ep11_urb *xcRB, + unsigned int *fcode) { unsigned int lfmt; static struct type6_hdr static_type6_ep11_hdr = { @@ -543,8 +548,8 @@ static int xcrb_msg_to_type6_ep11cprb_msgx(struct ap_message *ap_msg, msg->hdr.FromCardLen1 = xcRB->resp_len; /* Import CPRB data from the ioctl input parameter */ - if (copy_from_user(&(msg->cprbx.cprb_len), - (char __force __user *)xcRB->req, xcRB->req_len)) { + if (z_copy_from_user(userspace, &(msg->cprbx.cprb_len), + (char __force __user *)xcRB->req, xcRB->req_len)) { return -EFAULT; } @@ -569,6 +574,11 @@ static int xcrb_msg_to_type6_ep11cprb_msgx(struct ap_message *ap_msg, if (msg->cprbx.flags & 0x20) ap_msg->flags |= AP_MSG_FLAG_SPECIAL; +#ifdef CONFIG_ZCRYPT_DEBUG + if (ap_msg->fi.flags & AP_FI_FLAG_TOGGLE_SPECIAL) + ap_msg->flags ^= AP_MSG_FLAG_SPECIAL; +#endif + return 0; } @@ -650,23 +660,22 @@ static int convert_type86_ica(struct zcrypt_queue *zq, (service_rc == 8 && service_rs == 72) || (service_rc == 8 && service_rs == 770) || (service_rc == 12 && service_rs == 769)) { - ZCRYPT_DBF(DBF_DEBUG, - "device=%02x.%04x rc/rs=%d/%d => rc=EINVAL\n", - AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), - (int) service_rc, (int) service_rs); + ZCRYPT_DBF_WARN("dev=%02x.%04x rc/rs=%d/%d => rc=EINVAL\n", + AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), + (int) service_rc, (int) service_rs); return -EINVAL; } zq->online = 0; - pr_err("Cryptographic device %02x.%04x failed and was set offline\n", + pr_err("Crypto dev=%02x.%04x rc/rs=%d/%d online=0 rc=EAGAIN\n", AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid)); - ZCRYPT_DBF(DBF_ERR, - "device=%02x.%04x rc/rs=%d/%d => online=0 rc=EAGAIN\n", - AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), - (int) service_rc, (int) service_rs); - return -EAGAIN; /* repeat the request on a different device. */ + AP_QID_QUEUE(zq->queue->qid), + (int) service_rc, (int) service_rs); + ZCRYPT_DBF_ERR("dev=%02x.%04x rc/rs=%d/%d => online=0 rc=EAGAIN\n", + AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), + (int) service_rc, (int) service_rs); + return -EAGAIN; } data = msg->text; reply_len = msg->length - 2; @@ -707,7 +716,7 @@ static int convert_type86_ica(struct zcrypt_queue *zq, * * Returns 0 on success or -EINVAL, -EFAULT, -EAGAIN in case of an error. */ -static int convert_type86_xcrb(struct zcrypt_queue *zq, +static int convert_type86_xcrb(bool userspace, struct zcrypt_queue *zq, struct ap_message *reply, struct ica_xcRB *xcRB) { @@ -715,15 +724,15 @@ static int convert_type86_xcrb(struct zcrypt_queue *zq, char *data = reply->msg; /* Copy CPRB to user */ - if (copy_to_user(xcRB->reply_control_blk_addr, - data + msg->fmt2.offset1, msg->fmt2.count1)) + if (z_copy_to_user(userspace, xcRB->reply_control_blk_addr, + data + msg->fmt2.offset1, msg->fmt2.count1)) return -EFAULT; xcRB->reply_control_blk_length = msg->fmt2.count1; /* Copy data buffer to user */ if (msg->fmt2.count2) - if (copy_to_user(xcRB->reply_data_addr, - data + msg->fmt2.offset2, msg->fmt2.count2)) + if (z_copy_to_user(userspace, xcRB->reply_data_addr, + data + msg->fmt2.offset2, msg->fmt2.count2)) return -EFAULT; xcRB->reply_data_length = msg->fmt2.count2; return 0; @@ -738,7 +747,7 @@ static int convert_type86_xcrb(struct zcrypt_queue *zq, * * Returns 0 on success or -EINVAL, -EFAULT, -EAGAIN in case of an error. */ -static int convert_type86_ep11_xcrb(struct zcrypt_queue *zq, +static int convert_type86_ep11_xcrb(bool userspace, struct zcrypt_queue *zq, struct ap_message *reply, struct ep11_urb *xcRB) { @@ -749,8 +758,8 @@ static int convert_type86_ep11_xcrb(struct zcrypt_queue *zq, return -EINVAL; /* Copy response CPRB to user */ - if (copy_to_user((char __force __user *)xcRB->resp, - data + msg->fmt2.offset1, msg->fmt2.count1)) + if (z_copy_to_user(userspace, (char __force __user *)xcRB->resp, + data + msg->fmt2.offset1, msg->fmt2.count1)) return -EFAULT; xcRB->resp_len = msg->fmt2.count1; return 0; @@ -800,23 +809,24 @@ static int convert_response_ica(struct zcrypt_queue *zq, return convert_type86_ica(zq, reply, outputdata, outputdatalength); fallthrough; /* wrong cprb version is an unknown response */ - default: /* Unknown response type, this should NEVER EVER happen */ + default: + /* Unknown response type, this should NEVER EVER happen */ zq->online = 0; - pr_err("Cryptographic device %02x.%04x failed and was set offline\n", + pr_err("Crypto dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n", AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid)); - ZCRYPT_DBF(DBF_ERR, - "device=%02x.%04x rtype=0x%02x => online=0 rc=EAGAIN\n", - AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), - (int) msg->hdr.type); - return -EAGAIN; /* repeat the request on a different device. */ + AP_QID_QUEUE(zq->queue->qid), + (int) msg->hdr.type); + ZCRYPT_DBF_ERR("dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n", + AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), + (int) msg->hdr.type); + return -EAGAIN; } } -static int convert_response_xcrb(struct zcrypt_queue *zq, - struct ap_message *reply, - struct ica_xcRB *xcRB) +static int convert_response_xcrb(bool userspace, struct zcrypt_queue *zq, + struct ap_message *reply, + struct ica_xcRB *xcRB) { struct type86x_reply *msg = reply->msg; @@ -831,25 +841,25 @@ static int convert_response_xcrb(struct zcrypt_queue *zq, return convert_error(zq, reply); } if (msg->cprbx.cprb_ver_id == 0x02) - return convert_type86_xcrb(zq, reply, xcRB); + return convert_type86_xcrb(userspace, zq, reply, xcRB); fallthrough; /* wrong cprb version is an unknown response */ default: /* Unknown response type, this should NEVER EVER happen */ xcRB->status = 0x0008044DL; /* HDD_InvalidParm */ zq->online = 0; - pr_err("Cryptographic device %02x.%04x failed and was set offline\n", + pr_err("Crypto dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n", AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid)); - ZCRYPT_DBF(DBF_ERR, - "device=%02x.%04x rtype=0x%02x => online=0 rc=EAGAIN\n", - AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), - (int) msg->hdr.type); - return -EAGAIN; /* repeat the request on a different device. */ + AP_QID_QUEUE(zq->queue->qid), + (int) msg->hdr.type); + ZCRYPT_DBF_ERR("dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n", + AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), + (int) msg->hdr.type); + return -EAGAIN; } } -static int convert_response_ep11_xcrb(struct zcrypt_queue *zq, - struct ap_message *reply, struct ep11_urb *xcRB) +static int convert_response_ep11_xcrb(bool userspace, struct zcrypt_queue *zq, + struct ap_message *reply, struct ep11_urb *xcRB) { struct type86_ep11_reply *msg = reply->msg; @@ -861,19 +871,19 @@ static int convert_response_ep11_xcrb(struct zcrypt_queue *zq, if (msg->hdr.reply_code) return convert_error(zq, reply); if (msg->cprbx.cprb_ver_id == 0x04) - return convert_type86_ep11_xcrb(zq, reply, xcRB); + return convert_type86_ep11_xcrb(userspace, zq, reply, xcRB); fallthrough; /* wrong cprb version is an unknown resp */ default: /* Unknown response type, this should NEVER EVER happen */ zq->online = 0; - pr_err("Cryptographic device %02x.%04x failed and was set offline\n", + pr_err("Crypto dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n", AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid)); - ZCRYPT_DBF(DBF_ERR, - "device=%02x.%04x rtype=0x%02x => online=0 rc=EAGAIN\n", - AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), - (int) msg->hdr.type); - return -EAGAIN; /* repeat the request on a different device. */ + AP_QID_QUEUE(zq->queue->qid), + (int) msg->hdr.type); + ZCRYPT_DBF_ERR("dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n", + AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), + (int) msg->hdr.type); + return -EAGAIN; } } @@ -895,15 +905,15 @@ static int convert_response_rng(struct zcrypt_queue *zq, fallthrough; /* wrong cprb version is an unknown response */ default: /* Unknown response type, this should NEVER EVER happen */ zq->online = 0; - pr_err("Cryptographic device %02x.%04x failed and was set offline\n", + pr_err("Crypto dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n", AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid)); - ZCRYPT_DBF(DBF_ERR, - "device=%02x.%04x rtype=0x%02x => online=0 rc=EAGAIN\n", - AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), - (int) msg->hdr.type); - return -EAGAIN; /* repeat the request on a different device. */ + AP_QID_QUEUE(zq->queue->qid), + (int) msg->hdr.type); + ZCRYPT_DBF_ERR("dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n", + AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), + (int) msg->hdr.type); + return -EAGAIN; } } @@ -1007,39 +1017,42 @@ static atomic_t zcrypt_step = ATOMIC_INIT(0); * @mex: pointer to the modexpo request buffer */ static long zcrypt_msgtype6_modexpo(struct zcrypt_queue *zq, - struct ica_rsa_modexpo *mex) + struct ica_rsa_modexpo *mex, + struct ap_message *ap_msg) { - struct ap_message ap_msg; struct response_type resp_type = { .type = CEXXC_RESPONSE_TYPE_ICA, }; int rc; - ap_init_message(&ap_msg); - ap_msg.msg = (void *) get_zeroed_page(GFP_KERNEL); - if (!ap_msg.msg) + ap_msg->msg = (void *) get_zeroed_page(GFP_KERNEL); + if (!ap_msg->msg) return -ENOMEM; - ap_msg.receive = zcrypt_msgtype6_receive; - ap_msg.psmid = (((unsigned long long) current->pid) << 32) + - atomic_inc_return(&zcrypt_step); - ap_msg.private = &resp_type; - rc = ICAMEX_msg_to_type6MEX_msgX(zq, &ap_msg, mex); + ap_msg->receive = zcrypt_msgtype6_receive; + ap_msg->psmid = (((unsigned long long) current->pid) << 32) + + atomic_inc_return(&zcrypt_step); + ap_msg->private = &resp_type; + rc = ICAMEX_msg_to_type6MEX_msgX(zq, ap_msg, mex); if (rc) goto out_free; init_completion(&resp_type.work); - ap_queue_message(zq->queue, &ap_msg); + rc = ap_queue_message(zq->queue, ap_msg); + if (rc) + goto out_free; rc = wait_for_completion_interruptible(&resp_type.work); if (rc == 0) { - rc = ap_msg.rc; + rc = ap_msg->rc; if (rc == 0) - rc = convert_response_ica(zq, &ap_msg, + rc = convert_response_ica(zq, ap_msg, mex->outputdata, mex->outputdatalength); } else /* Signal pending. */ - ap_cancel_message(zq->queue, &ap_msg); + ap_cancel_message(zq->queue, ap_msg); out_free: - free_page((unsigned long) ap_msg.msg); + free_page((unsigned long) ap_msg->msg); + ap_msg->private = NULL; + ap_msg->msg = NULL; return rc; } @@ -1051,40 +1064,43 @@ out_free: * @crt: pointer to the modexpoc_crt request buffer */ static long zcrypt_msgtype6_modexpo_crt(struct zcrypt_queue *zq, - struct ica_rsa_modexpo_crt *crt) + struct ica_rsa_modexpo_crt *crt, + struct ap_message *ap_msg) { - struct ap_message ap_msg; struct response_type resp_type = { .type = CEXXC_RESPONSE_TYPE_ICA, }; int rc; - ap_init_message(&ap_msg); - ap_msg.msg = (void *) get_zeroed_page(GFP_KERNEL); - if (!ap_msg.msg) + ap_msg->msg = (void *) get_zeroed_page(GFP_KERNEL); + if (!ap_msg->msg) return -ENOMEM; - ap_msg.receive = zcrypt_msgtype6_receive; - ap_msg.psmid = (((unsigned long long) current->pid) << 32) + - atomic_inc_return(&zcrypt_step); - ap_msg.private = &resp_type; - rc = ICACRT_msg_to_type6CRT_msgX(zq, &ap_msg, crt); + ap_msg->receive = zcrypt_msgtype6_receive; + ap_msg->psmid = (((unsigned long long) current->pid) << 32) + + atomic_inc_return(&zcrypt_step); + ap_msg->private = &resp_type; + rc = ICACRT_msg_to_type6CRT_msgX(zq, ap_msg, crt); if (rc) goto out_free; init_completion(&resp_type.work); - ap_queue_message(zq->queue, &ap_msg); + rc = ap_queue_message(zq->queue, ap_msg); + if (rc) + goto out_free; rc = wait_for_completion_interruptible(&resp_type.work); if (rc == 0) { - rc = ap_msg.rc; + rc = ap_msg->rc; if (rc == 0) - rc = convert_response_ica(zq, &ap_msg, + rc = convert_response_ica(zq, ap_msg, crt->outputdata, crt->outputdatalength); } else { /* Signal pending. */ - ap_cancel_message(zq->queue, &ap_msg); + ap_cancel_message(zq->queue, ap_msg); } out_free: - free_page((unsigned long) ap_msg.msg); + free_page((unsigned long) ap_msg->msg); + ap_msg->private = NULL; + ap_msg->msg = NULL; return rc; } @@ -1095,9 +1111,9 @@ out_free: * by the caller with ap_init_message(). Also the caller has to * make sure ap_release_message() is always called even on failure. */ -unsigned int get_cprb_fc(struct ica_xcRB *xcRB, - struct ap_message *ap_msg, - unsigned int *func_code, unsigned short **dom) +unsigned int get_cprb_fc(bool userspace, struct ica_xcRB *xcRB, + struct ap_message *ap_msg, + unsigned int *func_code, unsigned short **dom) { struct response_type resp_type = { .type = CEXXC_RESPONSE_TYPE_XCRB, @@ -1112,7 +1128,7 @@ unsigned int get_cprb_fc(struct ica_xcRB *xcRB, ap_msg->private = kmemdup(&resp_type, sizeof(resp_type), GFP_KERNEL); if (!ap_msg->private) return -ENOMEM; - return XCRB_msg_to_type6CPRB_msgX(ap_msg, xcRB, func_code, dom); + return XCRB_msg_to_type6CPRB_msgX(userspace, ap_msg, xcRB, func_code, dom); } /** @@ -1122,24 +1138,26 @@ unsigned int get_cprb_fc(struct ica_xcRB *xcRB, * CEXxC device to the request distributor * @xcRB: pointer to the send_cprb request buffer */ -static long zcrypt_msgtype6_send_cprb(struct zcrypt_queue *zq, - struct ica_xcRB *xcRB, - struct ap_message *ap_msg) +static long zcrypt_msgtype6_send_cprb(bool userspace, struct zcrypt_queue *zq, + struct ica_xcRB *xcRB, + struct ap_message *ap_msg) { int rc; struct response_type *rtype = (struct response_type *)(ap_msg->private); init_completion(&rtype->work); - ap_queue_message(zq->queue, ap_msg); + rc = ap_queue_message(zq->queue, ap_msg); + if (rc) + goto out; rc = wait_for_completion_interruptible(&rtype->work); if (rc == 0) { rc = ap_msg->rc; if (rc == 0) - rc = convert_response_xcrb(zq, ap_msg, xcRB); + rc = convert_response_xcrb(userspace, zq, ap_msg, xcRB); } else /* Signal pending. */ ap_cancel_message(zq->queue, ap_msg); - +out: return rc; } @@ -1150,9 +1168,9 @@ static long zcrypt_msgtype6_send_cprb(struct zcrypt_queue *zq, * by the caller with ap_init_message(). Also the caller has to * make sure ap_release_message() is always called even on failure. */ -unsigned int get_ep11cprb_fc(struct ep11_urb *xcrb, - struct ap_message *ap_msg, - unsigned int *func_code) +unsigned int get_ep11cprb_fc(bool userspace, struct ep11_urb *xcrb, + struct ap_message *ap_msg, + unsigned int *func_code) { struct response_type resp_type = { .type = CEXXC_RESPONSE_TYPE_EP11, @@ -1167,7 +1185,7 @@ unsigned int get_ep11cprb_fc(struct ep11_urb *xcrb, ap_msg->private = kmemdup(&resp_type, sizeof(resp_type), GFP_KERNEL); if (!ap_msg->private) return -ENOMEM; - return xcrb_msg_to_type6_ep11cprb_msgx(ap_msg, xcrb, func_code); + return xcrb_msg_to_type6_ep11cprb_msgx(userspace, ap_msg, xcrb, func_code); } /** @@ -1177,7 +1195,7 @@ unsigned int get_ep11cprb_fc(struct ep11_urb *xcrb, * CEX4P device to the request distributor * @xcRB: pointer to the ep11 user request block */ -static long zcrypt_msgtype6_send_ep11_cprb(struct zcrypt_queue *zq, +static long zcrypt_msgtype6_send_ep11_cprb(bool userspace, struct zcrypt_queue *zq, struct ep11_urb *xcrb, struct ap_message *ap_msg) { @@ -1232,16 +1250,18 @@ static long zcrypt_msgtype6_send_ep11_cprb(struct zcrypt_queue *zq, } init_completion(&rtype->work); - ap_queue_message(zq->queue, ap_msg); + rc = ap_queue_message(zq->queue, ap_msg); + if (rc) + goto out; rc = wait_for_completion_interruptible(&rtype->work); if (rc == 0) { rc = ap_msg->rc; if (rc == 0) - rc = convert_response_ep11_xcrb(zq, ap_msg, xcrb); + rc = convert_response_ep11_xcrb(userspace, zq, ap_msg, xcrb); } else /* Signal pending. */ ap_cancel_message(zq->queue, ap_msg); - +out: return rc; } @@ -1293,7 +1313,9 @@ static long zcrypt_msgtype6_rng(struct zcrypt_queue *zq, msg->cprbx.domain = AP_QID_QUEUE(zq->queue->qid); init_completion(&rtype->work); - ap_queue_message(zq->queue, ap_msg); + rc = ap_queue_message(zq->queue, ap_msg); + if (rc) + goto out; rc = wait_for_completion_interruptible(&rtype->work); if (rc == 0) { rc = ap_msg->rc; @@ -1302,7 +1324,7 @@ static long zcrypt_msgtype6_rng(struct zcrypt_queue *zq, } else /* Signal pending. */ ap_cancel_message(zq->queue, ap_msg); - +out: return rc; } diff --git a/drivers/s390/crypto/zcrypt_msgtype6.h b/drivers/s390/crypto/zcrypt_msgtype6.h index 0de280a81dd4..0a0bf074206b 100644 --- a/drivers/s390/crypto/zcrypt_msgtype6.h +++ b/drivers/s390/crypto/zcrypt_msgtype6.h @@ -96,9 +96,9 @@ struct type86_fmt2_ext { unsigned int offset4; /* 0x00000000 */ } __packed; -unsigned int get_cprb_fc(struct ica_xcRB *, struct ap_message *, +unsigned int get_cprb_fc(bool userspace, struct ica_xcRB *, struct ap_message *, unsigned int *, unsigned short **); -unsigned int get_ep11cprb_fc(struct ep11_urb *, struct ap_message *, +unsigned int get_ep11cprb_fc(bool userspace, struct ep11_urb *, struct ap_message *, unsigned int *); unsigned int get_rng_fc(struct ap_message *, int *, unsigned int *); diff --git a/drivers/s390/crypto/zcrypt_queue.c b/drivers/s390/crypto/zcrypt_queue.c index 8bae6ad159a7..3c207066313c 100644 --- a/drivers/s390/crypto/zcrypt_queue.c +++ b/drivers/s390/crypto/zcrypt_queue.c @@ -40,22 +40,27 @@ static ssize_t online_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct zcrypt_queue *zq = to_ap_queue(dev)->private; + struct ap_queue *aq = to_ap_queue(dev); + struct zcrypt_queue *zq = aq->private; + int online = aq->config && zq->online ? 1 : 0; - return scnprintf(buf, PAGE_SIZE, "%d\n", zq->online); + return scnprintf(buf, PAGE_SIZE, "%d\n", online); } static ssize_t online_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct zcrypt_queue *zq = to_ap_queue(dev)->private; + struct ap_queue *aq = to_ap_queue(dev); + struct zcrypt_queue *zq = aq->private; struct zcrypt_card *zc = zq->zcard; int online; if (sscanf(buf, "%d\n", &online) != 1 || online < 0 || online > 1) return -EINVAL; + if (online && (!aq->config || !aq->card->config)) + return -ENODEV; if (online && !zc->online) return -EINVAL; zq->online = online; diff --git a/drivers/s390/scsi/zfcp_qdio.c b/drivers/s390/scsi/zfcp_qdio.c index e78d65bd46b1..a8a514074084 100644 --- a/drivers/s390/scsi/zfcp_qdio.c +++ b/drivers/s390/scsi/zfcp_qdio.c @@ -380,8 +380,6 @@ int zfcp_qdio_open(struct zfcp_qdio *qdio) &qdio->adapter->status); init_data.q_format = QDIO_ZFCP_QFMT; - memcpy(init_data.adapter_name, dev_name(&cdev->dev), 8); - ASCEBC(init_data.adapter_name, 8); init_data.qib_rflags = QIB_RFLAGS_ENABLE_DATA_DIV; if (enable_multibuffer) init_data.qdr_ac |= QDR_AC_MULTI_BUFFER_ENABLE; diff --git a/drivers/scsi/53c700.c b/drivers/scsi/53c700.c index 5117d90ccd9e..3242ff63986f 100644 --- a/drivers/scsi/53c700.c +++ b/drivers/scsi/53c700.c @@ -1506,10 +1506,8 @@ NCR_700_intr(int irq, void *dev_id) __u8 sstat0 = 0, dstat = 0; __u32 dsp; struct scsi_cmnd *SCp = hostdata->cmd; - enum NCR_700_Host_State state; handled = 1; - state = hostdata->state; SCp = hostdata->cmd; if(istat & SCSI_INT_PENDING) { @@ -1760,7 +1758,6 @@ NCR_700_queuecommand_lck(struct scsi_cmnd *SCp, void (*done)(struct scsi_cmnd *) struct NCR_700_Host_Parameters *hostdata = (struct NCR_700_Host_Parameters *)SCp->device->host->hostdata[0]; __u32 move_ins; - enum dma_data_direction direction; struct NCR_700_command_slot *slot; if(hostdata->command_slot_count >= NCR_700_COMMAND_SLOTS_PER_HOST) { @@ -1877,7 +1874,6 @@ NCR_700_queuecommand_lck(struct scsi_cmnd *SCp, void (*done)(struct scsi_cmnd *) } /* now build the scatter gather list */ - direction = SCp->sc_data_direction; if(move_ins != 0) { int i; int sg_count; diff --git a/drivers/scsi/arcmsr/arcmsr.h b/drivers/scsi/arcmsr/arcmsr.h index 9220bcf8388f..5d054d5c70a5 100644 --- a/drivers/scsi/arcmsr/arcmsr.h +++ b/drivers/scsi/arcmsr/arcmsr.h @@ -49,7 +49,7 @@ struct device_attribute; #define ARCMSR_MAX_OUTSTANDING_CMD 1024 #define ARCMSR_DEFAULT_OUTSTANDING_CMD 128 #define ARCMSR_MIN_OUTSTANDING_CMD 32 -#define ARCMSR_DRIVER_VERSION "v1.40.00.10-20190116" +#define ARCMSR_DRIVER_VERSION "v1.50.00.02-20200819" #define ARCMSR_SCSI_INITIATOR_ID 255 #define ARCMSR_MAX_XFER_SECTORS 512 #define ARCMSR_MAX_XFER_SECTORS_B 4096 @@ -80,6 +80,7 @@ struct device_attribute; #ifndef PCI_DEVICE_ID_ARECA_1884 #define PCI_DEVICE_ID_ARECA_1884 0x1884 #endif +#define PCI_DEVICE_ID_ARECA_1886 0x188A #define ARCMSR_HOURS (1000 * 60 * 60 * 4) #define ARCMSR_MINUTES (1000 * 60 * 60) /* @@ -436,6 +437,21 @@ struct FIRMWARE_INFO #define ARCMSR_HBEMU_DOORBELL_SYNC 0x100 #define ARCMSR_ARC188X_RESET_ADAPTER 0x00000004 #define ARCMSR_ARC1884_DiagWrite_ENABLE 0x00000080 + +/* +******************************************************************************* +** SPEC. for Areca Type F adapter +******************************************************************************* +*/ +#define ARCMSR_SIGNATURE_1886 0x188617D3 +// Doorbell and interrupt definition are same as Type E adapter +/* ARC-1886 doorbell sync */ +#define ARCMSR_HBFMU_DOORBELL_SYNC 0x100 +//set host rw buffer physical address at inbound message 0, 1 (low,high) +#define ARCMSR_HBFMU_DOORBELL_SYNC1 0x300 +#define ARCMSR_HBFMU_MESSAGE_FIRMWARE_OK 0x80000000 +#define ARCMSR_HBFMU_MESSAGE_NO_VOLUME_CHANGE 0x20000000 + /* ******************************************************************************* ** ARECA SCSI COMMAND DESCRIPTOR BLOCK size 0x1F8 (504) @@ -720,6 +736,80 @@ struct MessageUnit_E{ uint32_t msgcode_rwbuffer[256]; /*2200 23FF*/ }; +/* +********************************************************************* +** Messaging Unit (MU) of Type F processor(LSI) +********************************************************************* +*/ +struct MessageUnit_F { + uint32_t iobound_doorbell; /*0000 0003*/ + uint32_t write_sequence_3xxx; /*0004 0007*/ + uint32_t host_diagnostic_3xxx; /*0008 000B*/ + uint32_t posted_outbound_doorbell; /*000C 000F*/ + uint32_t master_error_attribute; /*0010 0013*/ + uint32_t master_error_address_low; /*0014 0017*/ + uint32_t master_error_address_high; /*0018 001B*/ + uint32_t hcb_size; /*001C 001F*/ + uint32_t inbound_doorbell; /*0020 0023*/ + uint32_t diagnostic_rw_data; /*0024 0027*/ + uint32_t diagnostic_rw_address_low; /*0028 002B*/ + uint32_t diagnostic_rw_address_high; /*002C 002F*/ + uint32_t host_int_status; /*0030 0033*/ + uint32_t host_int_mask; /*0034 0037*/ + uint32_t dcr_data; /*0038 003B*/ + uint32_t dcr_address; /*003C 003F*/ + uint32_t inbound_queueport; /*0040 0043*/ + uint32_t outbound_queueport; /*0044 0047*/ + uint32_t hcb_pci_address_low; /*0048 004B*/ + uint32_t hcb_pci_address_high; /*004C 004F*/ + uint32_t iop_int_status; /*0050 0053*/ + uint32_t iop_int_mask; /*0054 0057*/ + uint32_t iop_inbound_queue_port; /*0058 005B*/ + uint32_t iop_outbound_queue_port; /*005C 005F*/ + uint32_t inbound_free_list_index; /*0060 0063*/ + uint32_t inbound_post_list_index; /*0064 0067*/ + uint32_t reply_post_producer_index; /*0068 006B*/ + uint32_t reply_post_consumer_index; /*006C 006F*/ + uint32_t inbound_doorbell_clear; /*0070 0073*/ + uint32_t i2o_message_unit_control; /*0074 0077*/ + uint32_t last_used_message_source_address_low; /*0078 007B*/ + uint32_t last_used_message_source_address_high; /*007C 007F*/ + uint32_t pull_mode_data_byte_count[4]; /*0080 008F*/ + uint32_t message_dest_address_index; /*0090 0093*/ + uint32_t done_queue_not_empty_int_counter_timer; /*0094 0097*/ + uint32_t utility_A_int_counter_timer; /*0098 009B*/ + uint32_t outbound_doorbell; /*009C 009F*/ + uint32_t outbound_doorbell_clear; /*00A0 00A3*/ + uint32_t message_source_address_index; /*00A4 00A7*/ + uint32_t message_done_queue_index; /*00A8 00AB*/ + uint32_t reserved0; /*00AC 00AF*/ + uint32_t inbound_msgaddr0; /*00B0 00B3*/ + uint32_t inbound_msgaddr1; /*00B4 00B7*/ + uint32_t outbound_msgaddr0; /*00B8 00BB*/ + uint32_t outbound_msgaddr1; /*00BC 00BF*/ + uint32_t inbound_queueport_low; /*00C0 00C3*/ + uint32_t inbound_queueport_high; /*00C4 00C7*/ + uint32_t outbound_queueport_low; /*00C8 00CB*/ + uint32_t outbound_queueport_high; /*00CC 00CF*/ + uint32_t iop_inbound_queue_port_low; /*00D0 00D3*/ + uint32_t iop_inbound_queue_port_high; /*00D4 00D7*/ + uint32_t iop_outbound_queue_port_low; /*00D8 00DB*/ + uint32_t iop_outbound_queue_port_high; /*00DC 00DF*/ + uint32_t message_dest_queue_port_low; /*00E0 00E3*/ + uint32_t message_dest_queue_port_high; /*00E4 00E7*/ + uint32_t last_used_message_dest_address_low; /*00E8 00EB*/ + uint32_t last_used_message_dest_address_high; /*00EC 00EF*/ + uint32_t message_done_queue_base_address_low; /*00F0 00F3*/ + uint32_t message_done_queue_base_address_high; /*00F4 00F7*/ + uint32_t host_diagnostic; /*00F8 00FB*/ + uint32_t write_sequence; /*00FC 00FF*/ + uint32_t reserved1[46]; /*0100 01B7*/ + uint32_t reply_post_producer_index1; /*01B8 01BB*/ + uint32_t reply_post_consumer_index1; /*01BC 01BF*/ +}; + +#define MESG_RW_BUFFER_SIZE (256 * 3) + typedef struct deliver_completeQ { uint16_t cmdFlag; uint16_t cmdSMID; @@ -739,6 +829,7 @@ struct AdapterControlBlock #define ACB_ADAPTER_TYPE_C 0x00000002 /* hbc L IOP */ #define ACB_ADAPTER_TYPE_D 0x00000003 /* hbd M IOP */ #define ACB_ADAPTER_TYPE_E 0x00000004 /* hba L IOP */ +#define ACB_ADAPTER_TYPE_F 0x00000005 /* hba L IOP */ u32 ioqueue_size; struct pci_dev * pdev; struct Scsi_Host * host; @@ -760,10 +851,16 @@ struct AdapterControlBlock struct MessageUnit_C __iomem *pmuC; struct MessageUnit_D *pmuD; struct MessageUnit_E __iomem *pmuE; + struct MessageUnit_F __iomem *pmuF; }; /* message unit ATU inbound base address0 */ void __iomem *mem_base0; void __iomem *mem_base1; + //0x000 - COMPORT_IN (Host sent to ROC) + uint32_t *message_wbuffer; + //0x100 - COMPORT_OUT (ROC sent to Host) + uint32_t *message_rbuffer; + uint32_t *msgcode_rwbuffer; //0x200 - BIOS_AREA uint32_t acb_flags; u16 dev_id; uint8_t adapter_index; @@ -836,8 +933,6 @@ struct AdapterControlBlock #define FW_NORMAL 0x0000 #define FW_BOG 0x0001 #define FW_DEADLOCK 0x0010 - atomic_t rq_map_token; - atomic_t ante_token_value; uint32_t maxOutstanding; int vector_count; uint32_t maxFreeCCB; @@ -848,6 +943,7 @@ struct AdapterControlBlock uint32_t out_doorbell; uint32_t completionQ_entry; pCompletion_Q pCompletionQ; + uint32_t completeQ_size; };/* HW_DEVICE_EXTENSION */ /* ******************************************************************************* diff --git a/drivers/scsi/arcmsr/arcmsr_hba.c b/drivers/scsi/arcmsr/arcmsr_hba.c index ec895d0319f0..e4fdb473b990 100644 --- a/drivers/scsi/arcmsr/arcmsr_hba.c +++ b/drivers/scsi/arcmsr/arcmsr_hba.c @@ -133,6 +133,7 @@ static void arcmsr_hbaC_message_isr(struct AdapterControlBlock *pACB); static void arcmsr_hbaD_message_isr(struct AdapterControlBlock *acb); static void arcmsr_hbaE_message_isr(struct AdapterControlBlock *acb); static void arcmsr_hbaE_postqueue_isr(struct AdapterControlBlock *acb); +static void arcmsr_hbaF_postqueue_isr(struct AdapterControlBlock *acb); static void arcmsr_hardware_reset(struct AdapterControlBlock *acb); static const char *arcmsr_info(struct Scsi_Host *); static irqreturn_t arcmsr_interrupt(struct AdapterControlBlock *acb); @@ -209,6 +210,8 @@ static struct pci_device_id arcmsr_device_id_table[] = { .driver_data = ACB_ADAPTER_TYPE_C}, {PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1884), .driver_data = ACB_ADAPTER_TYPE_E}, + {PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1886), + .driver_data = ACB_ADAPTER_TYPE_F}, {0, 0}, /* Terminating entry */ }; MODULE_DEVICE_TABLE(pci, arcmsr_device_id_table); @@ -232,12 +235,12 @@ static void arcmsr_free_io_queue(struct AdapterControlBlock *acb) switch (acb->adapter_type) { case ACB_ADAPTER_TYPE_B: case ACB_ADAPTER_TYPE_D: - case ACB_ADAPTER_TYPE_E: { + case ACB_ADAPTER_TYPE_E: + case ACB_ADAPTER_TYPE_F: dma_free_coherent(&acb->pdev->dev, acb->ioqueue_size, acb->dma_coherent2, acb->dma_coherent_handle2); break; } - } } static bool arcmsr_remap_pciregion(struct AdapterControlBlock *acb) @@ -310,6 +313,19 @@ static bool arcmsr_remap_pciregion(struct AdapterControlBlock *acb) acb->out_doorbell = 0; break; } + case ACB_ADAPTER_TYPE_F: { + acb->pmuF = ioremap(pci_resource_start(pdev, 0), pci_resource_len(pdev, 0)); + if (!acb->pmuF) { + pr_notice("arcmsr%d: memory mapping region fail\n", + acb->host->host_no); + return false; + } + writel(0, &acb->pmuF->host_int_status); /* clear interrupt */ + writel(ARCMSR_HBFMU_DOORBELL_SYNC, &acb->pmuF->iobound_doorbell); + acb->in_doorbell = 0; + acb->out_doorbell = 0; + break; + } } return true; } @@ -317,26 +333,25 @@ static bool arcmsr_remap_pciregion(struct AdapterControlBlock *acb) static void arcmsr_unmap_pciregion(struct AdapterControlBlock *acb) { switch (acb->adapter_type) { - case ACB_ADAPTER_TYPE_A:{ + case ACB_ADAPTER_TYPE_A: iounmap(acb->pmuA); - } - break; - case ACB_ADAPTER_TYPE_B:{ + break; + case ACB_ADAPTER_TYPE_B: iounmap(acb->mem_base0); iounmap(acb->mem_base1); - } - - break; - case ACB_ADAPTER_TYPE_C:{ + break; + case ACB_ADAPTER_TYPE_C: iounmap(acb->pmuC); - } - break; + break; case ACB_ADAPTER_TYPE_D: iounmap(acb->mem_base0); break; case ACB_ADAPTER_TYPE_E: iounmap(acb->pmuE); break; + case ACB_ADAPTER_TYPE_F: + iounmap(acb->pmuF); + break; } } @@ -552,23 +567,20 @@ static void arcmsr_flush_adapter_cache(struct AdapterControlBlock *acb) { switch (acb->adapter_type) { - case ACB_ADAPTER_TYPE_A: { + case ACB_ADAPTER_TYPE_A: arcmsr_hbaA_flush_cache(acb); - } break; - - case ACB_ADAPTER_TYPE_B: { + case ACB_ADAPTER_TYPE_B: arcmsr_hbaB_flush_cache(acb); - } break; - case ACB_ADAPTER_TYPE_C: { + case ACB_ADAPTER_TYPE_C: arcmsr_hbaC_flush_cache(acb); - } break; case ACB_ADAPTER_TYPE_D: arcmsr_hbaD_flush_cache(acb); break; case ACB_ADAPTER_TYPE_E: + case ACB_ADAPTER_TYPE_F: arcmsr_hbaE_flush_cache(acb); break; } @@ -626,6 +638,27 @@ static void arcmsr_hbaD_assign_regAddr(struct AdapterControlBlock *acb) reg->msgcode_rwbuffer = MEM_BASE0(ARCMSR_ARC1214_MESSAGE_RWBUFFER); } +static void arcmsr_hbaF_assign_regAddr(struct AdapterControlBlock *acb) +{ + dma_addr_t host_buffer_dma; + struct MessageUnit_F __iomem *pmuF; + + memset(acb->dma_coherent2, 0xff, acb->completeQ_size); + acb->message_wbuffer = (uint32_t *)round_up((unsigned long)acb->dma_coherent2 + + acb->completeQ_size, 4); + acb->message_rbuffer = ((void *)acb->message_wbuffer) + 0x100; + acb->msgcode_rwbuffer = ((void *)acb->message_wbuffer) + 0x200; + memset((void *)acb->message_wbuffer, 0, MESG_RW_BUFFER_SIZE); + host_buffer_dma = round_up(acb->dma_coherent_handle2 + acb->completeQ_size, 4); + pmuF = acb->pmuF; + /* host buffer low address, bit0:1 all buffer active */ + writel(lower_32_bits(host_buffer_dma | 1), &pmuF->inbound_msgaddr0); + /* host buffer high address */ + writel(upper_32_bits(host_buffer_dma), &pmuF->inbound_msgaddr1); + /* set host buffer physical address */ + writel(ARCMSR_HBFMU_DOORBELL_SYNC1, &pmuF->iobound_doorbell); +} + static bool arcmsr_alloc_io_queue(struct AdapterControlBlock *acb) { bool rtn = true; @@ -679,6 +712,28 @@ static bool arcmsr_alloc_io_queue(struct AdapterControlBlock *acb) acb->doneq_index = 0; } break; + case ACB_ADAPTER_TYPE_F: { + uint32_t QueueDepth; + uint32_t depthTbl[] = {256, 512, 1024, 128, 64, 32}; + + arcmsr_wait_firmware_ready(acb); + QueueDepth = depthTbl[readl(&acb->pmuF->outbound_msgaddr1) & 7]; + acb->completeQ_size = sizeof(struct deliver_completeQ) * QueueDepth + 128; + acb->ioqueue_size = roundup(acb->completeQ_size + MESG_RW_BUFFER_SIZE, 32); + dma_coherent = dma_alloc_coherent(&pdev->dev, acb->ioqueue_size, + &dma_coherent_handle, GFP_KERNEL); + if (!dma_coherent) { + pr_notice("arcmsr%d: DMA allocation failed\n", acb->host->host_no); + return false; + } + acb->dma_coherent_handle2 = dma_coherent_handle; + acb->dma_coherent2 = dma_coherent; + acb->pCompletionQ = dma_coherent; + acb->completionQ_entry = acb->completeQ_size / sizeof(struct deliver_completeQ); + acb->doneq_index = 0; + arcmsr_hbaF_assign_regAddr(acb); + } + break; default: break; } @@ -713,7 +768,8 @@ static int arcmsr_alloc_ccb_pool(struct AdapterControlBlock *acb) acb->host->sg_tablesize = max_sg_entrys; roundup_ccbsize = roundup(sizeof(struct CommandControlBlock) + (max_sg_entrys - 1) * sizeof(struct SG64ENTRY), 32); acb->uncache_size = roundup_ccbsize * acb->maxFreeCCB; - acb->uncache_size += acb->ioqueue_size; + if (acb->adapter_type != ACB_ADAPTER_TYPE_F) + acb->uncache_size += acb->ioqueue_size; dma_coherent = dma_alloc_coherent(&pdev->dev, acb->uncache_size, &dma_coherent_handle, GFP_KERNEL); if(!dma_coherent){ printk(KERN_NOTICE "arcmsr%d: dma_alloc_coherent got error\n", acb->host->host_no); @@ -736,6 +792,7 @@ static int arcmsr_alloc_ccb_pool(struct AdapterControlBlock *acb) case ACB_ADAPTER_TYPE_C: case ACB_ADAPTER_TYPE_D: case ACB_ADAPTER_TYPE_E: + case ACB_ADAPTER_TYPE_F: ccb_tmp->cdb_phyaddr = cdb_phyaddr; break; } @@ -754,8 +811,10 @@ static int arcmsr_alloc_ccb_pool(struct AdapterControlBlock *acb) ccb_tmp = (struct CommandControlBlock *)((unsigned long)ccb_tmp + roundup_ccbsize); dma_coherent_handle = next_ccb_phy; } - acb->dma_coherent_handle2 = dma_coherent_handle; - acb->dma_coherent2 = ccb_tmp; + if (acb->adapter_type != ACB_ADAPTER_TYPE_F) { + acb->dma_coherent_handle2 = dma_coherent_handle; + acb->dma_coherent2 = ccb_tmp; + } switch (acb->adapter_type) { case ACB_ADAPTER_TYPE_B: acb->pmuB = (struct MessageUnit_B *)acb->dma_coherent2; @@ -785,7 +844,6 @@ static void arcmsr_message_isr_bh_fn(struct work_struct *work) struct scsi_device *psdev; char diff, temp; - acb->acb_flags &= ~ACB_F_MSG_GET_CONFIG; switch (acb->adapter_type) { case ACB_ADAPTER_TYPE_A: { struct MessageUnit_A __iomem *reg = acb->pmuA; @@ -822,8 +880,12 @@ static void arcmsr_message_isr_bh_fn(struct work_struct *work) devicemap = (char __iomem *)(®->msgcode_rwbuffer[21]); break; } + case ACB_ADAPTER_TYPE_F: { + signature = (uint32_t __iomem *)(&acb->msgcode_rwbuffer[0]); + devicemap = (char __iomem *)(&acb->msgcode_rwbuffer[21]); + break; + } } - atomic_inc(&acb->rq_map_token); if (readl(signature) != ARCMSR_SIGNATURE_GET_CONFIG) return; for (target = 0; target < ARCMSR_MAX_TARGETID - 1; @@ -854,6 +916,7 @@ static void arcmsr_message_isr_bh_fn(struct work_struct *work) devicemap++; acb_dev_map++; } + acb->acb_flags &= ~ACB_F_MSG_GET_CONFIG; } static int @@ -906,8 +969,6 @@ out_free_irq: static void arcmsr_init_get_devmap_timer(struct AdapterControlBlock *pacb) { INIT_WORK(&pacb->arcmsr_do_message_isr_bh, arcmsr_message_isr_bh_fn); - atomic_set(&pacb->rq_map_token, 16); - atomic_set(&pacb->ante_token_value, 16); pacb->fw_flag = FW_NORMAL; timer_setup(&pacb->eternal_timer, arcmsr_request_device_map, 0); pacb->eternal_timer.expires = jiffies + msecs_to_jiffies(6 * HZ); @@ -1009,7 +1070,8 @@ static int arcmsr_probe(struct pci_dev *pdev, const struct pci_device_id *id) if(!error){ goto free_hbb_mu; } - arcmsr_free_io_queue(acb); + if (acb->adapter_type != ACB_ADAPTER_TYPE_F) + arcmsr_free_io_queue(acb); error = arcmsr_alloc_ccb_pool(acb); if(error){ goto unmap_pci_region; @@ -1122,6 +1184,14 @@ static int arcmsr_resume(struct pci_dev *pdev) acb->out_doorbell = 0; acb->doneq_index = 0; break; + case ACB_ADAPTER_TYPE_F: + writel(0, &acb->pmuF->host_int_status); + writel(ARCMSR_HBFMU_DOORBELL_SYNC, &acb->pmuF->iobound_doorbell); + acb->in_doorbell = 0; + acb->out_doorbell = 0; + acb->doneq_index = 0; + arcmsr_hbaF_assign_regAddr(acb); + break; } arcmsr_iop_init(acb); arcmsr_init_get_devmap_timer(acb); @@ -1134,6 +1204,8 @@ controller_stop: controller_unregister: scsi_remove_host(host); arcmsr_free_ccb_pool(acb); + if (acb->adapter_type == ACB_ADAPTER_TYPE_F) + arcmsr_free_io_queue(acb); arcmsr_unmap_pciregion(acb); pci_release_regions(pdev); scsi_host_put(host); @@ -1213,25 +1285,20 @@ static uint8_t arcmsr_abort_allcmd(struct AdapterControlBlock *acb) { uint8_t rtnval = 0; switch (acb->adapter_type) { - case ACB_ADAPTER_TYPE_A: { + case ACB_ADAPTER_TYPE_A: rtnval = arcmsr_hbaA_abort_allcmd(acb); - } break; - - case ACB_ADAPTER_TYPE_B: { + case ACB_ADAPTER_TYPE_B: rtnval = arcmsr_hbaB_abort_allcmd(acb); - } break; - - case ACB_ADAPTER_TYPE_C: { + case ACB_ADAPTER_TYPE_C: rtnval = arcmsr_hbaC_abort_allcmd(acb); - } break; - case ACB_ADAPTER_TYPE_D: rtnval = arcmsr_hbaD_abort_allcmd(acb); break; case ACB_ADAPTER_TYPE_E: + case ACB_ADAPTER_TYPE_F: rtnval = arcmsr_hbaE_abort_allcmd(acb); break; } @@ -1307,7 +1374,8 @@ static u32 arcmsr_disable_outbound_ints(struct AdapterControlBlock *acb) writel(ARCMSR_ARC1214_ALL_INT_DISABLE, reg->pcief0_int_enable); } break; - case ACB_ADAPTER_TYPE_E: { + case ACB_ADAPTER_TYPE_E: + case ACB_ADAPTER_TYPE_F: { struct MessageUnit_E __iomem *reg = acb->pmuE; orig_mask = readl(®->host_int_mask); writel(orig_mask | ARCMSR_HBEMU_OUTBOUND_DOORBELL_ISR | ARCMSR_HBEMU_OUTBOUND_POSTQUEUE_ISR, ®->host_int_mask); @@ -1514,6 +1582,9 @@ static void arcmsr_done4abort_postqueue(struct AdapterControlBlock *acb) case ACB_ADAPTER_TYPE_E: arcmsr_hbaE_postqueue_isr(acb); break; + case ACB_ADAPTER_TYPE_F: + arcmsr_hbaF_postqueue_isr(acb); + break; } } @@ -1568,6 +1639,8 @@ static void arcmsr_free_pcidev(struct AdapterControlBlock *acb) pdev = acb->pdev; arcmsr_free_irq(pdev, acb); arcmsr_free_ccb_pool(acb); + if (acb->adapter_type == ACB_ADAPTER_TYPE_F) + arcmsr_free_io_queue(acb); arcmsr_unmap_pciregion(acb); pci_release_regions(pdev); scsi_host_put(host); @@ -1625,6 +1698,8 @@ static void arcmsr_remove(struct pci_dev *pdev) } arcmsr_free_irq(pdev, acb); arcmsr_free_ccb_pool(acb); + if (acb->adapter_type == ACB_ADAPTER_TYPE_F) + arcmsr_free_io_queue(acb); arcmsr_unmap_pciregion(acb); pci_release_regions(pdev); scsi_host_put(host); @@ -1702,7 +1777,8 @@ static void arcmsr_enable_outbound_ints(struct AdapterControlBlock *acb, writel(intmask_org | mask, reg->pcief0_int_enable); break; } - case ACB_ADAPTER_TYPE_E: { + case ACB_ADAPTER_TYPE_E: + case ACB_ADAPTER_TYPE_F: { struct MessageUnit_E __iomem *reg = acb->pmuE; mask = ~(ARCMSR_HBEMU_OUTBOUND_DOORBELL_ISR | ARCMSR_HBEMU_OUTBOUND_POSTQUEUE_ISR); @@ -1846,6 +1922,19 @@ static void arcmsr_post_ccb(struct AdapterControlBlock *acb, struct CommandContr writel(ccb_post_stamp, &pmu->inbound_queueport_low); break; } + case ACB_ADAPTER_TYPE_F: { + struct MessageUnit_F __iomem *pmu = acb->pmuF; + u32 ccb_post_stamp, arc_cdb_size; + + if (ccb->arc_cdb_size <= 0x300) + arc_cdb_size = (ccb->arc_cdb_size - 1) >> 6 | 1; + else + arc_cdb_size = (((ccb->arc_cdb_size + 0xff) >> 8) + 2) << 1 | 1; + ccb_post_stamp = (ccb->smid | arc_cdb_size); + writel(0, &pmu->inbound_queueport_high); + writel(ccb_post_stamp, &pmu->inbound_queueport_low); + break; + } } } @@ -1916,23 +2005,20 @@ static void arcmsr_hbaE_stop_bgrb(struct AdapterControlBlock *pACB) static void arcmsr_stop_adapter_bgrb(struct AdapterControlBlock *acb) { switch (acb->adapter_type) { - case ACB_ADAPTER_TYPE_A: { + case ACB_ADAPTER_TYPE_A: arcmsr_hbaA_stop_bgrb(acb); - } break; - - case ACB_ADAPTER_TYPE_B: { + case ACB_ADAPTER_TYPE_B: arcmsr_hbaB_stop_bgrb(acb); - } break; - case ACB_ADAPTER_TYPE_C: { + case ACB_ADAPTER_TYPE_C: arcmsr_hbaC_stop_bgrb(acb); - } break; case ACB_ADAPTER_TYPE_D: arcmsr_hbaD_stop_bgrb(acb); break; case ACB_ADAPTER_TYPE_E: + case ACB_ADAPTER_TYPE_F: arcmsr_hbaE_stop_bgrb(acb); break; } @@ -1951,7 +2037,6 @@ static void arcmsr_iop_message_read(struct AdapterControlBlock *acb) writel(ARCMSR_INBOUND_DRIVER_DATA_READ_OK, ®->inbound_doorbell); } break; - case ACB_ADAPTER_TYPE_B: { struct MessageUnit_B *reg = acb->pmuB; writel(ARCMSR_DRV2IOP_DATA_READ_OK, reg->drv2iop_doorbell); @@ -1969,7 +2054,8 @@ static void arcmsr_iop_message_read(struct AdapterControlBlock *acb) reg->inbound_doorbell); } break; - case ACB_ADAPTER_TYPE_E: { + case ACB_ADAPTER_TYPE_E: + case ACB_ADAPTER_TYPE_F: { struct MessageUnit_E __iomem *reg = acb->pmuE; acb->out_doorbell ^= ARCMSR_HBEMU_DRV2IOP_DATA_READ_OK; writel(acb->out_doorbell, ®->iobound_doorbell); @@ -2015,7 +2101,8 @@ static void arcmsr_iop_message_wrote(struct AdapterControlBlock *acb) reg->inbound_doorbell); } break; - case ACB_ADAPTER_TYPE_E: { + case ACB_ADAPTER_TYPE_E: + case ACB_ADAPTER_TYPE_F: { struct MessageUnit_E __iomem *reg = acb->pmuE; acb->out_doorbell ^= ARCMSR_HBEMU_DRV2IOP_DATA_WRITE_OK; writel(acb->out_doorbell, ®->iobound_doorbell); @@ -2034,7 +2121,6 @@ struct QBUFFER __iomem *arcmsr_get_iop_rqbuffer(struct AdapterControlBlock *acb) qbuffer = (struct QBUFFER __iomem *)®->message_rbuffer; } break; - case ACB_ADAPTER_TYPE_B: { struct MessageUnit_B *reg = acb->pmuB; qbuffer = (struct QBUFFER __iomem *)reg->message_rbuffer; @@ -2055,6 +2141,10 @@ struct QBUFFER __iomem *arcmsr_get_iop_rqbuffer(struct AdapterControlBlock *acb) qbuffer = (struct QBUFFER __iomem *)®->message_rbuffer; } break; + case ACB_ADAPTER_TYPE_F: { + qbuffer = (struct QBUFFER __iomem *)acb->message_rbuffer; + } + break; } return qbuffer; } @@ -2069,7 +2159,6 @@ static struct QBUFFER __iomem *arcmsr_get_iop_wqbuffer(struct AdapterControlBloc pqbuffer = (struct QBUFFER __iomem *) ®->message_wbuffer; } break; - case ACB_ADAPTER_TYPE_B: { struct MessageUnit_B *reg = acb->pmuB; pqbuffer = (struct QBUFFER __iomem *)reg->message_wbuffer; @@ -2090,6 +2179,9 @@ static struct QBUFFER __iomem *arcmsr_get_iop_wqbuffer(struct AdapterControlBloc pqbuffer = (struct QBUFFER __iomem *)®->message_wbuffer; } break; + case ACB_ADAPTER_TYPE_F: + pqbuffer = (struct QBUFFER __iomem *)acb->message_wbuffer; + break; } return pqbuffer; } @@ -2504,6 +2596,36 @@ static void arcmsr_hbaE_postqueue_isr(struct AdapterControlBlock *acb) spin_unlock_irqrestore(&acb->doneq_lock, flags); } +static void arcmsr_hbaF_postqueue_isr(struct AdapterControlBlock *acb) +{ + uint32_t doneq_index; + uint16_t cmdSMID; + int error; + struct MessageUnit_F __iomem *phbcmu; + struct CommandControlBlock *ccb; + unsigned long flags; + + spin_lock_irqsave(&acb->doneq_lock, flags); + doneq_index = acb->doneq_index; + phbcmu = acb->pmuF; + while (1) { + cmdSMID = acb->pCompletionQ[doneq_index].cmdSMID; + if (cmdSMID == 0xffff) + break; + ccb = acb->pccb_pool[cmdSMID]; + error = (acb->pCompletionQ[doneq_index].cmdFlag & + ARCMSR_CCBREPLY_FLAG_ERROR_MODE1) ? true : false; + arcmsr_drain_donequeue(acb, ccb, error); + acb->pCompletionQ[doneq_index].cmdSMID = 0xffff; + doneq_index++; + if (doneq_index >= acb->completionQ_entry) + doneq_index = 0; + } + acb->doneq_index = doneq_index; + writel(doneq_index, &phbcmu->reply_post_consumer_index); + spin_unlock_irqrestore(&acb->doneq_lock, flags); +} + /* ********************************************************************************** ** Handle a message interrupt @@ -2694,21 +2816,46 @@ static irqreturn_t arcmsr_hbaE_handle_isr(struct AdapterControlBlock *pACB) return IRQ_HANDLED; } +static irqreturn_t arcmsr_hbaF_handle_isr(struct AdapterControlBlock *pACB) +{ + uint32_t host_interrupt_status; + struct MessageUnit_F __iomem *phbcmu = pACB->pmuF; + + host_interrupt_status = readl(&phbcmu->host_int_status) & + (ARCMSR_HBEMU_OUTBOUND_POSTQUEUE_ISR | + ARCMSR_HBEMU_OUTBOUND_DOORBELL_ISR); + if (!host_interrupt_status) + return IRQ_NONE; + do { + /* MU post queue interrupts*/ + if (host_interrupt_status & ARCMSR_HBEMU_OUTBOUND_POSTQUEUE_ISR) + arcmsr_hbaF_postqueue_isr(pACB); + + /* MU ioctl transfer doorbell interrupts*/ + if (host_interrupt_status & ARCMSR_HBEMU_OUTBOUND_DOORBELL_ISR) + arcmsr_hbaE_doorbell_isr(pACB); + + host_interrupt_status = readl(&phbcmu->host_int_status); + } while (host_interrupt_status & (ARCMSR_HBEMU_OUTBOUND_POSTQUEUE_ISR | + ARCMSR_HBEMU_OUTBOUND_DOORBELL_ISR)); + return IRQ_HANDLED; +} + static irqreturn_t arcmsr_interrupt(struct AdapterControlBlock *acb) { switch (acb->adapter_type) { case ACB_ADAPTER_TYPE_A: return arcmsr_hbaA_handle_isr(acb); - break; case ACB_ADAPTER_TYPE_B: return arcmsr_hbaB_handle_isr(acb); - break; case ACB_ADAPTER_TYPE_C: return arcmsr_hbaC_handle_isr(acb); case ACB_ADAPTER_TYPE_D: return arcmsr_hbaD_handle_isr(acb); case ACB_ADAPTER_TYPE_E: return arcmsr_hbaE_handle_isr(acb); + case ACB_ADAPTER_TYPE_F: + return arcmsr_hbaF_handle_isr(acb); default: return IRQ_NONE; } @@ -3257,6 +3404,31 @@ static bool arcmsr_hbaE_get_config(struct AdapterControlBlock *pACB) return true; } +static bool arcmsr_hbaF_get_config(struct AdapterControlBlock *pACB) +{ + struct MessageUnit_F __iomem *reg = pACB->pmuF; + uint32_t intmask_org; + + /* disable all outbound interrupt */ + intmask_org = readl(®->host_int_mask); /* disable outbound message0 int */ + writel(intmask_org | ARCMSR_HBEMU_ALL_INTMASKENABLE, ®->host_int_mask); + /* wait firmware ready */ + arcmsr_wait_firmware_ready(pACB); + /* post "get config" instruction */ + writel(ARCMSR_INBOUND_MESG0_GET_CONFIG, ®->inbound_msgaddr0); + + pACB->out_doorbell ^= ARCMSR_HBEMU_DRV2IOP_MESSAGE_CMD_DONE; + writel(pACB->out_doorbell, ®->iobound_doorbell); + /* wait message ready */ + if (!arcmsr_hbaE_wait_msgint_ready(pACB)) { + pr_notice("arcmsr%d: wait get adapter firmware miscellaneous data timeout\n", + pACB->host->host_no); + return false; + } + arcmsr_get_adapter_config(pACB, pACB->msgcode_rwbuffer); + return true; +} + static bool arcmsr_get_firmware_spec(struct AdapterControlBlock *acb) { bool rtn = false; @@ -3277,6 +3449,9 @@ static bool arcmsr_get_firmware_spec(struct AdapterControlBlock *acb) case ACB_ADAPTER_TYPE_E: rtn = arcmsr_hbaE_get_config(acb); break; + case ACB_ADAPTER_TYPE_F: + rtn = arcmsr_hbaF_get_config(acb); + break; default: break; } @@ -3634,23 +3809,20 @@ static int arcmsr_polling_ccbdone(struct AdapterControlBlock *acb, int rtn = 0; switch (acb->adapter_type) { - case ACB_ADAPTER_TYPE_A: { + case ACB_ADAPTER_TYPE_A: rtn = arcmsr_hbaA_polling_ccbdone(acb, poll_ccb); - } break; - - case ACB_ADAPTER_TYPE_B: { + case ACB_ADAPTER_TYPE_B: rtn = arcmsr_hbaB_polling_ccbdone(acb, poll_ccb); - } break; - case ACB_ADAPTER_TYPE_C: { + case ACB_ADAPTER_TYPE_C: rtn = arcmsr_hbaC_polling_ccbdone(acb, poll_ccb); - } break; case ACB_ADAPTER_TYPE_D: rtn = arcmsr_hbaD_polling_ccbdone(acb, poll_ccb); break; case ACB_ADAPTER_TYPE_E: + case ACB_ADAPTER_TYPE_F: rtn = arcmsr_hbaE_polling_ccbdone(acb, poll_ccb); break; } @@ -3731,6 +3903,16 @@ static void arcmsr_set_iop_datetime(struct timer_list *t) writel(pacb->out_doorbell, ®->iobound_doorbell); break; } + case ACB_ADAPTER_TYPE_F: { + struct MessageUnit_F __iomem *reg = pacb->pmuF; + + pacb->msgcode_rwbuffer[0] = datetime.b.msg_time[0]; + pacb->msgcode_rwbuffer[1] = datetime.b.msg_time[1]; + writel(ARCMSR_INBOUND_MESG0_SYNC_TIMER, ®->inbound_msgaddr0); + pacb->out_doorbell ^= ARCMSR_HBEMU_DRV2IOP_MESSAGE_CMD_DONE; + writel(pacb->out_doorbell, ®->iobound_doorbell); + break; + } } if (sys_tz.tz_minuteswest) next_time = ARCMSR_HOURS; @@ -3756,6 +3938,7 @@ static int arcmsr_iop_confirm(struct AdapterControlBlock *acb) dma_coherent_handle = acb->dma_coherent_handle2; break; case ACB_ADAPTER_TYPE_E: + case ACB_ADAPTER_TYPE_F: dma_coherent_handle = acb->dma_coherent_handle + offsetof(struct CommandControlBlock, arcmsr_cdb); break; @@ -3873,11 +4056,8 @@ static int arcmsr_iop_confirm(struct AdapterControlBlock *acb) writel(cdb_phyaddr, ®->msgcode_rwbuffer[2]); writel(cdb_phyaddr_hi32, ®->msgcode_rwbuffer[3]); writel(acb->ccbsize, ®->msgcode_rwbuffer[4]); - dma_coherent_handle = acb->dma_coherent_handle2; - cdb_phyaddr = (uint32_t)(dma_coherent_handle & 0xffffffff); - cdb_phyaddr_hi32 = (uint32_t)((dma_coherent_handle >> 16) >> 16); - writel(cdb_phyaddr, ®->msgcode_rwbuffer[5]); - writel(cdb_phyaddr_hi32, ®->msgcode_rwbuffer[6]); + writel(lower_32_bits(acb->dma_coherent_handle2), ®->msgcode_rwbuffer[5]); + writel(upper_32_bits(acb->dma_coherent_handle2), ®->msgcode_rwbuffer[6]); writel(acb->ioqueue_size, ®->msgcode_rwbuffer[7]); writel(ARCMSR_INBOUND_MESG0_SET_CONFIG, ®->inbound_msgaddr0); acb->out_doorbell ^= ARCMSR_HBEMU_DRV2IOP_MESSAGE_CMD_DONE; @@ -3889,6 +4069,27 @@ static int arcmsr_iop_confirm(struct AdapterControlBlock *acb) } } break; + case ACB_ADAPTER_TYPE_F: { + struct MessageUnit_F __iomem *reg = acb->pmuF; + + acb->msgcode_rwbuffer[0] = ARCMSR_SIGNATURE_SET_CONFIG; + acb->msgcode_rwbuffer[1] = ARCMSR_SIGNATURE_1886; + acb->msgcode_rwbuffer[2] = cdb_phyaddr; + acb->msgcode_rwbuffer[3] = cdb_phyaddr_hi32; + acb->msgcode_rwbuffer[4] = acb->ccbsize; + acb->msgcode_rwbuffer[5] = lower_32_bits(acb->dma_coherent_handle2); + acb->msgcode_rwbuffer[6] = upper_32_bits(acb->dma_coherent_handle2); + acb->msgcode_rwbuffer[7] = acb->completeQ_size; + writel(ARCMSR_INBOUND_MESG0_SET_CONFIG, ®->inbound_msgaddr0); + acb->out_doorbell ^= ARCMSR_HBEMU_DRV2IOP_MESSAGE_CMD_DONE; + writel(acb->out_doorbell, ®->iobound_doorbell); + if (!arcmsr_hbaE_wait_msgint_ready(acb)) { + pr_notice("arcmsr%d: 'set command Q window' timeout\n", + acb->host->host_no); + return 1; + } + } + break; } return 0; } @@ -3937,7 +4138,8 @@ static void arcmsr_wait_firmware_ready(struct AdapterControlBlock *acb) ARCMSR_ARC1214_MESSAGE_FIRMWARE_OK) == 0); } break; - case ACB_ADAPTER_TYPE_E: { + case ACB_ADAPTER_TYPE_E: + case ACB_ADAPTER_TYPE_F: { struct MessageUnit_E __iomem *reg = acb->pmuE; do { if (!(acb->acb_flags & ACB_F_IOP_INITED)) @@ -3952,24 +4154,10 @@ static void arcmsr_wait_firmware_ready(struct AdapterControlBlock *acb) static void arcmsr_request_device_map(struct timer_list *t) { struct AdapterControlBlock *acb = from_timer(acb, t, eternal_timer); - if (unlikely(atomic_read(&acb->rq_map_token) == 0) || - (acb->acb_flags & ACB_F_BUS_RESET) || - (acb->acb_flags & ACB_F_ABORT)) { - mod_timer(&acb->eternal_timer, - jiffies + msecs_to_jiffies(6 * HZ)); + if (acb->acb_flags & (ACB_F_MSG_GET_CONFIG | ACB_F_BUS_RESET | ACB_F_ABORT)) { + mod_timer(&acb->eternal_timer, jiffies + msecs_to_jiffies(6 * HZ)); } else { acb->fw_flag = FW_NORMAL; - if (atomic_read(&acb->ante_token_value) == - atomic_read(&acb->rq_map_token)) { - atomic_set(&acb->rq_map_token, 16); - } - atomic_set(&acb->ante_token_value, - atomic_read(&acb->rq_map_token)); - if (atomic_dec_and_test(&acb->rq_map_token)) { - mod_timer(&acb->eternal_timer, jiffies + - msecs_to_jiffies(6 * HZ)); - return; - } switch (acb->adapter_type) { case ACB_ADAPTER_TYPE_A: { struct MessageUnit_A __iomem *reg = acb->pmuA; @@ -3999,10 +4187,23 @@ static void arcmsr_request_device_map(struct timer_list *t) writel(acb->out_doorbell, ®->iobound_doorbell); break; } + case ACB_ADAPTER_TYPE_F: { + struct MessageUnit_F __iomem *reg = acb->pmuF; + uint32_t outMsg1 = readl(®->outbound_msgaddr1); + + if (!(outMsg1 & ARCMSR_HBFMU_MESSAGE_FIRMWARE_OK) || + (outMsg1 & ARCMSR_HBFMU_MESSAGE_NO_VOLUME_CHANGE)) + goto nxt6s; + writel(ARCMSR_INBOUND_MESG0_GET_CONFIG, ®->inbound_msgaddr0); + acb->out_doorbell ^= ARCMSR_HBEMU_DRV2IOP_MESSAGE_CMD_DONE; + writel(acb->out_doorbell, ®->iobound_doorbell); + break; + } default: return; } acb->acb_flags |= ACB_F_MSG_GET_CONFIG; +nxt6s: mod_timer(&acb->eternal_timer, jiffies + msecs_to_jiffies(6 * HZ)); } } @@ -4084,6 +4285,7 @@ static void arcmsr_start_adapter_bgrb(struct AdapterControlBlock *acb) arcmsr_hbaD_start_bgrb(acb); break; case ACB_ADAPTER_TYPE_E: + case ACB_ADAPTER_TYPE_F: arcmsr_hbaE_start_bgrb(acb); break; } @@ -4163,7 +4365,8 @@ static void arcmsr_clear_doorbell_queue_buffer(struct AdapterControlBlock *acb) } } break; - case ACB_ADAPTER_TYPE_E: { + case ACB_ADAPTER_TYPE_E: + case ACB_ADAPTER_TYPE_F: { struct MessageUnit_E __iomem *reg = acb->pmuE; uint32_t i, tmp; @@ -4290,7 +4493,8 @@ static bool arcmsr_reset_in_progress(struct AdapterControlBlock *acb) true : false; } break; - case ACB_ADAPTER_TYPE_E:{ + case ACB_ADAPTER_TYPE_E: + case ACB_ADAPTER_TYPE_F:{ struct MessageUnit_E __iomem *reg = acb->pmuE; rtn = (readl(®->host_diagnostic_3xxx) & ARCMSR_ARC188X_RESET_ADAPTER) ? true : false; @@ -4389,8 +4593,6 @@ wait_reset_done: goto wait_reset_done; } arcmsr_iop_init(acb); - atomic_set(&acb->rq_map_token, 16); - atomic_set(&acb->ante_token_value, 16); acb->fw_flag = FW_NORMAL; mod_timer(&acb->eternal_timer, jiffies + msecs_to_jiffies(6 * HZ)); @@ -4399,8 +4601,6 @@ wait_reset_done: pr_notice("arcmsr: scsi bus reset eh returns with success\n"); } else { acb->acb_flags &= ~ACB_F_BUS_RESET; - atomic_set(&acb->rq_map_token, 16); - atomic_set(&acb->ante_token_value, 16); acb->fw_flag = FW_NORMAL; mod_timer(&acb->eternal_timer, jiffies + msecs_to_jiffies(6 * HZ)); @@ -4493,6 +4693,9 @@ static const char *arcmsr_info(struct Scsi_Host *host) case PCI_DEVICE_ID_ARECA_1884: type = "SAS/SATA"; break; + case PCI_DEVICE_ID_ARECA_1886: + type = "NVMe/SAS/SATA"; + break; default: type = "unknown"; raid6 = 0; diff --git a/drivers/scsi/bfa/bfad.c b/drivers/scsi/bfa/bfad.c index bc5d84f87d8f..440ef32be048 100644 --- a/drivers/scsi/bfa/bfad.c +++ b/drivers/scsi/bfa/bfad.c @@ -749,6 +749,7 @@ bfad_pci_init(struct pci_dev *pdev, struct bfad_s *bfad) if (bfad->pci_bar0_kva == NULL) { printk(KERN_ERR "Fail to map bar0\n"); + rc = -ENODEV; goto out_release_region; } diff --git a/drivers/scsi/cxlflash/ocxl_hw.c b/drivers/scsi/cxlflash/ocxl_hw.c index 7018cd802569..e4e0d767b98e 100644 --- a/drivers/scsi/cxlflash/ocxl_hw.c +++ b/drivers/scsi/cxlflash/ocxl_hw.c @@ -15,7 +15,8 @@ #include <linux/pseudo_fs.h> #include <linux/poll.h> #include <linux/sched/signal.h> - +#include <linux/interrupt.h> +#include <asm/xive.h> #include <misc/ocxl.h> #include <uapi/misc/cxl.h> @@ -180,7 +181,7 @@ static int afu_map_irq(u64 flags, struct ocxlflash_context *ctx, int num, struct ocxl_hw_afu *afu = ctx->hw_afu; struct device *dev = afu->dev; struct ocxlflash_irqs *irq; - void __iomem *vtrig; + struct xive_irq_data *xd; u32 virq; int rc = 0; @@ -204,15 +205,15 @@ static int afu_map_irq(u64 flags, struct ocxlflash_context *ctx, int num, goto err1; } - vtrig = ioremap(irq->ptrig, PAGE_SIZE); - if (unlikely(!vtrig)) { - dev_err(dev, "%s: Trigger page mapping failed\n", __func__); - rc = -ENOMEM; + xd = irq_get_handler_data(virq); + if (unlikely(!xd)) { + dev_err(dev, "%s: Can't get interrupt data\n", __func__); + rc = -ENXIO; goto err2; } irq->virq = virq; - irq->vtrig = vtrig; + irq->vtrig = xd->trig_mmio; out: return rc; err2: @@ -259,8 +260,6 @@ static void afu_unmap_irq(u64 flags, struct ocxlflash_context *ctx, int num, } irq = &ctx->irqs[num]; - if (irq->vtrig) - iounmap(irq->vtrig); if (irq_find_mapping(NULL, irq->hwirq)) { free_irq(irq->virq, cookie); @@ -615,7 +614,6 @@ static int alloc_afu_irqs(struct ocxlflash_context *ctx, int num) struct ocxl_hw_afu *afu = ctx->hw_afu; struct device *dev = afu->dev; struct ocxlflash_irqs *irqs; - u64 addr; int rc = 0; int hwirq; int i; @@ -640,7 +638,7 @@ static int alloc_afu_irqs(struct ocxlflash_context *ctx, int num) } for (i = 0; i < num; i++) { - rc = ocxl_link_irq_alloc(afu->link_token, &hwirq, &addr); + rc = ocxl_link_irq_alloc(afu->link_token, &hwirq); if (unlikely(rc)) { dev_err(dev, "%s: ocxl_link_irq_alloc failed rc=%d\n", __func__, rc); @@ -648,7 +646,6 @@ static int alloc_afu_irqs(struct ocxlflash_context *ctx, int num) } irqs[i].hwirq = hwirq; - irqs[i].ptrig = addr; } ctx->irqs = irqs; diff --git a/drivers/scsi/cxlflash/ocxl_hw.h b/drivers/scsi/cxlflash/ocxl_hw.h index fc6ad4f985de..f2fe88816bea 100644 --- a/drivers/scsi/cxlflash/ocxl_hw.h +++ b/drivers/scsi/cxlflash/ocxl_hw.h @@ -13,7 +13,6 @@ struct ocxlflash_irqs { int hwirq; u32 virq; - u64 ptrig; void __iomem *vtrig; }; diff --git a/drivers/scsi/dc395x.c b/drivers/scsi/dc395x.c index 0c251a3b99b7..fa16894d8758 100644 --- a/drivers/scsi/dc395x.c +++ b/drivers/scsi/dc395x.c @@ -4721,30 +4721,7 @@ static struct pci_driver dc395x_driver = { .probe = dc395x_init_one, .remove = dc395x_remove_one, }; - - -/** - * dc395x_module_init - Module initialization function - * - * Used by both module and built-in driver to initialise this driver. - **/ -static int __init dc395x_module_init(void) -{ - return pci_register_driver(&dc395x_driver); -} - - -/** - * dc395x_module_exit - Module cleanup function. - **/ -static void __exit dc395x_module_exit(void) -{ - pci_unregister_driver(&dc395x_driver); -} - - -module_init(dc395x_module_init); -module_exit(dc395x_module_exit); +module_pci_driver(dc395x_driver); MODULE_AUTHOR("C.L. Huang / Erich Chen / Kurt Garloff"); MODULE_DESCRIPTION("SCSI host adapter driver for Tekram TRM-S1040 based adapters: Tekram DC395 and DC315 series"); diff --git a/drivers/scsi/fcoe/fcoe_sysfs.c b/drivers/scsi/fcoe/fcoe_sysfs.c index 2cb7a8c93a15..ffef2c8eddc6 100644 --- a/drivers/scsi/fcoe/fcoe_sysfs.c +++ b/drivers/scsi/fcoe/fcoe_sysfs.c @@ -1053,16 +1053,10 @@ EXPORT_SYMBOL_GPL(fcoe_fcf_device_add); int __init fcoe_sysfs_setup(void) { - int error; - atomic_set(&ctlr_num, 0); atomic_set(&fcf_num, 0); - error = bus_register(&fcoe_bus_type); - if (error) - return error; - - return 0; + return bus_register(&fcoe_bus_type); } void __exit fcoe_sysfs_teardown(void) diff --git a/drivers/scsi/fnic/vnic_wq_copy.c b/drivers/scsi/fnic/vnic_wq_copy.c index 9eab7e7caf38..7b18635df7e6 100644 --- a/drivers/scsi/fnic/vnic_wq_copy.c +++ b/drivers/scsi/fnic/vnic_wq_copy.c @@ -79,8 +79,6 @@ int vnic_wq_copy_alloc(struct vnic_dev *vdev, struct vnic_wq_copy *wq, unsigned int index, unsigned int desc_count, unsigned int desc_size) { - int err; - wq->index = index; wq->vdev = vdev; wq->to_use_index = wq->to_clean_index = 0; @@ -92,11 +90,7 @@ int vnic_wq_copy_alloc(struct vnic_dev *vdev, struct vnic_wq_copy *wq, vnic_wq_copy_disable(wq); - err = vnic_dev_alloc_desc_ring(vdev, &wq->ring, desc_count, desc_size); - if (err) - return err; - - return 0; + return vnic_dev_alloc_desc_ring(vdev, &wq->ring, desc_count, desc_size); } void vnic_wq_copy_init(struct vnic_wq_copy *wq, unsigned int cq_index, diff --git a/drivers/scsi/gdth.c b/drivers/scsi/gdth.c index dc0e17729acf..5d801388680b 100644 --- a/drivers/scsi/gdth.c +++ b/drivers/scsi/gdth.c @@ -3168,81 +3168,6 @@ static inline void gdth_timer_init(void) } #endif -static void __init internal_setup(char *str,int *ints) -{ - int i; - char *cur_str, *argv; - - TRACE2(("internal_setup() str %s ints[0] %d\n", - str ? str:"NULL", ints ? ints[0]:0)); - - /* analyse string */ - argv = str; - while (argv && (cur_str = strchr(argv, ':'))) { - int val = 0, c = *++cur_str; - - if (c == 'n' || c == 'N') - val = 0; - else if (c == 'y' || c == 'Y') - val = 1; - else - val = (int)simple_strtoul(cur_str, NULL, 0); - - if (!strncmp(argv, "disable:", 8)) - disable = val; - else if (!strncmp(argv, "reserve_mode:", 13)) - reserve_mode = val; - else if (!strncmp(argv, "reverse_scan:", 13)) - reverse_scan = val; - else if (!strncmp(argv, "hdr_channel:", 12)) - hdr_channel = val; - else if (!strncmp(argv, "max_ids:", 8)) - max_ids = val; - else if (!strncmp(argv, "rescan:", 7)) - rescan = val; - else if (!strncmp(argv, "shared_access:", 14)) - shared_access = val; - else if (!strncmp(argv, "reserve_list:", 13)) { - reserve_list[0] = val; - for (i = 1; i < MAX_RES_ARGS; i++) { - cur_str = strchr(cur_str, ','); - if (!cur_str) - break; - if (!isdigit((int)*++cur_str)) { - --cur_str; - break; - } - reserve_list[i] = - (int)simple_strtoul(cur_str, NULL, 0); - } - if (!cur_str) - break; - argv = ++cur_str; - continue; - } - - if ((argv = strchr(argv, ','))) - ++argv; - } -} - -int __init option_setup(char *str) -{ - int ints[MAXHA]; - char *cur = str; - int i = 1; - - TRACE2(("option_setup() str %s\n", str ? str:"NULL")); - - while (cur && isdigit(*cur) && i < MAXHA) { - ints[i++] = simple_strtoul(cur, NULL, 0); - if ((cur = strchr(cur, ',')) != NULL) cur++; - } - - ints[0] = i - 1; - internal_setup(cur, ints); - return 1; -} static const char *gdth_ctr_name(gdth_ha_str *ha) { @@ -4317,5 +4242,81 @@ module_init(gdth_init); module_exit(gdth_exit); #ifndef MODULE +static void __init internal_setup(char *str,int *ints) +{ + int i; + char *cur_str, *argv; + + TRACE2(("internal_setup() str %s ints[0] %d\n", + str ? str:"NULL", ints ? ints[0]:0)); + + /* analyse string */ + argv = str; + while (argv && (cur_str = strchr(argv, ':'))) { + int val = 0, c = *++cur_str; + + if (c == 'n' || c == 'N') + val = 0; + else if (c == 'y' || c == 'Y') + val = 1; + else + val = (int)simple_strtoul(cur_str, NULL, 0); + + if (!strncmp(argv, "disable:", 8)) + disable = val; + else if (!strncmp(argv, "reserve_mode:", 13)) + reserve_mode = val; + else if (!strncmp(argv, "reverse_scan:", 13)) + reverse_scan = val; + else if (!strncmp(argv, "hdr_channel:", 12)) + hdr_channel = val; + else if (!strncmp(argv, "max_ids:", 8)) + max_ids = val; + else if (!strncmp(argv, "rescan:", 7)) + rescan = val; + else if (!strncmp(argv, "shared_access:", 14)) + shared_access = val; + else if (!strncmp(argv, "reserve_list:", 13)) { + reserve_list[0] = val; + for (i = 1; i < MAX_RES_ARGS; i++) { + cur_str = strchr(cur_str, ','); + if (!cur_str) + break; + if (!isdigit((int)*++cur_str)) { + --cur_str; + break; + } + reserve_list[i] = + (int)simple_strtoul(cur_str, NULL, 0); + } + if (!cur_str) + break; + argv = ++cur_str; + continue; + } + + if ((argv = strchr(argv, ','))) + ++argv; + } +} + +static int __init option_setup(char *str) +{ + int ints[MAXHA]; + char *cur = str; + int i = 1; + + TRACE2(("option_setup() str %s\n", str ? str:"NULL")); + + while (cur && isdigit(*cur) && i < MAXHA) { + ints[i++] = simple_strtoul(cur, NULL, 0); + if ((cur = strchr(cur, ',')) != NULL) cur++; + } + + ints[0] = i - 1; + internal_setup(cur, ints); + return 1; +} + __setup("gdth=", option_setup); #endif diff --git a/drivers/scsi/initio.c b/drivers/scsi/initio.c index 1d39628ac947..ca16ef45d8dc 100644 --- a/drivers/scsi/initio.c +++ b/drivers/scsi/initio.c @@ -2962,20 +2962,8 @@ static struct pci_driver initio_pci_driver = { .probe = initio_probe_one, .remove = initio_remove_one, }; - -static int __init initio_init_driver(void) -{ - return pci_register_driver(&initio_pci_driver); -} - -static void __exit initio_exit_driver(void) -{ - pci_unregister_driver(&initio_pci_driver); -} +module_pci_driver(initio_pci_driver); MODULE_DESCRIPTION("Initio INI-9X00U/UW SCSI device driver"); MODULE_AUTHOR("Initio Corporation"); MODULE_LICENSE("GPL"); - -module_init(initio_init_driver); -module_exit(initio_exit_driver); diff --git a/drivers/scsi/isci/remote_node_table.h b/drivers/scsi/isci/remote_node_table.h index 721ab982d2ac..0ddfdda2b248 100644 --- a/drivers/scsi/isci/remote_node_table.h +++ b/drivers/scsi/isci/remote_node_table.h @@ -61,7 +61,7 @@ /** * * - * Remote node sets are sets of remote node index in the remtoe node table The + * Remote node sets are sets of remote node index in the remote node table. The * SCU hardware requires that STP remote node entries take three consecutive * remote node index so the table is arranged in sets of three. The bits are * used as 0111 0111 to make a byte and the bits define the set of three remote diff --git a/drivers/scsi/myrb.c b/drivers/scsi/myrb.c index cbf1e8b091b9..5fa0f4ed6565 100644 --- a/drivers/scsi/myrb.c +++ b/drivers/scsi/myrb.c @@ -1050,7 +1050,7 @@ static int myrb_get_hba_config(struct myrb_hba *cb) enquiry2->fw.turn_id = 0; } snprintf(cb->fw_version, sizeof(cb->fw_version), - "%d.%02d-%c-%02d", + "%u.%02u-%c-%02u", enquiry2->fw.major_version, enquiry2->fw.minor_version, enquiry2->fw.firmware_type, @@ -2167,7 +2167,7 @@ static ssize_t ctlr_num_show(struct device *dev, struct Scsi_Host *shost = class_to_shost(dev); struct myrb_hba *cb = shost_priv(shost); - return snprintf(buf, 20, "%d\n", cb->ctlr_num); + return snprintf(buf, 20, "%u\n", cb->ctlr_num); } static DEVICE_ATTR_RO(ctlr_num); @@ -2732,7 +2732,6 @@ static int DAC960_LA_hw_init(struct pci_dev *pdev, DAC960_LA_disable_intr(base); DAC960_LA_ack_hw_mbox_status(base); udelay(1000); - timeout = 0; while (DAC960_LA_init_in_progress(base) && timeout < MYRB_MAILBOX_TIMEOUT) { if (DAC960_LA_read_error_status(base, &error, diff --git a/drivers/scsi/pm8001/pm8001_ctl.c b/drivers/scsi/pm8001/pm8001_ctl.c index 77c805db2724..3587f7c8a428 100644 --- a/drivers/scsi/pm8001/pm8001_ctl.c +++ b/drivers/scsi/pm8001/pm8001_ctl.c @@ -408,9 +408,10 @@ static ssize_t pm8001_ctl_ib_queue_log_show(struct device *cdev, int offset; char *str = buf; int start = 0; + u32 ib_offset = pm8001_ha->ib_offset; #define IB_MEMMAP(c) \ (*(u32 *)((u8 *)pm8001_ha-> \ - memoryMap.region[IB].virt_ptr + \ + memoryMap.region[ib_offset].virt_ptr + \ pm8001_ha->evtlog_ib_offset + (c))) for (offset = 0; offset < IB_OB_READ_TIMES; offset++) { @@ -442,9 +443,10 @@ static ssize_t pm8001_ctl_ob_queue_log_show(struct device *cdev, int offset; char *str = buf; int start = 0; + u32 ob_offset = pm8001_ha->ob_offset; #define OB_MEMMAP(c) \ (*(u32 *)((u8 *)pm8001_ha-> \ - memoryMap.region[OB].virt_ptr + \ + memoryMap.region[ob_offset].virt_ptr + \ pm8001_ha->evtlog_ob_offset + (c))) for (offset = 0; offset < IB_OB_READ_TIMES; offset++) { diff --git a/drivers/scsi/pm8001/pm8001_defs.h b/drivers/scsi/pm8001/pm8001_defs.h index 1c7f15fd69ce..501b574239e8 100644 --- a/drivers/scsi/pm8001/pm8001_defs.h +++ b/drivers/scsi/pm8001/pm8001_defs.h @@ -75,12 +75,10 @@ enum port_type { }; /* driver compile-time configuration */ -#define PM8001_MAX_CCB 256 /* max ccbs supported */ +#define PM8001_MAX_CCB 1024 /* max ccbs supported */ #define PM8001_MPI_QUEUE 1024 /* maximum mpi queue entries */ -#define PM8001_MAX_INB_NUM 1 -#define PM8001_MAX_OUTB_NUM 1 -#define PM8001_MAX_SPCV_INB_NUM 1 -#define PM8001_MAX_SPCV_OUTB_NUM 4 +#define PM8001_MAX_INB_NUM 64 +#define PM8001_MAX_OUTB_NUM 64 #define PM8001_CAN_QUEUE 508 /* SCSI Queue depth */ /* Inbound/Outbound queue size */ @@ -92,26 +90,27 @@ enum port_type { #define PM8001_MAX_PORTS 16 /* max. possible ports */ #define PM8001_MAX_DEVICES 2048 /* max supported device */ #define PM8001_MAX_MSIX_VEC 64 /* max msi-x int for spcv/ve */ +#define PM8001_RESERVE_SLOT 8 -#define USI_MAX_MEMCNT_BASE 5 -#define IB (USI_MAX_MEMCNT_BASE + 1) -#define CI (IB + PM8001_MAX_SPCV_INB_NUM) -#define OB (CI + PM8001_MAX_SPCV_INB_NUM) -#define PI (OB + PM8001_MAX_SPCV_OUTB_NUM) -#define USI_MAX_MEMCNT (PI + PM8001_MAX_SPCV_OUTB_NUM) #define CONFIG_SCSI_PM8001_MAX_DMA_SG 528 #define PM8001_MAX_DMA_SG CONFIG_SCSI_PM8001_MAX_DMA_SG + enum memory_region_num { AAP1 = 0x0, /* application acceleration processor */ IOP, /* IO processor */ NVMD, /* NVM device */ - DEV_MEM, /* memory for devices */ - CCB_MEM, /* memory for command control block */ FW_FLASH, /* memory for fw flash update */ - FORENSIC_MEM /* memory for fw forensic data */ + FORENSIC_MEM, /* memory for fw forensic data */ + USI_MAX_MEMCNT_BASE }; #define PM8001_EVENT_LOG_SIZE (128 * 1024) +/** + * maximum DMA memory regions(number of IBQ + number of IBQ CI + * + number of OBQ + number of OBQ PI) + */ +#define USI_MAX_MEMCNT (USI_MAX_MEMCNT_BASE + ((2 * PM8001_MAX_INB_NUM) \ + + (2 * PM8001_MAX_OUTB_NUM))) /*error code*/ enum mpi_err { MPI_IO_STATUS_SUCCESS = 0x0, diff --git a/drivers/scsi/pm8001/pm8001_hwi.c b/drivers/scsi/pm8001/pm8001_hwi.c index e9a939230b15..2b7b2954ec31 100644 --- a/drivers/scsi/pm8001/pm8001_hwi.c +++ b/drivers/scsi/pm8001/pm8001_hwi.c @@ -189,6 +189,10 @@ static void init_default_table_values(struct pm8001_hba_info *pm8001_ha) u32 offsetib, offsetob; void __iomem *addressib = pm8001_ha->inbnd_q_tbl_addr; void __iomem *addressob = pm8001_ha->outbnd_q_tbl_addr; + u32 ib_offset = pm8001_ha->ib_offset; + u32 ob_offset = pm8001_ha->ob_offset; + u32 ci_offset = pm8001_ha->ci_offset; + u32 pi_offset = pm8001_ha->pi_offset; pm8001_ha->main_cfg_tbl.pm8001_tbl.inbound_q_nppd_hppd = 0; pm8001_ha->main_cfg_tbl.pm8001_tbl.outbound_hw_event_pid0_3 = 0; @@ -223,19 +227,19 @@ static void init_default_table_values(struct pm8001_hba_info *pm8001_ha) pm8001_ha->inbnd_q_tbl[i].element_pri_size_cnt = PM8001_MPI_QUEUE | (pm8001_ha->iomb_size << 16) | (0x00<<30); pm8001_ha->inbnd_q_tbl[i].upper_base_addr = - pm8001_ha->memoryMap.region[IB + i].phys_addr_hi; + pm8001_ha->memoryMap.region[ib_offset + i].phys_addr_hi; pm8001_ha->inbnd_q_tbl[i].lower_base_addr = - pm8001_ha->memoryMap.region[IB + i].phys_addr_lo; + pm8001_ha->memoryMap.region[ib_offset + i].phys_addr_lo; pm8001_ha->inbnd_q_tbl[i].base_virt = - (u8 *)pm8001_ha->memoryMap.region[IB + i].virt_ptr; + (u8 *)pm8001_ha->memoryMap.region[ib_offset + i].virt_ptr; pm8001_ha->inbnd_q_tbl[i].total_length = - pm8001_ha->memoryMap.region[IB + i].total_len; + pm8001_ha->memoryMap.region[ib_offset + i].total_len; pm8001_ha->inbnd_q_tbl[i].ci_upper_base_addr = - pm8001_ha->memoryMap.region[CI + i].phys_addr_hi; + pm8001_ha->memoryMap.region[ci_offset + i].phys_addr_hi; pm8001_ha->inbnd_q_tbl[i].ci_lower_base_addr = - pm8001_ha->memoryMap.region[CI + i].phys_addr_lo; + pm8001_ha->memoryMap.region[ci_offset + i].phys_addr_lo; pm8001_ha->inbnd_q_tbl[i].ci_virt = - pm8001_ha->memoryMap.region[CI + i].virt_ptr; + pm8001_ha->memoryMap.region[ci_offset + i].virt_ptr; offsetib = i * 0x20; pm8001_ha->inbnd_q_tbl[i].pi_pci_bar = get_pci_bar_index(pm8001_mr32(addressib, @@ -249,21 +253,21 @@ static void init_default_table_values(struct pm8001_hba_info *pm8001_ha) pm8001_ha->outbnd_q_tbl[i].element_size_cnt = PM8001_MPI_QUEUE | (pm8001_ha->iomb_size << 16) | (0x01<<30); pm8001_ha->outbnd_q_tbl[i].upper_base_addr = - pm8001_ha->memoryMap.region[OB + i].phys_addr_hi; + pm8001_ha->memoryMap.region[ob_offset + i].phys_addr_hi; pm8001_ha->outbnd_q_tbl[i].lower_base_addr = - pm8001_ha->memoryMap.region[OB + i].phys_addr_lo; + pm8001_ha->memoryMap.region[ob_offset + i].phys_addr_lo; pm8001_ha->outbnd_q_tbl[i].base_virt = - (u8 *)pm8001_ha->memoryMap.region[OB + i].virt_ptr; + (u8 *)pm8001_ha->memoryMap.region[ob_offset + i].virt_ptr; pm8001_ha->outbnd_q_tbl[i].total_length = - pm8001_ha->memoryMap.region[OB + i].total_len; + pm8001_ha->memoryMap.region[ob_offset + i].total_len; pm8001_ha->outbnd_q_tbl[i].pi_upper_base_addr = - pm8001_ha->memoryMap.region[PI + i].phys_addr_hi; + pm8001_ha->memoryMap.region[pi_offset + i].phys_addr_hi; pm8001_ha->outbnd_q_tbl[i].pi_lower_base_addr = - pm8001_ha->memoryMap.region[PI + i].phys_addr_lo; + pm8001_ha->memoryMap.region[pi_offset + i].phys_addr_lo; pm8001_ha->outbnd_q_tbl[i].interrup_vec_cnt_delay = 0 | (10 << 16) | (i << 24); pm8001_ha->outbnd_q_tbl[i].pi_virt = - pm8001_ha->memoryMap.region[PI + i].virt_ptr; + pm8001_ha->memoryMap.region[pi_offset + i].virt_ptr; offsetob = i * 0x24; pm8001_ha->outbnd_q_tbl[i].ci_pci_bar = get_pci_bar_index(pm8001_mr32(addressob, @@ -4371,8 +4375,7 @@ static int pm8001_chip_ssp_io_req(struct pm8001_hba_info *pm8001_ha, /* fill in PRD (scatter/gather) table, if any */ if (task->num_scatter > 1) { pm8001_chip_make_sg(task->scatter, ccb->n_elem, ccb->buf_prd); - phys_addr = ccb->ccb_dma_handle + - offsetof(struct pm8001_ccb_info, buf_prd[0]); + phys_addr = ccb->ccb_dma_handle; ssp_cmd.addr_low = cpu_to_le32(lower_32_bits(phys_addr)); ssp_cmd.addr_high = cpu_to_le32(upper_32_bits(phys_addr)); ssp_cmd.esgl = cpu_to_le32(1<<31); @@ -4445,8 +4448,7 @@ static int pm8001_chip_sata_req(struct pm8001_hba_info *pm8001_ha, /* fill in PRD (scatter/gather) table, if any */ if (task->num_scatter > 1) { pm8001_chip_make_sg(task->scatter, ccb->n_elem, ccb->buf_prd); - phys_addr = ccb->ccb_dma_handle + - offsetof(struct pm8001_ccb_info, buf_prd[0]); + phys_addr = ccb->ccb_dma_handle; sata_cmd.addr_low = lower_32_bits(phys_addr); sata_cmd.addr_high = upper_32_bits(phys_addr); sata_cmd.esgl = cpu_to_le32(1 << 31); diff --git a/drivers/scsi/pm8001/pm8001_init.c b/drivers/scsi/pm8001/pm8001_init.c index 20fa96cbc9d3..3cf3e58b6979 100644 --- a/drivers/scsi/pm8001/pm8001_init.c +++ b/drivers/scsi/pm8001/pm8001_init.c @@ -56,6 +56,7 @@ MODULE_PARM_DESC(link_rate, "Enable link rate.\n" " 8: Link rate 12.0G\n"); static struct scsi_transport_template *pm8001_stt; +static int pm8001_init_ccb_tag(struct pm8001_hba_info *, struct Scsi_Host *, struct pci_dev *); /* * chip info structure to identify chip key functionality as @@ -264,12 +265,36 @@ static u32 pm8001_request_irq(struct pm8001_hba_info *pm8001_ha); static int pm8001_alloc(struct pm8001_hba_info *pm8001_ha, const struct pci_device_id *ent) { - int i; + int i, count = 0, rc = 0; + u32 ci_offset, ib_offset, ob_offset, pi_offset; + struct inbound_queue_table *circularQ; + spin_lock_init(&pm8001_ha->lock); spin_lock_init(&pm8001_ha->bitmap_lock); PM8001_INIT_DBG(pm8001_ha, pm8001_printk("pm8001_alloc: PHY:%x\n", pm8001_ha->chip->n_phy)); + + /* Setup Interrupt */ + rc = pm8001_setup_irq(pm8001_ha); + if (rc) { + PM8001_FAIL_DBG(pm8001_ha, pm8001_printk( + "pm8001_setup_irq failed [ret: %d]\n", rc)); + goto err_out_shost; + } + /* Request Interrupt */ + rc = pm8001_request_irq(pm8001_ha); + if (rc) + goto err_out_shost; + + count = pm8001_ha->max_q_num; + /* Queues are chosen based on the number of cores/msix availability */ + ib_offset = pm8001_ha->ib_offset = USI_MAX_MEMCNT_BASE; + ci_offset = pm8001_ha->ci_offset = ib_offset + count; + ob_offset = pm8001_ha->ob_offset = ci_offset + count; + pi_offset = pm8001_ha->pi_offset = ob_offset + count; + pm8001_ha->max_memcnt = pi_offset + count; + for (i = 0; i < pm8001_ha->chip->n_phy; i++) { pm8001_phy_init(pm8001_ha, i); pm8001_ha->port[i].wide_port_phymap = 0; @@ -278,9 +303,6 @@ static int pm8001_alloc(struct pm8001_hba_info *pm8001_ha, INIT_LIST_HEAD(&pm8001_ha->port[i].list); } - pm8001_ha->tags = kzalloc(PM8001_MAX_CCB, GFP_KERNEL); - if (!pm8001_ha->tags) - goto err_out; /* MPI Memory region 1 for AAP Event Log for fw */ pm8001_ha->memoryMap.region[AAP1].num_elements = 1; pm8001_ha->memoryMap.region[AAP1].element_size = PM8001_EVENT_LOG_SIZE; @@ -293,54 +315,62 @@ static int pm8001_alloc(struct pm8001_hba_info *pm8001_ha, pm8001_ha->memoryMap.region[IOP].total_len = PM8001_EVENT_LOG_SIZE; pm8001_ha->memoryMap.region[IOP].alignment = 32; - for (i = 0; i < PM8001_MAX_SPCV_INB_NUM; i++) { + for (i = 0; i < count; i++) { + circularQ = &pm8001_ha->inbnd_q_tbl[i]; + spin_lock_init(&circularQ->iq_lock); /* MPI Memory region 3 for consumer Index of inbound queues */ - pm8001_ha->memoryMap.region[CI+i].num_elements = 1; - pm8001_ha->memoryMap.region[CI+i].element_size = 4; - pm8001_ha->memoryMap.region[CI+i].total_len = 4; - pm8001_ha->memoryMap.region[CI+i].alignment = 4; + pm8001_ha->memoryMap.region[ci_offset+i].num_elements = 1; + pm8001_ha->memoryMap.region[ci_offset+i].element_size = 4; + pm8001_ha->memoryMap.region[ci_offset+i].total_len = 4; + pm8001_ha->memoryMap.region[ci_offset+i].alignment = 4; if ((ent->driver_data) != chip_8001) { /* MPI Memory region 5 inbound queues */ - pm8001_ha->memoryMap.region[IB+i].num_elements = + pm8001_ha->memoryMap.region[ib_offset+i].num_elements = PM8001_MPI_QUEUE; - pm8001_ha->memoryMap.region[IB+i].element_size = 128; - pm8001_ha->memoryMap.region[IB+i].total_len = + pm8001_ha->memoryMap.region[ib_offset+i].element_size + = 128; + pm8001_ha->memoryMap.region[ib_offset+i].total_len = PM8001_MPI_QUEUE * 128; - pm8001_ha->memoryMap.region[IB+i].alignment = 128; + pm8001_ha->memoryMap.region[ib_offset+i].alignment + = 128; } else { - pm8001_ha->memoryMap.region[IB+i].num_elements = + pm8001_ha->memoryMap.region[ib_offset+i].num_elements = PM8001_MPI_QUEUE; - pm8001_ha->memoryMap.region[IB+i].element_size = 64; - pm8001_ha->memoryMap.region[IB+i].total_len = + pm8001_ha->memoryMap.region[ib_offset+i].element_size + = 64; + pm8001_ha->memoryMap.region[ib_offset+i].total_len = PM8001_MPI_QUEUE * 64; - pm8001_ha->memoryMap.region[IB+i].alignment = 64; + pm8001_ha->memoryMap.region[ib_offset+i].alignment = 64; } } - for (i = 0; i < PM8001_MAX_SPCV_OUTB_NUM; i++) { + for (i = 0; i < count; i++) { /* MPI Memory region 4 for producer Index of outbound queues */ - pm8001_ha->memoryMap.region[PI+i].num_elements = 1; - pm8001_ha->memoryMap.region[PI+i].element_size = 4; - pm8001_ha->memoryMap.region[PI+i].total_len = 4; - pm8001_ha->memoryMap.region[PI+i].alignment = 4; + pm8001_ha->memoryMap.region[pi_offset+i].num_elements = 1; + pm8001_ha->memoryMap.region[pi_offset+i].element_size = 4; + pm8001_ha->memoryMap.region[pi_offset+i].total_len = 4; + pm8001_ha->memoryMap.region[pi_offset+i].alignment = 4; if (ent->driver_data != chip_8001) { /* MPI Memory region 6 Outbound queues */ - pm8001_ha->memoryMap.region[OB+i].num_elements = + pm8001_ha->memoryMap.region[ob_offset+i].num_elements = PM8001_MPI_QUEUE; - pm8001_ha->memoryMap.region[OB+i].element_size = 128; - pm8001_ha->memoryMap.region[OB+i].total_len = + pm8001_ha->memoryMap.region[ob_offset+i].element_size + = 128; + pm8001_ha->memoryMap.region[ob_offset+i].total_len = PM8001_MPI_QUEUE * 128; - pm8001_ha->memoryMap.region[OB+i].alignment = 128; + pm8001_ha->memoryMap.region[ob_offset+i].alignment + = 128; } else { /* MPI Memory region 6 Outbound queues */ - pm8001_ha->memoryMap.region[OB+i].num_elements = + pm8001_ha->memoryMap.region[ob_offset+i].num_elements = PM8001_MPI_QUEUE; - pm8001_ha->memoryMap.region[OB+i].element_size = 64; - pm8001_ha->memoryMap.region[OB+i].total_len = + pm8001_ha->memoryMap.region[ob_offset+i].element_size + = 64; + pm8001_ha->memoryMap.region[ob_offset+i].total_len = PM8001_MPI_QUEUE * 64; - pm8001_ha->memoryMap.region[OB+i].alignment = 64; + pm8001_ha->memoryMap.region[ob_offset+i].alignment = 64; } } @@ -348,19 +378,6 @@ static int pm8001_alloc(struct pm8001_hba_info *pm8001_ha, pm8001_ha->memoryMap.region[NVMD].num_elements = 1; pm8001_ha->memoryMap.region[NVMD].element_size = 4096; pm8001_ha->memoryMap.region[NVMD].total_len = 4096; - /* Memory region for devices*/ - pm8001_ha->memoryMap.region[DEV_MEM].num_elements = 1; - pm8001_ha->memoryMap.region[DEV_MEM].element_size = PM8001_MAX_DEVICES * - sizeof(struct pm8001_device); - pm8001_ha->memoryMap.region[DEV_MEM].total_len = PM8001_MAX_DEVICES * - sizeof(struct pm8001_device); - - /* Memory region for ccb_info*/ - pm8001_ha->memoryMap.region[CCB_MEM].num_elements = 1; - pm8001_ha->memoryMap.region[CCB_MEM].element_size = PM8001_MAX_CCB * - sizeof(struct pm8001_ccb_info); - pm8001_ha->memoryMap.region[CCB_MEM].total_len = PM8001_MAX_CCB * - sizeof(struct pm8001_ccb_info); /* Memory region for fw flash */ pm8001_ha->memoryMap.region[FW_FLASH].total_len = 4096; @@ -369,7 +386,7 @@ static int pm8001_alloc(struct pm8001_hba_info *pm8001_ha, pm8001_ha->memoryMap.region[FORENSIC_MEM].total_len = 0x10000; pm8001_ha->memoryMap.region[FORENSIC_MEM].element_size = 0x10000; pm8001_ha->memoryMap.region[FORENSIC_MEM].alignment = 0x10000; - for (i = 0; i < USI_MAX_MEMCNT; i++) { + for (i = 0; i < pm8001_ha->max_memcnt; i++) { if (pm8001_mem_alloc(pm8001_ha->pdev, &pm8001_ha->memoryMap.region[i].virt_ptr, &pm8001_ha->memoryMap.region[i].phys_addr, @@ -384,27 +401,36 @@ static int pm8001_alloc(struct pm8001_hba_info *pm8001_ha, } } - pm8001_ha->devices = pm8001_ha->memoryMap.region[DEV_MEM].virt_ptr; + /* Memory region for devices*/ + pm8001_ha->devices = kzalloc(PM8001_MAX_DEVICES + * sizeof(struct pm8001_device), GFP_KERNEL); + if (!pm8001_ha->devices) { + rc = -ENOMEM; + goto err_out_nodev; + } for (i = 0; i < PM8001_MAX_DEVICES; i++) { pm8001_ha->devices[i].dev_type = SAS_PHY_UNUSED; pm8001_ha->devices[i].id = i; pm8001_ha->devices[i].device_id = PM8001_MAX_DEVICES; pm8001_ha->devices[i].running_req = 0; } - pm8001_ha->ccb_info = pm8001_ha->memoryMap.region[CCB_MEM].virt_ptr; - for (i = 0; i < PM8001_MAX_CCB; i++) { - pm8001_ha->ccb_info[i].ccb_dma_handle = - pm8001_ha->memoryMap.region[CCB_MEM].phys_addr + - i * sizeof(struct pm8001_ccb_info); - pm8001_ha->ccb_info[i].task = NULL; - pm8001_ha->ccb_info[i].ccb_tag = 0xffffffff; - pm8001_ha->ccb_info[i].device = NULL; - ++pm8001_ha->tags_num; - } pm8001_ha->flags = PM8001F_INIT_TIME; /* Initialize tags */ pm8001_tag_init(pm8001_ha); return 0; + +err_out_shost: + scsi_remove_host(pm8001_ha->shost); +err_out_nodev: + for (i = 0; i < pm8001_ha->max_memcnt; i++) { + if (pm8001_ha->memoryMap.region[i].virt_ptr != NULL) { + pci_free_consistent(pm8001_ha->pdev, + (pm8001_ha->memoryMap.region[i].total_len + + pm8001_ha->memoryMap.region[i].alignment), + pm8001_ha->memoryMap.region[i].virt_ptr, + pm8001_ha->memoryMap.region[i].phys_addr); + } + } err_out: return 1; } @@ -899,7 +925,8 @@ static int pm8001_configure_phy_settings(struct pm8001_hba_info *pm8001_ha) static u32 pm8001_setup_msix(struct pm8001_hba_info *pm8001_ha) { u32 number_of_intr; - int rc; + int rc, cpu_online_count; + unsigned int allocated_irq_vectors; /* SPCv controllers supports 64 msi-x */ if (pm8001_ha->chip_id == chip_8001) { @@ -908,13 +935,21 @@ static u32 pm8001_setup_msix(struct pm8001_hba_info *pm8001_ha) number_of_intr = PM8001_MAX_MSIX_VEC; } + cpu_online_count = num_online_cpus(); + number_of_intr = min_t(int, cpu_online_count, number_of_intr); rc = pci_alloc_irq_vectors(pm8001_ha->pdev, number_of_intr, number_of_intr, PCI_IRQ_MSIX); - number_of_intr = rc; + allocated_irq_vectors = rc; if (rc < 0) return rc; + + /* Assigns the number of interrupts */ + number_of_intr = min_t(int, allocated_irq_vectors, number_of_intr); pm8001_ha->number_of_intr = number_of_intr; + /* Maximum queue number updating in HBA structure */ + pm8001_ha->max_q_num = number_of_intr; + PM8001_INIT_DBG(pm8001_ha, pm8001_printk( "pci_alloc_irq_vectors request ret:%d no of intr %d\n", rc, pm8001_ha->number_of_intr)); @@ -1069,13 +1104,6 @@ static int pm8001_pci_probe(struct pci_dev *pdev, rc = -ENOMEM; goto err_out_free; } - /* Setup Interrupt */ - rc = pm8001_setup_irq(pm8001_ha); - if (rc) { - PM8001_FAIL_DBG(pm8001_ha, pm8001_printk( - "pm8001_setup_irq failed [ret: %d]\n", rc)); - goto err_out_shost; - } PM8001_CHIP_DISP->chip_soft_rst(pm8001_ha); rc = PM8001_CHIP_DISP->chip_init(pm8001_ha); @@ -1085,16 +1113,13 @@ static int pm8001_pci_probe(struct pci_dev *pdev, goto err_out_ha_free; } + rc = pm8001_init_ccb_tag(pm8001_ha, shost, pdev); + if (rc) + goto err_out_enable; + rc = scsi_add_host(shost, &pdev->dev); if (rc) goto err_out_ha_free; - /* Request Interrupt */ - rc = pm8001_request_irq(pm8001_ha); - if (rc) { - PM8001_FAIL_DBG(pm8001_ha, pm8001_printk( - "pm8001_request_irq failed [ret: %d]\n", rc)); - goto err_out_shost; - } PM8001_CHIP_DISP->interrupt_enable(pm8001_ha, 0); if (pm8001_ha->chip_id != chip_8001) { @@ -1137,6 +1162,60 @@ err_out_enable: return rc; } +/* + * pm8001_init_ccb_tag - allocate memory to CCB and tag. + * @pm8001_ha: our hba card information. + * @shost: scsi host which has been allocated outside. + */ +static int +pm8001_init_ccb_tag(struct pm8001_hba_info *pm8001_ha, struct Scsi_Host *shost, + struct pci_dev *pdev) +{ + int i = 0; + u32 max_out_io, ccb_count; + u32 can_queue; + + max_out_io = pm8001_ha->main_cfg_tbl.pm80xx_tbl.max_out_io; + ccb_count = min_t(int, PM8001_MAX_CCB, max_out_io); + + /* Update to the scsi host*/ + can_queue = ccb_count - PM8001_RESERVE_SLOT; + shost->can_queue = can_queue; + + pm8001_ha->tags = kzalloc(ccb_count, GFP_KERNEL); + if (!pm8001_ha->tags) + goto err_out; + + /* Memory region for ccb_info*/ + pm8001_ha->ccb_info = (struct pm8001_ccb_info *) + kcalloc(ccb_count, sizeof(struct pm8001_ccb_info), GFP_KERNEL); + if (!pm8001_ha->ccb_info) { + PM8001_FAIL_DBG(pm8001_ha, pm8001_printk + ("Unable to allocate memory for ccb\n")); + goto err_out_noccb; + } + for (i = 0; i < ccb_count; i++) { + pm8001_ha->ccb_info[i].buf_prd = pci_alloc_consistent(pdev, + sizeof(struct pm8001_prd) * PM8001_MAX_DMA_SG, + &pm8001_ha->ccb_info[i].ccb_dma_handle); + if (!pm8001_ha->ccb_info[i].buf_prd) { + PM8001_FAIL_DBG(pm8001_ha, pm8001_printk + ("pm80xx: ccb prd memory allocation error\n")); + goto err_out; + } + pm8001_ha->ccb_info[i].task = NULL; + pm8001_ha->ccb_info[i].ccb_tag = 0xffffffff; + pm8001_ha->ccb_info[i].device = NULL; + ++pm8001_ha->tags_num; + } + return 0; + +err_out_noccb: + kfree(pm8001_ha->devices); +err_out: + return -ENOMEM; +} + static void pm8001_pci_remove(struct pci_dev *pdev) { struct sas_ha_struct *sha = pci_get_drvdata(pdev); diff --git a/drivers/scsi/pm8001/pm8001_sas.h b/drivers/scsi/pm8001/pm8001_sas.h index ae7ba9b3c4bc..95663e138083 100644 --- a/drivers/scsi/pm8001/pm8001_sas.h +++ b/drivers/scsi/pm8001/pm8001_sas.h @@ -58,7 +58,7 @@ #include "pm8001_defs.h" #define DRV_NAME "pm80xx" -#define DRV_VERSION "0.1.39" +#define DRV_VERSION "0.1.40" #define PM8001_FAIL_LOGGING 0x01 /* Error message logging */ #define PM8001_INIT_LOGGING 0x02 /* driver init logging */ #define PM8001_DISC_LOGGING 0x04 /* discovery layer logging */ @@ -315,7 +315,7 @@ struct pm8001_ccb_info { u32 ccb_tag; dma_addr_t ccb_dma_handle; struct pm8001_device *device; - struct pm8001_prd buf_prd[PM8001_MAX_DMA_SG]; + struct pm8001_prd *buf_prd; struct fw_control_ex *fw_control_context; u8 open_retry; }; @@ -468,6 +468,7 @@ struct inbound_queue_table { u32 reserved; __le32 consumer_index; u32 producer_idx; + spinlock_t iq_lock; }; struct outbound_queue_table { u32 element_size_cnt; @@ -524,8 +525,8 @@ struct pm8001_hba_info { void __iomem *fatal_tbl_addr; /*MPI IVT Table Addr */ union main_cfg_table main_cfg_tbl; union general_status_table gs_tbl; - struct inbound_queue_table inbnd_q_tbl[PM8001_MAX_SPCV_INB_NUM]; - struct outbound_queue_table outbnd_q_tbl[PM8001_MAX_SPCV_OUTB_NUM]; + struct inbound_queue_table inbnd_q_tbl[PM8001_MAX_INB_NUM]; + struct outbound_queue_table outbnd_q_tbl[PM8001_MAX_OUTB_NUM]; struct sas_phy_attribute_table phy_attr_table; /* MPI SAS PHY attributes */ u8 sas_addr[SAS_ADDR_SIZE]; @@ -561,6 +562,12 @@ struct pm8001_hba_info { u32 reset_in_progress; u32 non_fatal_count; u32 non_fatal_read_length; + u32 max_q_num; + u32 ib_offset; + u32 ob_offset; + u32 ci_offset; + u32 pi_offset; + u32 max_memcnt; }; struct pm8001_work { diff --git a/drivers/scsi/pm8001/pm80xx_hwi.c b/drivers/scsi/pm8001/pm80xx_hwi.c index b42f41d1ed49..7593f248afb2 100644 --- a/drivers/scsi/pm8001/pm80xx_hwi.c +++ b/drivers/scsi/pm8001/pm80xx_hwi.c @@ -720,7 +720,7 @@ static void read_inbnd_queue_table(struct pm8001_hba_info *pm8001_ha) { int i; void __iomem *address = pm8001_ha->inbnd_q_tbl_addr; - for (i = 0; i < PM8001_MAX_SPCV_INB_NUM; i++) { + for (i = 0; i < PM8001_MAX_INB_NUM; i++) { u32 offset = i * 0x20; pm8001_ha->inbnd_q_tbl[i].pi_pci_bar = get_pci_bar_index(pm8001_mr32(address, @@ -738,7 +738,7 @@ static void read_outbnd_queue_table(struct pm8001_hba_info *pm8001_ha) { int i; void __iomem *address = pm8001_ha->outbnd_q_tbl_addr; - for (i = 0; i < PM8001_MAX_SPCV_OUTB_NUM; i++) { + for (i = 0; i < PM8001_MAX_OUTB_NUM; i++) { u32 offset = i * 0x24; pm8001_ha->outbnd_q_tbl[i].ci_pci_bar = get_pci_bar_index(pm8001_mr32(address, @@ -758,6 +758,10 @@ static void init_default_table_values(struct pm8001_hba_info *pm8001_ha) u32 offsetib, offsetob; void __iomem *addressib = pm8001_ha->inbnd_q_tbl_addr; void __iomem *addressob = pm8001_ha->outbnd_q_tbl_addr; + u32 ib_offset = pm8001_ha->ib_offset; + u32 ob_offset = pm8001_ha->ob_offset; + u32 ci_offset = pm8001_ha->ci_offset; + u32 pi_offset = pm8001_ha->pi_offset; pm8001_ha->main_cfg_tbl.pm80xx_tbl.upper_event_log_addr = pm8001_ha->memoryMap.region[AAP1].phys_addr_hi; @@ -778,23 +782,23 @@ static void init_default_table_values(struct pm8001_hba_info *pm8001_ha) /* Disable end to end CRC checking */ pm8001_ha->main_cfg_tbl.pm80xx_tbl.crc_core_dump = (0x1 << 16); - for (i = 0; i < PM8001_MAX_SPCV_INB_NUM; i++) { + for (i = 0; i < pm8001_ha->max_q_num; i++) { pm8001_ha->inbnd_q_tbl[i].element_pri_size_cnt = PM8001_MPI_QUEUE | (pm8001_ha->iomb_size << 16) | (0x00<<30); pm8001_ha->inbnd_q_tbl[i].upper_base_addr = - pm8001_ha->memoryMap.region[IB + i].phys_addr_hi; + pm8001_ha->memoryMap.region[ib_offset + i].phys_addr_hi; pm8001_ha->inbnd_q_tbl[i].lower_base_addr = - pm8001_ha->memoryMap.region[IB + i].phys_addr_lo; + pm8001_ha->memoryMap.region[ib_offset + i].phys_addr_lo; pm8001_ha->inbnd_q_tbl[i].base_virt = - (u8 *)pm8001_ha->memoryMap.region[IB + i].virt_ptr; + (u8 *)pm8001_ha->memoryMap.region[ib_offset + i].virt_ptr; pm8001_ha->inbnd_q_tbl[i].total_length = - pm8001_ha->memoryMap.region[IB + i].total_len; + pm8001_ha->memoryMap.region[ib_offset + i].total_len; pm8001_ha->inbnd_q_tbl[i].ci_upper_base_addr = - pm8001_ha->memoryMap.region[CI + i].phys_addr_hi; + pm8001_ha->memoryMap.region[ci_offset + i].phys_addr_hi; pm8001_ha->inbnd_q_tbl[i].ci_lower_base_addr = - pm8001_ha->memoryMap.region[CI + i].phys_addr_lo; + pm8001_ha->memoryMap.region[ci_offset + i].phys_addr_lo; pm8001_ha->inbnd_q_tbl[i].ci_virt = - pm8001_ha->memoryMap.region[CI + i].virt_ptr; + pm8001_ha->memoryMap.region[ci_offset + i].virt_ptr; offsetib = i * 0x20; pm8001_ha->inbnd_q_tbl[i].pi_pci_bar = get_pci_bar_index(pm8001_mr32(addressib, @@ -809,25 +813,25 @@ static void init_default_table_values(struct pm8001_hba_info *pm8001_ha) pm8001_ha->inbnd_q_tbl[i].pi_pci_bar, pm8001_ha->inbnd_q_tbl[i].pi_offset)); } - for (i = 0; i < PM8001_MAX_SPCV_OUTB_NUM; i++) { + for (i = 0; i < pm8001_ha->max_q_num; i++) { pm8001_ha->outbnd_q_tbl[i].element_size_cnt = PM8001_MPI_QUEUE | (pm8001_ha->iomb_size << 16) | (0x01<<30); pm8001_ha->outbnd_q_tbl[i].upper_base_addr = - pm8001_ha->memoryMap.region[OB + i].phys_addr_hi; + pm8001_ha->memoryMap.region[ob_offset + i].phys_addr_hi; pm8001_ha->outbnd_q_tbl[i].lower_base_addr = - pm8001_ha->memoryMap.region[OB + i].phys_addr_lo; + pm8001_ha->memoryMap.region[ob_offset + i].phys_addr_lo; pm8001_ha->outbnd_q_tbl[i].base_virt = - (u8 *)pm8001_ha->memoryMap.region[OB + i].virt_ptr; + (u8 *)pm8001_ha->memoryMap.region[ob_offset + i].virt_ptr; pm8001_ha->outbnd_q_tbl[i].total_length = - pm8001_ha->memoryMap.region[OB + i].total_len; + pm8001_ha->memoryMap.region[ob_offset + i].total_len; pm8001_ha->outbnd_q_tbl[i].pi_upper_base_addr = - pm8001_ha->memoryMap.region[PI + i].phys_addr_hi; + pm8001_ha->memoryMap.region[pi_offset + i].phys_addr_hi; pm8001_ha->outbnd_q_tbl[i].pi_lower_base_addr = - pm8001_ha->memoryMap.region[PI + i].phys_addr_lo; + pm8001_ha->memoryMap.region[pi_offset + i].phys_addr_lo; /* interrupt vector based on oq */ pm8001_ha->outbnd_q_tbl[i].interrup_vec_cnt_delay = (i << 24); pm8001_ha->outbnd_q_tbl[i].pi_virt = - pm8001_ha->memoryMap.region[PI + i].virt_ptr; + pm8001_ha->memoryMap.region[pi_offset + i].virt_ptr; offsetob = i * 0x24; pm8001_ha->outbnd_q_tbl[i].ci_pci_bar = get_pci_bar_index(pm8001_mr32(addressob, @@ -871,7 +875,7 @@ static void update_main_config_table(struct pm8001_hba_info *pm8001_ha) pm8001_ha->main_cfg_tbl.pm80xx_tbl.pcs_event_log_severity); /* Update Fatal error interrupt vector */ pm8001_ha->main_cfg_tbl.pm80xx_tbl.fatal_err_interrupt |= - ((pm8001_ha->number_of_intr - 1) << 8); + ((pm8001_ha->max_q_num - 1) << 8); pm8001_mw32(address, MAIN_FATAL_ERROR_INTERRUPT, pm8001_ha->main_cfg_tbl.pm80xx_tbl.fatal_err_interrupt); PM8001_DEV_DBG(pm8001_ha, pm8001_printk( @@ -1010,8 +1014,12 @@ static int mpi_init_check(struct pm8001_hba_info *pm8001_ha) value &= SPCv_MSGU_CFG_TABLE_UPDATE; } while ((value != 0) && (--max_wait_count)); - if (!max_wait_count) - return -1; + if (!max_wait_count) { + /* additional check */ + PM8001_FAIL_DBG(pm8001_ha, pm8001_printk( + "Inb doorbell clear not toggled[value:%x]\n", value)); + return -EBUSY; + } /* check the MPI-State for initialization upto 100ms*/ max_wait_count = 100 * 1000;/* 100 msec */ do { @@ -1022,12 +1030,12 @@ static int mpi_init_check(struct pm8001_hba_info *pm8001_ha) } while ((GST_MPI_STATE_INIT != (gst_len_mpistate & GST_MPI_STATE_MASK)) && (--max_wait_count)); if (!max_wait_count) - return -1; + return -EBUSY; /* check MPI Initialization error */ gst_len_mpistate = gst_len_mpistate >> 16; if (0x0000 != gst_len_mpistate) - return -1; + return -EBUSY; return 0; } @@ -1469,11 +1477,10 @@ static int pm80xx_chip_init(struct pm8001_hba_info *pm8001_ha) /* update main config table ,inbound table and outbound table */ update_main_config_table(pm8001_ha); - for (i = 0; i < PM8001_MAX_SPCV_INB_NUM; i++) + for (i = 0; i < pm8001_ha->max_q_num; i++) { update_inbnd_queue_table(pm8001_ha, i); - for (i = 0; i < PM8001_MAX_SPCV_OUTB_NUM; i++) update_outbnd_queue_table(pm8001_ha, i); - + } /* notify firmware update finished and check initialization status */ if (0 == mpi_init_check(pm8001_ha)) { PM8001_INIT_DBG(pm8001_ha, @@ -4191,7 +4198,7 @@ static int process_oq(struct pm8001_hba_info *pm8001_ha, u8 vec) unsigned long flags; u32 regval; - if (vec == (pm8001_ha->number_of_intr - 1)) { + if (vec == (pm8001_ha->max_q_num - 1)) { regval = pm8001_cr32(pm8001_ha, 0, MSGU_SCRATCH_PAD_1); if ((regval & SCRATCH_PAD_MIPSALL_READY) != SCRATCH_PAD_MIPSALL_READY) { @@ -4274,6 +4281,7 @@ static int pm80xx_chip_smp_req(struct pm8001_hba_info *pm8001_ha, char *preq_dma_addr = NULL; __le64 tmp_addr; u32 i, length; + unsigned long flags; memset(&smp_cmd, 0, sizeof(smp_cmd)); /* @@ -4369,8 +4377,10 @@ static int pm80xx_chip_smp_req(struct pm8001_hba_info *pm8001_ha, build_smp_cmd(pm8001_dev->device_id, smp_cmd.tag, &smp_cmd, pm8001_ha->smp_exp_mode, length); + spin_lock_irqsave(&circularQ->iq_lock, flags); rc = pm8001_mpi_build_cmd(pm8001_ha, circularQ, opc, &smp_cmd, sizeof(smp_cmd), 0); + spin_unlock_irqrestore(&circularQ->iq_lock, flags); if (rc) goto err_out_2; return 0; @@ -4434,7 +4444,8 @@ static int pm80xx_chip_ssp_io_req(struct pm8001_hba_info *pm8001_ha, u64 phys_addr, start_addr, end_addr; u32 end_addr_high, end_addr_low; struct inbound_queue_table *circularQ; - u32 q_index; + unsigned long flags; + u32 q_index, cpu_id; u32 opc = OPC_INB_SSPINIIOSTART; memset(&ssp_cmd, 0, sizeof(ssp_cmd)); memcpy(ssp_cmd.ssp_iu.lun, task->ssp_task.LUN, 8); @@ -4453,7 +4464,8 @@ static int pm80xx_chip_ssp_io_req(struct pm8001_hba_info *pm8001_ha, ssp_cmd.ssp_iu.efb_prio_attr |= (task->ssp_task.task_attr & 7); memcpy(ssp_cmd.ssp_iu.cdb, task->ssp_task.cmd->cmnd, task->ssp_task.cmd->cmd_len); - q_index = (u32) (pm8001_dev->id & 0x00ffffff) % PM8001_MAX_INB_NUM; + cpu_id = smp_processor_id(); + q_index = (u32) (cpu_id) % (pm8001_ha->max_q_num); circularQ = &pm8001_ha->inbnd_q_tbl[q_index]; /* Check if encryption is set */ @@ -4471,8 +4483,7 @@ static int pm80xx_chip_ssp_io_req(struct pm8001_hba_info *pm8001_ha, if (task->num_scatter > 1) { pm8001_chip_make_sg(task->scatter, ccb->n_elem, ccb->buf_prd); - phys_addr = ccb->ccb_dma_handle + - offsetof(struct pm8001_ccb_info, buf_prd[0]); + phys_addr = ccb->ccb_dma_handle; ssp_cmd.enc_addr_low = cpu_to_le32(lower_32_bits(phys_addr)); ssp_cmd.enc_addr_high = @@ -4501,9 +4512,7 @@ static int pm80xx_chip_ssp_io_req(struct pm8001_hba_info *pm8001_ha, end_addr_high, end_addr_low)); pm8001_chip_make_sg(task->scatter, 1, ccb->buf_prd); - phys_addr = ccb->ccb_dma_handle + - offsetof(struct pm8001_ccb_info, - buf_prd[0]); + phys_addr = ccb->ccb_dma_handle; ssp_cmd.enc_addr_low = cpu_to_le32(lower_32_bits(phys_addr)); ssp_cmd.enc_addr_high = @@ -4531,8 +4540,7 @@ static int pm80xx_chip_ssp_io_req(struct pm8001_hba_info *pm8001_ha, if (task->num_scatter > 1) { pm8001_chip_make_sg(task->scatter, ccb->n_elem, ccb->buf_prd); - phys_addr = ccb->ccb_dma_handle + - offsetof(struct pm8001_ccb_info, buf_prd[0]); + phys_addr = ccb->ccb_dma_handle; ssp_cmd.addr_low = cpu_to_le32(lower_32_bits(phys_addr)); ssp_cmd.addr_high = @@ -4560,9 +4568,7 @@ static int pm80xx_chip_ssp_io_req(struct pm8001_hba_info *pm8001_ha, end_addr_high, end_addr_low)); pm8001_chip_make_sg(task->scatter, 1, ccb->buf_prd); - phys_addr = ccb->ccb_dma_handle + - offsetof(struct pm8001_ccb_info, - buf_prd[0]); + phys_addr = ccb->ccb_dma_handle; ssp_cmd.addr_low = cpu_to_le32(lower_32_bits(phys_addr)); ssp_cmd.addr_high = @@ -4576,9 +4582,10 @@ static int pm80xx_chip_ssp_io_req(struct pm8001_hba_info *pm8001_ha, ssp_cmd.esgl = 0; } } - q_index = (u32) (pm8001_dev->id & 0x00ffffff) % PM8001_MAX_OUTB_NUM; + spin_lock_irqsave(&circularQ->iq_lock, flags); ret = pm8001_mpi_build_cmd(pm8001_ha, circularQ, opc, &ssp_cmd, sizeof(ssp_cmd), q_index); + spin_unlock_irqrestore(&circularQ->iq_lock, flags); return ret; } @@ -4590,7 +4597,7 @@ static int pm80xx_chip_sata_req(struct pm8001_hba_info *pm8001_ha, struct pm8001_device *pm8001_ha_dev = dev->lldd_dev; u32 tag = ccb->ccb_tag; int ret; - u32 q_index; + u32 q_index, cpu_id; struct sata_start_req sata_cmd; u32 hdr_tag, ncg_tag = 0; u64 phys_addr, start_addr, end_addr; @@ -4601,7 +4608,8 @@ static int pm80xx_chip_sata_req(struct pm8001_hba_info *pm8001_ha, unsigned long flags; u32 opc = OPC_INB_SATA_HOST_OPSTART; memset(&sata_cmd, 0, sizeof(sata_cmd)); - q_index = (u32) (pm8001_ha_dev->id & 0x00ffffff) % PM8001_MAX_INB_NUM; + cpu_id = smp_processor_id(); + q_index = (u32) (cpu_id) % (pm8001_ha->max_q_num); circularQ = &pm8001_ha->inbnd_q_tbl[q_index]; if (task->data_dir == DMA_NONE) { @@ -4652,8 +4660,7 @@ static int pm80xx_chip_sata_req(struct pm8001_hba_info *pm8001_ha, if (task->num_scatter > 1) { pm8001_chip_make_sg(task->scatter, ccb->n_elem, ccb->buf_prd); - phys_addr = ccb->ccb_dma_handle + - offsetof(struct pm8001_ccb_info, buf_prd[0]); + phys_addr = ccb->ccb_dma_handle; sata_cmd.enc_addr_low = lower_32_bits(phys_addr); sata_cmd.enc_addr_high = upper_32_bits(phys_addr); sata_cmd.enc_esgl = cpu_to_le32(1 << 31); @@ -4678,9 +4685,7 @@ static int pm80xx_chip_sata_req(struct pm8001_hba_info *pm8001_ha, end_addr_high, end_addr_low)); pm8001_chip_make_sg(task->scatter, 1, ccb->buf_prd); - phys_addr = ccb->ccb_dma_handle + - offsetof(struct pm8001_ccb_info, - buf_prd[0]); + phys_addr = ccb->ccb_dma_handle; sata_cmd.enc_addr_low = lower_32_bits(phys_addr); sata_cmd.enc_addr_high = @@ -4718,8 +4723,7 @@ static int pm80xx_chip_sata_req(struct pm8001_hba_info *pm8001_ha, if (task->num_scatter > 1) { pm8001_chip_make_sg(task->scatter, ccb->n_elem, ccb->buf_prd); - phys_addr = ccb->ccb_dma_handle + - offsetof(struct pm8001_ccb_info, buf_prd[0]); + phys_addr = ccb->ccb_dma_handle; sata_cmd.addr_low = lower_32_bits(phys_addr); sata_cmd.addr_high = upper_32_bits(phys_addr); sata_cmd.esgl = cpu_to_le32(1 << 31); @@ -4744,9 +4748,7 @@ static int pm80xx_chip_sata_req(struct pm8001_hba_info *pm8001_ha, end_addr_high, end_addr_low)); pm8001_chip_make_sg(task->scatter, 1, ccb->buf_prd); - phys_addr = ccb->ccb_dma_handle + - offsetof(struct pm8001_ccb_info, - buf_prd[0]); + phys_addr = ccb->ccb_dma_handle; sata_cmd.addr_low = lower_32_bits(phys_addr); sata_cmd.addr_high = @@ -4817,9 +4819,10 @@ static int pm80xx_chip_sata_req(struct pm8001_hba_info *pm8001_ha, } } } - q_index = (u32) (pm8001_ha_dev->id & 0x00ffffff) % PM8001_MAX_OUTB_NUM; + spin_lock_irqsave(&circularQ->iq_lock, flags); ret = pm8001_mpi_build_cmd(pm8001_ha, circularQ, opc, &sata_cmd, sizeof(sata_cmd), q_index); + spin_unlock_irqrestore(&circularQ->iq_lock, flags); return ret; } diff --git a/drivers/scsi/qla2xxx/qla_dfs.c b/drivers/scsi/qla2xxx/qla_dfs.c index f89ad3274412..d5ebcf7d70ff 100644 --- a/drivers/scsi/qla2xxx/qla_dfs.c +++ b/drivers/scsi/qla2xxx/qla_dfs.c @@ -170,20 +170,7 @@ qla2x00_dfs_tgt_sess_show(struct seq_file *s, void *unused) return 0; } -static int -qla2x00_dfs_tgt_sess_open(struct inode *inode, struct file *file) -{ - scsi_qla_host_t *vha = inode->i_private; - - return single_open(file, qla2x00_dfs_tgt_sess_show, vha); -} - -static const struct file_operations dfs_tgt_sess_ops = { - .open = qla2x00_dfs_tgt_sess_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(qla2x00_dfs_tgt_sess); static int qla2x00_dfs_tgt_port_database_show(struct seq_file *s, void *unused) @@ -239,20 +226,7 @@ out_free_id_list: return 0; } -static int -qla2x00_dfs_tgt_port_database_open(struct inode *inode, struct file *file) -{ - scsi_qla_host_t *vha = inode->i_private; - - return single_open(file, qla2x00_dfs_tgt_port_database_show, vha); -} - -static const struct file_operations dfs_tgt_port_database_ops = { - .open = qla2x00_dfs_tgt_port_database_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(qla2x00_dfs_tgt_port_database); static int qla_dfs_fw_resource_cnt_show(struct seq_file *s, void *unused) @@ -301,20 +275,7 @@ qla_dfs_fw_resource_cnt_show(struct seq_file *s, void *unused) return 0; } -static int -qla_dfs_fw_resource_cnt_open(struct inode *inode, struct file *file) -{ - struct scsi_qla_host *vha = inode->i_private; - - return single_open(file, qla_dfs_fw_resource_cnt_show, vha); -} - -static const struct file_operations dfs_fw_resource_cnt_ops = { - .open = qla_dfs_fw_resource_cnt_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(qla_dfs_fw_resource_cnt); static int qla_dfs_tgt_counters_show(struct seq_file *s, void *unused) @@ -391,20 +352,7 @@ qla_dfs_tgt_counters_show(struct seq_file *s, void *unused) return 0; } -static int -qla_dfs_tgt_counters_open(struct inode *inode, struct file *file) -{ - struct scsi_qla_host *vha = inode->i_private; - - return single_open(file, qla_dfs_tgt_counters_show, vha); -} - -static const struct file_operations dfs_tgt_counters_ops = { - .open = qla_dfs_tgt_counters_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(qla_dfs_tgt_counters); static int qla2x00_dfs_fce_show(struct seq_file *s, void *unused) @@ -606,19 +554,19 @@ create_dir: create_nodes: ha->dfs_fw_resource_cnt = debugfs_create_file("fw_resource_count", - S_IRUSR, ha->dfs_dir, vha, &dfs_fw_resource_cnt_ops); + S_IRUSR, ha->dfs_dir, vha, &qla_dfs_fw_resource_cnt_fops); ha->dfs_tgt_counters = debugfs_create_file("tgt_counters", S_IRUSR, - ha->dfs_dir, vha, &dfs_tgt_counters_ops); + ha->dfs_dir, vha, &qla_dfs_tgt_counters_fops); ha->tgt.dfs_tgt_port_database = debugfs_create_file("tgt_port_database", - S_IRUSR, ha->dfs_dir, vha, &dfs_tgt_port_database_ops); + S_IRUSR, ha->dfs_dir, vha, &qla2x00_dfs_tgt_port_database_fops); ha->dfs_fce = debugfs_create_file("fce", S_IRUSR, ha->dfs_dir, vha, &dfs_fce_ops); ha->tgt.dfs_tgt_sess = debugfs_create_file("tgt_sess", - S_IRUSR, ha->dfs_dir, vha, &dfs_tgt_sess_ops); + S_IRUSR, ha->dfs_dir, vha, &qla2x00_dfs_tgt_sess_fops); if (IS_QLA27XX(ha) || IS_QLA83XX(ha) || IS_QLA28XX(ha)) { ha->tgt.dfs_naqp = debugfs_create_file("naqp", diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c index a39b1a885053..a24b82de4aab 100644 --- a/drivers/scsi/qla2xxx/qla_isr.c +++ b/drivers/scsi/qla2xxx/qla_isr.c @@ -1838,6 +1838,7 @@ qla24xx_mbx_iocb_entry(scsi_qla_host_t *vha, struct req_que *req, struct mbx_24xx_entry *pkt) { const char func[] = "MBX-IOCB2"; + struct qla_hw_data *ha = vha->hw; srb_t *sp; struct srb_iocb *si; u16 sz, i; @@ -1847,6 +1848,18 @@ qla24xx_mbx_iocb_entry(scsi_qla_host_t *vha, struct req_que *req, if (!sp) return; + if (sp->type == SRB_SCSI_CMD || + sp->type == SRB_NVME_CMD || + sp->type == SRB_TM_CMD) { + ql_log(ql_log_warn, vha, 0x509d, + "Inconsistent event entry type %d\n", sp->type); + if (IS_P3P_TYPE(ha)) + set_bit(FCOE_CTX_RESET_NEEDED, &vha->dpc_flags); + else + set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags); + return; + } + si = &sp->u.iocb_cmd; sz = min(ARRAY_SIZE(pkt->mb), ARRAY_SIZE(sp->u.iocb_cmd.u.mbx.in_mb)); @@ -3399,32 +3412,6 @@ void qla24xx_nvme_ls4_iocb(struct scsi_qla_host *vha, sp->done(sp, comp_status); } -static void qla24xx_process_mbx_iocb_response(struct scsi_qla_host *vha, - struct rsp_que *rsp, struct sts_entry_24xx *pkt) -{ - struct qla_hw_data *ha = vha->hw; - srb_t *sp; - static const char func[] = "MBX-IOCB2"; - - sp = qla2x00_get_sp_from_handle(vha, func, rsp->req, pkt); - if (!sp) - return; - - if (sp->type == SRB_SCSI_CMD || - sp->type == SRB_NVME_CMD || - sp->type == SRB_TM_CMD) { - ql_log(ql_log_warn, vha, 0x509d, - "Inconsistent event entry type %d\n", sp->type); - if (IS_P3P_TYPE(ha)) - set_bit(FCOE_CTX_RESET_NEEDED, &vha->dpc_flags); - else - set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags); - return; - } - - qla24xx_mbx_iocb_entry(vha, rsp->req, (struct mbx_24xx_entry *)pkt); -} - /** * qla24xx_process_response_queue() - Process response queue entries. * @vha: SCSI driver HA context @@ -3534,7 +3521,8 @@ process_err: (struct abort_entry_24xx *)pkt); break; case MBX_IOCB_TYPE: - qla24xx_process_mbx_iocb_response(vha, rsp, pkt); + qla24xx_mbx_iocb_entry(vha, rsp->req, + (struct mbx_24xx_entry *)pkt); break; case VP_CTRL_IOCB_TYPE: qla_ctrlvp_completed(vha, rsp->req, diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c index b4f22dbe7175..1f9005125313 100644 --- a/drivers/scsi/qla2xxx/qla_nvme.c +++ b/drivers/scsi/qla2xxx/qla_nvme.c @@ -541,7 +541,7 @@ static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport, fc_port_t *fcport; struct srb_iocb *nvme; struct scsi_qla_host *vha; - int rval = -ENODEV; + int rval; srb_t *sp; struct qla_qpair *qpair = hw_queue_handle; struct nvme_private *priv = fd->private; @@ -549,19 +549,19 @@ static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport, if (!priv) { /* nvme association has been torn down */ - return rval; + return -ENODEV; } fcport = qla_rport->fcport; if (!qpair || !fcport || (qpair && !qpair->fw_started) || (fcport && fcport->deleted)) - return rval; + return -ENODEV; vha = fcport->vha; if (!(fcport->nvme_flag & NVME_FLAG_REGISTERED)) - return rval; + return -ENODEV; if (test_bit(ABORT_ISP_ACTIVE, &vha->dpc_flags) || (qpair && !qpair->fw_started) || fcport->deleted) diff --git a/drivers/scsi/qla2xxx/qla_nx2.c b/drivers/scsi/qla2xxx/qla_nx2.c index 3a415b12dcec..01ccd4526707 100644 --- a/drivers/scsi/qla2xxx/qla_nx2.c +++ b/drivers/scsi/qla2xxx/qla_nx2.c @@ -659,7 +659,7 @@ static int qla8044_poll_reg(struct scsi_qla_host *vha, uint32_t addr, int duration, uint32_t test_mask, uint32_t test_result) { - uint32_t value; + uint32_t value = 0; int timeout_error; uint8_t retries; int ret_val = QLA_SUCCESS; diff --git a/drivers/scsi/qla4xxx/ql4_nx.c b/drivers/scsi/qla4xxx/ql4_nx.c index fd3aabb6a190..f1767b21076f 100644 --- a/drivers/scsi/qla4xxx/ql4_nx.c +++ b/drivers/scsi/qla4xxx/ql4_nx.c @@ -3225,7 +3225,7 @@ static void qla4_8xxx_uevent_emit(struct scsi_qla_host *ha, u32 code) switch (code) { case QL4_UEVENT_CODE_FW_DUMP: - snprintf(event_string, sizeof(event_string), "FW_DUMP=%ld", + snprintf(event_string, sizeof(event_string), "FW_DUMP=%lu", ha->host_no); break; default: diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 97ff31ed2a44..ab676ce2d051 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -293,21 +293,6 @@ int __scsi_execute(struct scsi_device *sdev, const unsigned char *cmd, } EXPORT_SYMBOL(__scsi_execute); -/** - * scsi_init_cmd_errh - Initialize cmd fields related to error handling. - * @cmd: command that is ready to be queued. - * - * This function has the job of initializing a number of fields related to error - * handling. Typically this will be called once for each command, as required. - */ -static void scsi_init_cmd_errh(struct scsi_cmnd *cmd) -{ - scsi_set_resid(cmd, 0); - memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE); - if (cmd->cmd_len == 0) - cmd->cmd_len = scsi_command_size(cmd->cmnd); -} - /* * Wake up the error handler if necessary. Avoid as follows that the error * handler is not woken up if host in-flight requests number == @@ -530,7 +515,7 @@ static void scsi_uninit_cmd(struct scsi_cmnd *cmd) } } -static void scsi_free_sgtables(struct scsi_cmnd *cmd) +void scsi_free_sgtables(struct scsi_cmnd *cmd) { if (cmd->sdb.table.nents) sg_free_table_chained(&cmd->sdb.table, @@ -539,6 +524,7 @@ static void scsi_free_sgtables(struct scsi_cmnd *cmd) sg_free_table_chained(&cmd->prot_sdb->table, SCSI_INLINE_PROT_SG_CNT); } +EXPORT_SYMBOL_GPL(scsi_free_sgtables); static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd) { @@ -998,7 +984,7 @@ static inline bool scsi_cmd_needs_dma_drain(struct scsi_device *sdev, } /** - * scsi_init_io - SCSI I/O initialization function. + * scsi_alloc_sgtables - allocate S/G tables for a command * @cmd: command descriptor we wish to initialize * * Returns: @@ -1006,7 +992,7 @@ static inline bool scsi_cmd_needs_dma_drain(struct scsi_device *sdev, * * BLK_STS_RESOURCE - if the failure is retryable * * BLK_STS_IOERR - if the failure is fatal */ -blk_status_t scsi_init_io(struct scsi_cmnd *cmd) +blk_status_t scsi_alloc_sgtables(struct scsi_cmnd *cmd) { struct scsi_device *sdev = cmd->device; struct request *rq = cmd->request; @@ -1098,7 +1084,7 @@ out_free_sgtables: scsi_free_sgtables(cmd); return ret; } -EXPORT_SYMBOL(scsi_init_io); +EXPORT_SYMBOL(scsi_alloc_sgtables); /** * scsi_initialize_rq - initialize struct scsi_cmnd partially @@ -1186,7 +1172,7 @@ static blk_status_t scsi_setup_scsi_cmnd(struct scsi_device *sdev, * submit a request without an attached bio. */ if (req->bio) { - blk_status_t ret = scsi_init_io(cmd); + blk_status_t ret = scsi_alloc_sgtables(cmd); if (unlikely(ret != BLK_STS_OK)) return ret; } else { @@ -1196,58 +1182,16 @@ static blk_status_t scsi_setup_scsi_cmnd(struct scsi_device *sdev, } cmd->cmd_len = scsi_req(req)->cmd_len; + if (cmd->cmd_len == 0) + cmd->cmd_len = scsi_command_size(cmd->cmnd); cmd->cmnd = scsi_req(req)->cmd; cmd->transfersize = blk_rq_bytes(req); cmd->allowed = scsi_req(req)->retries; return BLK_STS_OK; } -/* - * Setup a normal block command. These are simple request from filesystems - * that still need to be translated to SCSI CDBs from the ULD. - */ -static blk_status_t scsi_setup_fs_cmnd(struct scsi_device *sdev, - struct request *req) -{ - struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); - - if (unlikely(sdev->handler && sdev->handler->prep_fn)) { - blk_status_t ret = sdev->handler->prep_fn(sdev, req); - if (ret != BLK_STS_OK) - return ret; - } - - cmd->cmnd = scsi_req(req)->cmd = scsi_req(req)->__cmd; - memset(cmd->cmnd, 0, BLK_MAX_CDB); - return scsi_cmd_to_driver(cmd)->init_command(cmd); -} - -static blk_status_t scsi_setup_cmnd(struct scsi_device *sdev, - struct request *req) -{ - struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); - blk_status_t ret; - - if (!blk_rq_bytes(req)) - cmd->sc_data_direction = DMA_NONE; - else if (rq_data_dir(req) == WRITE) - cmd->sc_data_direction = DMA_TO_DEVICE; - else - cmd->sc_data_direction = DMA_FROM_DEVICE; - - if (blk_rq_is_scsi(req)) - ret = scsi_setup_scsi_cmnd(sdev, req); - else - ret = scsi_setup_fs_cmnd(sdev, req); - - if (ret != BLK_STS_OK) - scsi_free_sgtables(cmd); - - return ret; -} - static blk_status_t -scsi_prep_state_check(struct scsi_device *sdev, struct request *req) +scsi_device_state_check(struct scsi_device *sdev, struct request *req) { switch (sdev->sdev_state) { case SDEV_OFFLINE: @@ -1589,7 +1533,7 @@ static unsigned int scsi_mq_inline_sgl_size(struct Scsi_Host *shost) sizeof(struct scatterlist); } -static blk_status_t scsi_mq_prep_fn(struct request *req) +static blk_status_t scsi_prepare_cmd(struct request *req) { struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); struct scsi_device *sdev = req->q->queuedata; @@ -1601,6 +1545,10 @@ static blk_status_t scsi_mq_prep_fn(struct request *req) cmd->request = req; cmd->tag = req->tag; cmd->prot_op = SCSI_PROT_NORMAL; + if (blk_rq_bytes(req)) + cmd->sc_data_direction = rq_dma_dir(req); + else + cmd->sc_data_direction = DMA_NONE; sg = (void *)cmd + sizeof(struct scsi_cmnd) + shost->hostt->cmd_size; cmd->sdb.table.sgl = sg; @@ -1612,9 +1560,23 @@ static blk_status_t scsi_mq_prep_fn(struct request *req) (struct scatterlist *)(cmd->prot_sdb + 1); } - blk_mq_start_request(req); + /* + * Special handling for passthrough commands, which don't go to the ULP + * at all: + */ + if (blk_rq_is_scsi(req)) + return scsi_setup_scsi_cmnd(sdev, req); + + if (sdev->handler && sdev->handler->prep_fn) { + blk_status_t ret = sdev->handler->prep_fn(sdev, req); - return scsi_setup_cmnd(sdev, req); + if (ret != BLK_STS_OK) + return ret; + } + + cmd->cmnd = scsi_req(req)->cmd = scsi_req(req)->__cmd; + memset(cmd->cmnd, 0, BLK_MAX_CDB); + return scsi_cmd_to_driver(cmd)->init_command(cmd); } static void scsi_mq_done(struct scsi_cmnd *cmd) @@ -1680,7 +1642,7 @@ static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx, * commands. */ if (unlikely(sdev->sdev_state != SDEV_RUNNING)) { - ret = scsi_prep_state_check(sdev, req); + ret = scsi_device_state_check(sdev, req); if (ret != BLK_STS_OK) goto out_put_budget; } @@ -1692,13 +1654,12 @@ static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx, goto out_dec_target_busy; if (!(req->rq_flags & RQF_DONTPREP)) { - ret = scsi_mq_prep_fn(req); + ret = scsi_prepare_cmd(req); if (ret != BLK_STS_OK) goto out_dec_host_busy; req->rq_flags |= RQF_DONTPREP; } else { clear_bit(SCMD_STATE_COMPLETE, &cmd->state); - blk_mq_start_request(req); } cmd->flags &= SCMD_PRESERVED_FLAGS; @@ -1707,9 +1668,11 @@ static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx, if (bd->last) cmd->flags |= SCMD_LAST; - scsi_init_cmd_errh(cmd); + scsi_set_resid(cmd, 0); + memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE); cmd->scsi_done = scsi_mq_done; + blk_mq_start_request(req); reason = scsi_dispatch_cmd(cmd); if (reason) { scsi_set_blocked(cmd, reason); @@ -1970,7 +1933,6 @@ struct scsi_device *scsi_device_from_queue(struct request_queue *q) return sdev; } -EXPORT_SYMBOL_GPL(scsi_device_from_queue); /** * scsi_block_requests - Utility function used by low-level drivers to prevent diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index 83c4d95756a9..656bcf4940d6 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -902,7 +902,7 @@ static blk_status_t sd_setup_unmap_cmnd(struct scsi_cmnd *cmd) cmd->transfersize = data_len; rq->timeout = SD_TIMEOUT; - return scsi_init_io(cmd); + return scsi_alloc_sgtables(cmd); } static blk_status_t sd_setup_write_same16_cmnd(struct scsi_cmnd *cmd, @@ -934,7 +934,7 @@ static blk_status_t sd_setup_write_same16_cmnd(struct scsi_cmnd *cmd, cmd->transfersize = data_len; rq->timeout = unmap ? SD_TIMEOUT : SD_WRITE_SAME_TIMEOUT; - return scsi_init_io(cmd); + return scsi_alloc_sgtables(cmd); } static blk_status_t sd_setup_write_same10_cmnd(struct scsi_cmnd *cmd, @@ -966,7 +966,7 @@ static blk_status_t sd_setup_write_same10_cmnd(struct scsi_cmnd *cmd, cmd->transfersize = data_len; rq->timeout = unmap ? SD_TIMEOUT : SD_WRITE_SAME_TIMEOUT; - return scsi_init_io(cmd); + return scsi_alloc_sgtables(cmd); } static blk_status_t sd_setup_write_zeroes_cmnd(struct scsi_cmnd *cmd) @@ -1107,7 +1107,7 @@ static blk_status_t sd_setup_write_same_cmnd(struct scsi_cmnd *cmd) * knows how much to actually write. */ rq->__data_len = sdp->sector_size; - ret = scsi_init_io(cmd); + ret = scsi_alloc_sgtables(cmd); rq->__data_len = blk_rq_bytes(rq); return ret; @@ -1226,23 +1226,24 @@ static blk_status_t sd_setup_read_write_cmnd(struct scsi_cmnd *cmd) unsigned int dif; bool dix; - ret = scsi_init_io(cmd); + ret = scsi_alloc_sgtables(cmd); if (ret != BLK_STS_OK) return ret; + ret = BLK_STS_IOERR; if (!scsi_device_online(sdp) || sdp->changed) { scmd_printk(KERN_ERR, cmd, "device offline or changed\n"); - return BLK_STS_IOERR; + goto fail; } if (blk_rq_pos(rq) + blk_rq_sectors(rq) > get_capacity(rq->rq_disk)) { scmd_printk(KERN_ERR, cmd, "access beyond end of device\n"); - return BLK_STS_IOERR; + goto fail; } if ((blk_rq_pos(rq) & mask) || (blk_rq_sectors(rq) & mask)) { scmd_printk(KERN_ERR, cmd, "request not aligned to the logical block size\n"); - return BLK_STS_IOERR; + goto fail; } /* @@ -1264,7 +1265,7 @@ static blk_status_t sd_setup_read_write_cmnd(struct scsi_cmnd *cmd) if (req_op(rq) == REQ_OP_ZONE_APPEND) { ret = sd_zbc_prepare_zone_append(cmd, &lba, nr_blocks); if (ret) - return ret; + goto fail; } fua = rq->cmd_flags & REQ_FUA ? 0x8 : 0; @@ -1292,7 +1293,7 @@ static blk_status_t sd_setup_read_write_cmnd(struct scsi_cmnd *cmd) } if (unlikely(ret != BLK_STS_OK)) - return ret; + goto fail; /* * We shouldn't disconnect in the middle of a sector, so with a dumb @@ -1316,10 +1317,12 @@ static blk_status_t sd_setup_read_write_cmnd(struct scsi_cmnd *cmd) blk_rq_sectors(rq))); /* - * This indicates that the command is ready from our end to be - * queued. + * This indicates that the command is ready from our end to be queued. */ return BLK_STS_OK; +fail: + scsi_free_sgtables(cmd); + return ret; } static blk_status_t sd_init_command(struct scsi_cmnd *cmd) diff --git a/drivers/scsi/snic/vnic_cq.c b/drivers/scsi/snic/vnic_cq.c index 4c8e64e4fba6..3455dd7e73f4 100644 --- a/drivers/scsi/snic/vnic_cq.c +++ b/drivers/scsi/snic/vnic_cq.c @@ -31,8 +31,6 @@ void svnic_cq_free(struct vnic_cq *cq) int svnic_cq_alloc(struct vnic_dev *vdev, struct vnic_cq *cq, unsigned int index, unsigned int desc_count, unsigned int desc_size) { - int err; - cq->index = index; cq->vdev = vdev; @@ -43,11 +41,7 @@ int svnic_cq_alloc(struct vnic_dev *vdev, struct vnic_cq *cq, return -EINVAL; } - err = svnic_dev_alloc_desc_ring(vdev, &cq->ring, desc_count, desc_size); - if (err) - return err; - - return 0; + return svnic_dev_alloc_desc_ring(vdev, &cq->ring, desc_count, desc_size); } void svnic_cq_init(struct vnic_cq *cq, unsigned int flow_control_enable, diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c index 2b43c0f97442..fd4b582110b2 100644 --- a/drivers/scsi/sr.c +++ b/drivers/scsi/sr.c @@ -392,15 +392,11 @@ static blk_status_t sr_init_command(struct scsi_cmnd *SCpnt) struct request *rq = SCpnt->request; blk_status_t ret; - ret = scsi_init_io(SCpnt); + ret = scsi_alloc_sgtables(SCpnt); if (ret != BLK_STS_OK) - goto out; + return ret; cd = scsi_cd(rq->rq_disk); - /* from here on until we're complete, any goto out - * is used for a killable error condition */ - ret = BLK_STS_IOERR; - SCSI_LOG_HLQUEUE(1, scmd_printk(KERN_INFO, SCpnt, "Doing sr request, block = %d\n", block)); @@ -507,14 +503,15 @@ static blk_status_t sr_init_command(struct scsi_cmnd *SCpnt) SCpnt->transfersize = cd->device->sector_size; SCpnt->underflow = this_count << 9; SCpnt->allowed = MAX_RETRIES; + SCpnt->cmd_len = 10; /* - * This indicates that the command is ready from our end to be - * queued. + * This indicates that the command is ready from our end to be queued. */ - ret = BLK_STS_OK; + return BLK_STS_OK; out: - return ret; + scsi_free_sgtables(SCpnt); + return BLK_STS_IOERR; } static void sr_revalidate_disk(struct scsi_cd *cd) diff --git a/drivers/scsi/sym53c8xx_2/sym_hipd.c b/drivers/scsi/sym53c8xx_2/sym_hipd.c index cc11daa1222b..a9fe092a4906 100644 --- a/drivers/scsi/sym53c8xx_2/sym_hipd.c +++ b/drivers/scsi/sym53c8xx_2/sym_hipd.c @@ -5656,7 +5656,7 @@ int sym_hcb_attach(struct Scsi_Host *shost, struct sym_fw *fw, struct sym_nvram /* * Allocate the array of lists of CCBs hashed by DSA. */ - np->ccbh = kcalloc(CCB_HASH_SIZE, sizeof(struct sym_ccb **), GFP_KERNEL); + np->ccbh = kcalloc(CCB_HASH_SIZE, sizeof(*np->ccbh), GFP_KERNEL); if (!np->ccbh) goto attach_failed; diff --git a/drivers/soc/actions/owl-sps-helper.c b/drivers/soc/actions/owl-sps-helper.c index 291a206d6f04..e3f36603dd53 100644 --- a/drivers/soc/actions/owl-sps-helper.c +++ b/drivers/soc/actions/owl-sps-helper.c @@ -10,6 +10,7 @@ #include <linux/delay.h> #include <linux/io.h> +#include <linux/soc/actions/owl-sps.h> #define OWL_SPS_PG_CTL 0x0 diff --git a/drivers/soc/amlogic/meson-ee-pwrc.c b/drivers/soc/amlogic/meson-ee-pwrc.c index 43665b77aa9e..5164a4dc2352 100644 --- a/drivers/soc/amlogic/meson-ee-pwrc.c +++ b/drivers/soc/amlogic/meson-ee-pwrc.c @@ -15,6 +15,7 @@ #include <linux/reset.h> #include <linux/clk.h> #include <dt-bindings/power/meson8-power.h> +#include <dt-bindings/power/meson-axg-power.h> #include <dt-bindings/power/meson-g12a-power.h> #include <dt-bindings/power/meson-gxbb-power.h> #include <dt-bindings/power/meson-sm1-power.h> @@ -134,6 +135,11 @@ static struct meson_ee_pwrc_top_domain sm1_pwrc_ge2d = SM1_EE_PD(19); { __reg, BIT(14) }, \ { __reg, BIT(15) } +static struct meson_ee_pwrc_mem_domain axg_pwrc_mem_vpu[] = { + VPU_MEMPD(HHI_VPU_MEM_PD_REG0), + VPU_HHI_MEMPD(HHI_MEM_PD_REG0), +}; + static struct meson_ee_pwrc_mem_domain g12a_pwrc_mem_vpu[] = { VPU_MEMPD(HHI_VPU_MEM_PD_REG0), VPU_MEMPD(HHI_VPU_MEM_PD_REG1), @@ -190,6 +196,10 @@ static struct meson_ee_pwrc_mem_domain sm1_pwrc_mem_ge2d[] = { { HHI_MEM_PD_REG0, GENMASK(25, 18) }, }; +static struct meson_ee_pwrc_mem_domain axg_pwrc_mem_audio[] = { + { HHI_MEM_PD_REG0, GENMASK(5, 4) }, +}; + static struct meson_ee_pwrc_mem_domain sm1_pwrc_mem_audio[] = { { HHI_MEM_PD_REG0, GENMASK(5, 4) }, { HHI_AUDIO_MEM_PD_REG0, GENMASK(1, 0) }, @@ -231,6 +241,13 @@ static struct meson_ee_pwrc_mem_domain sm1_pwrc_mem_audio[] = { static bool pwrc_ee_get_power(struct meson_ee_pwrc_domain *pwrc_domain); +static struct meson_ee_pwrc_domain_desc axg_pwrc_domains[] = { + [PWRC_AXG_VPU_ID] = VPU_PD("VPU", &gx_pwrc_vpu, axg_pwrc_mem_vpu, + pwrc_ee_get_power, 5, 2), + [PWRC_AXG_ETHERNET_MEM_ID] = MEM_PD("ETH", meson_pwrc_mem_eth), + [PWRC_AXG_AUDIO_ID] = MEM_PD("AUDIO", axg_pwrc_mem_audio), +}; + static struct meson_ee_pwrc_domain_desc g12a_pwrc_domains[] = { [PWRC_G12A_VPU_ID] = VPU_PD("VPU", &gx_pwrc_vpu, g12a_pwrc_mem_vpu, pwrc_ee_get_power, 11, 2), @@ -433,8 +450,8 @@ static int meson_ee_pwrc_init_domain(struct platform_device *pdev, if (ret) return ret; - ret = pm_genpd_init(&dom->base, &pm_domain_always_on_gov, - false); + dom->base.flags = GENPD_FLAG_ALWAYS_ON; + ret = pm_genpd_init(&dom->base, NULL, false); if (ret) return ret; } else { @@ -529,6 +546,11 @@ static struct meson_ee_pwrc_domain_data meson_ee_g12a_pwrc_data = { .domains = g12a_pwrc_domains, }; +static struct meson_ee_pwrc_domain_data meson_ee_axg_pwrc_data = { + .count = ARRAY_SIZE(axg_pwrc_domains), + .domains = axg_pwrc_domains, +}; + static struct meson_ee_pwrc_domain_data meson_ee_gxbb_pwrc_data = { .count = ARRAY_SIZE(gxbb_pwrc_domains), .domains = gxbb_pwrc_domains, @@ -563,6 +585,10 @@ static const struct of_device_id meson_ee_pwrc_match_table[] = { .data = &meson_ee_m8b_pwrc_data, }, { + .compatible = "amlogic,meson-axg-pwrc", + .data = &meson_ee_axg_pwrc_data, + }, + { .compatible = "amlogic,meson-gxbb-pwrc", .data = &meson_ee_gxbb_pwrc_data, }, diff --git a/drivers/soc/amlogic/meson-gx-pwrc-vpu.c b/drivers/soc/amlogic/meson-gx-pwrc-vpu.c index 511b6856225d..21b4bc811c00 100644 --- a/drivers/soc/amlogic/meson-gx-pwrc-vpu.c +++ b/drivers/soc/amlogic/meson-gx-pwrc-vpu.c @@ -339,8 +339,8 @@ static int meson_gx_pwrc_vpu_probe(struct platform_device *pdev) return ret; } - pm_genpd_init(&vpu_pd->genpd, &pm_domain_always_on_gov, - powered_off); + vpu_pd->genpd.flags = GENPD_FLAG_ALWAYS_ON; + pm_genpd_init(&vpu_pd->genpd, NULL, powered_off); return of_genpd_add_provider_simple(pdev->dev.of_node, &vpu_pd->genpd); diff --git a/drivers/soc/bcm/Kconfig b/drivers/soc/bcm/Kconfig index 648e32693b7e..24f92a6e882a 100644 --- a/drivers/soc/bcm/Kconfig +++ b/drivers/soc/bcm/Kconfig @@ -22,6 +22,15 @@ config RASPBERRYPI_POWER This enables support for the RPi power domains which can be enabled or disabled via the RPi firmware. +config SOC_BCM63XX + bool "Broadcom 63xx SoC drivers" + depends on BMIPS_GENERIC || COMPILE_TEST + help + Enables drivers for the Broadcom 63xx series of chips. + Drivers can be enabled individually within this menu. + + If unsure, say N. + config SOC_BRCMSTB bool "Broadcom STB SoC drivers" depends on ARM || ARM64 || BMIPS_GENERIC || COMPILE_TEST @@ -33,6 +42,7 @@ config SOC_BRCMSTB If unsure, say N. +source "drivers/soc/bcm/bcm63xx/Kconfig" source "drivers/soc/bcm/brcmstb/Kconfig" endmenu diff --git a/drivers/soc/bcm/Makefile b/drivers/soc/bcm/Makefile index d92268a829a9..7bc90e0bd773 100644 --- a/drivers/soc/bcm/Makefile +++ b/drivers/soc/bcm/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_BCM2835_POWER) += bcm2835-power.o obj-$(CONFIG_RASPBERRYPI_POWER) += raspberrypi-power.o +obj-$(CONFIG_SOC_BCM63XX) += bcm63xx/ obj-$(CONFIG_SOC_BRCMSTB) += brcmstb/ diff --git a/drivers/soc/bcm/bcm63xx/Kconfig b/drivers/soc/bcm/bcm63xx/Kconfig new file mode 100644 index 000000000000..16f648a6c70a --- /dev/null +++ b/drivers/soc/bcm/bcm63xx/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only +if SOC_BCM63XX + +config BCM63XX_POWER + bool "BCM63xx power domain driver" + depends on BMIPS_GENERIC || (COMPILE_TEST && OF) + select PM_GENERIC_DOMAINS if PM + help + This enables support for the BCM63xx power domains controller on + BCM6318, BCM6328, BCM6362 and BCM63268 SoCs. + +endif # SOC_BCM63XX diff --git a/drivers/soc/bcm/bcm63xx/Makefile b/drivers/soc/bcm/bcm63xx/Makefile new file mode 100644 index 000000000000..0710d5e018cc --- /dev/null +++ b/drivers/soc/bcm/bcm63xx/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_BCM63XX_POWER) += bcm63xx-power.o diff --git a/drivers/soc/bcm/bcm63xx/bcm63xx-power.c b/drivers/soc/bcm/bcm63xx/bcm63xx-power.c new file mode 100644 index 000000000000..515fe182dc34 --- /dev/null +++ b/drivers/soc/bcm/bcm63xx/bcm63xx-power.c @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * BCM63xx Power Domain Controller Driver + * + * Copyright (C) 2020 Álvaro Fernández Rojas <noltari@gmail.com> + */ + +#include <dt-bindings/soc/bcm6318-pm.h> +#include <dt-bindings/soc/bcm6328-pm.h> +#include <dt-bindings/soc/bcm6362-pm.h> +#include <dt-bindings/soc/bcm63268-pm.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/pm_domain.h> +#include <linux/of.h> +#include <linux/of_device.h> + +struct bcm63xx_power_dev { + struct generic_pm_domain genpd; + struct bcm63xx_power *power; + uint32_t mask; +}; + +struct bcm63xx_power { + void __iomem *base; + spinlock_t lock; + struct bcm63xx_power_dev *dev; + struct genpd_onecell_data genpd_data; + struct generic_pm_domain **genpd; +}; + +struct bcm63xx_power_data { + const char * const name; + uint8_t bit; + unsigned int flags; +}; + +static int bcm63xx_power_get_state(struct bcm63xx_power_dev *pmd, bool *is_on) +{ + struct bcm63xx_power *power = pmd->power; + + if (!pmd->mask) { + *is_on = false; + return -EINVAL; + } + + *is_on = !(__raw_readl(power->base) & pmd->mask); + + return 0; +} + +static int bcm63xx_power_set_state(struct bcm63xx_power_dev *pmd, bool on) +{ + struct bcm63xx_power *power = pmd->power; + unsigned long flags; + uint32_t val; + + if (!pmd->mask) + return -EINVAL; + + spin_lock_irqsave(&power->lock, flags); + val = __raw_readl(power->base); + if (on) + val &= ~pmd->mask; + else + val |= pmd->mask; + __raw_writel(val, power->base); + spin_unlock_irqrestore(&power->lock, flags); + + return 0; +} + +static int bcm63xx_power_on(struct generic_pm_domain *genpd) +{ + struct bcm63xx_power_dev *pmd = container_of(genpd, + struct bcm63xx_power_dev, genpd); + + return bcm63xx_power_set_state(pmd, true); +} + +static int bcm63xx_power_off(struct generic_pm_domain *genpd) +{ + struct bcm63xx_power_dev *pmd = container_of(genpd, + struct bcm63xx_power_dev, genpd); + + return bcm63xx_power_set_state(pmd, false); +} + +static int bcm63xx_power_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct resource *res; + const struct bcm63xx_power_data *entry, *table; + struct bcm63xx_power *power; + unsigned int ndom; + uint8_t max_bit = 0; + int ret; + + power = devm_kzalloc(dev, sizeof(*power), GFP_KERNEL); + if (!power) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + power->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(power->base)) + return PTR_ERR(power->base); + + table = of_device_get_match_data(dev); + if (!table) + return -EINVAL; + + power->genpd_data.num_domains = 0; + ndom = 0; + for (entry = table; entry->name; entry++) { + max_bit = max(max_bit, entry->bit); + ndom++; + } + + if (!ndom) + return -ENODEV; + + power->genpd_data.num_domains = max_bit + 1; + + power->dev = devm_kcalloc(dev, power->genpd_data.num_domains, + sizeof(struct bcm63xx_power_dev), + GFP_KERNEL); + if (!power->dev) + return -ENOMEM; + + power->genpd = devm_kcalloc(dev, power->genpd_data.num_domains, + sizeof(struct generic_pm_domain *), + GFP_KERNEL); + if (!power->genpd) + return -ENOMEM; + + power->genpd_data.domains = power->genpd; + + ndom = 0; + for (entry = table; entry->name; entry++) { + struct bcm63xx_power_dev *pmd = &power->dev[ndom]; + bool is_on; + + pmd->power = power; + pmd->mask = BIT(entry->bit); + pmd->genpd.name = entry->name; + pmd->genpd.flags = entry->flags; + + ret = bcm63xx_power_get_state(pmd, &is_on); + if (ret) + dev_warn(dev, "unable to get current state for %s\n", + pmd->genpd.name); + + pmd->genpd.power_on = bcm63xx_power_on; + pmd->genpd.power_off = bcm63xx_power_off; + + pm_genpd_init(&pmd->genpd, NULL, !is_on); + power->genpd[entry->bit] = &pmd->genpd; + + ndom++; + } + + spin_lock_init(&power->lock); + + ret = of_genpd_add_provider_onecell(np, &power->genpd_data); + if (ret) { + dev_err(dev, "failed to register genpd driver: %d\n", ret); + return ret; + } + + dev_info(dev, "registered %u power domains\n", ndom); + + return 0; +} + +static const struct bcm63xx_power_data bcm6318_power_domains[] = { + { + .name = "pcie", + .bit = BCM6318_POWER_DOMAIN_PCIE, + }, { + .name = "usb", + .bit = BCM6318_POWER_DOMAIN_USB, + }, { + .name = "ephy0", + .bit = BCM6318_POWER_DOMAIN_EPHY0, + }, { + .name = "ephy1", + .bit = BCM6318_POWER_DOMAIN_EPHY1, + }, { + .name = "ephy2", + .bit = BCM6318_POWER_DOMAIN_EPHY2, + }, { + .name = "ephy3", + .bit = BCM6318_POWER_DOMAIN_EPHY3, + }, { + .name = "ldo2p5", + .bit = BCM6318_POWER_DOMAIN_LDO2P5, + .flags = GENPD_FLAG_ALWAYS_ON, + }, { + .name = "ldo2p9", + .bit = BCM6318_POWER_DOMAIN_LDO2P9, + .flags = GENPD_FLAG_ALWAYS_ON, + }, { + .name = "sw1p0", + .bit = BCM6318_POWER_DOMAIN_SW1P0, + .flags = GENPD_FLAG_ALWAYS_ON, + }, { + .name = "pad", + .bit = BCM6318_POWER_DOMAIN_PAD, + .flags = GENPD_FLAG_ALWAYS_ON, + }, { + /* sentinel */ + }, +}; + +static const struct bcm63xx_power_data bcm6328_power_domains[] = { + { + .name = "adsl2-mips", + .bit = BCM6328_POWER_DOMAIN_ADSL2_MIPS, + }, { + .name = "adsl2-phy", + .bit = BCM6328_POWER_DOMAIN_ADSL2_PHY, + }, { + .name = "adsl2-afe", + .bit = BCM6328_POWER_DOMAIN_ADSL2_AFE, + }, { + .name = "sar", + .bit = BCM6328_POWER_DOMAIN_SAR, + }, { + .name = "pcm", + .bit = BCM6328_POWER_DOMAIN_PCM, + }, { + .name = "usbd", + .bit = BCM6328_POWER_DOMAIN_USBD, + }, { + .name = "usbh", + .bit = BCM6328_POWER_DOMAIN_USBH, + }, { + .name = "pcie", + .bit = BCM6328_POWER_DOMAIN_PCIE, + }, { + .name = "robosw", + .bit = BCM6328_POWER_DOMAIN_ROBOSW, + }, { + .name = "ephy", + .bit = BCM6328_POWER_DOMAIN_EPHY, + }, { + /* sentinel */ + }, +}; + +static const struct bcm63xx_power_data bcm6362_power_domains[] = { + { + .name = "sar", + .bit = BCM6362_POWER_DOMAIN_SAR, + }, { + .name = "ipsec", + .bit = BCM6362_POWER_DOMAIN_IPSEC, + }, { + .name = "mips", + .bit = BCM6362_POWER_DOMAIN_MIPS, + .flags = GENPD_FLAG_ALWAYS_ON, + }, { + .name = "dect", + .bit = BCM6362_POWER_DOMAIN_DECT, + }, { + .name = "usbh", + .bit = BCM6362_POWER_DOMAIN_USBH, + }, { + .name = "usbd", + .bit = BCM6362_POWER_DOMAIN_USBD, + }, { + .name = "robosw", + .bit = BCM6362_POWER_DOMAIN_ROBOSW, + }, { + .name = "pcm", + .bit = BCM6362_POWER_DOMAIN_PCM, + }, { + .name = "periph", + .bit = BCM6362_POWER_DOMAIN_PERIPH, + .flags = GENPD_FLAG_ALWAYS_ON, + }, { + .name = "adsl-phy", + .bit = BCM6362_POWER_DOMAIN_ADSL_PHY, + }, { + .name = "gmii-pads", + .bit = BCM6362_POWER_DOMAIN_GMII_PADS, + }, { + .name = "fap", + .bit = BCM6362_POWER_DOMAIN_FAP, + }, { + .name = "pcie", + .bit = BCM6362_POWER_DOMAIN_PCIE, + }, { + .name = "wlan-pads", + .bit = BCM6362_POWER_DOMAIN_WLAN_PADS, + }, { + /* sentinel */ + }, +}; + +static const struct bcm63xx_power_data bcm63268_power_domains[] = { + { + .name = "sar", + .bit = BCM63268_POWER_DOMAIN_SAR, + }, { + .name = "ipsec", + .bit = BCM63268_POWER_DOMAIN_IPSEC, + }, { + .name = "mips", + .bit = BCM63268_POWER_DOMAIN_MIPS, + .flags = GENPD_FLAG_ALWAYS_ON, + }, { + .name = "dect", + .bit = BCM63268_POWER_DOMAIN_DECT, + }, { + .name = "usbh", + .bit = BCM63268_POWER_DOMAIN_USBH, + }, { + .name = "usbd", + .bit = BCM63268_POWER_DOMAIN_USBD, + }, { + .name = "robosw", + .bit = BCM63268_POWER_DOMAIN_ROBOSW, + }, { + .name = "pcm", + .bit = BCM63268_POWER_DOMAIN_PCM, + }, { + .name = "periph", + .bit = BCM63268_POWER_DOMAIN_PERIPH, + .flags = GENPD_FLAG_ALWAYS_ON, + }, { + .name = "vdsl-phy", + .bit = BCM63268_POWER_DOMAIN_VDSL_PHY, + }, { + .name = "vdsl-mips", + .bit = BCM63268_POWER_DOMAIN_VDSL_MIPS, + }, { + .name = "fap", + .bit = BCM63268_POWER_DOMAIN_FAP, + }, { + .name = "pcie", + .bit = BCM63268_POWER_DOMAIN_PCIE, + }, { + .name = "wlan-pads", + .bit = BCM63268_POWER_DOMAIN_WLAN_PADS, + }, { + /* sentinel */ + }, +}; + +static const struct of_device_id bcm63xx_power_of_match[] = { + { + .compatible = "brcm,bcm6318-power-controller", + .data = &bcm6318_power_domains, + }, { + .compatible = "brcm,bcm6328-power-controller", + .data = &bcm6328_power_domains, + }, { + .compatible = "brcm,bcm6362-power-controller", + .data = &bcm6362_power_domains, + }, { + .compatible = "brcm,bcm63268-power-controller", + .data = &bcm63268_power_domains, + }, { + /* sentinel */ + } +}; + +static struct platform_driver bcm63xx_power_driver = { + .driver = { + .name = "bcm63xx-power-controller", + .of_match_table = bcm63xx_power_of_match, + }, + .probe = bcm63xx_power_probe, +}; +builtin_platform_driver(bcm63xx_power_driver); diff --git a/drivers/soc/bcm/brcmstb/biuctrl.c b/drivers/soc/bcm/brcmstb/biuctrl.c index 61731e01f94b..7f8dc302ae6e 100644 --- a/drivers/soc/bcm/brcmstb/biuctrl.c +++ b/drivers/soc/bcm/brcmstb/biuctrl.c @@ -13,6 +13,22 @@ #include <linux/syscore_ops.h> #include <linux/soc/brcmstb/brcmstb.h> +#define RACENPREF_MASK 0x3 +#define RACPREFINST_SHIFT 0 +#define RACENINST_SHIFT 2 +#define RACPREFDATA_SHIFT 4 +#define RACENDATA_SHIFT 6 +#define RAC_CPU_SHIFT 8 +#define RACCFG_MASK 0xff +#define DPREF_LINE_2_SHIFT 24 +#define DPREF_LINE_2_MASK 0xff + +/* Bitmask to enable instruction and data prefetching with a 256-bytes stride */ +#define RAC_DATA_INST_EN_MASK (1 << RACPREFINST_SHIFT | \ + RACENPREF_MASK << RACENINST_SHIFT | \ + 1 << RACPREFDATA_SHIFT | \ + RACENPREF_MASK << RACENDATA_SHIFT) + #define CPU_CREDIT_REG_MCPx_WR_PAIRING_EN_MASK 0x70000000 #define CPU_CREDIT_REG_MCPx_READ_CRED_MASK 0xf #define CPU_CREDIT_REG_MCPx_WRITE_CRED_MASK 0xf @@ -31,11 +47,21 @@ static void __iomem *cpubiuctrl_base; static bool mcp_wr_pairing_en; static const int *cpubiuctrl_regs; +enum cpubiuctrl_regs { + CPU_CREDIT_REG = 0, + CPU_MCP_FLOW_REG, + CPU_WRITEBACK_CTRL_REG, + RAC_CONFIG0_REG, + RAC_CONFIG1_REG, + NUM_CPU_BIUCTRL_REGS, +}; + static inline u32 cbc_readl(int reg) { int offset = cpubiuctrl_regs[reg]; - if (offset == -1) + if (offset == -1 || + (IS_ENABLED(CONFIG_CACHE_B15_RAC) && reg >= RAC_CONFIG0_REG)) return (u32)-1; return readl_relaxed(cpubiuctrl_base + offset); @@ -45,22 +71,19 @@ static inline void cbc_writel(u32 val, int reg) { int offset = cpubiuctrl_regs[reg]; - if (offset == -1) + if (offset == -1 || + (IS_ENABLED(CONFIG_CACHE_B15_RAC) && reg >= RAC_CONFIG0_REG)) return; writel(val, cpubiuctrl_base + offset); } -enum cpubiuctrl_regs { - CPU_CREDIT_REG = 0, - CPU_MCP_FLOW_REG, - CPU_WRITEBACK_CTRL_REG -}; - static const int b15_cpubiuctrl_regs[] = { [CPU_CREDIT_REG] = 0x184, [CPU_MCP_FLOW_REG] = -1, [CPU_WRITEBACK_CTRL_REG] = -1, + [RAC_CONFIG0_REG] = -1, + [RAC_CONFIG1_REG] = -1, }; /* Odd cases, e.g: 7260A0 */ @@ -68,22 +91,26 @@ static const int b53_cpubiuctrl_no_wb_regs[] = { [CPU_CREDIT_REG] = 0x0b0, [CPU_MCP_FLOW_REG] = 0x0b4, [CPU_WRITEBACK_CTRL_REG] = -1, + [RAC_CONFIG0_REG] = 0x78, + [RAC_CONFIG1_REG] = 0x7c, }; static const int b53_cpubiuctrl_regs[] = { [CPU_CREDIT_REG] = 0x0b0, [CPU_MCP_FLOW_REG] = 0x0b4, [CPU_WRITEBACK_CTRL_REG] = 0x22c, + [RAC_CONFIG0_REG] = 0x78, + [RAC_CONFIG1_REG] = 0x7c, }; static const int a72_cpubiuctrl_regs[] = { [CPU_CREDIT_REG] = 0x18, [CPU_MCP_FLOW_REG] = 0x1c, [CPU_WRITEBACK_CTRL_REG] = 0x20, + [RAC_CONFIG0_REG] = 0x08, + [RAC_CONFIG1_REG] = 0x0c, }; -#define NUM_CPU_BIUCTRL_REGS 3 - static int __init mcp_write_pairing_set(void) { u32 creds = 0; @@ -110,6 +137,8 @@ static int __init mcp_write_pairing_set(void) static const u32 a72_b53_mach_compat[] = { 0x7211, 0x7216, + 0x72164, + 0x72165, 0x7255, 0x7260, 0x7268, @@ -117,6 +146,61 @@ static const u32 a72_b53_mach_compat[] = { 0x7278, }; +/* The read-ahead cache present in the Brahma-B53 CPU is a special piece of + * hardware after the integrated L2 cache of the B53 CPU complex whose purpose + * is to prefetch instruction and/or data with a line size of either 64 bytes + * or 256 bytes. The rationale is that the data-bus of the CPU interface is + * optimized for 256-byte transactions, and enabling the read-ahead cache + * provides a significant performance boost (typically twice the performance + * for a memcpy benchmark application). + * + * The read-ahead cache is transparent for Virtual Address cache maintenance + * operations: IC IVAU, DC IVAC, DC CVAC, DC CVAU and DC CIVAC. So no special + * handling is needed for the DMA API above and beyond what is included in the + * arm64 implementation. + * + * In addition, since the Point of Unification is typically between L1 and L2 + * for the Brahma-B53 processor no special read-ahead cache handling is needed + * for the IC IALLU and IC IALLUIS cache maintenance operations. + * + * However, it is not possible to specify the cache level (L3) for the cache + * maintenance instructions operating by set/way to operate on the read-ahead + * cache. The read-ahead cache will maintain coherency when inner cache lines + * are cleaned by set/way, but if it is necessary to invalidate inner cache + * lines by set/way to maintain coherency with system masters operating on + * shared memory that does not have hardware support for coherency, then it + * will also be necessary to explicitly invalidate the read-ahead cache. + */ +static void __init a72_b53_rac_enable_all(struct device_node *np) +{ + unsigned int cpu; + u32 enable = 0, pref_dist, shift; + + if (IS_ENABLED(CONFIG_CACHE_B15_RAC)) + return; + + if (WARN(num_possible_cpus() > 4, "RAC only supports 4 CPUs\n")) + return; + + pref_dist = cbc_readl(RAC_CONFIG1_REG); + for_each_possible_cpu(cpu) { + shift = cpu * RAC_CPU_SHIFT + RACPREFDATA_SHIFT; + enable |= RAC_DATA_INST_EN_MASK << (cpu * RAC_CPU_SHIFT); + if (cpubiuctrl_regs == a72_cpubiuctrl_regs) { + enable &= ~(RACENPREF_MASK << shift); + enable |= 3 << shift; + pref_dist |= 1 << (cpu + DPREF_LINE_2_SHIFT); + } + } + + cbc_writel(enable, RAC_CONFIG0_REG); + cbc_writel(pref_dist, RAC_CONFIG1_REG); + + pr_info("%pOF: Broadcom %s read-ahead cache\n", + np, cpubiuctrl_regs == a72_cpubiuctrl_regs ? + "Cortex-A72" : "Brahma-B53"); +} + static void __init mcp_a72_b53_set(void) { unsigned int i; @@ -262,6 +346,7 @@ static int __init brcmstb_biuctrl_init(void) return ret; } + a72_b53_rac_enable_all(np); mcp_a72_b53_set(); #ifdef CONFIG_PM_SLEEP register_syscore_ops(&brcmstb_cpu_credit_syscore_ops); diff --git a/drivers/soc/fsl/dpio/qbman-portal.c b/drivers/soc/fsl/dpio/qbman-portal.c index 0ab85bfb116f..659b4a570d5b 100644 --- a/drivers/soc/fsl/dpio/qbman-portal.c +++ b/drivers/soc/fsl/dpio/qbman-portal.c @@ -647,7 +647,6 @@ int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s, const uint32_t *cl = (uint32_t *)d; uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; int i, num_enqueued = 0; - uint64_t addr_cena; spin_lock(&s->access_spinlock); half_mask = (s->eqcr.pi_ci_mask>>1); @@ -701,7 +700,6 @@ int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s, /* Flush all the cacheline without load/store in between */ eqcr_pi = s->eqcr.pi; - addr_cena = (size_t)s->addr_cena; for (i = 0; i < num_enqueued; i++) eqcr_pi++; s->eqcr.pi = eqcr_pi & full_mask; diff --git a/drivers/soc/fsl/qbman/bman.c b/drivers/soc/fsl/qbman/bman.c index f4fb527d8301..c5dd026fe889 100644 --- a/drivers/soc/fsl/qbman/bman.c +++ b/drivers/soc/fsl/qbman/bman.c @@ -660,7 +660,7 @@ int bm_shutdown_pool(u32 bpid) } done: put_affine_portal(); - return 0; + return err; } struct gen_pool *bm_bpalloc; diff --git a/drivers/soc/fsl/qbman/qman_test_api.c b/drivers/soc/fsl/qbman/qman_test_api.c index 2895d062cf51..7066b2f1467c 100644 --- a/drivers/soc/fsl/qbman/qman_test_api.c +++ b/drivers/soc/fsl/qbman/qman_test_api.c @@ -86,7 +86,7 @@ static void fd_inc(struct qm_fd *fd) len--; qm_fd_set_param(fd, fmt, off, len); - fd->cmd = cpu_to_be32(be32_to_cpu(fd->cmd) + 1); + be32_add_cpu(&fd->cmd, 1); } /* The only part of the 'fd' we can't memcmp() is the ppid */ diff --git a/drivers/soc/fsl/qe/ucc.c b/drivers/soc/fsl/qe/ucc.c index cac0fb7693a0..21dbcd787cd5 100644 --- a/drivers/soc/fsl/qe/ucc.c +++ b/drivers/soc/fsl/qe/ucc.c @@ -523,7 +523,7 @@ int ucc_set_tdm_rxtx_clk(u32 tdm_num, enum qe_clock clock, qe_mux_reg = &qe_immr->qmx; - if (tdm_num > 7 || tdm_num < 0) + if (tdm_num > 7) return -EINVAL; /* The communications direction must be RX or TX */ diff --git a/drivers/soc/imx/gpcv2.c b/drivers/soc/imx/gpcv2.c index 6cf8a7a412bd..db7e7fc321b1 100644 --- a/drivers/soc/imx/gpcv2.c +++ b/drivers/soc/imx/gpcv2.c @@ -487,22 +487,17 @@ static int imx_pgc_domain_probe(struct platform_device *pdev) domain->regulator = devm_regulator_get_optional(domain->dev, "power"); if (IS_ERR(domain->regulator)) { - if (PTR_ERR(domain->regulator) != -ENODEV) { - if (PTR_ERR(domain->regulator) != -EPROBE_DEFER) - dev_err(domain->dev, "Failed to get domain's regulator\n"); - return PTR_ERR(domain->regulator); - } + if (PTR_ERR(domain->regulator) != -ENODEV) + return dev_err_probe(domain->dev, PTR_ERR(domain->regulator), + "Failed to get domain's regulator\n"); } else if (domain->voltage) { regulator_set_voltage(domain->regulator, domain->voltage, domain->voltage); } ret = imx_pgc_get_clocks(domain); - if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(domain->dev, "Failed to get domain's clocks\n"); - return ret; - } + if (ret) + return dev_err_probe(domain->dev, ret, "Failed to get domain's clocks\n"); ret = pm_genpd_init(&domain->genpd, NULL, true); if (ret) { diff --git a/drivers/soc/mediatek/mtk-cmdq-helper.c b/drivers/soc/mediatek/mtk-cmdq-helper.c index dc644cfb6419..505651b0d715 100644 --- a/drivers/soc/mediatek/mtk-cmdq-helper.c +++ b/drivers/soc/mediatek/mtk-cmdq-helper.c @@ -13,11 +13,16 @@ #define CMDQ_POLL_ENABLE_MASK BIT(0) #define CMDQ_EOC_IRQ_EN BIT(0) #define CMDQ_REG_TYPE 1 +#define CMDQ_JUMP_RELATIVE 1 struct cmdq_instruction { union { u32 value; u32 mask; + struct { + u16 arg_c; + u16 src_reg; + }; }; union { u16 offset; @@ -223,15 +228,104 @@ int cmdq_pkt_write_mask(struct cmdq_pkt *pkt, u8 subsys, } EXPORT_SYMBOL(cmdq_pkt_write_mask); -int cmdq_pkt_wfe(struct cmdq_pkt *pkt, u16 event) +int cmdq_pkt_read_s(struct cmdq_pkt *pkt, u16 high_addr_reg_idx, u16 addr_low, + u16 reg_idx) +{ + struct cmdq_instruction inst = {}; + + inst.op = CMDQ_CODE_READ_S; + inst.dst_t = CMDQ_REG_TYPE; + inst.sop = high_addr_reg_idx; + inst.reg_dst = reg_idx; + inst.src_reg = addr_low; + + return cmdq_pkt_append_command(pkt, inst); +} +EXPORT_SYMBOL(cmdq_pkt_read_s); + +int cmdq_pkt_write_s(struct cmdq_pkt *pkt, u16 high_addr_reg_idx, + u16 addr_low, u16 src_reg_idx) +{ + struct cmdq_instruction inst = {}; + + inst.op = CMDQ_CODE_WRITE_S; + inst.src_t = CMDQ_REG_TYPE; + inst.sop = high_addr_reg_idx; + inst.offset = addr_low; + inst.src_reg = src_reg_idx; + + return cmdq_pkt_append_command(pkt, inst); +} +EXPORT_SYMBOL(cmdq_pkt_write_s); + +int cmdq_pkt_write_s_mask(struct cmdq_pkt *pkt, u16 high_addr_reg_idx, + u16 addr_low, u16 src_reg_idx, u32 mask) +{ + struct cmdq_instruction inst = {}; + int err; + + inst.op = CMDQ_CODE_MASK; + inst.mask = ~mask; + err = cmdq_pkt_append_command(pkt, inst); + if (err < 0) + return err; + + inst.mask = 0; + inst.op = CMDQ_CODE_WRITE_S_MASK; + inst.src_t = CMDQ_REG_TYPE; + inst.sop = high_addr_reg_idx; + inst.offset = addr_low; + inst.src_reg = src_reg_idx; + + return cmdq_pkt_append_command(pkt, inst); +} +EXPORT_SYMBOL(cmdq_pkt_write_s_mask); + +int cmdq_pkt_write_s_value(struct cmdq_pkt *pkt, u8 high_addr_reg_idx, + u16 addr_low, u32 value) +{ + struct cmdq_instruction inst = {}; + + inst.op = CMDQ_CODE_WRITE_S; + inst.sop = high_addr_reg_idx; + inst.offset = addr_low; + inst.value = value; + + return cmdq_pkt_append_command(pkt, inst); +} +EXPORT_SYMBOL(cmdq_pkt_write_s_value); + +int cmdq_pkt_write_s_mask_value(struct cmdq_pkt *pkt, u8 high_addr_reg_idx, + u16 addr_low, u32 value, u32 mask) +{ + struct cmdq_instruction inst = {}; + int err; + + inst.op = CMDQ_CODE_MASK; + inst.mask = ~mask; + err = cmdq_pkt_append_command(pkt, inst); + if (err < 0) + return err; + + inst.op = CMDQ_CODE_WRITE_S_MASK; + inst.sop = high_addr_reg_idx; + inst.offset = addr_low; + inst.value = value; + + return cmdq_pkt_append_command(pkt, inst); +} +EXPORT_SYMBOL(cmdq_pkt_write_s_mask_value); + +int cmdq_pkt_wfe(struct cmdq_pkt *pkt, u16 event, bool clear) { struct cmdq_instruction inst = { {0} }; + u32 clear_option = clear ? CMDQ_WFE_UPDATE : 0; if (event >= CMDQ_MAX_EVENT) return -EINVAL; inst.op = CMDQ_CODE_WFE; - inst.value = CMDQ_WFE_OPTION; + inst.value = CMDQ_WFE_OPTION | clear_option; inst.event = event; return cmdq_pkt_append_command(pkt, inst); @@ -315,6 +409,18 @@ int cmdq_pkt_assign(struct cmdq_pkt *pkt, u16 reg_idx, u32 value) } EXPORT_SYMBOL(cmdq_pkt_assign); +int cmdq_pkt_jump(struct cmdq_pkt *pkt, dma_addr_t addr) +{ + struct cmdq_instruction inst = {}; + + inst.op = CMDQ_CODE_JUMP; + inst.offset = CMDQ_JUMP_RELATIVE; + inst.value = addr >> + cmdq_get_shift_pa(((struct cmdq_client *)pkt->cl)->chan); + return cmdq_pkt_append_command(pkt, inst); +} +EXPORT_SYMBOL(cmdq_pkt_jump); + int cmdq_pkt_finalize(struct cmdq_pkt *pkt) { struct cmdq_instruction inst = { {0} }; @@ -329,7 +435,8 @@ int cmdq_pkt_finalize(struct cmdq_pkt *pkt) /* JUMP to end */ inst.op = CMDQ_CODE_JUMP; - inst.value = CMDQ_JUMP_PASS; + inst.value = CMDQ_JUMP_PASS >> + cmdq_get_shift_pa(((struct cmdq_client *)pkt->cl)->chan); err = cmdq_pkt_append_command(pkt, inst); return err; diff --git a/drivers/soc/mediatek/mtk-infracfg.c b/drivers/soc/mediatek/mtk-infracfg.c index 341c7ac250e3..4a123796aad3 100644 --- a/drivers/soc/mediatek/mtk-infracfg.c +++ b/drivers/soc/mediatek/mtk-infracfg.c @@ -19,7 +19,7 @@ /** * mtk_infracfg_set_bus_protection - enable bus protection - * @regmap: The infracfg regmap + * @infracfg: The infracfg regmap * @mask: The mask containing the protection bits to be enabled. * @reg_update: The boolean flag determines to set the protection bits * by regmap_update_bits with enable register(PROTECTEN) or @@ -50,7 +50,7 @@ int mtk_infracfg_set_bus_protection(struct regmap *infracfg, u32 mask, /** * mtk_infracfg_clear_bus_protection - disable bus protection - * @regmap: The infracfg regmap + * @infracfg: The infracfg regmap * @mask: The mask containing the protection bits to be disabled. * @reg_update: The boolean flag determines to clear the protection bits * by regmap_update_bits with enable register(PROTECTEN) or diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig index 3dc3e3d61ea3..6a3b69b43ad5 100644 --- a/drivers/soc/qcom/Kconfig +++ b/drivers/soc/qcom/Kconfig @@ -26,6 +26,22 @@ config QCOM_COMMAND_DB resource on a RPM-hardened platform must use this database to get SoC specific identifier and information for the shared resources. +config QCOM_CPR + tristate "QCOM Core Power Reduction (CPR) support" + depends on ARCH_QCOM && HAS_IOMEM + select PM_OPP + select REGMAP + help + Say Y here to enable support for the CPR hardware found on Qualcomm + SoCs like QCS404. + + This driver populates CPU OPPs tables and makes adjustments to the + tables based on feedback from the CPR hardware. If you want to do + CPUfrequency scaling say Y here. + + To compile this driver as a module, choose M here: the module will + be called qcom-cpr + config QCOM_GENI_SE tristate "QCOM GENI Serial Engine Driver" depends on ARCH_QCOM || COMPILE_TEST diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile index 93392d9dc7f7..ad675a6593d0 100644 --- a/drivers/soc/qcom/Makefile +++ b/drivers/soc/qcom/Makefile @@ -3,6 +3,7 @@ CFLAGS_rpmh-rsc.o := -I$(src) obj-$(CONFIG_QCOM_AOSS_QMP) += qcom_aoss.o obj-$(CONFIG_QCOM_GENI_SE) += qcom-geni-se.o obj-$(CONFIG_QCOM_COMMAND_DB) += cmd-db.o +obj-$(CONFIG_QCOM_CPR) += cpr.o obj-$(CONFIG_QCOM_GSBI) += qcom_gsbi.o obj-$(CONFIG_QCOM_MDT_LOADER) += mdt_loader.o obj-$(CONFIG_QCOM_OCMEM) += ocmem.o diff --git a/drivers/soc/qcom/apr.c b/drivers/soc/qcom/apr.c index 1f35b097c635..7abfc8c4fdc7 100644 --- a/drivers/soc/qcom/apr.c +++ b/drivers/soc/qcom/apr.c @@ -328,7 +328,7 @@ static int of_apr_add_pd_lookups(struct device *dev) pds = pdr_add_lookup(apr->pdr, service_name, service_path); if (IS_ERR(pds) && PTR_ERR(pds) != -EALREADY) { - dev_err(dev, "pdr add lookup failed: %d\n", ret); + dev_err(dev, "pdr add lookup failed: %ld\n", PTR_ERR(pds)); return PTR_ERR(pds); } } diff --git a/drivers/power/avs/qcom-cpr.c b/drivers/soc/qcom/cpr.c index b24cc77d1889..b24cc77d1889 100644 --- a/drivers/power/avs/qcom-cpr.c +++ b/drivers/soc/qcom/cpr.c diff --git a/drivers/soc/qcom/llcc-qcom.c b/drivers/soc/qcom/llcc-qcom.c index 429b5a60a1ba..70fbe70c6213 100644 --- a/drivers/soc/qcom/llcc-qcom.c +++ b/drivers/soc/qcom/llcc-qcom.c @@ -387,7 +387,6 @@ static int qcom_llcc_remove(struct platform_device *pdev) static struct regmap *qcom_llcc_init_mmio(struct platform_device *pdev, const char *name) { - struct resource *res; void __iomem *base; struct regmap_config llcc_regmap_config = { .reg_bits = 32, @@ -396,11 +395,7 @@ static struct regmap *qcom_llcc_init_mmio(struct platform_device *pdev, .fast_io = true, }; - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name); - if (!res) - return ERR_PTR(-ENODEV); - - base = devm_ioremap_resource(&pdev->dev, res); + base = devm_platform_ioremap_resource_byname(pdev, name); if (IS_ERR(base)) return ERR_CAST(base); diff --git a/drivers/soc/qcom/pdr_internal.h b/drivers/soc/qcom/pdr_internal.h index 15b5002e4127..ab9ae8cdfa54 100644 --- a/drivers/soc/qcom/pdr_internal.h +++ b/drivers/soc/qcom/pdr_internal.h @@ -185,7 +185,7 @@ struct qmi_elem_info servreg_get_domain_list_resp_ei[] = { .data_type = QMI_STRUCT, .elem_len = SERVREG_DOMAIN_LIST_LENGTH, .elem_size = sizeof(struct servreg_location_entry), - .array_type = NO_ARRAY, + .array_type = VAR_LEN_ARRAY, .tlv_type = 0x12, .offset = offsetof(struct servreg_get_domain_list_resp, domain_list), diff --git a/drivers/soc/qcom/rpmh-internal.h b/drivers/soc/qcom/rpmh-internal.h index ef60e790a750..344ba687c13b 100644 --- a/drivers/soc/qcom/rpmh-internal.h +++ b/drivers/soc/qcom/rpmh-internal.h @@ -8,6 +8,7 @@ #define __RPM_INTERNAL_H__ #include <linux/bitmap.h> +#include <linux/wait.h> #include <soc/qcom/tcs.h> #define TCS_TYPE_NR 4 @@ -106,6 +107,8 @@ struct rpmh_ctrlr { * @lock: Synchronize state of the controller. If RPMH's cache * lock will also be held, the order is: drv->lock then * cache_lock. + * @tcs_wait: Wait queue used to wait for @tcs_in_use to free up a + * slot * @client: Handle to the DRV's client. */ struct rsc_drv { @@ -118,6 +121,7 @@ struct rsc_drv { struct tcs_group tcs[TCS_TYPE_NR]; DECLARE_BITMAP(tcs_in_use, MAX_TCS_NR); spinlock_t lock; + wait_queue_head_t tcs_wait; struct rpmh_ctrlr client; }; diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c index ae6675782581..a297911afe57 100644 --- a/drivers/soc/qcom/rpmh-rsc.c +++ b/drivers/soc/qcom/rpmh-rsc.c @@ -19,6 +19,7 @@ #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/spinlock.h> +#include <linux/wait.h> #include <soc/qcom/cmd-db.h> #include <soc/qcom/tcs.h> @@ -453,6 +454,7 @@ skip: if (!drv->tcs[ACTIVE_TCS].num_tcs) enable_tcs_irq(drv, i, false); spin_unlock(&drv->lock); + wake_up(&drv->tcs_wait); if (req) rpmh_tx_done(req, err); } @@ -571,73 +573,34 @@ static int find_free_tcs(struct tcs_group *tcs) } /** - * tcs_write() - Store messages into a TCS right now, or return -EBUSY. + * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active. * @drv: The controller. + * @tcs: The tcs_group used for ACTIVE_ONLY transfers. * @msg: The data to be sent. * - * Grabs a TCS for ACTIVE_ONLY transfers and writes the messages to it. + * Claims a tcs in the given tcs_group while making sure that no existing cmd + * is in flight that would conflict with the one in @msg. * - * If there are no free TCSes for ACTIVE_ONLY transfers or if a command for - * the same address is already transferring returns -EBUSY which means the - * client should retry shortly. + * Context: Must be called with the drv->lock held since that protects + * tcs_in_use. * - * Return: 0 on success, -EBUSY if client should retry, or an error. - * Client should have interrupts enabled for a bit before retrying. + * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight + * or the tcs_group is full. */ -static int tcs_write(struct rsc_drv *drv, const struct tcs_request *msg) +static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs, + const struct tcs_request *msg) { - struct tcs_group *tcs; - int tcs_id; - unsigned long flags; int ret; - tcs = get_tcs_for_msg(drv, msg); - if (IS_ERR(tcs)) - return PTR_ERR(tcs); - - spin_lock_irqsave(&drv->lock, flags); /* * The h/w does not like if we send a request to the same address, * when one is already in-flight or being processed. */ ret = check_for_req_inflight(drv, tcs, msg); if (ret) - goto unlock; - - ret = find_free_tcs(tcs); - if (ret < 0) - goto unlock; - tcs_id = ret; - - tcs->req[tcs_id - tcs->offset] = msg; - set_bit(tcs_id, drv->tcs_in_use); - if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) { - /* - * Clear previously programmed WAKE commands in selected - * repurposed TCS to avoid triggering them. tcs->slots will be - * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate() - */ - write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0); - write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, 0); - enable_tcs_irq(drv, tcs_id, true); - } - spin_unlock_irqrestore(&drv->lock, flags); - - /* - * These two can be done after the lock is released because: - * - We marked "tcs_in_use" under lock. - * - Once "tcs_in_use" has been marked nobody else could be writing - * to these registers until the interrupt goes off. - * - The interrupt can't go off until we trigger w/ the last line - * of __tcs_set_trigger() below. - */ - __tcs_buffer_write(drv, tcs_id, 0, msg); - __tcs_set_trigger(drv, tcs_id, true); + return ret; - return 0; -unlock: - spin_unlock_irqrestore(&drv->lock, flags); - return ret; + return find_free_tcs(tcs); } /** @@ -664,18 +627,47 @@ unlock: */ int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) { - int ret; + struct tcs_group *tcs; + int tcs_id; + unsigned long flags; - do { - ret = tcs_write(drv, msg); - if (ret == -EBUSY) { - pr_info_ratelimited("TCS Busy, retrying RPMH message send: addr=%#x\n", - msg->cmds[0].addr); - udelay(10); - } - } while (ret == -EBUSY); + tcs = get_tcs_for_msg(drv, msg); + if (IS_ERR(tcs)) + return PTR_ERR(tcs); - return ret; + spin_lock_irqsave(&drv->lock, flags); + + /* Wait forever for a free tcs. It better be there eventually! */ + wait_event_lock_irq(drv->tcs_wait, + (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0, + drv->lock); + + tcs->req[tcs_id - tcs->offset] = msg; + set_bit(tcs_id, drv->tcs_in_use); + if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) { + /* + * Clear previously programmed WAKE commands in selected + * repurposed TCS to avoid triggering them. tcs->slots will be + * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate() + */ + write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0); + write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, 0); + enable_tcs_irq(drv, tcs_id, true); + } + spin_unlock_irqrestore(&drv->lock, flags); + + /* + * These two can be done after the lock is released because: + * - We marked "tcs_in_use" under lock. + * - Once "tcs_in_use" has been marked nobody else could be writing + * to these registers until the interrupt goes off. + * - The interrupt can't go off until we trigger w/ the last line + * of __tcs_set_trigger() below. + */ + __tcs_buffer_write(drv, tcs_id, 0, msg); + __tcs_set_trigger(drv, tcs_id, true); + + return 0; } /** @@ -983,6 +975,7 @@ static int rpmh_rsc_probe(struct platform_device *pdev) return ret; spin_lock_init(&drv->lock); + init_waitqueue_head(&drv->tcs_wait); bitmap_zero(drv->tcs_in_use, MAX_TCS_NR); irq = platform_get_irq(pdev, drv->id); diff --git a/drivers/soc/qcom/socinfo.c b/drivers/soc/qcom/socinfo.c index b25d0f7dac9e..b44ede48decc 100644 --- a/drivers/soc/qcom/socinfo.c +++ b/drivers/soc/qcom/socinfo.c @@ -194,6 +194,7 @@ static const struct soc_id soc_id[] = { { 186, "MSM8674" }, { 194, "MSM8974PRO" }, { 206, "MSM8916" }, + { 207, "MSM8994" }, { 208, "APQ8074-AA" }, { 209, "APQ8074-AB" }, { 210, "APQ8074PRO" }, @@ -214,6 +215,8 @@ static const struct soc_id soc_id[] = { { 248, "MSM8216" }, { 249, "MSM8116" }, { 250, "MSM8616" }, + { 251, "MSM8992" }, + { 253, "APQ8094" }, { 291, "APQ8096" }, { 305, "MSM8996SG" }, { 310, "MSM8996AU" }, @@ -223,6 +226,8 @@ static const struct soc_id soc_id[] = { { 321, "SDM845" }, { 341, "SDA845" }, { 356, "SM8250" }, + { 402, "IPQ6018" }, + { 425, "SC7180" }, }; static const char *socinfo_machine(struct device *dev, unsigned int id) diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig index 30984659df90..b70bbc38efc6 100644 --- a/drivers/soc/renesas/Kconfig +++ b/drivers/soc/renesas/Kconfig @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -config SOC_RENESAS +menuconfig SOC_RENESAS bool "Renesas SoC driver support" if COMPILE_TEST && !ARCH_RENESAS default y if ARCH_RENESAS select SOC_BUS @@ -49,126 +49,126 @@ if ARM && ARCH_RENESAS #comment "Renesas ARM SoCs System Type" config ARCH_EMEV2 - bool "Emma Mobile EV2" + bool "ARM32 Platform support for Emma Mobile EV2" select HAVE_ARM_SCU if SMP select SYS_SUPPORTS_EM_STI -config ARCH_R7S72100 - bool "RZ/A1H (R7S72100)" - select ARM_ERRATA_754322 - select PM - select PM_GENERIC_DOMAINS - select RENESAS_OSTM - select RENESAS_RZA1_IRQC - select SYS_SUPPORTS_SH_MTU2 +config ARCH_R8A7794 + bool "ARM32 Platform support for R-Car E2" + select ARCH_RCAR_GEN2 + select ARM_ERRATA_814220 + select SYSC_R8A7794 -config ARCH_R7S9210 - bool "RZ/A2 (R7S9210)" - select PM - select PM_GENERIC_DOMAINS - select RENESAS_OSTM - select RENESAS_RZA1_IRQC +config ARCH_R8A7779 + bool "ARM32 Platform support for R-Car H1" + select ARCH_RCAR_GEN1 + select ARM_ERRATA_754322 + select ARM_GLOBAL_TIMER + select HAVE_ARM_SCU if SMP + select HAVE_ARM_TWD if SMP + select SYSC_R8A7779 -config ARCH_R8A73A4 - bool "R-Mobile APE6 (R8A73A40)" - select ARCH_RMOBILE +config ARCH_R8A7790 + bool "ARM32 Platform support for R-Car H2" + select ARCH_RCAR_GEN2 select ARM_ERRATA_798181 if SMP select ARM_ERRATA_814220 - select HAVE_ARM_ARCH_TIMER - select RENESAS_IRQC + select I2C + select SYSC_R8A7790 -config ARCH_R8A7740 - bool "R-Mobile A1 (R8A77400)" - select ARCH_RMOBILE +config ARCH_R8A7778 + bool "ARM32 Platform support for R-Car M1A" + select ARCH_RCAR_GEN1 select ARM_ERRATA_754322 - select RENESAS_INTC_IRQPIN -config ARCH_R8A7742 - bool "RZ/G1H (R8A77420)" +config ARCH_R8A7793 + bool "ARM32 Platform support for R-Car M2-N" select ARCH_RCAR_GEN2 select ARM_ERRATA_798181 if SMP - select ARM_ERRATA_814220 - select SYSC_R8A7742 + select I2C + select SYSC_R8A7791 -config ARCH_R8A7743 - bool "RZ/G1M (R8A77430)" +config ARCH_R8A7791 + bool "ARM32 Platform support for R-Car M2-W" select ARCH_RCAR_GEN2 select ARM_ERRATA_798181 if SMP - select SYSC_R8A7743 + select I2C + select SYSC_R8A7791 -config ARCH_R8A7744 - bool "RZ/G1N (R8A77440)" +config ARCH_R8A7792 + bool "ARM32 Platform support for R-Car V2H" select ARCH_RCAR_GEN2 select ARM_ERRATA_798181 if SMP - select SYSC_R8A7743 + select SYSC_R8A7792 -config ARCH_R8A7745 - bool "RZ/G1E (R8A77450)" - select ARCH_RCAR_GEN2 - select ARM_ERRATA_814220 - select SYSC_R8A7745 +config ARCH_R8A7740 + bool "ARM32 Platform support for R-Mobile A1" + select ARCH_RMOBILE + select ARM_ERRATA_754322 + select RENESAS_INTC_IRQPIN -config ARCH_R8A77470 - bool "RZ/G1C (R8A77470)" - select ARCH_RCAR_GEN2 +config ARCH_R8A73A4 + bool "ARM32 Platform support for R-Mobile APE6" + select ARCH_RMOBILE + select ARM_ERRATA_798181 if SMP select ARM_ERRATA_814220 - select SYSC_R8A77470 + select HAVE_ARM_ARCH_TIMER + select RENESAS_IRQC -config ARCH_R8A7778 - bool "R-Car M1A (R8A77781)" - select ARCH_RCAR_GEN1 +config ARCH_R7S72100 + bool "ARM32 Platform support for RZ/A1H" select ARM_ERRATA_754322 + select PM + select PM_GENERIC_DOMAINS + select RENESAS_OSTM + select RENESAS_RZA1_IRQC + select SYS_SUPPORTS_SH_MTU2 -config ARCH_R8A7779 - bool "R-Car H1 (R8A77790)" - select ARCH_RCAR_GEN1 - select ARM_ERRATA_754322 - select ARM_GLOBAL_TIMER - select HAVE_ARM_SCU if SMP - select HAVE_ARM_TWD if SMP - select SYSC_R8A7779 +config ARCH_R7S9210 + bool "ARM32 Platform support for RZ/A2" + select PM + select PM_GENERIC_DOMAINS + select RENESAS_OSTM + select RENESAS_RZA1_IRQC -config ARCH_R8A7790 - bool "R-Car H2 (R8A77900)" +config ARCH_R8A77470 + bool "ARM32 Platform support for RZ/G1C" select ARCH_RCAR_GEN2 - select ARM_ERRATA_798181 if SMP select ARM_ERRATA_814220 - select I2C - select SYSC_R8A7790 + select SYSC_R8A77470 -config ARCH_R8A7791 - bool "R-Car M2-W (R8A77910)" +config ARCH_R8A7745 + bool "ARM32 Platform support for RZ/G1E" select ARCH_RCAR_GEN2 - select ARM_ERRATA_798181 if SMP - select I2C - select SYSC_R8A7791 + select ARM_ERRATA_814220 + select SYSC_R8A7745 -config ARCH_R8A7792 - bool "R-Car V2H (R8A77920)" +config ARCH_R8A7742 + bool "ARM32 Platform support for RZ/G1H" select ARCH_RCAR_GEN2 select ARM_ERRATA_798181 if SMP - select SYSC_R8A7792 + select ARM_ERRATA_814220 + select SYSC_R8A7742 -config ARCH_R8A7793 - bool "R-Car M2-N (R8A7793)" +config ARCH_R8A7743 + bool "ARM32 Platform support for RZ/G1M" select ARCH_RCAR_GEN2 select ARM_ERRATA_798181 if SMP - select I2C - select SYSC_R8A7791 + select SYSC_R8A7743 -config ARCH_R8A7794 - bool "R-Car E2 (R8A77940)" +config ARCH_R8A7744 + bool "ARM32 Platform support for RZ/G1N" select ARCH_RCAR_GEN2 - select ARM_ERRATA_814220 - select SYSC_R8A7794 + select ARM_ERRATA_798181 if SMP + select SYSC_R8A7743 config ARCH_R9A06G032 - bool "RZ/N1D (R9A06G032)" + bool "ARM32 Platform support for RZ/N1D" select ARCH_RZN1 select ARM_ERRATA_814220 config ARCH_SH73A0 - bool "SH-Mobile AG5 (R8A73A00)" + bool "ARM32 Platform support for SH-Mobile AG5" select ARCH_RMOBILE select ARM_ERRATA_754322 select ARM_GLOBAL_TIMER @@ -180,193 +180,201 @@ endif # ARM if ARM64 -config ARCH_R8A774A1 - bool "Renesas RZ/G2M SoC Platform" - select ARCH_RCAR_GEN3 - select SYSC_R8A774A1 - help - This enables support for the Renesas RZ/G2M SoC. - -config ARCH_R8A774B1 - bool "Renesas RZ/G2N SoC Platform" - select ARCH_RCAR_GEN3 - select SYSC_R8A774B1 - help - This enables support for the Renesas RZ/G2N SoC. - -config ARCH_R8A774C0 - bool "Renesas RZ/G2E SoC Platform" +config ARCH_R8A77995 + bool "ARM64 Platform support for R-Car D3" select ARCH_RCAR_GEN3 - select SYSC_R8A774C0 + select SYSC_R8A77995 help - This enables support for the Renesas RZ/G2E SoC. + This enables support for the Renesas R-Car D3 SoC. -config ARCH_R8A774E1 - bool "Renesas RZ/G2H SoC Platform" +config ARCH_R8A77990 + bool "ARM64 Platform support for R-Car E3" select ARCH_RCAR_GEN3 - select SYSC_R8A774E1 + select SYSC_R8A77990 help - This enables support for the Renesas RZ/G2H SoC. + This enables support for the Renesas R-Car E3 SoC. config ARCH_R8A77950 - bool "Renesas R-Car H3 ES1.x SoC Platform" + bool "ARM64 Platform support for R-Car H3 ES1.x" select ARCH_RCAR_GEN3 select SYSC_R8A7795 help This enables support for the Renesas R-Car H3 SoC (revision 1.x). config ARCH_R8A77951 - bool "Renesas R-Car H3 ES2.0+ SoC Platform" + bool "ARM64 Platform support for R-Car H3 ES2.0+" select ARCH_RCAR_GEN3 select SYSC_R8A7795 help This enables support for the Renesas R-Car H3 SoC (revisions 2.0 and later). +config ARCH_R8A77965 + bool "ARM64 Platform support for R-Car M3-N" + select ARCH_RCAR_GEN3 + select SYSC_R8A77965 + help + This enables support for the Renesas R-Car M3-N SoC. + config ARCH_R8A77960 - bool "Renesas R-Car M3-W SoC Platform" + bool "ARM64 Platform support for R-Car M3-W" select ARCH_RCAR_GEN3 select SYSC_R8A77960 help This enables support for the Renesas R-Car M3-W SoC. config ARCH_R8A77961 - bool "Renesas R-Car M3-W+ SoC Platform" + bool "ARM64 Platform support for R-Car M3-W+" select ARCH_RCAR_GEN3 select SYSC_R8A77961 help This enables support for the Renesas R-Car M3-W+ SoC. -config ARCH_R8A77965 - bool "Renesas R-Car M3-N SoC Platform" +config ARCH_R8A77980 + bool "ARM64 Platform support for R-Car V3H" select ARCH_RCAR_GEN3 - select SYSC_R8A77965 + select SYSC_R8A77980 help - This enables support for the Renesas R-Car M3-N SoC. + This enables support for the Renesas R-Car V3H SoC. config ARCH_R8A77970 - bool "Renesas R-Car V3M SoC Platform" + bool "ARM64 Platform support for R-Car V3M" select ARCH_RCAR_GEN3 select SYSC_R8A77970 help This enables support for the Renesas R-Car V3M SoC. -config ARCH_R8A77980 - bool "Renesas R-Car V3H SoC Platform" +config ARCH_R8A779A0 + bool "ARM64 Platform support for R-Car V3U" select ARCH_RCAR_GEN3 - select SYSC_R8A77980 + select SYSC_R8A779A0 help - This enables support for the Renesas R-Car V3H SoC. + This enables support for the Renesas R-Car V3U SoC. -config ARCH_R8A77990 - bool "Renesas R-Car E3 SoC Platform" +config ARCH_R8A774C0 + bool "ARM64 Platform support for RZ/G2E" select ARCH_RCAR_GEN3 - select SYSC_R8A77990 + select SYSC_R8A774C0 help - This enables support for the Renesas R-Car E3 SoC. + This enables support for the Renesas RZ/G2E SoC. -config ARCH_R8A77995 - bool "Renesas R-Car D3 SoC Platform" +config ARCH_R8A774E1 + bool "ARM64 Platform support for RZ/G2H" select ARCH_RCAR_GEN3 - select SYSC_R8A77995 + select SYSC_R8A774E1 help - This enables support for the Renesas R-Car D3 SoC. + This enables support for the Renesas RZ/G2H SoC. + +config ARCH_R8A774A1 + bool "ARM64 Platform support for RZ/G2M" + select ARCH_RCAR_GEN3 + select SYSC_R8A774A1 + help + This enables support for the Renesas RZ/G2M SoC. + +config ARCH_R8A774B1 + bool "ARM64 Platform support for RZ/G2N" + select ARCH_RCAR_GEN3 + select SYSC_R8A774B1 + help + This enables support for the Renesas RZ/G2N SoC. endif # ARM64 -# SoC -config SYSC_R8A7742 - bool "RZ/G1H System Controller support" if COMPILE_TEST - select SYSC_RCAR +config RST_RCAR + bool "Reset Controller support for R-Car" if COMPILE_TEST -config SYSC_R8A7743 - bool "RZ/G1M System Controller support" if COMPILE_TEST +config SYSC_RCAR + bool "System Controller support for R-Car" if COMPILE_TEST + +config SYSC_R8A77995 + bool "System Controller support for R-Car D3" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A7745 - bool "RZ/G1E System Controller support" if COMPILE_TEST +config SYSC_R8A7794 + bool "System Controller support for R-Car E2" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A77470 - bool "RZ/G1C System Controller support" if COMPILE_TEST +config SYSC_R8A77990 + bool "System Controller support for R-Car E3" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A774A1 - bool "RZ/G2M System Controller support" if COMPILE_TEST +config SYSC_R8A7779 + bool "System Controller support for R-Car H1" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A774B1 - bool "RZ/G2N System Controller support" if COMPILE_TEST +config SYSC_R8A7790 + bool "System Controller support for R-Car H2" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A774C0 - bool "RZ/G2E System Controller support" if COMPILE_TEST +config SYSC_R8A7795 + bool "System Controller support for R-Car H3" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A774E1 - bool "RZ/G2H System Controller support" if COMPILE_TEST +config SYSC_R8A7791 + bool "System Controller support for R-Car M2-W/N" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A7779 - bool "R-Car H1 System Controller support" if COMPILE_TEST +config SYSC_R8A77965 + bool "System Controller support for R-Car M3-N" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A7790 - bool "R-Car H2 System Controller support" if COMPILE_TEST +config SYSC_R8A77960 + bool "System Controller support for R-Car M3-W" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A7791 - bool "R-Car M2-W/N System Controller support" if COMPILE_TEST +config SYSC_R8A77961 + bool "System Controller support for R-Car M3-W+" if COMPILE_TEST select SYSC_RCAR config SYSC_R8A7792 - bool "R-Car V2H System Controller support" if COMPILE_TEST + bool "System Controller support for R-Car V2H" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A7794 - bool "R-Car E2 System Controller support" if COMPILE_TEST +config SYSC_R8A77980 + bool "System Controller support for R-Car V3H" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A7795 - bool "R-Car H3 System Controller support" if COMPILE_TEST +config SYSC_R8A77970 + bool "System Controller support for R-Car V3M" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A77960 - bool "R-Car M3-W System Controller support" if COMPILE_TEST - select SYSC_RCAR +config SYSC_R8A779A0 + bool "System Controller support for R-Car V3U" if COMPILE_TEST -config SYSC_R8A77961 - bool "R-Car M3-W+ System Controller support" if COMPILE_TEST - select SYSC_RCAR +config SYSC_RMOBILE + bool "System Controller support for R-Mobile" if COMPILE_TEST -config SYSC_R8A77965 - bool "R-Car M3-N System Controller support" if COMPILE_TEST +config SYSC_R8A77470 + bool "System Controller support for RZ/G1C" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A77970 - bool "R-Car V3M System Controller support" if COMPILE_TEST +config SYSC_R8A7745 + bool "System Controller support for RZ/G1E" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A77980 - bool "R-Car V3H System Controller support" if COMPILE_TEST +config SYSC_R8A7742 + bool "System Controller support for RZ/G1H" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A77990 - bool "R-Car E3 System Controller support" if COMPILE_TEST +config SYSC_R8A7743 + bool "System Controller support for RZ/G1M" if COMPILE_TEST select SYSC_RCAR -config SYSC_R8A77995 - bool "R-Car D3 System Controller support" if COMPILE_TEST +config SYSC_R8A774C0 + bool "System Controller support for RZ/G2E" if COMPILE_TEST select SYSC_RCAR -# Family -config RST_RCAR - bool "R-Car Reset Controller support" if COMPILE_TEST +config SYSC_R8A774E1 + bool "System Controller support for RZ/G2H" if COMPILE_TEST + select SYSC_RCAR -config SYSC_RCAR - bool "R-Car System Controller support" if COMPILE_TEST +config SYSC_R8A774A1 + bool "System Controller support for RZ/G2M" if COMPILE_TEST + select SYSC_RCAR -config SYSC_RMOBILE - bool "R-Mobile System Controller support" if COMPILE_TEST +config SYSC_R8A774B1 + bool "System Controller support for RZ/G2N" if COMPILE_TEST + select SYSC_RCAR endif # SOC_RENESAS diff --git a/drivers/soc/renesas/Makefile b/drivers/soc/renesas/Makefile index 10a399fc486a..9b29bed2a597 100644 --- a/drivers/soc/renesas/Makefile +++ b/drivers/soc/renesas/Makefile @@ -24,6 +24,7 @@ obj-$(CONFIG_SYSC_R8A77970) += r8a77970-sysc.o obj-$(CONFIG_SYSC_R8A77980) += r8a77980-sysc.o obj-$(CONFIG_SYSC_R8A77990) += r8a77990-sysc.o obj-$(CONFIG_SYSC_R8A77995) += r8a77995-sysc.o +obj-$(CONFIG_SYSC_R8A779A0) += r8a779a0-sysc.o ifdef CONFIG_SMP obj-$(CONFIG_ARCH_R9A06G032) += r9a06g032-smp.o endif diff --git a/drivers/soc/renesas/r8a779a0-sysc.c b/drivers/soc/renesas/r8a779a0-sysc.c new file mode 100644 index 000000000000..d464ffa1be33 --- /dev/null +++ b/drivers/soc/renesas/r8a779a0-sysc.c @@ -0,0 +1,448 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Renesas R-Car V3U System Controller + * + * Copyright (C) 2020 Renesas Electronics Corp. + */ + +#include <linux/bits.h> +#include <linux/clk/renesas.h> +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/of_address.h> +#include <linux/pm_domain.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/types.h> + +#include <dt-bindings/power/r8a779a0-sysc.h> + +/* + * Power Domain flags + */ +#define PD_CPU BIT(0) /* Area contains main CPU core */ +#define PD_SCU BIT(1) /* Area contains SCU and L2 cache */ +#define PD_NO_CR BIT(2) /* Area lacks PWR{ON,OFF}CR registers */ + +#define PD_CPU_NOCR PD_CPU | PD_NO_CR /* CPU area lacks CR */ +#define PD_ALWAYS_ON PD_NO_CR /* Always-on area */ + +/* + * Description of a Power Area + */ +struct r8a779a0_sysc_area { + const char *name; + u8 pdr; /* PDRn */ + int parent; /* -1 if none */ + unsigned int flags; /* See PD_* */ +}; + +/* + * SoC-specific Power Area Description + */ +struct r8a779a0_sysc_info { + const struct r8a779a0_sysc_area *areas; + unsigned int num_areas; +}; + +static struct r8a779a0_sysc_area r8a779a0_areas[] __initdata = { + { "always-on", R8A779A0_PD_ALWAYS_ON, -1, PD_ALWAYS_ON }, + { "a3e0", R8A779A0_PD_A3E0, R8A779A0_PD_ALWAYS_ON, PD_SCU }, + { "a3e1", R8A779A0_PD_A3E1, R8A779A0_PD_ALWAYS_ON, PD_SCU }, + { "a2e0d0", R8A779A0_PD_A2E0D0, R8A779A0_PD_A3E0, PD_SCU }, + { "a2e0d1", R8A779A0_PD_A2E0D1, R8A779A0_PD_A3E0, PD_SCU }, + { "a2e1d0", R8A779A0_PD_A2E1D0, R8A779A0_PD_A3E1, PD_SCU }, + { "a2e1d1", R8A779A0_PD_A2E1D1, R8A779A0_PD_A3E1, PD_SCU }, + { "a1e0d0c0", R8A779A0_PD_A1E0D0C0, R8A779A0_PD_A2E0D0, PD_CPU_NOCR }, + { "a1e0d0c1", R8A779A0_PD_A1E0D0C1, R8A779A0_PD_A2E0D0, PD_CPU_NOCR }, + { "a1e0d1c0", R8A779A0_PD_A1E0D1C0, R8A779A0_PD_A2E0D1, PD_CPU_NOCR }, + { "a1e0d1c1", R8A779A0_PD_A1E0D1C1, R8A779A0_PD_A2E0D1, PD_CPU_NOCR }, + { "a1e1d0c0", R8A779A0_PD_A1E1D0C0, R8A779A0_PD_A2E1D0, PD_CPU_NOCR }, + { "a1e1d0c1", R8A779A0_PD_A1E1D0C1, R8A779A0_PD_A2E1D0, PD_CPU_NOCR }, + { "a1e1d1c0", R8A779A0_PD_A1E1D1C0, R8A779A0_PD_A2E1D1, PD_CPU_NOCR }, + { "a1e1d1c1", R8A779A0_PD_A1E1D1C1, R8A779A0_PD_A2E1D1, PD_CPU_NOCR }, + { "3dg-a", R8A779A0_PD_3DG_A, R8A779A0_PD_ALWAYS_ON }, + { "3dg-b", R8A779A0_PD_3DG_B, R8A779A0_PD_3DG_A }, + { "a3vip0", R8A779A0_PD_A3VIP0, R8A779A0_PD_ALWAYS_ON }, + { "a3vip1", R8A779A0_PD_A3VIP1, R8A779A0_PD_ALWAYS_ON }, + { "a3vip3", R8A779A0_PD_A3VIP3, R8A779A0_PD_ALWAYS_ON }, + { "a3vip2", R8A779A0_PD_A3VIP2, R8A779A0_PD_ALWAYS_ON }, + { "a3isp01", R8A779A0_PD_A3ISP01, R8A779A0_PD_ALWAYS_ON }, + { "a3isp23", R8A779A0_PD_A3ISP23, R8A779A0_PD_ALWAYS_ON }, + { "a3ir", R8A779A0_PD_A3IR, R8A779A0_PD_ALWAYS_ON }, + { "a2cn0", R8A779A0_PD_A2CN0, R8A779A0_PD_A3IR }, + { "a2imp01", R8A779A0_PD_A2IMP01, R8A779A0_PD_A3IR }, + { "a2dp0", R8A779A0_PD_A2DP0, R8A779A0_PD_A3IR }, + { "a2cv0", R8A779A0_PD_A2CV0, R8A779A0_PD_A3IR }, + { "a2cv1", R8A779A0_PD_A2CV1, R8A779A0_PD_A3IR }, + { "a2cv4", R8A779A0_PD_A2CV4, R8A779A0_PD_A3IR }, + { "a2cv6", R8A779A0_PD_A2CV6, R8A779A0_PD_A3IR }, + { "a2cn2", R8A779A0_PD_A2CN2, R8A779A0_PD_A3IR }, + { "a2imp23", R8A779A0_PD_A2IMP23, R8A779A0_PD_A3IR }, + { "a2dp1", R8A779A0_PD_A2DP0, R8A779A0_PD_A3IR }, + { "a2cv2", R8A779A0_PD_A2CV0, R8A779A0_PD_A3IR }, + { "a2cv3", R8A779A0_PD_A2CV1, R8A779A0_PD_A3IR }, + { "a2cv5", R8A779A0_PD_A2CV4, R8A779A0_PD_A3IR }, + { "a2cv7", R8A779A0_PD_A2CV6, R8A779A0_PD_A3IR }, + { "a2cn1", R8A779A0_PD_A2CN1, R8A779A0_PD_A3IR }, + { "a1cnn0", R8A779A0_PD_A1CNN0, R8A779A0_PD_A2CN0 }, + { "a1cnn2", R8A779A0_PD_A1CNN2, R8A779A0_PD_A2CN2 }, + { "a1dsp0", R8A779A0_PD_A1DSP0, R8A779A0_PD_A2CN2 }, + { "a1cnn1", R8A779A0_PD_A1CNN1, R8A779A0_PD_A2CN1 }, + { "a1dsp1", R8A779A0_PD_A1DSP1, R8A779A0_PD_A2CN1 }, +}; + +static const struct r8a779a0_sysc_info r8a779a0_sysc_info __initconst = { + .areas = r8a779a0_areas, + .num_areas = ARRAY_SIZE(r8a779a0_areas), +}; + +/* SYSC Common */ +#define SYSCSR 0x000 /* SYSC Status Register */ +#define SYSCPONSR(x) (0x800 + ((x) * 0x4)) /* Power-ON Status Register 0 */ +#define SYSCPOFFSR(x) (0x808 + ((x) * 0x4)) /* Power-OFF Status Register */ +#define SYSCISCR(x) (0x810 + ((x) * 0x4)) /* Interrupt Status/Clear Register */ +#define SYSCIER(x) (0x820 + ((x) * 0x4)) /* Interrupt Enable Register */ +#define SYSCIMR(x) (0x830 + ((x) * 0x4)) /* Interrupt Mask Register */ + +/* Power Domain Registers */ +#define PDRSR(n) (0x1000 + ((n) * 0x40)) +#define PDRONCR(n) (0x1004 + ((n) * 0x40)) +#define PDROFFCR(n) (0x1008 + ((n) * 0x40)) +#define PDRESR(n) (0x100C + ((n) * 0x40)) + +/* PWRON/PWROFF */ +#define PWRON_PWROFF BIT(0) /* Power-ON/OFF request */ + +/* PDRESR */ +#define PDRESR_ERR BIT(0) + +/* PDRSR */ +#define PDRSR_OFF BIT(0) /* Power-OFF state */ +#define PDRSR_ON BIT(4) /* Power-ON state */ +#define PDRSR_OFF_STATE BIT(8) /* Processing Power-OFF sequence */ +#define PDRSR_ON_STATE BIT(12) /* Processing Power-ON sequence */ + +#define SYSCSR_BUSY GENMASK(1, 0) /* All bit sets is not busy */ + +#define SYSCSR_TIMEOUT 10000 +#define SYSCSR_DELAY_US 10 + +#define PDRESR_RETRIES 1000 +#define PDRESR_DELAY_US 10 + +#define SYSCISR_TIMEOUT 10000 +#define SYSCISR_DELAY_US 10 + +#define NUM_DOMAINS_EACH_REG BITS_PER_TYPE(u32) + +static void __iomem *r8a779a0_sysc_base; +static DEFINE_SPINLOCK(r8a779a0_sysc_lock); /* SMP CPUs + I/O devices */ + +static int r8a779a0_sysc_pwr_on_off(u8 pdr, bool on) +{ + unsigned int reg_offs; + u32 val; + int ret; + + if (on) + reg_offs = PDRONCR(pdr); + else + reg_offs = PDROFFCR(pdr); + + /* Wait until SYSC is ready to accept a power request */ + ret = readl_poll_timeout_atomic(r8a779a0_sysc_base + SYSCSR, val, + (val & SYSCSR_BUSY) == SYSCSR_BUSY, + SYSCSR_DELAY_US, SYSCSR_TIMEOUT); + if (ret < 0) + return -EAGAIN; + + /* Submit power shutoff or power resume request */ + iowrite32(PWRON_PWROFF, r8a779a0_sysc_base + reg_offs); + + return 0; +} + +static int clear_irq_flags(unsigned int reg_idx, unsigned int isr_mask) +{ + u32 val; + int ret; + + iowrite32(isr_mask, r8a779a0_sysc_base + SYSCISCR(reg_idx)); + + ret = readl_poll_timeout_atomic(r8a779a0_sysc_base + SYSCISCR(reg_idx), + val, !(val & isr_mask), + SYSCISR_DELAY_US, SYSCISR_TIMEOUT); + if (ret < 0) { + pr_err("\n %s : Can not clear IRQ flags in SYSCISCR", __func__); + return -EIO; + } + + return 0; +} + +static int r8a779a0_sysc_power(u8 pdr, bool on) +{ + unsigned int isr_mask; + unsigned int reg_idx, bit_idx; + unsigned int status; + unsigned long flags; + int ret = 0; + u32 val; + int k; + + spin_lock_irqsave(&r8a779a0_sysc_lock, flags); + + reg_idx = pdr / NUM_DOMAINS_EACH_REG; + bit_idx = pdr % NUM_DOMAINS_EACH_REG; + + isr_mask = BIT(bit_idx); + + /* + * The interrupt source needs to be enabled, but masked, to prevent the + * CPU from receiving it. + */ + iowrite32(ioread32(r8a779a0_sysc_base + SYSCIER(reg_idx)) | isr_mask, + r8a779a0_sysc_base + SYSCIER(reg_idx)); + iowrite32(ioread32(r8a779a0_sysc_base + SYSCIMR(reg_idx)) | isr_mask, + r8a779a0_sysc_base + SYSCIMR(reg_idx)); + + ret = clear_irq_flags(reg_idx, isr_mask); + if (ret) + goto out; + + /* Submit power shutoff or resume request until it was accepted */ + for (k = 0; k < PDRESR_RETRIES; k++) { + ret = r8a779a0_sysc_pwr_on_off(pdr, on); + if (ret) + goto out; + + status = ioread32(r8a779a0_sysc_base + PDRESR(pdr)); + if (!(status & PDRESR_ERR)) + break; + + udelay(PDRESR_DELAY_US); + } + + if (k == PDRESR_RETRIES) { + ret = -EIO; + goto out; + } + + /* Wait until the power shutoff or resume request has completed * */ + ret = readl_poll_timeout_atomic(r8a779a0_sysc_base + SYSCISCR(reg_idx), + val, (val & isr_mask), + SYSCISR_DELAY_US, SYSCISR_TIMEOUT); + if (ret < 0) { + ret = -EIO; + goto out; + } + + /* Clear interrupt flags */ + ret = clear_irq_flags(reg_idx, isr_mask); + if (ret) + goto out; + + out: + spin_unlock_irqrestore(&r8a779a0_sysc_lock, flags); + + pr_debug("sysc power %s domain %d: %08x -> %d\n", on ? "on" : "off", + pdr, ioread32(r8a779a0_sysc_base + SYSCISCR(reg_idx)), ret); + return ret; +} + +static bool r8a779a0_sysc_power_is_off(u8 pdr) +{ + unsigned int st; + + st = ioread32(r8a779a0_sysc_base + PDRSR(pdr)); + + if (st & PDRSR_OFF) + return true; + + return false; +} + +struct r8a779a0_sysc_pd { + struct generic_pm_domain genpd; + u8 pdr; + unsigned int flags; + char name[]; +}; + +static inline struct r8a779a0_sysc_pd *to_r8a779a0_pd(struct generic_pm_domain *d) +{ + return container_of(d, struct r8a779a0_sysc_pd, genpd); +} + +static int r8a779a0_sysc_pd_power_off(struct generic_pm_domain *genpd) +{ + struct r8a779a0_sysc_pd *pd = to_r8a779a0_pd(genpd); + + pr_debug("%s: %s\n", __func__, genpd->name); + return r8a779a0_sysc_power(pd->pdr, false); +} + +static int r8a779a0_sysc_pd_power_on(struct generic_pm_domain *genpd) +{ + struct r8a779a0_sysc_pd *pd = to_r8a779a0_pd(genpd); + + pr_debug("%s: %s\n", __func__, genpd->name); + return r8a779a0_sysc_power(pd->pdr, true); +} + +static int __init r8a779a0_sysc_pd_setup(struct r8a779a0_sysc_pd *pd) +{ + struct generic_pm_domain *genpd = &pd->genpd; + const char *name = pd->genpd.name; + int error; + + if (pd->flags & PD_CPU) { + /* + * This domain contains a CPU core and therefore it should + * only be turned off if the CPU is not in use. + */ + pr_debug("PM domain %s contains %s\n", name, "CPU"); + genpd->flags |= GENPD_FLAG_ALWAYS_ON; + } else if (pd->flags & PD_SCU) { + /* + * This domain contains an SCU and cache-controller, and + * therefore it should only be turned off if the CPU cores are + * not in use. + */ + pr_debug("PM domain %s contains %s\n", name, "SCU"); + genpd->flags |= GENPD_FLAG_ALWAYS_ON; + } else if (pd->flags & PD_NO_CR) { + /* + * This domain cannot be turned off. + */ + genpd->flags |= GENPD_FLAG_ALWAYS_ON; + } + + if (!(pd->flags & (PD_CPU | PD_SCU))) { + /* Enable Clock Domain for I/O devices */ + genpd->flags |= GENPD_FLAG_PM_CLK | GENPD_FLAG_ACTIVE_WAKEUP; + genpd->attach_dev = cpg_mssr_attach_dev; + genpd->detach_dev = cpg_mssr_detach_dev; + } + + genpd->power_off = r8a779a0_sysc_pd_power_off; + genpd->power_on = r8a779a0_sysc_pd_power_on; + + if (pd->flags & (PD_CPU | PD_NO_CR)) { + /* Skip CPUs (handled by SMP code) and areas without control */ + pr_debug("%s: Not touching %s\n", __func__, genpd->name); + goto finalize; + } + + if (!r8a779a0_sysc_power_is_off(pd->pdr)) { + pr_debug("%s: %s is already powered\n", __func__, genpd->name); + goto finalize; + } + + r8a779a0_sysc_power(pd->pdr, true); + +finalize: + error = pm_genpd_init(genpd, &simple_qos_governor, false); + if (error) + pr_err("Failed to init PM domain %s: %d\n", name, error); + + return error; +} + +static const struct of_device_id r8a779a0_sysc_matches[] __initconst = { + { .compatible = "renesas,r8a779a0-sysc", .data = &r8a779a0_sysc_info }, + { /* sentinel */ } +}; + +struct r8a779a0_pm_domains { + struct genpd_onecell_data onecell_data; + struct generic_pm_domain *domains[R8A779A0_PD_ALWAYS_ON + 1]; +}; + +static struct genpd_onecell_data *r8a779a0_sysc_onecell_data; + +static int __init r8a779a0_sysc_pd_init(void) +{ + const struct r8a779a0_sysc_info *info; + const struct of_device_id *match; + struct r8a779a0_pm_domains *domains; + struct device_node *np; + void __iomem *base; + unsigned int i; + int error; + + np = of_find_matching_node_and_match(NULL, r8a779a0_sysc_matches, &match); + if (!np) + return -ENODEV; + + info = match->data; + + base = of_iomap(np, 0); + if (!base) { + pr_warn("%pOF: Cannot map regs\n", np); + error = -ENOMEM; + goto out_put; + } + + r8a779a0_sysc_base = base; + + domains = kzalloc(sizeof(*domains), GFP_KERNEL); + if (!domains) { + error = -ENOMEM; + goto out_put; + } + + domains->onecell_data.domains = domains->domains; + domains->onecell_data.num_domains = ARRAY_SIZE(domains->domains); + r8a779a0_sysc_onecell_data = &domains->onecell_data; + + for (i = 0; i < info->num_areas; i++) { + const struct r8a779a0_sysc_area *area = &info->areas[i]; + struct r8a779a0_sysc_pd *pd; + + if (!area->name) { + /* Skip NULLified area */ + continue; + } + + pd = kzalloc(sizeof(*pd) + strlen(area->name) + 1, GFP_KERNEL); + if (!pd) { + error = -ENOMEM; + goto out_put; + } + + strcpy(pd->name, area->name); + pd->genpd.name = pd->name; + pd->pdr = area->pdr; + pd->flags = area->flags; + + error = r8a779a0_sysc_pd_setup(pd); + if (error) + goto out_put; + + domains->domains[area->pdr] = &pd->genpd; + + if (area->parent < 0) + continue; + + error = pm_genpd_add_subdomain(domains->domains[area->parent], + &pd->genpd); + if (error) { + pr_warn("Failed to add PM subdomain %s to parent %u\n", + area->name, area->parent); + goto out_put; + } + } + + error = of_genpd_add_provider_onecell(np, &domains->onecell_data); + +out_put: + of_node_put(np); + return error; +} +early_initcall(r8a779a0_sysc_pd_init); diff --git a/drivers/soc/renesas/rcar-rst.c b/drivers/soc/renesas/rcar-rst.c index a932015ce9c1..8a1e402ea799 100644 --- a/drivers/soc/renesas/rcar-rst.c +++ b/drivers/soc/renesas/rcar-rst.c @@ -37,6 +37,10 @@ static const struct rst_config rcar_rst_gen3 __initconst = { .modemr = 0x60, }; +static const struct rst_config rcar_rst_r8a779a0 __initconst = { + .modemr = 0x00, /* MODEMR0 and it has CPG related bits */ +}; + static const struct of_device_id rcar_rst_matches[] __initconst = { /* RZ/G1 is handled like R-Car Gen2 */ { .compatible = "renesas,r8a7742-rst", .data = &rcar_rst_gen2 }, @@ -67,6 +71,8 @@ static const struct of_device_id rcar_rst_matches[] __initconst = { { .compatible = "renesas,r8a77980-rst", .data = &rcar_rst_gen3 }, { .compatible = "renesas,r8a77990-rst", .data = &rcar_rst_gen3 }, { .compatible = "renesas,r8a77995-rst", .data = &rcar_rst_gen3 }, + /* R-Car V3U */ + { .compatible = "renesas,r8a779a0-rst", .data = &rcar_rst_r8a779a0 }, { /* sentinel */ } }; diff --git a/drivers/soc/renesas/renesas-soc.c b/drivers/soc/renesas/renesas-soc.c index f815a6a8b88b..0f8eff4a641a 100644 --- a/drivers/soc/renesas/renesas-soc.c +++ b/drivers/soc/renesas/renesas-soc.c @@ -200,6 +200,11 @@ static const struct renesas_soc soc_rcar_d3 __initconst __maybe_unused = { .id = 0x58, }; +static const struct renesas_soc soc_rcar_v3u __initconst __maybe_unused = { + .family = &fam_rcar_gen3, + .id = 0x59, +}; + static const struct renesas_soc soc_shmobile_ag5 __initconst __maybe_unused = { .family = &fam_shmobile, .id = 0x37, @@ -291,6 +296,9 @@ static const struct of_device_id renesas_socs[] __initconst = { #ifdef CONFIG_ARCH_R8A77995 { .compatible = "renesas,r8a77995", .data = &soc_rcar_d3 }, #endif +#ifdef CONFIG_ARCH_R8A779A0 + { .compatible = "renesas,r8a779a0", .data = &soc_rcar_v3u }, +#endif #ifdef CONFIG_ARCH_SH73A0 { .compatible = "renesas,sh73a0", .data = &soc_shmobile_ag5 }, #endif diff --git a/drivers/soc/rockchip/Kconfig b/drivers/soc/rockchip/Kconfig index b71b73bf5fc5..2c13bf4dd5db 100644 --- a/drivers/soc/rockchip/Kconfig +++ b/drivers/soc/rockchip/Kconfig @@ -14,6 +14,14 @@ config ROCKCHIP_GRF In a lot of cases there also need to be default settings initialized to make some of them conform to expectations of the kernel. +config ROCKCHIP_IODOMAIN + tristate "Rockchip IO domain support" + depends on OF + help + Say y here to enable support io domains on Rockchip SoCs. It is + necessary for the io domain setting of the SoC to match the + voltage supplied by the regulators. + config ROCKCHIP_PM_DOMAINS bool "Rockchip generic power domain" depends on PM diff --git a/drivers/soc/rockchip/Makefile b/drivers/soc/rockchip/Makefile index afca0a4c4b72..875032f7344e 100644 --- a/drivers/soc/rockchip/Makefile +++ b/drivers/soc/rockchip/Makefile @@ -3,4 +3,5 @@ # Rockchip Soc drivers # obj-$(CONFIG_ROCKCHIP_GRF) += grf.o +obj-$(CONFIG_ROCKCHIP_IODOMAIN) += io-domain.o obj-$(CONFIG_ROCKCHIP_PM_DOMAINS) += pm_domains.o diff --git a/drivers/power/avs/rockchip-io-domain.c b/drivers/soc/rockchip/io-domain.c index eece97f97ef8..eece97f97ef8 100644 --- a/drivers/power/avs/rockchip-io-domain.c +++ b/drivers/soc/rockchip/io-domain.c diff --git a/drivers/soc/samsung/Kconfig b/drivers/soc/samsung/Kconfig index 264185664594..fc7f48a92288 100644 --- a/drivers/soc/samsung/Kconfig +++ b/drivers/soc/samsung/Kconfig @@ -35,7 +35,54 @@ config EXYNOS_PMU_ARM_DRIVERS config EXYNOS_PM_DOMAINS bool "Exynos PM domains" if COMPILE_TEST - depends on PM_GENERIC_DOMAINS || COMPILE_TEST + depends on (ARCH_EXYNOS && PM_GENERIC_DOMAINS) || COMPILE_TEST + +config SAMSUNG_PM_DEBUG + bool "Samsung PM Suspend debug" + depends on PM && DEBUG_KERNEL + depends on PLAT_S3C24XX || ARCH_S3C64XX || ARCH_S5PV210 + depends on DEBUG_S3C24XX_UART || DEBUG_S3C2410_UART + depends on DEBUG_LL && MMU + help + Say Y here if you want verbose debugging from the PM Suspend and + Resume code. See <file:Documentation/arm/samsung-s3c24xx/suspend.rst> + for more information. + +config S3C_PM_DEBUG_LED_SMDK + bool "SMDK LED suspend/resume debugging" + depends on PM && (MACH_SMDK6410) + help + Say Y here to enable the use of the SMDK LEDs on the baseboard + for debugging of the state of the suspend and resume process. + + Note, this currently only works for S3C64XX based SMDK boards. + +config SAMSUNG_PM_CHECK + bool "S3C2410 PM Suspend Memory CRC" + depends on PM && (PLAT_S3C24XX || ARCH_S3C64XX || ARCH_S5PV210) + select CRC32 + help + Enable the PM code's memory area checksum over sleep. This option + will generate CRCs of all blocks of memory, and store them before + going to sleep. The blocks are then checked on resume for any + errors. + + Note, this can take several seconds depending on memory size + and CPU speed. + + See <file:Documentation/arm/samsung-s3c24xx/suspend.rst> + +config SAMSUNG_PM_CHECK_CHUNKSIZE + int "S3C2410 PM Suspend CRC Chunksize (KiB)" + depends on PM && SAMSUNG_PM_CHECK + default 64 + help + Set the chunksize in Kilobytes of the CRC for checking memory + corruption over suspend and resume. A smaller value will mean that + the CRC data block will take more memory, but will identify any + faults with better precision. + + See <file:Documentation/arm/samsung-s3c24xx/suspend.rst> config EXYNOS_REGULATOR_COUPLER bool "Exynos SoC Regulator Coupler" if COMPILE_TEST diff --git a/drivers/soc/samsung/Makefile b/drivers/soc/samsung/Makefile index ecc3a32f6406..59e8e9453f27 100644 --- a/drivers/soc/samsung/Makefile +++ b/drivers/soc/samsung/Makefile @@ -10,3 +10,6 @@ obj-$(CONFIG_EXYNOS_PMU_ARM_DRIVERS) += exynos3250-pmu.o exynos4-pmu.o \ exynos5250-pmu.o exynos5420-pmu.o obj-$(CONFIG_EXYNOS_PM_DOMAINS) += pm_domains.o obj-$(CONFIG_EXYNOS_REGULATOR_COUPLER) += exynos-regulator-coupler.o + +obj-$(CONFIG_SAMSUNG_PM_CHECK) += s3c-pm-check.o +obj-$(CONFIG_SAMSUNG_PM_DEBUG) += s3c-pm-debug.o diff --git a/drivers/soc/samsung/s3c-pm-check.c b/drivers/soc/samsung/s3c-pm-check.c new file mode 100644 index 000000000000..ff3e099fc208 --- /dev/null +++ b/drivers/soc/samsung/s3c-pm-check.c @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// originally in linux/arch/arm/plat-s3c24xx/pm.c +// +// Copyright (c) 2004-2008 Simtec Electronics +// http://armlinux.simtec.co.uk +// Ben Dooks <ben@simtec.co.uk> +// +// S3C Power Mangament - suspend/resume memory corruption check. + +#include <linux/kernel.h> +#include <linux/suspend.h> +#include <linux/init.h> +#include <linux/crc32.h> +#include <linux/ioport.h> +#include <linux/slab.h> + +#include <linux/soc/samsung/s3c-pm.h> + +#if CONFIG_SAMSUNG_PM_CHECK_CHUNKSIZE < 1 +#error CONFIG_SAMSUNG_PM_CHECK_CHUNKSIZE must be a positive non-zero value +#endif + +/* suspend checking code... + * + * this next area does a set of crc checks over all the installed + * memory, so the system can verify if the resume was ok. + * + * CONFIG_SAMSUNG_PM_CHECK_CHUNKSIZE defines the block-size for the CRC, + * increasing it will mean that the area corrupted will be less easy to spot, + * and reducing the size will cause the CRC save area to grow +*/ + +#define CHECK_CHUNKSIZE (CONFIG_SAMSUNG_PM_CHECK_CHUNKSIZE * 1024) + +static u32 crc_size; /* size needed for the crc block */ +static u32 *crcs; /* allocated over suspend/resume */ + +typedef u32 *(run_fn_t)(struct resource *ptr, u32 *arg); + +/* s3c_pm_run_res + * + * go through the given resource list, and look for system ram +*/ + +static void s3c_pm_run_res(struct resource *ptr, run_fn_t fn, u32 *arg) +{ + while (ptr != NULL) { + if (ptr->child != NULL) + s3c_pm_run_res(ptr->child, fn, arg); + + if ((ptr->flags & IORESOURCE_SYSTEM_RAM) + == IORESOURCE_SYSTEM_RAM) { + S3C_PMDBG("Found system RAM at %08lx..%08lx\n", + (unsigned long)ptr->start, + (unsigned long)ptr->end); + arg = (fn)(ptr, arg); + } + + ptr = ptr->sibling; + } +} + +static void s3c_pm_run_sysram(run_fn_t fn, u32 *arg) +{ + s3c_pm_run_res(&iomem_resource, fn, arg); +} + +static u32 *s3c_pm_countram(struct resource *res, u32 *val) +{ + u32 size = (u32)resource_size(res); + + size += CHECK_CHUNKSIZE-1; + size /= CHECK_CHUNKSIZE; + + S3C_PMDBG("Area %08lx..%08lx, %d blocks\n", + (unsigned long)res->start, (unsigned long)res->end, size); + + *val += size * sizeof(u32); + return val; +} + +/* s3c_pm_prepare_check + * + * prepare the necessary information for creating the CRCs. This + * must be done before the final save, as it will require memory + * allocating, and thus touching bits of the kernel we do not + * know about. +*/ + +void s3c_pm_check_prepare(void) +{ + crc_size = 0; + + s3c_pm_run_sysram(s3c_pm_countram, &crc_size); + + S3C_PMDBG("s3c_pm_prepare_check: %u checks needed\n", crc_size); + + crcs = kmalloc(crc_size+4, GFP_KERNEL); + if (crcs == NULL) + printk(KERN_ERR "Cannot allocated CRC save area\n"); +} + +static u32 *s3c_pm_makecheck(struct resource *res, u32 *val) +{ + unsigned long addr, left; + + for (addr = res->start; addr < res->end; + addr += CHECK_CHUNKSIZE) { + left = res->end - addr; + + if (left > CHECK_CHUNKSIZE) + left = CHECK_CHUNKSIZE; + + *val = crc32_le(~0, phys_to_virt(addr), left); + val++; + } + + return val; +} + +/* s3c_pm_check_store + * + * compute the CRC values for the memory blocks before the final + * sleep. +*/ + +void s3c_pm_check_store(void) +{ + if (crcs != NULL) + s3c_pm_run_sysram(s3c_pm_makecheck, crcs); +} + +/* in_region + * + * return TRUE if the area defined by ptr..ptr+size contains the + * what..what+whatsz +*/ + +static inline int in_region(void *ptr, int size, void *what, size_t whatsz) +{ + if ((what+whatsz) < ptr) + return 0; + + if (what > (ptr+size)) + return 0; + + return 1; +} + +/** + * s3c_pm_runcheck() - helper to check a resource on restore. + * @res: The resource to check + * @vak: Pointer to list of CRC32 values to check. + * + * Called from the s3c_pm_check_restore() via s3c_pm_run_sysram(), this + * function runs the given memory resource checking it against the stored + * CRC to ensure that memory is restored. The function tries to skip as + * many of the areas used during the suspend process. + */ +static u32 *s3c_pm_runcheck(struct resource *res, u32 *val) +{ + unsigned long addr; + unsigned long left; + void *stkpage; + void *ptr; + u32 calc; + + stkpage = (void *)((u32)&calc & ~PAGE_MASK); + + for (addr = res->start; addr < res->end; + addr += CHECK_CHUNKSIZE) { + left = res->end - addr; + + if (left > CHECK_CHUNKSIZE) + left = CHECK_CHUNKSIZE; + + ptr = phys_to_virt(addr); + + if (in_region(ptr, left, stkpage, 4096)) { + S3C_PMDBG("skipping %08lx, has stack in\n", addr); + goto skip_check; + } + + if (in_region(ptr, left, crcs, crc_size)) { + S3C_PMDBG("skipping %08lx, has crc block in\n", addr); + goto skip_check; + } + + /* calculate and check the checksum */ + + calc = crc32_le(~0, ptr, left); + if (calc != *val) { + printk(KERN_ERR "Restore CRC error at " + "%08lx (%08x vs %08x)\n", addr, calc, *val); + + S3C_PMDBG("Restore CRC error at %08lx (%08x vs %08x)\n", + addr, calc, *val); + } + + skip_check: + val++; + } + + return val; +} + +/** + * s3c_pm_check_restore() - memory check called on resume + * + * check the CRCs after the restore event and free the memory used + * to hold them +*/ +void s3c_pm_check_restore(void) +{ + if (crcs != NULL) + s3c_pm_run_sysram(s3c_pm_runcheck, crcs); +} + +/** + * s3c_pm_check_cleanup() - free memory resources + * + * Free the resources that where allocated by the suspend + * memory check code. We do this separately from the + * s3c_pm_check_restore() function as we cannot call any + * functions that might sleep during that resume. + */ +void s3c_pm_check_cleanup(void) +{ + kfree(crcs); + crcs = NULL; +} + diff --git a/drivers/soc/samsung/s3c-pm-debug.c b/drivers/soc/samsung/s3c-pm-debug.c new file mode 100644 index 000000000000..b5ce0e9a41e5 --- /dev/null +++ b/drivers/soc/samsung/s3c-pm-debug.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (C) 2013 Samsung Electronics Co., Ltd. +// Tomasz Figa <t.figa@samsung.com> +// Copyright (C) 2008 Openmoko, Inc. +// Copyright (C) 2004-2008 Simtec Electronics +// Ben Dooks <ben@simtec.co.uk> +// http://armlinux.simtec.co.uk/ +// +// Samsung common power management (suspend to RAM) debug support + +#include <linux/serial_core.h> +#include <linux/serial_s3c.h> +#include <linux/io.h> + +#include <asm/mach/map.h> + +#include <linux/soc/samsung/s3c-pm.h> + +static struct pm_uart_save uart_save; + +extern void printascii(const char *); + +void s3c_pm_dbg(const char *fmt, ...) +{ + va_list va; + char buff[256]; + + va_start(va, fmt); + vsnprintf(buff, sizeof(buff), fmt, va); + va_end(va); + + printascii(buff); +} + +static inline void __iomem *s3c_pm_uart_base(void) +{ + unsigned long paddr; + unsigned long vaddr; + + debug_ll_addr(&paddr, &vaddr); + + return (void __iomem *)vaddr; +} + +void s3c_pm_save_uarts(bool is_s3c2410) +{ + void __iomem *regs = s3c_pm_uart_base(); + struct pm_uart_save *save = &uart_save; + + save->ulcon = __raw_readl(regs + S3C2410_ULCON); + save->ucon = __raw_readl(regs + S3C2410_UCON); + save->ufcon = __raw_readl(regs + S3C2410_UFCON); + save->umcon = __raw_readl(regs + S3C2410_UMCON); + save->ubrdiv = __raw_readl(regs + S3C2410_UBRDIV); + + if (!is_s3c2410) + save->udivslot = __raw_readl(regs + S3C2443_DIVSLOT); + + S3C_PMDBG("UART[%p]: ULCON=%04x, UCON=%04x, UFCON=%04x, UBRDIV=%04x\n", + regs, save->ulcon, save->ucon, save->ufcon, save->ubrdiv); +} + +void s3c_pm_restore_uarts(bool is_s3c2410) +{ + void __iomem *regs = s3c_pm_uart_base(); + struct pm_uart_save *save = &uart_save; + + s3c_pm_arch_update_uart(regs, save); + + __raw_writel(save->ulcon, regs + S3C2410_ULCON); + __raw_writel(save->ucon, regs + S3C2410_UCON); + __raw_writel(save->ufcon, regs + S3C2410_UFCON); + __raw_writel(save->umcon, regs + S3C2410_UMCON); + __raw_writel(save->ubrdiv, regs + S3C2410_UBRDIV); + + if (!is_s3c2410) + __raw_writel(save->udivslot, regs + S3C2443_DIVSLOT); +} diff --git a/drivers/soc/sunxi/sunxi_sram.c b/drivers/soc/sunxi/sunxi_sram.c index 1b0d50f36349..d4c7bd59429e 100644 --- a/drivers/soc/sunxi/sunxi_sram.c +++ b/drivers/soc/sunxi/sunxi_sram.c @@ -194,7 +194,7 @@ static const struct sunxi_sram_data *sunxi_sram_of_parse(struct device_node *nod if (!data) { ret = -EINVAL; goto err; - }; + } for (func = data->func; func->func; func++) { if (val == func->val) { diff --git a/drivers/soc/tegra/Kconfig b/drivers/soc/tegra/Kconfig index 6bc603d0b9d9..976dee036470 100644 --- a/drivers/soc/tegra/Kconfig +++ b/drivers/soc/tegra/Kconfig @@ -119,6 +119,16 @@ config ARCH_TEGRA_194_SOC help Enable support for the NVIDIA Tegra194 SoC. +config ARCH_TEGRA_234_SOC + bool "NVIDIA Tegra234 SoC" + select MAILBOX + select TEGRA_BPMP + select TEGRA_HSP_MBOX + select TEGRA_IVC + select SOC_TEGRA_PMC + help + Enable support for the NVIDIA Tegra234 SoC. + endif endif diff --git a/drivers/soc/tegra/fuse/fuse-tegra.c b/drivers/soc/tegra/fuse/fuse-tegra.c index d1f8dd0289e6..94b60a692b51 100644 --- a/drivers/soc/tegra/fuse/fuse-tegra.c +++ b/drivers/soc/tegra/fuse/fuse-tegra.c @@ -49,6 +49,9 @@ static struct tegra_fuse *fuse = &(struct tegra_fuse) { }; static const struct of_device_id tegra_fuse_match[] = { +#ifdef CONFIG_ARCH_TEGRA_234_SOC + { .compatible = "nvidia,tegra234-efuse", .data = &tegra234_fuse_soc }, +#endif #ifdef CONFIG_ARCH_TEGRA_194_SOC { .compatible = "nvidia,tegra194-efuse", .data = &tegra194_fuse_soc }, #endif @@ -326,7 +329,8 @@ const struct attribute_group tegra_soc_attr_group = { .attrs = tegra_soc_attr, }; -#ifdef CONFIG_ARCH_TEGRA_194_SOC +#if IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \ + IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC) static ssize_t platform_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -336,7 +340,7 @@ static ssize_t platform_show(struct device *dev, struct device_attribute *attr, * platform type is silicon and all other non-zero values indicate * the type of simulation platform is being used. */ - return sprintf(buf, "%d\n", (tegra_read_chipid() >> 20) & 0xf); + return sprintf(buf, "%d\n", tegra_get_platform()); } static DEVICE_ATTR_RO(platform); diff --git a/drivers/soc/tegra/fuse/fuse-tegra30.c b/drivers/soc/tegra/fuse/fuse-tegra30.c index 85accef41fa1..9ea7f0168457 100644 --- a/drivers/soc/tegra/fuse/fuse-tegra30.c +++ b/drivers/soc/tegra/fuse/fuse-tegra30.c @@ -356,3 +356,33 @@ const struct tegra_fuse_soc tegra194_fuse_soc = { .soc_attr_group = &tegra194_soc_attr_group, }; #endif + +#if defined(CONFIG_ARCH_TEGRA_234_SOC) +static const struct nvmem_cell_lookup tegra234_fuse_lookups[] = { + { + .nvmem_name = "fuse", + .cell_name = "xusb-pad-calibration", + .dev_id = "3520000.padctl", + .con_id = "calibration", + }, { + .nvmem_name = "fuse", + .cell_name = "xusb-pad-calibration-ext", + .dev_id = "3520000.padctl", + .con_id = "calibration-ext", + }, +}; + +static const struct tegra_fuse_info tegra234_fuse_info = { + .read = tegra30_fuse_read, + .size = 0x300, + .spare = 0x280, +}; + +const struct tegra_fuse_soc tegra234_fuse_soc = { + .init = tegra30_fuse_init, + .info = &tegra234_fuse_info, + .lookups = tegra234_fuse_lookups, + .num_lookups = ARRAY_SIZE(tegra234_fuse_lookups), + .soc_attr_group = &tegra194_soc_attr_group, +}; +#endif diff --git a/drivers/soc/tegra/fuse/fuse.h b/drivers/soc/tegra/fuse/fuse.h index 9d4fc315a007..e057a58e2060 100644 --- a/drivers/soc/tegra/fuse/fuse.h +++ b/drivers/soc/tegra/fuse/fuse.h @@ -115,9 +115,17 @@ extern const struct tegra_fuse_soc tegra210_fuse_soc; extern const struct tegra_fuse_soc tegra186_fuse_soc; #endif +#if IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \ + IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC) +extern const struct attribute_group tegra194_soc_attr_group; +#endif + #ifdef CONFIG_ARCH_TEGRA_194_SOC extern const struct tegra_fuse_soc tegra194_fuse_soc; -extern const struct attribute_group tegra194_soc_attr_group; +#endif + +#ifdef CONFIG_ARCH_TEGRA_234_SOC +extern const struct tegra_fuse_soc tegra234_fuse_soc; #endif #endif diff --git a/drivers/soc/tegra/fuse/tegra-apbmisc.c b/drivers/soc/tegra/fuse/tegra-apbmisc.c index 8e416ad91ee2..cee207d10024 100644 --- a/drivers/soc/tegra/fuse/tegra-apbmisc.c +++ b/drivers/soc/tegra/fuse/tegra-apbmisc.c @@ -47,6 +47,31 @@ u8 tegra_get_minor_rev(void) return (tegra_read_chipid() >> 16) & 0xf; } +u8 tegra_get_platform(void) +{ + return (tegra_read_chipid() >> 20) & 0xf; +} + +bool tegra_is_silicon(void) +{ + switch (tegra_get_chip_id()) { + case TEGRA194: + case TEGRA234: + if (tegra_get_platform() == 0) + return true; + + return false; + } + + /* + * Chips prior to Tegra194 have a different way of determining whether + * they are silicon or not. Since we never supported simulation on the + * older Tegra chips, don't bother extracting the information and just + * report that we're running on silicon. + */ + return true; +} + u32 tegra_read_straps(void) { WARN(!chipid, "Tegra ABP MISC not yet available\n"); @@ -70,6 +95,7 @@ static const struct of_device_id apbmisc_match[] __initconst = { { .compatible = "nvidia,tegra20-apbmisc", }, { .compatible = "nvidia,tegra186-misc", }, { .compatible = "nvidia,tegra194-misc", }, + { .compatible = "nvidia,tegra234-misc", }, {}, }; diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c index b0bba8ab75bb..df9a5ca8c99c 100644 --- a/drivers/soc/tegra/pmc.c +++ b/drivers/soc/tegra/pmc.c @@ -336,45 +336,6 @@ struct tegra_pmc_soc { bool has_blink_output; }; -static const char * const tegra186_reset_sources[] = { - "SYS_RESET", - "AOWDT", - "MCCPLEXWDT", - "BPMPWDT", - "SCEWDT", - "SPEWDT", - "APEWDT", - "BCCPLEXWDT", - "SENSOR", - "AOTAG", - "VFSENSOR", - "SWREST", - "SC7", - "HSM", - "CORESIGHT" -}; - -static const char * const tegra186_reset_levels[] = { - "L0", "L1", "L2", "WARM" -}; - -static const char * const tegra30_reset_sources[] = { - "POWER_ON_RESET", - "WATCHDOG", - "SENSOR", - "SW_MAIN", - "LP0" -}; - -static const char * const tegra210_reset_sources[] = { - "POWER_ON_RESET", - "WATCHDOG", - "SENSOR", - "SW_MAIN", - "LP0", - "AOTAG" -}; - /** * struct tegra_pmc - NVIDIA Tegra PMC * @dev: pointer to PMC device structure @@ -2771,6 +2732,14 @@ static const u8 tegra30_cpu_powergates[] = { TEGRA_POWERGATE_CPU3, }; +static const char * const tegra30_reset_sources[] = { + "POWER_ON_RESET", + "WATCHDOG", + "SENSOR", + "SW_MAIN", + "LP0" +}; + static const struct tegra_pmc_soc tegra30_pmc_soc = { .num_powergates = ARRAY_SIZE(tegra30_powergates), .powergates = tegra30_powergates, @@ -3048,6 +3017,15 @@ static const struct pinctrl_pin_desc tegra210_pin_descs[] = { TEGRA210_IO_PAD_TABLE(TEGRA_IO_PIN_DESC) }; +static const char * const tegra210_reset_sources[] = { + "POWER_ON_RESET", + "WATCHDOG", + "SENSOR", + "SW_MAIN", + "LP0", + "AOTAG" +}; + static const struct tegra_wake_event tegra210_wake_events[] = { TEGRA_WAKE_IRQ("rtc", 16, 2), TEGRA_WAKE_IRQ("pmu", 51, 86), @@ -3180,6 +3158,28 @@ static void tegra186_pmc_setup_irq_polarity(struct tegra_pmc *pmc, iounmap(wake); } +static const char * const tegra186_reset_sources[] = { + "SYS_RESET", + "AOWDT", + "MCCPLEXWDT", + "BPMPWDT", + "SCEWDT", + "SPEWDT", + "APEWDT", + "BCCPLEXWDT", + "SENSOR", + "AOTAG", + "VFSENSOR", + "SWREST", + "SC7", + "HSM", + "CORESIGHT" +}; + +static const char * const tegra186_reset_levels[] = { + "L0", "L1", "L2", "WARM" +}; + static const struct tegra_wake_event tegra186_wake_events[] = { TEGRA_WAKE_IRQ("pmu", 24, 209), TEGRA_WAKE_GPIO("power", 29, 1, TEGRA186_AON_GPIO(FF, 0)), @@ -3349,7 +3349,75 @@ static const struct tegra_pmc_soc tegra194_pmc_soc = { .has_blink_output = false, }; +static const struct tegra_pmc_regs tegra234_pmc_regs = { + .scratch0 = 0x2000, + .dpd_req = 0, + .dpd_status = 0, + .dpd2_req = 0, + .dpd2_status = 0, + .rst_status = 0x70, + .rst_source_shift = 0x2, + .rst_source_mask = 0xfc, + .rst_level_shift = 0x0, + .rst_level_mask = 0x3, +}; + +static const char * const tegra234_reset_sources[] = { + "SYS_RESET_N", + "AOWDT", + "BCCPLEXWDT", + "BPMPWDT", + "SCEWDT", + "SPEWDT", + "APEWDT", + "LCCPLEXWDT", + "SENSOR", + "AOTAG", + "VFSENSOR", + "MAINSWRST", + "SC7", + "HSM", + "CSITE", + "RCEWDT", + "PVA0WDT", + "PVA1WDT", + "L1A_ASYNC", + "BPMPBOOT", + "FUSECRC", +}; + +static const struct tegra_pmc_soc tegra234_pmc_soc = { + .num_powergates = 0, + .powergates = NULL, + .num_cpu_powergates = 0, + .cpu_powergates = NULL, + .has_tsense_reset = false, + .has_gpu_clamps = false, + .needs_mbist_war = false, + .has_impl_33v_pwr = true, + .maybe_tz_only = false, + .num_io_pads = 0, + .io_pads = NULL, + .num_pin_descs = 0, + .pin_descs = NULL, + .regs = &tegra234_pmc_regs, + .init = NULL, + .setup_irq_polarity = tegra186_pmc_setup_irq_polarity, + .irq_set_wake = tegra186_pmc_irq_set_wake, + .irq_set_type = tegra186_pmc_irq_set_type, + .reset_sources = tegra234_reset_sources, + .num_reset_sources = ARRAY_SIZE(tegra234_reset_sources), + .reset_levels = tegra186_reset_levels, + .num_reset_levels = ARRAY_SIZE(tegra186_reset_levels), + .num_wake_events = 0, + .wake_events = NULL, + .pmc_clks_data = NULL, + .num_pmc_clks = 0, + .has_blink_output = false, +}; + static const struct of_device_id tegra_pmc_match[] = { + { .compatible = "nvidia,tegra234-pmc", .data = &tegra234_pmc_soc }, { .compatible = "nvidia,tegra194-pmc", .data = &tegra194_pmc_soc }, { .compatible = "nvidia,tegra186-pmc", .data = &tegra186_pmc_soc }, { .compatible = "nvidia,tegra210-pmc", .data = &tegra210_pmc_soc }, diff --git a/drivers/soc/ti/Kconfig b/drivers/soc/ti/Kconfig index e192fb788836..f5b82ffa637b 100644 --- a/drivers/soc/ti/Kconfig +++ b/drivers/soc/ti/Kconfig @@ -101,6 +101,17 @@ config TI_K3_SOCINFO platforms to provide information about the SoC family and variant to user space. +config TI_PRUSS + tristate "TI PRU-ICSS Subsystem Platform drivers" + depends on SOC_AM33XX || SOC_AM43XX || SOC_DRA7XX || ARCH_KEYSTONE || ARCH_K3 + select MFD_SYSCON + help + TI PRU-ICSS Subsystem platform specific support. + + Say Y or M here to support the Programmable Realtime Unit (PRU) + processors on various TI SoCs. It's safe to say N here if you're + not interested in the PRU or if you are unsure. + endif # SOC_TI config TI_SCI_INTA_MSI_DOMAIN diff --git a/drivers/soc/ti/Makefile b/drivers/soc/ti/Makefile index 1110e5c98685..cc3c972fad2e 100644 --- a/drivers/soc/ti/Makefile +++ b/drivers/soc/ti/Makefile @@ -12,3 +12,5 @@ obj-$(CONFIG_TI_SCI_PM_DOMAINS) += ti_sci_pm_domains.o obj-$(CONFIG_TI_SCI_INTA_MSI_DOMAIN) += ti_sci_inta_msi.o obj-$(CONFIG_TI_K3_RINGACC) += k3-ringacc.o obj-$(CONFIG_TI_K3_SOCINFO) += k3-socinfo.o +obj-$(CONFIG_TI_PRUSS) += pruss.o +obj-$(CONFIG_POWER_AVS_OMAP) += smartreflex.o diff --git a/drivers/soc/ti/k3-ringacc.c b/drivers/soc/ti/k3-ringacc.c index 6dcc21dde0cb..1147dc4c1d59 100644 --- a/drivers/soc/ti/k3-ringacc.c +++ b/drivers/soc/ti/k3-ringacc.c @@ -10,6 +10,7 @@ #include <linux/init.h> #include <linux/of.h> #include <linux/platform_device.h> +#include <linux/sys_soc.h> #include <linux/soc/ti/k3-ringacc.h> #include <linux/soc/ti/ti_sci_protocol.h> #include <linux/soc/ti/ti_sci_inta_msi.h> @@ -208,6 +209,15 @@ struct k3_ringacc { const struct k3_ringacc_ops *ops; }; +/** + * struct k3_ringacc - Rings accelerator SoC data + * + * @dma_ring_reset_quirk: DMA reset w/a enable + */ +struct k3_ringacc_soc_data { + unsigned dma_ring_reset_quirk:1; +}; + static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring) { return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES - @@ -1051,9 +1061,6 @@ static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc) return ret; } - ringacc->dma_ring_reset_quirk = - of_property_read_bool(node, "ti,dma-ring-reset-quirk"); - ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci"); if (IS_ERR(ringacc->tisci)) { ret = PTR_ERR(ringacc->tisci); @@ -1084,9 +1091,22 @@ static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc) ringacc->rm_gp_range); } +static const struct k3_ringacc_soc_data k3_ringacc_soc_data_sr1 = { + .dma_ring_reset_quirk = 1, +}; + +static const struct soc_device_attribute k3_ringacc_socinfo[] = { + { .family = "AM65X", + .revision = "SR1.0", + .data = &k3_ringacc_soc_data_sr1 + }, + {/* sentinel */} +}; + static int k3_ringacc_init(struct platform_device *pdev, struct k3_ringacc *ringacc) { + const struct soc_device_attribute *soc; void __iomem *base_fifo, *base_rt; struct device *dev = &pdev->dev; struct resource *res; @@ -1103,6 +1123,13 @@ static int k3_ringacc_init(struct platform_device *pdev, if (ret) return ret; + soc = soc_device_match(k3_ringacc_socinfo); + if (soc && soc->data) { + const struct k3_ringacc_soc_data *soc_data = soc->data; + + ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk; + } + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt"); base_rt = devm_ioremap_resource(dev, res); if (IS_ERR(base_rt)) diff --git a/drivers/soc/ti/k3-socinfo.c b/drivers/soc/ti/k3-socinfo.c index af0ba5288e58..bbbc2d2b7091 100644 --- a/drivers/soc/ti/k3-socinfo.c +++ b/drivers/soc/ti/k3-socinfo.c @@ -39,6 +39,7 @@ static const struct k3_soc_id { } k3_soc_ids[] = { { 0xBB5A, "AM65X" }, { 0xBB64, "J721E" }, + { 0xBB6D, "J7200" }, }; static int diff --git a/drivers/soc/ti/knav_dma.c b/drivers/soc/ti/knav_dma.c index 6285cd8efb21..8c863ecb1c60 100644 --- a/drivers/soc/ti/knav_dma.c +++ b/drivers/soc/ti/knav_dma.c @@ -355,7 +355,7 @@ static void dma_debug_show_devices(struct seq_file *s, } } -static int dma_debug_show(struct seq_file *s, void *v) +static int knav_dma_debug_show(struct seq_file *s, void *v) { struct knav_dma_device *dma; @@ -370,17 +370,7 @@ static int dma_debug_show(struct seq_file *s, void *v) return 0; } -static int knav_dma_debug_open(struct inode *inode, struct file *file) -{ - return single_open(file, dma_debug_show, NULL); -} - -static const struct file_operations knav_dma_debug_ops = { - .open = knav_dma_debug_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(knav_dma_debug); static int of_channel_match_helper(struct device_node *np, const char *name, const char **dma_instance) @@ -778,7 +768,7 @@ static int knav_dma_probe(struct platform_device *pdev) } debugfs_create_file("knav_dma", S_IFREG | S_IRUGO, NULL, NULL, - &knav_dma_debug_ops); + &knav_dma_debug_fops); device_ready = true; return ret; diff --git a/drivers/soc/ti/knav_qmss_queue.c b/drivers/soc/ti/knav_qmss_queue.c index aa071d96ef36..a460f201bf8e 100644 --- a/drivers/soc/ti/knav_qmss_queue.c +++ b/drivers/soc/ti/knav_qmss_queue.c @@ -478,17 +478,7 @@ static int knav_queue_debug_show(struct seq_file *s, void *v) return 0; } -static int knav_queue_debug_open(struct inode *inode, struct file *file) -{ - return single_open(file, knav_queue_debug_show, NULL); -} - -static const struct file_operations knav_queue_debug_ops = { - .open = knav_queue_debug_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(knav_queue_debug); static inline int knav_queue_pdsp_wait(u32 * __iomem addr, unsigned timeout, u32 flags) @@ -1878,7 +1868,7 @@ static int knav_queue_probe(struct platform_device *pdev) } debugfs_create_file("qmss", S_IFREG | S_IRUGO, NULL, NULL, - &knav_queue_debug_ops); + &knav_queue_debug_fops); device_ready = true; return 0; diff --git a/drivers/soc/ti/omap_prm.c b/drivers/soc/ti/omap_prm.c index c9b3f9ebf0bb..980b04c38fd9 100644 --- a/drivers/soc/ti/omap_prm.c +++ b/drivers/soc/ti/omap_prm.c @@ -10,14 +10,39 @@ #include <linux/device.h> #include <linux/io.h> #include <linux/iopoll.h> +#include <linux/module.h> #include <linux/of.h> #include <linux/of_device.h> #include <linux/platform_device.h> +#include <linux/pm_domain.h> #include <linux/reset-controller.h> #include <linux/delay.h> #include <linux/platform_data/ti-prm.h> +enum omap_prm_domain_mode { + OMAP_PRMD_OFF, + OMAP_PRMD_RETENTION, + OMAP_PRMD_ON_INACTIVE, + OMAP_PRMD_ON_ACTIVE, +}; + +struct omap_prm_domain_map { + unsigned int usable_modes; /* Mask of hardware supported modes */ + unsigned long statechange:1; /* Optional low-power state change */ + unsigned long logicretstate:1; /* Optional logic off mode */ +}; + +struct omap_prm_domain { + struct device *dev; + struct omap_prm *prm; + struct generic_pm_domain pd; + u16 pwrstctrl; + u16 pwrstst; + const struct omap_prm_domain_map *cap; + u32 pwrstctrl_saved; +}; + struct omap_rst_map { s8 rst; s8 st; @@ -27,6 +52,9 @@ struct omap_prm_data { u32 base; const char *name; const char *clkdm_name; + u16 pwrstctrl; + u16 pwrstst; + const struct omap_prm_domain_map *dmap; u16 rstctrl; u16 rstst; const struct omap_rst_map *rstmap; @@ -36,6 +64,7 @@ struct omap_prm_data { struct omap_prm { const struct omap_prm_data *data; void __iomem *base; + struct omap_prm_domain *prmd; }; struct omap_reset_data { @@ -47,6 +76,7 @@ struct omap_reset_data { struct device *dev; }; +#define genpd_to_prm_domain(gpd) container_of(gpd, struct omap_prm_domain, pd) #define to_omap_reset_data(p) container_of((p), struct omap_reset_data, rcdev) #define OMAP_MAX_RESETS 8 @@ -58,6 +88,39 @@ struct omap_reset_data { #define OMAP_PRM_HAS_RESETS (OMAP_PRM_HAS_RSTCTRL | OMAP_PRM_HAS_RSTST) +#define PRM_STATE_MAX_WAIT 10000 +#define PRM_LOGICRETSTATE BIT(2) +#define PRM_LOWPOWERSTATECHANGE BIT(4) +#define PRM_POWERSTATE_MASK OMAP_PRMD_ON_ACTIVE + +#define PRM_ST_INTRANSITION BIT(20) + +static const struct omap_prm_domain_map omap_prm_all = { + .usable_modes = BIT(OMAP_PRMD_ON_ACTIVE) | BIT(OMAP_PRMD_ON_INACTIVE) | + BIT(OMAP_PRMD_RETENTION) | BIT(OMAP_PRMD_OFF), + .statechange = 1, + .logicretstate = 1, +}; + +static const struct omap_prm_domain_map omap_prm_noinact = { + .usable_modes = BIT(OMAP_PRMD_ON_ACTIVE) | BIT(OMAP_PRMD_RETENTION) | + BIT(OMAP_PRMD_OFF), + .statechange = 1, + .logicretstate = 1, +}; + +static const struct omap_prm_domain_map omap_prm_nooff = { + .usable_modes = BIT(OMAP_PRMD_ON_ACTIVE) | BIT(OMAP_PRMD_ON_INACTIVE) | + BIT(OMAP_PRMD_RETENTION), + .statechange = 1, + .logicretstate = 1, +}; + +static const struct omap_prm_domain_map omap_prm_onoff_noauto = { + .usable_modes = BIT(OMAP_PRMD_ON_ACTIVE) | BIT(OMAP_PRMD_OFF), + .statechange = 1, +}; + static const struct omap_rst_map rst_map_0[] = { { .rst = 0, .st = 0 }, { .rst = -1 }, @@ -78,6 +141,10 @@ static const struct omap_rst_map rst_map_012[] = { static const struct omap_prm_data omap4_prm_data[] = { { .name = "tesla", .base = 0x4a306400, .rstctrl = 0x10, .rstst = 0x14, .rstmap = rst_map_01 }, + { + .name = "abe", .base = 0x4a306500, + .pwrstctrl = 0, .pwrstst = 0x4, .dmap = &omap_prm_all, + }, { .name = "core", .base = 0x4a306700, .rstctrl = 0x210, .rstst = 0x214, .clkdm_name = "ducati", .rstmap = rst_map_012 }, { .name = "ivahd", .base = 0x4a306f00, .rstctrl = 0x10, .rstst = 0x14, .rstmap = rst_map_012 }, { .name = "device", .base = 0x4a307b00, .rstctrl = 0x0, .rstst = 0x4, .rstmap = rst_map_01, .flags = OMAP_PRM_HAS_RSTCTRL | OMAP_PRM_HAS_NO_CLKDM }, @@ -86,6 +153,10 @@ static const struct omap_prm_data omap4_prm_data[] = { static const struct omap_prm_data omap5_prm_data[] = { { .name = "dsp", .base = 0x4ae06400, .rstctrl = 0x10, .rstst = 0x14, .rstmap = rst_map_01 }, + { + .name = "abe", .base = 0x4ae06500, + .pwrstctrl = 0, .pwrstst = 0x4, .dmap = &omap_prm_nooff, + }, { .name = "core", .base = 0x4ae06700, .rstctrl = 0x210, .rstst = 0x214, .clkdm_name = "ipu", .rstmap = rst_map_012 }, { .name = "iva", .base = 0x4ae07200, .rstctrl = 0x10, .rstst = 0x14, .rstmap = rst_map_012 }, { .name = "device", .base = 0x4ae07c00, .rstctrl = 0x0, .rstst = 0x4, .rstmap = rst_map_01, .flags = OMAP_PRM_HAS_RSTCTRL | OMAP_PRM_HAS_NO_CLKDM }, @@ -119,7 +190,11 @@ static const struct omap_prm_data am3_prm_data[] = { { .name = "per", .base = 0x44e00c00, .rstctrl = 0x0, .rstmap = am3_per_rst_map, .flags = OMAP_PRM_HAS_RSTCTRL, .clkdm_name = "pruss_ocp" }, { .name = "wkup", .base = 0x44e00d00, .rstctrl = 0x0, .rstst = 0xc, .rstmap = am3_wkup_rst_map, .flags = OMAP_PRM_HAS_RSTCTRL | OMAP_PRM_HAS_NO_CLKDM }, { .name = "device", .base = 0x44e00f00, .rstctrl = 0x0, .rstst = 0x8, .rstmap = rst_map_01, .flags = OMAP_PRM_HAS_RSTCTRL | OMAP_PRM_HAS_NO_CLKDM }, - { .name = "gfx", .base = 0x44e01100, .rstctrl = 0x4, .rstst = 0x14, .rstmap = rst_map_0, .clkdm_name = "gfx_l3" }, + { + .name = "gfx", .base = 0x44e01100, + .pwrstctrl = 0, .pwrstst = 0x10, .dmap = &omap_prm_noinact, + .rstctrl = 0x4, .rstst = 0x14, .rstmap = rst_map_0, .clkdm_name = "gfx_l3", + }, { }, }; @@ -135,7 +210,11 @@ static const struct omap_rst_map am4_device_rst_map[] = { }; static const struct omap_prm_data am4_prm_data[] = { - { .name = "gfx", .base = 0x44df0400, .rstctrl = 0x10, .rstst = 0x14, .rstmap = rst_map_0, .clkdm_name = "gfx_l3" }, + { + .name = "gfx", .base = 0x44df0400, + .pwrstctrl = 0, .pwrstst = 0x4, .dmap = &omap_prm_onoff_noauto, + .rstctrl = 0x10, .rstst = 0x14, .rstmap = rst_map_0, .clkdm_name = "gfx_l3", + }, { .name = "per", .base = 0x44df0800, .rstctrl = 0x10, .rstst = 0x14, .rstmap = am4_per_rst_map, .clkdm_name = "pruss_ocp" }, { .name = "wkup", .base = 0x44df2000, .rstctrl = 0x10, .rstst = 0x14, .rstmap = am3_wkup_rst_map, .flags = OMAP_PRM_HAS_NO_CLKDM }, { .name = "device", .base = 0x44df4000, .rstctrl = 0x0, .rstst = 0x4, .rstmap = am4_device_rst_map, .flags = OMAP_PRM_HAS_RSTCTRL | OMAP_PRM_HAS_NO_CLKDM }, @@ -151,6 +230,180 @@ static const struct of_device_id omap_prm_id_table[] = { { }, }; +#ifdef DEBUG +static void omap_prm_domain_show_state(struct omap_prm_domain *prmd, + const char *desc) +{ + dev_dbg(prmd->dev, "%s %s: %08x/%08x\n", + prmd->pd.name, desc, + readl_relaxed(prmd->prm->base + prmd->pwrstctrl), + readl_relaxed(prmd->prm->base + prmd->pwrstst)); +} +#else +static inline void omap_prm_domain_show_state(struct omap_prm_domain *prmd, + const char *desc) +{ +} +#endif + +static int omap_prm_domain_power_on(struct generic_pm_domain *domain) +{ + struct omap_prm_domain *prmd; + int ret; + u32 v; + + prmd = genpd_to_prm_domain(domain); + if (!prmd->cap) + return 0; + + omap_prm_domain_show_state(prmd, "on: previous state"); + + if (prmd->pwrstctrl_saved) + v = prmd->pwrstctrl_saved; + else + v = readl_relaxed(prmd->prm->base + prmd->pwrstctrl); + + writel_relaxed(v | OMAP_PRMD_ON_ACTIVE, + prmd->prm->base + prmd->pwrstctrl); + + /* wait for the transition bit to get cleared */ + ret = readl_relaxed_poll_timeout(prmd->prm->base + prmd->pwrstst, + v, !(v & PRM_ST_INTRANSITION), 1, + PRM_STATE_MAX_WAIT); + if (ret) + dev_err(prmd->dev, "%s: %s timed out\n", + prmd->pd.name, __func__); + + omap_prm_domain_show_state(prmd, "on: new state"); + + return ret; +} + +/* No need to check for holes in the mask for the lowest mode */ +static int omap_prm_domain_find_lowest(struct omap_prm_domain *prmd) +{ + return __ffs(prmd->cap->usable_modes); +} + +static int omap_prm_domain_power_off(struct generic_pm_domain *domain) +{ + struct omap_prm_domain *prmd; + int ret; + u32 v; + + prmd = genpd_to_prm_domain(domain); + if (!prmd->cap) + return 0; + + omap_prm_domain_show_state(prmd, "off: previous state"); + + v = readl_relaxed(prmd->prm->base + prmd->pwrstctrl); + prmd->pwrstctrl_saved = v; + + v &= ~PRM_POWERSTATE_MASK; + v |= omap_prm_domain_find_lowest(prmd); + + if (prmd->cap->statechange) + v |= PRM_LOWPOWERSTATECHANGE; + if (prmd->cap->logicretstate) + v &= ~PRM_LOGICRETSTATE; + else + v |= PRM_LOGICRETSTATE; + + writel_relaxed(v, prmd->prm->base + prmd->pwrstctrl); + + /* wait for the transition bit to get cleared */ + ret = readl_relaxed_poll_timeout(prmd->prm->base + prmd->pwrstst, + v, !(v & PRM_ST_INTRANSITION), 1, + PRM_STATE_MAX_WAIT); + if (ret) + dev_warn(prmd->dev, "%s: %s timed out\n", + __func__, prmd->pd.name); + + omap_prm_domain_show_state(prmd, "off: new state"); + + return 0; +} + +static int omap_prm_domain_attach_dev(struct generic_pm_domain *domain, + struct device *dev) +{ + struct generic_pm_domain_data *genpd_data; + struct of_phandle_args pd_args; + struct omap_prm_domain *prmd; + struct device_node *np; + int ret; + + prmd = genpd_to_prm_domain(domain); + np = dev->of_node; + + ret = of_parse_phandle_with_args(np, "power-domains", + "#power-domain-cells", 0, &pd_args); + if (ret < 0) + return ret; + + if (pd_args.args_count != 0) + dev_warn(dev, "%s: unusupported #power-domain-cells: %i\n", + prmd->pd.name, pd_args.args_count); + + genpd_data = dev_gpd_data(dev); + genpd_data->data = NULL; + + return 0; +} + +static void omap_prm_domain_detach_dev(struct generic_pm_domain *domain, + struct device *dev) +{ + struct generic_pm_domain_data *genpd_data; + + genpd_data = dev_gpd_data(dev); + genpd_data->data = NULL; +} + +static int omap_prm_domain_init(struct device *dev, struct omap_prm *prm) +{ + struct omap_prm_domain *prmd; + struct device_node *np = dev->of_node; + const struct omap_prm_data *data; + const char *name; + int error; + + if (!of_find_property(dev->of_node, "#power-domain-cells", NULL)) + return 0; + + of_node_put(dev->of_node); + + prmd = devm_kzalloc(dev, sizeof(*prmd), GFP_KERNEL); + if (!prmd) + return -ENOMEM; + + data = prm->data; + name = devm_kasprintf(dev, GFP_KERNEL, "prm_%s", + data->name); + + prmd->dev = dev; + prmd->prm = prm; + prmd->cap = prmd->prm->data->dmap; + prmd->pwrstctrl = prmd->prm->data->pwrstctrl; + prmd->pwrstst = prmd->prm->data->pwrstst; + + prmd->pd.name = name; + prmd->pd.power_on = omap_prm_domain_power_on; + prmd->pd.power_off = omap_prm_domain_power_off; + prmd->pd.attach_dev = omap_prm_domain_attach_dev; + prmd->pd.detach_dev = omap_prm_domain_detach_dev; + + pm_genpd_init(&prmd->pd, NULL, true); + error = of_genpd_add_provider_simple(np, &prmd->pd); + if (error) + pm_genpd_remove(&prmd->pd); + else + prm->prmd = prmd; + + return error; +} + static bool _is_valid_reset(struct omap_reset_data *reset, unsigned long id) { if (reset->mask & BIT(id)) @@ -351,6 +604,7 @@ static int omap_prm_probe(struct platform_device *pdev) const struct omap_prm_data *data; struct omap_prm *prm; const struct of_device_id *match; + int ret; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res) @@ -378,7 +632,21 @@ static int omap_prm_probe(struct platform_device *pdev) if (IS_ERR(prm->base)) return PTR_ERR(prm->base); - return omap_prm_reset_init(pdev, prm); + ret = omap_prm_domain_init(&pdev->dev, prm); + if (ret) + return ret; + + ret = omap_prm_reset_init(pdev, prm); + if (ret) + goto err_domain; + + return 0; + +err_domain: + of_genpd_del_provider(pdev->dev.of_node); + pm_genpd_remove(&prm->prmd->pd); + + return ret; } static struct platform_driver omap_prm_driver = { diff --git a/drivers/soc/ti/pm33xx.c b/drivers/soc/ti/pm33xx.c index de0123ec8ad6..d2f5e7001a93 100644 --- a/drivers/soc/ti/pm33xx.c +++ b/drivers/soc/ti/pm33xx.c @@ -16,6 +16,7 @@ #include <linux/module.h> #include <linux/nvmem-consumer.h> #include <linux/of.h> +#include <linux/of_address.h> #include <linux/platform_data/pm33xx.h> #include <linux/platform_device.h> #include <linux/rtc.h> @@ -39,6 +40,8 @@ #define GIC_INT_SET_PENDING_BASE 0x200 #define AM43XX_GIC_DIST_BASE 0x48241000 +static void __iomem *rtc_base_virt; +static struct clk *rtc_fck; static u32 rtc_magic_val; static int (*am33xx_do_wfi_sram)(unsigned long unused); @@ -90,7 +93,7 @@ static int am33xx_push_sram_idle(void) ro_sram_data.amx3_pm_sram_data_virt = ocmcram_location_data; ro_sram_data.amx3_pm_sram_data_phys = gen_pool_virt_to_phys(sram_pool_data, ocmcram_location_data); - ro_sram_data.rtc_base_virt = pm_ops->get_rtc_base_addr(); + ro_sram_data.rtc_base_virt = rtc_base_virt; /* Save physical address to calculate resume offset during pm init */ am33xx_do_wfi_sram_phys = gen_pool_virt_to_phys(sram_pool, @@ -158,7 +161,7 @@ static struct wkup_m3_wakeup_src rtc_wake_src(void) { u32 i; - i = __raw_readl(pm_ops->get_rtc_base_addr() + 0x44) & 0x40; + i = __raw_readl(rtc_base_virt + 0x44) & 0x40; if (i) { retrigger_irq = rtc_alarm_wakeup.irq_nr; @@ -177,13 +180,24 @@ static int am33xx_rtc_only_idle(unsigned long wfi_flags) return 0; } +/* + * Note that the RTC module clock must be re-enabled only for rtc+ddr suspend. + * And looks like the module can stay in SYSC_IDLE_SMART_WKUP mode configured + * by the interconnect code just fine for both rtc+ddr suspend and retention + * suspend. + */ static int am33xx_pm_suspend(suspend_state_t suspend_state) { int i, ret = 0; if (suspend_state == PM_SUSPEND_MEM && pm_ops->check_off_mode_enable()) { - pm_ops->prepare_rtc_suspend(); + ret = clk_prepare_enable(rtc_fck); + if (ret) { + dev_err(pm33xx_dev, "Failed to enable clock: %i\n", ret); + return ret; + } + pm_ops->save_context(); suspend_wfi_flags |= WFI_FLAG_RTC_ONLY; clk_save_context(); @@ -236,7 +250,7 @@ static int am33xx_pm_suspend(suspend_state_t suspend_state) } if (suspend_state == PM_SUSPEND_MEM && pm_ops->check_off_mode_enable()) - pm_ops->prepare_rtc_resume(); + clk_disable_unprepare(rtc_fck); return ret; } @@ -425,14 +439,28 @@ static int am33xx_pm_rtc_setup(void) struct device_node *np; unsigned long val = 0; struct nvmem_device *nvmem; + int error; np = of_find_node_by_name(NULL, "rtc"); if (of_device_is_available(np)) { + /* RTC interconnect target module clock */ + rtc_fck = of_clk_get_by_name(np->parent, "fck"); + if (IS_ERR(rtc_fck)) + return PTR_ERR(rtc_fck); + + rtc_base_virt = of_iomap(np, 0); + if (!rtc_base_virt) { + pr_warn("PM: could not iomap rtc"); + error = -ENODEV; + goto err_clk_put; + } + omap_rtc = rtc_class_open("rtc0"); if (!omap_rtc) { pr_warn("PM: rtc0 not available"); - return -EPROBE_DEFER; + error = -EPROBE_DEFER; + goto err_iounmap; } nvmem = devm_nvmem_device_get(&omap_rtc->dev, @@ -454,6 +482,13 @@ static int am33xx_pm_rtc_setup(void) } return 0; + +err_iounmap: + iounmap(rtc_base_virt); +err_clk_put: + clk_put(rtc_fck); + + return error; } static int am33xx_pm_probe(struct platform_device *pdev) @@ -544,6 +579,8 @@ static int am33xx_pm_remove(struct platform_device *pdev) suspend_set_ops(NULL); wkup_m3_ipc_put(m3_ipc); am33xx_pm_free_sram(); + iounmap(rtc_base_virt); + clk_put(rtc_fck); return 0; } diff --git a/drivers/soc/ti/pruss.c b/drivers/soc/ti/pruss.c new file mode 100644 index 000000000000..cc0b4ad7a3d3 --- /dev/null +++ b/drivers/soc/ti/pruss.c @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PRU-ICSS platform driver for various TI SoCs + * + * Copyright (C) 2014-2020 Texas Instruments Incorporated - http://www.ti.com/ + * Author(s): + * Suman Anna <s-anna@ti.com> + * Andrew F. Davis <afd@ti.com> + */ + +#include <linux/clk-provider.h> +#include <linux/dma-mapping.h> +#include <linux/io.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/pm_runtime.h> +#include <linux/pruss_driver.h> +#include <linux/regmap.h> +#include <linux/slab.h> + +/** + * struct pruss_private_data - PRUSS driver private data + * @has_no_sharedram: flag to indicate the absence of PRUSS Shared Data RAM + * @has_core_mux_clock: flag to indicate the presence of PRUSS core clock + */ +struct pruss_private_data { + bool has_no_sharedram; + bool has_core_mux_clock; +}; + +static void pruss_of_free_clk_provider(void *data) +{ + struct device_node *clk_mux_np = data; + + of_clk_del_provider(clk_mux_np); + of_node_put(clk_mux_np); +} + +static int pruss_clk_mux_setup(struct pruss *pruss, struct clk *clk_mux, + char *mux_name, struct device_node *clks_np) +{ + struct device_node *clk_mux_np; + struct device *dev = pruss->dev; + char *clk_mux_name; + unsigned int num_parents; + const char **parent_names; + void __iomem *reg; + u32 reg_offset; + int ret; + + clk_mux_np = of_get_child_by_name(clks_np, mux_name); + if (!clk_mux_np) { + dev_err(dev, "%pOF is missing its '%s' node\n", clks_np, + mux_name); + return -ENODEV; + } + + num_parents = of_clk_get_parent_count(clk_mux_np); + if (num_parents < 1) { + dev_err(dev, "mux-clock %pOF must have parents\n", clk_mux_np); + ret = -EINVAL; + goto put_clk_mux_np; + } + + parent_names = devm_kcalloc(dev, sizeof(*parent_names), num_parents, + GFP_KERNEL); + if (!parent_names) { + ret = -ENOMEM; + goto put_clk_mux_np; + } + + of_clk_parent_fill(clk_mux_np, parent_names, num_parents); + + clk_mux_name = devm_kasprintf(dev, GFP_KERNEL, "%s.%pOFn", + dev_name(dev), clk_mux_np); + if (!clk_mux_name) { + ret = -ENOMEM; + goto put_clk_mux_np; + } + + ret = of_property_read_u32(clk_mux_np, "reg", ®_offset); + if (ret) + goto put_clk_mux_np; + + reg = pruss->cfg_base + reg_offset; + + clk_mux = clk_register_mux(NULL, clk_mux_name, parent_names, + num_parents, 0, reg, 0, 1, 0, NULL); + if (IS_ERR(clk_mux)) { + ret = PTR_ERR(clk_mux); + goto put_clk_mux_np; + } + + ret = devm_add_action_or_reset(dev, (void(*)(void *))clk_unregister_mux, + clk_mux); + if (ret) { + dev_err(dev, "failed to add clkmux unregister action %d", ret); + goto put_clk_mux_np; + } + + ret = of_clk_add_provider(clk_mux_np, of_clk_src_simple_get, clk_mux); + if (ret) + goto put_clk_mux_np; + + ret = devm_add_action_or_reset(dev, pruss_of_free_clk_provider, + clk_mux_np); + if (ret) { + dev_err(dev, "failed to add clkmux free action %d", ret); + goto put_clk_mux_np; + } + + return 0; + +put_clk_mux_np: + of_node_put(clk_mux_np); + return ret; +} + +static int pruss_clk_init(struct pruss *pruss, struct device_node *cfg_node) +{ + const struct pruss_private_data *data; + struct device_node *clks_np; + struct device *dev = pruss->dev; + int ret = 0; + + data = of_device_get_match_data(dev); + if (IS_ERR(data)) + return -ENODEV; + + clks_np = of_get_child_by_name(cfg_node, "clocks"); + if (!clks_np) { + dev_err(dev, "%pOF is missing its 'clocks' node\n", clks_np); + return -ENODEV; + } + + if (data && data->has_core_mux_clock) { + ret = pruss_clk_mux_setup(pruss, pruss->core_clk_mux, + "coreclk-mux", clks_np); + if (ret) { + dev_err(dev, "failed to setup coreclk-mux\n"); + goto put_clks_node; + } + } + + ret = pruss_clk_mux_setup(pruss, pruss->iep_clk_mux, "iepclk-mux", + clks_np); + if (ret) { + dev_err(dev, "failed to setup iepclk-mux\n"); + goto put_clks_node; + } + +put_clks_node: + of_node_put(clks_np); + + return ret; +} + +static struct regmap_config regmap_conf = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, +}; + +static int pruss_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev_of_node(dev); + struct device_node *child; + struct pruss *pruss; + struct resource res; + int ret, i, index; + const struct pruss_private_data *data; + const char *mem_names[PRUSS_MEM_MAX] = { "dram0", "dram1", "shrdram2" }; + + data = of_device_get_match_data(&pdev->dev); + if (IS_ERR(data)) { + dev_err(dev, "missing private data\n"); + return -ENODEV; + } + + ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(dev, "failed to set the DMA coherent mask"); + return ret; + } + + pruss = devm_kzalloc(dev, sizeof(*pruss), GFP_KERNEL); + if (!pruss) + return -ENOMEM; + + pruss->dev = dev; + + child = of_get_child_by_name(np, "memories"); + if (!child) { + dev_err(dev, "%pOF is missing its 'memories' node\n", child); + return -ENODEV; + } + + for (i = 0; i < PRUSS_MEM_MAX; i++) { + /* + * On AM437x one of two PRUSS units don't contain Shared RAM, + * skip it + */ + if (data && data->has_no_sharedram && i == PRUSS_MEM_SHRD_RAM2) + continue; + + index = of_property_match_string(child, "reg-names", + mem_names[i]); + if (index < 0) { + of_node_put(child); + return index; + } + + if (of_address_to_resource(child, index, &res)) { + of_node_put(child); + return -EINVAL; + } + + pruss->mem_regions[i].va = devm_ioremap(dev, res.start, + resource_size(&res)); + if (!pruss->mem_regions[i].va) { + dev_err(dev, "failed to parse and map memory resource %d %s\n", + i, mem_names[i]); + of_node_put(child); + return -ENOMEM; + } + pruss->mem_regions[i].pa = res.start; + pruss->mem_regions[i].size = resource_size(&res); + + dev_dbg(dev, "memory %8s: pa %pa size 0x%zx va %pK\n", + mem_names[i], &pruss->mem_regions[i].pa, + pruss->mem_regions[i].size, pruss->mem_regions[i].va); + } + of_node_put(child); + + platform_set_drvdata(pdev, pruss); + + pm_runtime_enable(dev); + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + dev_err(dev, "couldn't enable module\n"); + pm_runtime_put_noidle(dev); + goto rpm_disable; + } + + child = of_get_child_by_name(np, "cfg"); + if (!child) { + dev_err(dev, "%pOF is missing its 'cfg' node\n", child); + ret = -ENODEV; + goto rpm_put; + } + + if (of_address_to_resource(child, 0, &res)) { + ret = -ENOMEM; + goto node_put; + } + + pruss->cfg_base = devm_ioremap(dev, res.start, resource_size(&res)); + if (!pruss->cfg_base) { + ret = -ENOMEM; + goto node_put; + } + + regmap_conf.name = kasprintf(GFP_KERNEL, "%pOFn@%llx", child, + (u64)res.start); + regmap_conf.max_register = resource_size(&res) - 4; + + pruss->cfg_regmap = devm_regmap_init_mmio(dev, pruss->cfg_base, + ®map_conf); + kfree(regmap_conf.name); + if (IS_ERR(pruss->cfg_regmap)) { + dev_err(dev, "regmap_init_mmio failed for cfg, ret = %ld\n", + PTR_ERR(pruss->cfg_regmap)); + ret = PTR_ERR(pruss->cfg_regmap); + goto node_put; + } + + ret = pruss_clk_init(pruss, child); + if (ret) { + dev_err(dev, "failed to setup coreclk-mux\n"); + goto node_put; + } + + ret = devm_of_platform_populate(dev); + if (ret) { + dev_err(dev, "failed to register child devices\n"); + goto node_put; + } + + of_node_put(child); + + return 0; + +node_put: + of_node_put(child); +rpm_put: + pm_runtime_put_sync(dev); +rpm_disable: + pm_runtime_disable(dev); + return ret; +} + +static int pruss_remove(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + + devm_of_platform_depopulate(dev); + + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); + + return 0; +} + +/* instance-specific driver private data */ +static const struct pruss_private_data am437x_pruss1_data = { + .has_no_sharedram = false, +}; + +static const struct pruss_private_data am437x_pruss0_data = { + .has_no_sharedram = true, +}; + +static const struct pruss_private_data am65x_j721e_pruss_data = { + .has_core_mux_clock = true, +}; + +static const struct of_device_id pruss_of_match[] = { + { .compatible = "ti,am3356-pruss" }, + { .compatible = "ti,am4376-pruss0", .data = &am437x_pruss0_data, }, + { .compatible = "ti,am4376-pruss1", .data = &am437x_pruss1_data, }, + { .compatible = "ti,am5728-pruss" }, + { .compatible = "ti,k2g-pruss" }, + { .compatible = "ti,am654-icssg", .data = &am65x_j721e_pruss_data, }, + { .compatible = "ti,j721e-icssg", .data = &am65x_j721e_pruss_data, }, + {}, +}; +MODULE_DEVICE_TABLE(of, pruss_of_match); + +static struct platform_driver pruss_driver = { + .driver = { + .name = "pruss", + .of_match_table = pruss_of_match, + }, + .probe = pruss_probe, + .remove = pruss_remove, +}; +module_platform_driver(pruss_driver); + +MODULE_AUTHOR("Suman Anna <s-anna@ti.com>"); +MODULE_DESCRIPTION("PRU-ICSS Subsystem Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/power/avs/smartreflex.c b/drivers/soc/ti/smartreflex.c index 5376f3d22f31..5376f3d22f31 100644 --- a/drivers/power/avs/smartreflex.c +++ b/drivers/soc/ti/smartreflex.c diff --git a/drivers/soc/ti/ti_sci_pm_domains.c b/drivers/soc/ti/ti_sci_pm_domains.c index 8c2a2f23982c..af2126d2b2ff 100644 --- a/drivers/soc/ti/ti_sci_pm_domains.c +++ b/drivers/soc/ti/ti_sci_pm_domains.c @@ -9,7 +9,6 @@ #include <linux/err.h> #include <linux/module.h> -#include <linux/mutex.h> #include <linux/of.h> #include <linux/platform_device.h> #include <linux/pm_domain.h> @@ -18,150 +17,95 @@ #include <dt-bindings/soc/ti,sci_pm_domain.h> /** - * struct ti_sci_genpd_dev_data: holds data needed for every device attached - * to this genpd - * @idx: index of the device that identifies it with the system - * control processor. - * @exclusive: Permissions for exclusive request or shared request of the - * device. + * struct ti_sci_genpd_provider: holds common TI SCI genpd provider data + * @ti_sci: handle to TI SCI protocol driver that provides ops to + * communicate with system control processor. + * @dev: pointer to dev for the driver for devm allocs + * @pd_list: list of all the power domains on the device + * @data: onecell data for genpd core */ -struct ti_sci_genpd_dev_data { - int idx; - u8 exclusive; +struct ti_sci_genpd_provider { + const struct ti_sci_handle *ti_sci; + struct device *dev; + struct list_head pd_list; + struct genpd_onecell_data data; }; /** * struct ti_sci_pm_domain: TI specific data needed for power domain - * @ti_sci: handle to TI SCI protocol driver that provides ops to - * communicate with system control processor. - * @dev: pointer to dev for the driver for devm allocs + * @idx: index of the device that identifies it with the system + * control processor. + * @exclusive: Permissions for exclusive request or shared request of the + * device. * @pd: generic_pm_domain for use with the genpd framework + * @node: link for the genpd list + * @parent: link to the parent TI SCI genpd provider */ struct ti_sci_pm_domain { - const struct ti_sci_handle *ti_sci; - struct device *dev; + int idx; + u8 exclusive; struct generic_pm_domain pd; + struct list_head node; + struct ti_sci_genpd_provider *parent; }; #define genpd_to_ti_sci_pd(gpd) container_of(gpd, struct ti_sci_pm_domain, pd) -/** - * ti_sci_dev_id(): get prepopulated ti_sci id from struct dev - * @dev: pointer to device associated with this genpd - * - * Returns device_id stored from ti,sci_id property - */ -static int ti_sci_dev_id(struct device *dev) -{ - struct generic_pm_domain_data *genpd_data = dev_gpd_data(dev); - struct ti_sci_genpd_dev_data *sci_dev_data = genpd_data->data; - - return sci_dev_data->idx; -} - -static u8 is_ti_sci_dev_exclusive(struct device *dev) -{ - struct generic_pm_domain_data *genpd_data = dev_gpd_data(dev); - struct ti_sci_genpd_dev_data *sci_dev_data = genpd_data->data; - - return sci_dev_data->exclusive; -} - -/** - * ti_sci_dev_to_sci_handle(): get pointer to ti_sci_handle - * @dev: pointer to device associated with this genpd - * - * Returns ti_sci_handle to be used to communicate with system - * control processor. +/* + * ti_sci_pd_power_off(): genpd power down hook + * @domain: pointer to the powerdomain to power off */ -static const struct ti_sci_handle *ti_sci_dev_to_sci_handle(struct device *dev) +static int ti_sci_pd_power_off(struct generic_pm_domain *domain) { - struct generic_pm_domain *pd = pd_to_genpd(dev->pm_domain); - struct ti_sci_pm_domain *ti_sci_genpd = genpd_to_ti_sci_pd(pd); + struct ti_sci_pm_domain *pd = genpd_to_ti_sci_pd(domain); + const struct ti_sci_handle *ti_sci = pd->parent->ti_sci; - return ti_sci_genpd->ti_sci; + return ti_sci->ops.dev_ops.put_device(ti_sci, pd->idx); } -/** - * ti_sci_dev_start(): genpd device start hook called to turn device on - * @dev: pointer to device associated with this genpd to be powered on +/* + * ti_sci_pd_power_on(): genpd power up hook + * @domain: pointer to the powerdomain to power on */ -static int ti_sci_dev_start(struct device *dev) +static int ti_sci_pd_power_on(struct generic_pm_domain *domain) { - const struct ti_sci_handle *ti_sci = ti_sci_dev_to_sci_handle(dev); - int idx = ti_sci_dev_id(dev); + struct ti_sci_pm_domain *pd = genpd_to_ti_sci_pd(domain); + const struct ti_sci_handle *ti_sci = pd->parent->ti_sci; - if (is_ti_sci_dev_exclusive(dev)) - return ti_sci->ops.dev_ops.get_device_exclusive(ti_sci, idx); + if (pd->exclusive) + return ti_sci->ops.dev_ops.get_device_exclusive(ti_sci, + pd->idx); else - return ti_sci->ops.dev_ops.get_device(ti_sci, idx); + return ti_sci->ops.dev_ops.get_device(ti_sci, pd->idx); } -/** - * ti_sci_dev_stop(): genpd device stop hook called to turn device off - * @dev: pointer to device associated with this genpd to be powered off +/* + * ti_sci_pd_xlate(): translation service for TI SCI genpds + * @genpdspec: DT identification data for the genpd + * @data: genpd core data for all the powerdomains on the device */ -static int ti_sci_dev_stop(struct device *dev) +static struct generic_pm_domain *ti_sci_pd_xlate( + struct of_phandle_args *genpdspec, + void *data) { - const struct ti_sci_handle *ti_sci = ti_sci_dev_to_sci_handle(dev); - int idx = ti_sci_dev_id(dev); + struct genpd_onecell_data *genpd_data = data; + unsigned int idx = genpdspec->args[0]; - return ti_sci->ops.dev_ops.put_device(ti_sci, idx); -} + if (genpdspec->args_count < 2) + return ERR_PTR(-EINVAL); -static int ti_sci_pd_attach_dev(struct generic_pm_domain *domain, - struct device *dev) -{ - struct device_node *np = dev->of_node; - struct of_phandle_args pd_args; - struct ti_sci_pm_domain *ti_sci_genpd = genpd_to_ti_sci_pd(domain); - const struct ti_sci_handle *ti_sci = ti_sci_genpd->ti_sci; - struct ti_sci_genpd_dev_data *sci_dev_data; - struct generic_pm_domain_data *genpd_data; - int idx, ret = 0; - - ret = of_parse_phandle_with_args(np, "power-domains", - "#power-domain-cells", 0, &pd_args); - if (ret < 0) - return ret; - - if (pd_args.args_count != 1 && pd_args.args_count != 2) - return -EINVAL; - - idx = pd_args.args[0]; - - /* - * Check the validity of the requested idx, if the index is not valid - * the PMMC will return a NAK here and we will not allocate it. - */ - ret = ti_sci->ops.dev_ops.is_valid(ti_sci, idx); - if (ret) - return -EINVAL; - - sci_dev_data = kzalloc(sizeof(*sci_dev_data), GFP_KERNEL); - if (!sci_dev_data) - return -ENOMEM; + if (idx >= genpd_data->num_domains) { + pr_err("%s: invalid domain index %u\n", __func__, idx); + return ERR_PTR(-EINVAL); + } - sci_dev_data->idx = idx; - /* Enable the exclusive permissions by default */ - sci_dev_data->exclusive = TI_SCI_PD_EXCLUSIVE; - if (pd_args.args_count == 2) - sci_dev_data->exclusive = pd_args.args[1] & 0x1; + if (!genpd_data->domains[idx]) + return ERR_PTR(-ENOENT); - genpd_data = dev_gpd_data(dev); - genpd_data->data = sci_dev_data; + genpd_to_ti_sci_pd(genpd_data->domains[idx])->exclusive = + genpdspec->args[1]; - return 0; -} - -static void ti_sci_pd_detach_dev(struct generic_pm_domain *domain, - struct device *dev) -{ - struct generic_pm_domain_data *genpd_data = dev_gpd_data(dev); - struct ti_sci_genpd_dev_data *sci_dev_data = genpd_data->data; - - kfree(sci_dev_data); - genpd_data->data = NULL; + return genpd_data->domains[idx]; } static const struct of_device_id ti_sci_pm_domain_matches[] = { @@ -173,33 +117,80 @@ MODULE_DEVICE_TABLE(of, ti_sci_pm_domain_matches); static int ti_sci_pm_domain_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct device_node *np = dev->of_node; - struct ti_sci_pm_domain *ti_sci_pd; + struct ti_sci_genpd_provider *pd_provider; + struct ti_sci_pm_domain *pd; + struct device_node *np = NULL; + struct of_phandle_args args; int ret; + u32 max_id = 0; + int index; - ti_sci_pd = devm_kzalloc(dev, sizeof(*ti_sci_pd), GFP_KERNEL); - if (!ti_sci_pd) + pd_provider = devm_kzalloc(dev, sizeof(*pd_provider), GFP_KERNEL); + if (!pd_provider) return -ENOMEM; - ti_sci_pd->ti_sci = devm_ti_sci_get_handle(dev); - if (IS_ERR(ti_sci_pd->ti_sci)) - return PTR_ERR(ti_sci_pd->ti_sci); + pd_provider->ti_sci = devm_ti_sci_get_handle(dev); + if (IS_ERR(pd_provider->ti_sci)) + return PTR_ERR(pd_provider->ti_sci); + + pd_provider->dev = dev; + + INIT_LIST_HEAD(&pd_provider->pd_list); + + /* Find highest device ID used for power domains */ + while (1) { + np = of_find_node_with_property(np, "power-domains"); + if (!np) + break; + + index = 0; + + while (1) { + ret = of_parse_phandle_with_args(np, "power-domains", + "#power-domain-cells", + index, &args); + if (ret) + break; + + if (args.args_count >= 1 && args.np == dev->of_node) { + if (args.args[0] > max_id) + max_id = args.args[0]; + + pd = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL); + if (!pd) + return -ENOMEM; + + pd->pd.name = devm_kasprintf(dev, GFP_KERNEL, + "pd:%d", + args.args[0]); + if (!pd->pd.name) + return -ENOMEM; - ti_sci_pd->dev = dev; + pd->pd.power_off = ti_sci_pd_power_off; + pd->pd.power_on = ti_sci_pd_power_on; + pd->idx = args.args[0]; + pd->parent = pd_provider; - ti_sci_pd->pd.name = "ti_sci_pd"; + pm_genpd_init(&pd->pd, NULL, true); - ti_sci_pd->pd.attach_dev = ti_sci_pd_attach_dev; - ti_sci_pd->pd.detach_dev = ti_sci_pd_detach_dev; + list_add(&pd->node, &pd_provider->pd_list); + } + index++; + } + } - ti_sci_pd->pd.dev_ops.start = ti_sci_dev_start; - ti_sci_pd->pd.dev_ops.stop = ti_sci_dev_stop; + pd_provider->data.domains = + devm_kcalloc(dev, max_id + 1, + sizeof(*pd_provider->data.domains), + GFP_KERNEL); - pm_genpd_init(&ti_sci_pd->pd, NULL, true); + pd_provider->data.num_domains = max_id + 1; + pd_provider->data.xlate = ti_sci_pd_xlate; - ret = of_genpd_add_provider_simple(np, &ti_sci_pd->pd); + list_for_each_entry(pd, &pd_provider->pd_list, node) + pd_provider->data.domains[pd->idx] = &pd->pd; - return ret; + return of_genpd_add_provider_onecell(dev->of_node, &pd_provider->data); } static struct platform_driver ti_sci_pm_domains_driver = { diff --git a/drivers/soc/versatile/soc-integrator.c b/drivers/soc/versatile/soc-integrator.c index 7dcf77ccd31e..bab4ad87aa75 100644 --- a/drivers/soc/versatile/soc-integrator.c +++ b/drivers/soc/versatile/soc-integrator.c @@ -100,7 +100,7 @@ ATTRIBUTE_GROUPS(integrator); static int __init integrator_soc_init(void) { - static struct regmap *syscon_regmap; + struct regmap *syscon_regmap; struct soc_device *soc_dev; struct soc_device_attribute *soc_dev_attr; struct device_node *np; diff --git a/drivers/soc/xilinx/zynqmp_power.c b/drivers/soc/xilinx/zynqmp_power.c index 31ff49fcd078..c556623dae02 100644 --- a/drivers/soc/xilinx/zynqmp_power.c +++ b/drivers/soc/xilinx/zynqmp_power.c @@ -205,7 +205,7 @@ static int zynqmp_pm_probe(struct platform_device *pdev) rx_chan = mbox_request_channel_byname(client, "rx"); if (IS_ERR(rx_chan)) { dev_err(&pdev->dev, "Failed to request rx channel\n"); - return IS_ERR(rx_chan); + return PTR_ERR(rx_chan); } } else if (of_find_property(pdev->dev.of_node, "interrupts", NULL)) { irq = platform_get_irq(pdev, 0); diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index d2c976e55b8b..5cff60de8e83 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -709,7 +709,7 @@ config SPI_S3C24XX_FIQ config SPI_S3C64XX tristate "Samsung S3C64XX series type SPI" - depends on (PLAT_SAMSUNG || ARCH_EXYNOS || COMPILE_TEST) + depends on (PLAT_SAMSUNG || ARCH_S5PV210 || ARCH_EXYNOS || COMPILE_TEST) help SPI driver for Samsung S3C64XX and newer SoCs. diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile index 21dc75842aca..6fea5821662e 100644 --- a/drivers/spi/Makefile +++ b/drivers/spi/Makefile @@ -98,7 +98,6 @@ obj-$(CONFIG_SPI_RPCIF) += spi-rpc-if.o obj-$(CONFIG_SPI_RSPI) += spi-rspi.o obj-$(CONFIG_SPI_S3C24XX) += spi-s3c24xx-hw.o spi-s3c24xx-hw-y := spi-s3c24xx.o -spi-s3c24xx-hw-$(CONFIG_SPI_S3C24XX_FIQ) += spi-s3c24xx-fiq.o obj-$(CONFIG_SPI_S3C64XX) += spi-s3c64xx.o obj-$(CONFIG_SPI_SC18IS602) += spi-sc18is602.o obj-$(CONFIG_SPI_SH) += spi-sh.o diff --git a/drivers/spi/spi-s3c24xx-fiq.S b/drivers/spi/spi-s3c24xx-fiq.S deleted file mode 100644 index e95d6282109e..000000000000 --- a/drivers/spi/spi-s3c24xx-fiq.S +++ /dev/null @@ -1,113 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* linux/drivers/spi/spi_s3c24xx_fiq.S - * - * Copyright 2009 Simtec Electronics - * Ben Dooks <ben@simtec.co.uk> - * - * S3C24XX SPI - FIQ pseudo-DMA transfer code -*/ - -#include <linux/linkage.h> -#include <asm/assembler.h> - -#include <mach/map.h> -#include <mach/regs-irq.h> -#include <plat/regs-spi.h> - -#include "spi-s3c24xx-fiq.h" - - .text - - @ entry to these routines is as follows, with the register names - @ defined in fiq.h so that they can be shared with the C files which - @ setup the calling registers. - @ - @ fiq_rirq The base of the IRQ registers to find S3C2410_SRCPND - @ fiq_rtmp Temporary register to hold tx/rx data - @ fiq_rspi The base of the SPI register block - @ fiq_rtx The tx buffer pointer - @ fiq_rrx The rx buffer pointer - @ fiq_rcount The number of bytes to move - - @ each entry starts with a word entry of how long it is - @ and an offset to the irq acknowledgment word - -ENTRY(s3c24xx_spi_fiq_rx) -s3c24xx_spi_fix_rx: - .word fiq_rx_end - fiq_rx_start - .word fiq_rx_irq_ack - fiq_rx_start -fiq_rx_start: - ldr fiq_rtmp, fiq_rx_irq_ack - str fiq_rtmp, [ fiq_rirq, # S3C2410_SRCPND - S3C24XX_VA_IRQ ] - - ldrb fiq_rtmp, [ fiq_rspi, # S3C2410_SPRDAT ] - strb fiq_rtmp, [ fiq_rrx ], #1 - - mov fiq_rtmp, #0xff - strb fiq_rtmp, [ fiq_rspi, # S3C2410_SPTDAT ] - - subs fiq_rcount, fiq_rcount, #1 - subnes pc, lr, #4 @@ return, still have work to do - - @@ set IRQ controller so that next op will trigger IRQ - mov fiq_rtmp, #0 - str fiq_rtmp, [ fiq_rirq, # S3C2410_INTMOD - S3C24XX_VA_IRQ ] - subs pc, lr, #4 - -fiq_rx_irq_ack: - .word 0 -fiq_rx_end: - -ENTRY(s3c24xx_spi_fiq_txrx) -s3c24xx_spi_fiq_txrx: - .word fiq_txrx_end - fiq_txrx_start - .word fiq_txrx_irq_ack - fiq_txrx_start -fiq_txrx_start: - - ldrb fiq_rtmp, [ fiq_rspi, # S3C2410_SPRDAT ] - strb fiq_rtmp, [ fiq_rrx ], #1 - - ldr fiq_rtmp, fiq_txrx_irq_ack - str fiq_rtmp, [ fiq_rirq, # S3C2410_SRCPND - S3C24XX_VA_IRQ ] - - ldrb fiq_rtmp, [ fiq_rtx ], #1 - strb fiq_rtmp, [ fiq_rspi, # S3C2410_SPTDAT ] - - subs fiq_rcount, fiq_rcount, #1 - subnes pc, lr, #4 @@ return, still have work to do - - mov fiq_rtmp, #0 - str fiq_rtmp, [ fiq_rirq, # S3C2410_INTMOD - S3C24XX_VA_IRQ ] - subs pc, lr, #4 - -fiq_txrx_irq_ack: - .word 0 - -fiq_txrx_end: - -ENTRY(s3c24xx_spi_fiq_tx) -s3c24xx_spi_fix_tx: - .word fiq_tx_end - fiq_tx_start - .word fiq_tx_irq_ack - fiq_tx_start -fiq_tx_start: - ldrb fiq_rtmp, [ fiq_rspi, # S3C2410_SPRDAT ] - - ldr fiq_rtmp, fiq_tx_irq_ack - str fiq_rtmp, [ fiq_rirq, # S3C2410_SRCPND - S3C24XX_VA_IRQ ] - - ldrb fiq_rtmp, [ fiq_rtx ], #1 - strb fiq_rtmp, [ fiq_rspi, # S3C2410_SPTDAT ] - - subs fiq_rcount, fiq_rcount, #1 - subnes pc, lr, #4 @@ return, still have work to do - - mov fiq_rtmp, #0 - str fiq_rtmp, [ fiq_rirq, # S3C2410_INTMOD - S3C24XX_VA_IRQ ] - subs pc, lr, #4 - -fiq_tx_irq_ack: - .word 0 - -fiq_tx_end: - - .end diff --git a/drivers/spi/spi-s3c24xx-fiq.h b/drivers/spi/spi-s3c24xx-fiq.h deleted file mode 100644 index 7786b0ea56ec..000000000000 --- a/drivers/spi/spi-s3c24xx-fiq.h +++ /dev/null @@ -1,23 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* linux/drivers/spi/spi_s3c24xx_fiq.h - * - * Copyright 2009 Simtec Electronics - * Ben Dooks <ben@simtec.co.uk> - * - * S3C24XX SPI - FIQ pseudo-DMA transfer support -*/ - -/* We have R8 through R13 to play with */ - -#ifdef __ASSEMBLY__ -#define __REG_NR(x) r##x -#else -#define __REG_NR(x) (x) -#endif - -#define fiq_rspi __REG_NR(8) -#define fiq_rtmp __REG_NR(9) -#define fiq_rrx __REG_NR(10) -#define fiq_rtx __REG_NR(11) -#define fiq_rcount __REG_NR(12) -#define fiq_rirq __REG_NR(13) diff --git a/drivers/spi/spi-s3c24xx-regs.h b/drivers/spi/spi-s3c24xx-regs.h new file mode 100644 index 000000000000..f51464ab5677 --- /dev/null +++ b/drivers/spi/spi-s3c24xx-regs.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2004 Fetron GmbH + * + * S3C2410 SPI register definition + */ + +#ifndef __SPI_S3C2410_H +#define __SPI_S3C2410_H + +#define S3C2410_SPCON (0x00) + +#define S3C2410_SPCON_SMOD_DMA (2 << 5) /* DMA mode */ +#define S3C2410_SPCON_SMOD_INT (1 << 5) /* interrupt mode */ +#define S3C2410_SPCON_SMOD_POLL (0 << 5) /* polling mode */ +#define S3C2410_SPCON_ENSCK (1 << 4) /* Enable SCK */ +#define S3C2410_SPCON_MSTR (1 << 3) /* Master:1, Slave:0 select */ +#define S3C2410_SPCON_CPOL_HIGH (1 << 2) /* Clock polarity select */ +#define S3C2410_SPCON_CPOL_LOW (0 << 2) /* Clock polarity select */ + +#define S3C2410_SPCON_CPHA_FMTB (1 << 1) /* Clock Phase Select */ +#define S3C2410_SPCON_CPHA_FMTA (0 << 1) /* Clock Phase Select */ + +#define S3C2410_SPSTA (0x04) + +#define S3C2410_SPSTA_DCOL (1 << 2) /* Data Collision Error */ +#define S3C2410_SPSTA_MULD (1 << 1) /* Multi Master Error */ +#define S3C2410_SPSTA_READY (1 << 0) /* Data Tx/Rx ready */ +#define S3C2412_SPSTA_READY_ORG (1 << 3) + +#define S3C2410_SPPIN (0x08) + +#define S3C2410_SPPIN_ENMUL (1 << 2) /* Multi Master Error detect */ +#define S3C2410_SPPIN_RESERVED (1 << 1) +#define S3C2410_SPPIN_KEEP (1 << 0) /* Master Out keep */ + +#define S3C2410_SPPRE (0x0C) +#define S3C2410_SPTDAT (0x10) +#define S3C2410_SPRDAT (0x14) + +#endif /* __SPI_S3C2410_H */ diff --git a/drivers/spi/spi-s3c24xx.c b/drivers/spi/spi-s3c24xx.c index 7742170fca91..d6f51695ca5b 100644 --- a/drivers/spi/spi-s3c24xx.c +++ b/drivers/spi/spi-s3c24xx.c @@ -19,13 +19,12 @@ #include <linux/spi/spi.h> #include <linux/spi/spi_bitbang.h> #include <linux/spi/s3c24xx.h> +#include <linux/spi/s3c24xx-fiq.h> #include <linux/module.h> -#include <plat/regs-spi.h> - #include <asm/fiq.h> -#include "spi-s3c24xx-fiq.h" +#include "spi-s3c24xx-regs.h" /** * struct s3c24xx_spi_devstate - per device data @@ -230,21 +229,6 @@ struct spi_fiq_code { u8 data[]; }; -extern struct spi_fiq_code s3c24xx_spi_fiq_txrx; -extern struct spi_fiq_code s3c24xx_spi_fiq_tx; -extern struct spi_fiq_code s3c24xx_spi_fiq_rx; - -/** - * ack_bit - turn IRQ into IRQ acknowledgement bit - * @irq: The interrupt number - * - * Returns the bit to write to the interrupt acknowledge register. - */ -static inline u32 ack_bit(unsigned int irq) -{ - return 1 << (irq - IRQ_EINT0); -} - /** * s3c24xx_spi_tryfiq - attempt to claim and setup FIQ for transfer * @hw: The hardware state. @@ -261,6 +245,7 @@ static void s3c24xx_spi_tryfiq(struct s3c24xx_spi *hw) struct pt_regs regs; enum spi_fiq_mode mode; struct spi_fiq_code *code; + u32 *ack_ptr = NULL; int ret; if (!hw->fiq_claimed) { @@ -283,13 +268,10 @@ static void s3c24xx_spi_tryfiq(struct s3c24xx_spi *hw) regs.uregs[fiq_rrx] = (long)hw->rx; regs.uregs[fiq_rtx] = (long)hw->tx + 1; regs.uregs[fiq_rcount] = hw->len - 1; - regs.uregs[fiq_rirq] = (long)S3C24XX_VA_IRQ; set_fiq_regs(®s); if (hw->fiq_mode != mode) { - u32 *ack_ptr; - hw->fiq_mode = mode; switch (mode) { @@ -309,12 +291,10 @@ static void s3c24xx_spi_tryfiq(struct s3c24xx_spi *hw) BUG_ON(!code); ack_ptr = (u32 *)&code->data[code->ack_offset]; - *ack_ptr = ack_bit(hw->irq); - set_fiq_handler(&code->data, code->length); } - s3c24xx_set_fiq(hw->irq, true); + s3c24xx_set_fiq(hw->irq, ack_ptr, true); hw->fiq_mode = mode; hw->fiq_inuse = 1; diff --git a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c index 376205e97a89..0295e2e32d79 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c +++ b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c @@ -1570,7 +1570,7 @@ static int atomisp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *i spin_lock_init(&isp->lock); /* This is not a true PCI device on SoC, so the delay is not needed. */ - pdev->d3_delay = 0; + pdev->d3hot_delay = 0; pci_set_drvdata(pdev, isp); diff --git a/drivers/target/target_core_rd.c b/drivers/target/target_core_rd.c index 408bd975170b..bf936bbeccfe 100644 --- a/drivers/target/target_core_rd.c +++ b/drivers/target/target_core_rd.c @@ -131,7 +131,7 @@ static int rd_allocate_sgl_table(struct rd_dev *rd_dev, struct rd_dev_sg_table * if (sg_per_table < total_sg_needed) chain_entry = 1; - sg = kcalloc(sg_per_table + chain_entry, sizeof(*sg), + sg = kmalloc_array(sg_per_table + chain_entry, sizeof(*sg), GFP_KERNEL); if (!sg) return -ENOMEM; diff --git a/drivers/tee/optee/core.c b/drivers/tee/optee/core.c index b373b1b08b6d..cf4718c6d35d 100644 --- a/drivers/tee/optee/core.c +++ b/drivers/tee/optee/core.c @@ -216,6 +216,8 @@ static void optee_get_version(struct tee_device *teedev, if (optee->sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) v.gen_caps |= TEE_GEN_CAP_REG_MEM; + if (optee->sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL) + v.gen_caps |= TEE_GEN_CAP_MEMREF_NULL; *vers = v; } @@ -262,6 +264,11 @@ static int optee_open(struct tee_context *ctx) mutex_init(&ctxdata->mutex); INIT_LIST_HEAD(&ctxdata->sess_list); + if (optee->sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL) + ctx->cap_memref_null = true; + else + ctx->cap_memref_null = false; + ctx->data = ctxdata; return 0; } diff --git a/drivers/tee/optee/optee_msg.h b/drivers/tee/optee/optee_msg.h index 795bc19ae17a..7b2d919da2ac 100644 --- a/drivers/tee/optee/optee_msg.h +++ b/drivers/tee/optee/optee_msg.h @@ -419,4 +419,25 @@ struct optee_msg_arg { */ #define OPTEE_MSG_RPC_CMD_SHM_FREE 7 +/* + * Access a device on an i2c bus + * + * [in] param[0].u.value.a mode: RD(0), WR(1) + * [in] param[0].u.value.b i2c adapter + * [in] param[0].u.value.c i2c chip + * + * [in] param[1].u.value.a i2c control flags + * + * [in/out] memref[2] buffer to exchange the transfer data + * with the secure world + * + * [out] param[3].u.value.a bytes transferred by the driver + */ +#define OPTEE_MSG_RPC_CMD_I2C_TRANSFER 21 +/* I2C master transfer modes */ +#define OPTEE_MSG_RPC_CMD_I2C_TRANSFER_RD 0 +#define OPTEE_MSG_RPC_CMD_I2C_TRANSFER_WR 1 +/* I2C master control flags */ +#define OPTEE_MSG_RPC_CMD_I2C_FLAGS_TEN_BIT BIT(0) + #endif /* _OPTEE_MSG_H */ diff --git a/drivers/tee/optee/optee_private.h b/drivers/tee/optee/optee_private.h index 8b71839a357e..e25b216a14ef 100644 --- a/drivers/tee/optee/optee_private.h +++ b/drivers/tee/optee/optee_private.h @@ -17,6 +17,7 @@ /* Some Global Platform error codes used in this driver */ #define TEEC_SUCCESS 0x00000000 #define TEEC_ERROR_BAD_PARAMETERS 0xFFFF0006 +#define TEEC_ERROR_NOT_SUPPORTED 0xFFFF000A #define TEEC_ERROR_COMMUNICATION 0xFFFF000E #define TEEC_ERROR_OUT_OF_MEMORY 0xFFFF000C #define TEEC_ERROR_SHORT_BUFFER 0xFFFF0010 diff --git a/drivers/tee/optee/optee_smc.h b/drivers/tee/optee/optee_smc.h index c72122d9c997..777ad54d4c2c 100644 --- a/drivers/tee/optee/optee_smc.h +++ b/drivers/tee/optee/optee_smc.h @@ -215,6 +215,9 @@ struct optee_smc_get_shm_config_result { */ #define OPTEE_SMC_SEC_CAP_DYNAMIC_SHM BIT(2) +/* Secure world supports Shared Memory with a NULL buffer reference */ +#define OPTEE_SMC_SEC_CAP_MEMREF_NULL BIT(4) + #define OPTEE_SMC_FUNCID_EXCHANGE_CAPABILITIES 9 #define OPTEE_SMC_EXCHANGE_CAPABILITIES \ OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_EXCHANGE_CAPABILITIES) diff --git a/drivers/tee/optee/rpc.c b/drivers/tee/optee/rpc.c index b4ade54d1f28..1e3614e4798f 100644 --- a/drivers/tee/optee/rpc.c +++ b/drivers/tee/optee/rpc.c @@ -7,6 +7,7 @@ #include <linux/delay.h> #include <linux/device.h> +#include <linux/i2c.h> #include <linux/slab.h> #include <linux/tee_drv.h> #include "optee_private.h" @@ -49,6 +50,97 @@ bad: arg->ret = TEEC_ERROR_BAD_PARAMETERS; } +#if IS_REACHABLE(CONFIG_I2C) +static void handle_rpc_func_cmd_i2c_transfer(struct tee_context *ctx, + struct optee_msg_arg *arg) +{ + struct i2c_client client = { 0 }; + struct tee_param *params; + size_t i; + int ret = -EOPNOTSUPP; + u8 attr[] = { + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT, + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT, + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT, + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT, + }; + + if (arg->num_params != ARRAY_SIZE(attr)) { + arg->ret = TEEC_ERROR_BAD_PARAMETERS; + return; + } + + params = kmalloc_array(arg->num_params, sizeof(struct tee_param), + GFP_KERNEL); + if (!params) { + arg->ret = TEEC_ERROR_OUT_OF_MEMORY; + return; + } + + if (optee_from_msg_param(params, arg->num_params, arg->params)) + goto bad; + + for (i = 0; i < arg->num_params; i++) { + if (params[i].attr != attr[i]) + goto bad; + } + + client.adapter = i2c_get_adapter(params[0].u.value.b); + if (!client.adapter) + goto bad; + + if (params[1].u.value.a & OPTEE_MSG_RPC_CMD_I2C_FLAGS_TEN_BIT) { + if (!i2c_check_functionality(client.adapter, + I2C_FUNC_10BIT_ADDR)) { + i2c_put_adapter(client.adapter); + goto bad; + } + + client.flags = I2C_CLIENT_TEN; + } + + client.addr = params[0].u.value.c; + snprintf(client.name, I2C_NAME_SIZE, "i2c%d", client.adapter->nr); + + switch (params[0].u.value.a) { + case OPTEE_MSG_RPC_CMD_I2C_TRANSFER_RD: + ret = i2c_master_recv(&client, params[2].u.memref.shm->kaddr, + params[2].u.memref.size); + break; + case OPTEE_MSG_RPC_CMD_I2C_TRANSFER_WR: + ret = i2c_master_send(&client, params[2].u.memref.shm->kaddr, + params[2].u.memref.size); + break; + default: + i2c_put_adapter(client.adapter); + goto bad; + } + + if (ret < 0) { + arg->ret = TEEC_ERROR_COMMUNICATION; + } else { + params[3].u.value.a = ret; + if (optee_to_msg_param(arg->params, arg->num_params, params)) + arg->ret = TEEC_ERROR_BAD_PARAMETERS; + else + arg->ret = TEEC_SUCCESS; + } + + i2c_put_adapter(client.adapter); + kfree(params); + return; +bad: + kfree(params); + arg->ret = TEEC_ERROR_BAD_PARAMETERS; +} +#else +static void handle_rpc_func_cmd_i2c_transfer(struct tee_context *ctx, + struct optee_msg_arg *arg) +{ + arg->ret = TEEC_ERROR_NOT_SUPPORTED; +} +#endif + static struct wq_entry *wq_entry_get(struct optee_wait_queue *wq, u32 key) { struct wq_entry *w; @@ -382,6 +474,9 @@ static void handle_rpc_func_cmd(struct tee_context *ctx, struct optee *optee, case OPTEE_MSG_RPC_CMD_SHM_FREE: handle_rpc_func_cmd_shm_free(ctx, arg); break; + case OPTEE_MSG_RPC_CMD_I2C_TRANSFER: + handle_rpc_func_cmd_i2c_transfer(ctx, arg); + break; default: handle_rpc_supp_cmd(ctx, arg); } diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c index 64637e09a095..f53bf336c0a2 100644 --- a/drivers/tee/tee_core.c +++ b/drivers/tee/tee_core.c @@ -383,25 +383,38 @@ static int params_from_user(struct tee_context *ctx, struct tee_param *params, case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT: case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT: /* - * If we fail to get a pointer to a shared memory - * object (and increase the ref count) from an - * identifier we return an error. All pointers that - * has been added in params have an increased ref - * count. It's the callers responibility to do - * tee_shm_put() on all resolved pointers. + * If a NULL pointer is passed to a TA in the TEE, + * the ip.c IOCTL parameters is set to TEE_MEMREF_NULL + * indicating a NULL memory reference. */ - shm = tee_shm_get_from_id(ctx, ip.c); - if (IS_ERR(shm)) - return PTR_ERR(shm); - - /* - * Ensure offset + size does not overflow offset - * and does not overflow the size of the referred - * shared memory object. - */ - if ((ip.a + ip.b) < ip.a || - (ip.a + ip.b) > shm->size) { - tee_shm_put(shm); + if (ip.c != TEE_MEMREF_NULL) { + /* + * If we fail to get a pointer to a shared + * memory object (and increase the ref count) + * from an identifier we return an error. All + * pointers that has been added in params have + * an increased ref count. It's the callers + * responibility to do tee_shm_put() on all + * resolved pointers. + */ + shm = tee_shm_get_from_id(ctx, ip.c); + if (IS_ERR(shm)) + return PTR_ERR(shm); + + /* + * Ensure offset + size does not overflow + * offset and does not overflow the size of + * the referred shared memory object. + */ + if ((ip.a + ip.b) < ip.a || + (ip.a + ip.b) > shm->size) { + tee_shm_put(shm); + return -EINVAL; + } + } else if (ctx->cap_memref_null) { + /* Pass NULL pointer to OP-TEE */ + shm = NULL; + } else { return -EINVAL; } @@ -917,7 +930,6 @@ struct tee_device *tee_device_alloc(const struct tee_desc *teedesc, cdev_init(&teedev->cdev, &tee_fops); teedev->cdev.owner = teedesc->owner; - teedev->cdev.kobj.parent = &teedev->dev.kobj; dev_set_drvdata(&teedev->dev, driver_data); device_initialize(&teedev->dev); @@ -963,9 +975,7 @@ static struct attribute *tee_dev_attrs[] = { NULL }; -static const struct attribute_group tee_dev_group = { - .attrs = tee_dev_attrs, -}; +ATTRIBUTE_GROUPS(tee_dev); /** * tee_device_register() - Registers a TEE device @@ -985,39 +995,19 @@ int tee_device_register(struct tee_device *teedev) return -EINVAL; } - rc = cdev_add(&teedev->cdev, teedev->dev.devt, 1); - if (rc) { - dev_err(&teedev->dev, - "unable to cdev_add() %s, major %d, minor %d, err=%d\n", - teedev->name, MAJOR(teedev->dev.devt), - MINOR(teedev->dev.devt), rc); - return rc; - } + teedev->dev.groups = tee_dev_groups; - rc = device_add(&teedev->dev); + rc = cdev_device_add(&teedev->cdev, &teedev->dev); if (rc) { dev_err(&teedev->dev, - "unable to device_add() %s, major %d, minor %d, err=%d\n", + "unable to cdev_device_add() %s, major %d, minor %d, err=%d\n", teedev->name, MAJOR(teedev->dev.devt), MINOR(teedev->dev.devt), rc); - goto err_device_add; - } - - rc = sysfs_create_group(&teedev->dev.kobj, &tee_dev_group); - if (rc) { - dev_err(&teedev->dev, - "failed to create sysfs attributes, err=%d\n", rc); - goto err_sysfs_create_group; + return rc; } teedev->flags |= TEE_DEVICE_FLAG_REGISTERED; return 0; - -err_sysfs_create_group: - device_del(&teedev->dev); -err_device_add: - cdev_del(&teedev->cdev); - return rc; } EXPORT_SYMBOL_GPL(tee_device_register); @@ -1060,11 +1050,8 @@ void tee_device_unregister(struct tee_device *teedev) if (!teedev) return; - if (teedev->flags & TEE_DEVICE_FLAG_REGISTERED) { - sysfs_remove_group(&teedev->dev.kobj, &tee_dev_group); - cdev_del(&teedev->cdev); - device_del(&teedev->dev); - } + if (teedev->flags & TEE_DEVICE_FLAG_REGISTERED) + cdev_device_del(&teedev->cdev, &teedev->dev); tee_device_put(teedev); wait_for_completion(&teedev->c_no_users); diff --git a/drivers/tee/tee_shm.c b/drivers/tee/tee_shm.c index 827ac3d0fea9..00472f5ce22e 100644 --- a/drivers/tee/tee_shm.c +++ b/drivers/tee/tee_shm.c @@ -12,6 +12,22 @@ #include <linux/uio.h> #include "tee_private.h" +static void release_registered_pages(struct tee_shm *shm) +{ + if (shm->pages) { + if (shm->flags & TEE_SHM_USER_MAPPED) { + unpin_user_pages(shm->pages, shm->num_pages); + } else { + size_t n; + + for (n = 0; n < shm->num_pages; n++) + put_page(shm->pages[n]); + } + + kfree(shm->pages); + } +} + static void tee_shm_release(struct tee_shm *shm) { struct tee_device *teedev = shm->ctx->teedev; @@ -32,17 +48,13 @@ static void tee_shm_release(struct tee_shm *shm) poolm->ops->free(poolm, shm); } else if (shm->flags & TEE_SHM_REGISTER) { - size_t n; int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm); if (rc) dev_err(teedev->dev.parent, "unregister shm %p failed: %d", shm, rc); - for (n = 0; n < shm->num_pages; n++) - put_page(shm->pages[n]); - - kfree(shm->pages); + release_registered_pages(shm); } teedev_ctx_put(shm->ctx); @@ -228,7 +240,7 @@ struct tee_shm *tee_shm_register(struct tee_context *ctx, unsigned long addr, } if (flags & TEE_SHM_USER_MAPPED) { - rc = get_user_pages_fast(start, num_pages, FOLL_WRITE, + rc = pin_user_pages_fast(start, num_pages, FOLL_WRITE, shm->pages); } else { struct kvec *kiov; @@ -292,18 +304,12 @@ struct tee_shm *tee_shm_register(struct tee_context *ctx, unsigned long addr, return shm; err: if (shm) { - size_t n; - if (shm->id >= 0) { mutex_lock(&teedev->mutex); idr_remove(&teedev->idr, shm->id); mutex_unlock(&teedev->mutex); } - if (shm->pages) { - for (n = 0; n < shm->num_pages; n++) - put_page(shm->pages[n]); - kfree(shm->pages); - } + release_registered_pages(shm); } kfree(shm); teedev_ctx_put(ctx); diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index b668224f906d..7edc8dc6bbab 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -346,13 +346,13 @@ config RCAR_THERMAL thermal framework. config RCAR_GEN3_THERMAL - tristate "Renesas R-Car Gen3 thermal driver" + tristate "Renesas R-Car Gen3 and RZ/G2 thermal driver" depends on ARCH_RENESAS || COMPILE_TEST depends on HAS_IOMEM depends on OF help - Enable this to plug the R-Car Gen3 thermal sensor driver into the Linux - thermal framework. + Enable this to plug the R-Car Gen3 or RZ/G2 thermal sensor driver into + the Linux thermal framework. config KIRKWOOD_THERMAL tristate "Temperature sensor on Marvell Kirkwood SoCs" diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c index 6cf23a54e853..cc2959f22f01 100644 --- a/drivers/thermal/cpufreq_cooling.c +++ b/drivers/thermal/cpufreq_cooling.c @@ -182,7 +182,6 @@ static u32 get_dynamic_power(struct cpufreq_cooling_device *cpufreq_cdev, /** * cpufreq_get_requested_power() - get the current power * @cdev: &thermal_cooling_device pointer - * @tz: a valid thermal zone device pointer * @power: pointer in which to store the resulting power * * Calculate the current power consumption of the cpus in milliwatts @@ -203,7 +202,6 @@ static u32 get_dynamic_power(struct cpufreq_cooling_device *cpufreq_cdev, * Return: 0 on success, -E* if getting the static power failed. */ static int cpufreq_get_requested_power(struct thermal_cooling_device *cdev, - struct thermal_zone_device *tz, u32 *power) { unsigned long freq; @@ -253,7 +251,6 @@ static int cpufreq_get_requested_power(struct thermal_cooling_device *cdev, /** * cpufreq_state2power() - convert a cpu cdev state to power consumed * @cdev: &thermal_cooling_device pointer - * @tz: a valid thermal zone device pointer * @state: cooling device state to be converted * @power: pointer in which to store the resulting power * @@ -266,7 +263,6 @@ static int cpufreq_get_requested_power(struct thermal_cooling_device *cdev, * when calculating the static power. */ static int cpufreq_state2power(struct thermal_cooling_device *cdev, - struct thermal_zone_device *tz, unsigned long state, u32 *power) { unsigned int freq, num_cpus, idx; @@ -288,7 +284,6 @@ static int cpufreq_state2power(struct thermal_cooling_device *cdev, /** * cpufreq_power2state() - convert power to a cooling device state * @cdev: &thermal_cooling_device pointer - * @tz: a valid thermal zone device pointer * @power: power in milliwatts to be converted * @state: pointer in which to store the resulting state * @@ -306,8 +301,7 @@ static int cpufreq_state2power(struct thermal_cooling_device *cdev, * device. */ static int cpufreq_power2state(struct thermal_cooling_device *cdev, - struct thermal_zone_device *tz, u32 power, - unsigned long *state) + u32 power, unsigned long *state) { unsigned int target_freq; u32 last_load, normalised_power; diff --git a/drivers/thermal/cpuidle_cooling.c b/drivers/thermal/cpuidle_cooling.c index 78e3e8238116..7ecab4b16b29 100644 --- a/drivers/thermal/cpuidle_cooling.c +++ b/drivers/thermal/cpuidle_cooling.c @@ -30,7 +30,7 @@ static DEFINE_IDA(cpuidle_ida); /** * cpuidle_cooling_runtime - Running time computation - * @idle_duration_us: the idle cooling device + * @idle_duration_us: CPU idle time to inject in microseconds * @state: a percentile based number * * The running duration is computed from the idle injection duration diff --git a/drivers/thermal/devfreq_cooling.c b/drivers/thermal/devfreq_cooling.c index a12d29096229..dfab49a67252 100644 --- a/drivers/thermal/devfreq_cooling.c +++ b/drivers/thermal/devfreq_cooling.c @@ -229,7 +229,6 @@ static inline unsigned long get_total_power(struct devfreq_cooling_device *dfc, static int devfreq_cooling_get_requested_power(struct thermal_cooling_device *cdev, - struct thermal_zone_device *tz, u32 *power) { struct devfreq_cooling_device *dfc = cdev->devdata; @@ -289,7 +288,6 @@ fail: } static int devfreq_cooling_state2power(struct thermal_cooling_device *cdev, - struct thermal_zone_device *tz, unsigned long state, u32 *power) { @@ -308,7 +306,6 @@ static int devfreq_cooling_state2power(struct thermal_cooling_device *cdev, } static int devfreq_cooling_power2state(struct thermal_cooling_device *cdev, - struct thermal_zone_device *tz, u32 power, unsigned long *state) { struct devfreq_cooling_device *dfc = cdev->devdata; diff --git a/drivers/thermal/gov_power_allocator.c b/drivers/thermal/gov_power_allocator.c index 5cb518d8f156..ab0be26f0816 100644 --- a/drivers/thermal/gov_power_allocator.c +++ b/drivers/thermal/gov_power_allocator.c @@ -96,7 +96,7 @@ static u32 estimate_sustainable_power(struct thermal_zone_device *tz) if (instance->trip != params->trip_max_desired_temperature) continue; - if (power_actor_get_min_power(cdev, tz, &min_power)) + if (power_actor_get_min_power(cdev, &min_power)) continue; sustainable_power += min_power; @@ -388,7 +388,7 @@ static int allocate_power(struct thermal_zone_device *tz, if (!cdev_is_power_actor(cdev)) continue; - if (cdev->ops->get_requested_power(cdev, tz, &req_power[i])) + if (cdev->ops->get_requested_power(cdev, &req_power[i])) continue; if (!total_weight) @@ -398,7 +398,7 @@ static int allocate_power(struct thermal_zone_device *tz, weighted_req_power[i] = frac_to_int(weight * req_power[i]); - if (power_actor_get_max_power(cdev, tz, &max_power[i])) + if (power_actor_get_max_power(cdev, &max_power[i])) continue; total_req_power += req_power[i]; diff --git a/drivers/thermal/imx8mm_thermal.c b/drivers/thermal/imx8mm_thermal.c index f5124f14cf81..a1e4f9bb4cb0 100644 --- a/drivers/thermal/imx8mm_thermal.c +++ b/drivers/thermal/imx8mm_thermal.c @@ -146,13 +146,9 @@ static int imx8mm_tmu_probe(struct platform_device *pdev) return PTR_ERR(tmu->base); tmu->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(tmu->clk)) { - ret = PTR_ERR(tmu->clk); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, - "failed to get tmu clock: %d\n", ret); - return ret; - } + if (IS_ERR(tmu->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(tmu->clk), + "failed to get tmu clock\n"); ret = clk_prepare_enable(tmu->clk); if (ret) { diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c index 3f74ab4c1ab9..2c7473d86a59 100644 --- a/drivers/thermal/imx_thermal.c +++ b/drivers/thermal/imx_thermal.c @@ -716,14 +716,9 @@ static int imx_thermal_probe(struct platform_device *pdev) if (of_find_property(pdev->dev.of_node, "nvmem-cells", NULL)) { ret = imx_init_from_nvmem_cells(pdev); - if (ret) { - if (ret == -EPROBE_DEFER) - return ret; - - dev_err(&pdev->dev, "failed to init from nvmem: %d\n", - ret); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "failed to init from nvmem\n"); } else { ret = imx_init_from_tempmon_data(pdev); if (ret) { @@ -746,14 +741,9 @@ static int imx_thermal_probe(struct platform_device *pdev) data->socdata->power_down_mask); ret = imx_thermal_register_legacy_cooling(data); - if (ret) { - if (ret == -EPROBE_DEFER) - return ret; - - dev_err(&pdev->dev, - "failed to register cpufreq cooling device: %d\n", ret); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "failed to register cpufreq cooling device\n"); data->thermal_clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(data->thermal_clk)) { diff --git a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c index 4f5859d4c780..0966551cbaaa 100644 --- a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c +++ b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c @@ -14,6 +14,7 @@ #define INT3400_THERMAL_TABLE_CHANGED 0x83 #define INT3400_ODVP_CHANGED 0x88 +#define INT3400_KEEP_ALIVE 0xA0 enum int3400_thermal_uuid { INT3400_THERMAL_PASSIVE_1, @@ -83,8 +84,33 @@ static struct bin_attribute *data_attributes[] = { NULL, }; +static ssize_t imok_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct int3400_thermal_priv *priv = dev_get_drvdata(dev); + acpi_status status; + int input, ret; + + ret = kstrtouint(buf, 10, &input); + if (ret) + return ret; + status = acpi_execute_simple_method(priv->adev->handle, "IMOK", input); + if (ACPI_FAILURE(status)) + return -EIO; + + return count; +} + +static DEVICE_ATTR_WO(imok); + +static struct attribute *imok_attr[] = { + &dev_attr_imok.attr, + NULL +}; + static const struct attribute_group data_attribute_group = { .bin_attrs = data_attributes, + .attrs = imok_attr, }; static ssize_t available_uuids_show(struct device *dev, @@ -349,30 +375,33 @@ static void int3400_notify(acpi_handle handle, { struct int3400_thermal_priv *priv = data; char *thermal_prop[5]; + int therm_event; if (!priv) return; switch (event) { case INT3400_THERMAL_TABLE_CHANGED: - thermal_prop[0] = kasprintf(GFP_KERNEL, "NAME=%s", - priv->thermal->type); - thermal_prop[1] = kasprintf(GFP_KERNEL, "TEMP=%d", - priv->thermal->temperature); - thermal_prop[2] = kasprintf(GFP_KERNEL, "TRIP="); - thermal_prop[3] = kasprintf(GFP_KERNEL, "EVENT=%d", - THERMAL_TABLE_CHANGED); - thermal_prop[4] = NULL; - kobject_uevent_env(&priv->thermal->device.kobj, KOBJ_CHANGE, - thermal_prop); + therm_event = THERMAL_TABLE_CHANGED; + break; + case INT3400_KEEP_ALIVE: + therm_event = THERMAL_EVENT_KEEP_ALIVE; break; case INT3400_ODVP_CHANGED: evaluate_odvp(priv); + therm_event = THERMAL_DEVICE_POWER_CAPABILITY_CHANGED; break; default: /* Ignore unknown notification codes sent to INT3400 device */ - break; + return; } + + thermal_prop[0] = kasprintf(GFP_KERNEL, "NAME=%s", priv->thermal->type); + thermal_prop[1] = kasprintf(GFP_KERNEL, "TEMP=%d", priv->thermal->temperature); + thermal_prop[2] = kasprintf(GFP_KERNEL, "TRIP="); + thermal_prop[3] = kasprintf(GFP_KERNEL, "EVENT=%d", therm_event); + thermal_prop[4] = NULL; + kobject_uevent_env(&priv->thermal->device.kobj, KOBJ_CHANGE, thermal_prop); } static int int3400_thermal_get_temp(struct thermal_zone_device *thermal, diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c index 787710bb88fe..5c2a13bf249c 100644 --- a/drivers/thermal/rcar_thermal.c +++ b/drivers/thermal/rcar_thermal.c @@ -546,11 +546,11 @@ static int rcar_thermal_probe(struct platform_device *pdev) if (ret < 0) goto error_unregister; - if (chip->use_of_thermal) + if (chip->use_of_thermal) { priv->zone = devm_thermal_zone_of_sensor_register( dev, i, priv, &rcar_thermal_zone_of_ops); - else { + } else { priv->zone = thermal_zone_device_register( "rcar_thermal", 1, 0, priv, diff --git a/drivers/thermal/st/Kconfig b/drivers/thermal/st/Kconfig index 3c3b695cc3e9..58ece381956b 100644 --- a/drivers/thermal/st/Kconfig +++ b/drivers/thermal/st/Kconfig @@ -23,5 +23,5 @@ config STM32_THERMAL help Support for thermal framework on STMicroelectronics STM32 series of SoCs. This thermal driver allows to access to general thermal framework - functionalities and to acces to SoC sensor functionalities. This + functionalities and to access to SoC sensor functionalities. This configuration is fully dependent of MACH_STM32MP157. diff --git a/drivers/thermal/st/stm_thermal.c b/drivers/thermal/st/stm_thermal.c index 331e2b768df5..5fd3fb8912a6 100644 --- a/drivers/thermal/st/stm_thermal.c +++ b/drivers/thermal/st/stm_thermal.c @@ -446,14 +446,9 @@ thermal_unprepare: #ifdef CONFIG_PM_SLEEP static int stm_thermal_suspend(struct device *dev) { - int ret; struct stm_thermal_sensor *sensor = dev_get_drvdata(dev); - ret = stm_thermal_sensor_off(sensor); - if (ret) - return ret; - - return 0; + return stm_thermal_sensor_off(sensor); } static int stm_thermal_resume(struct device *dev) diff --git a/drivers/thermal/sun8i_thermal.c b/drivers/thermal/sun8i_thermal.c index 74d73be16496..f8b13071a6f4 100644 --- a/drivers/thermal/sun8i_thermal.c +++ b/drivers/thermal/sun8i_thermal.c @@ -244,7 +244,7 @@ static int sun50i_h6_ths_calibrate(struct ths_device *tmdev, ft_temp = (caldata[0] & FT_TEMP_MASK) * 100; for (i = 0; i < tmdev->chip->sensor_num; i++) { - int sensor_reg = caldata[i + 1]; + int sensor_reg = caldata[i + 1] & TEMP_CALIB_MASK; int cdata, offset; int sensor_temp = tmdev->chip->calc_temp(tmdev, i, sensor_reg); @@ -590,6 +590,19 @@ static const struct ths_thermal_chip sun50i_a64_ths = { .calc_temp = sun8i_ths_calc_temp, }; +static const struct ths_thermal_chip sun50i_a100_ths = { + .sensor_num = 3, + .has_bus_clk_reset = true, + .ft_deviation = 8000, + .offset = 187744, + .scale = 672, + .temp_data_base = SUN50I_H6_THS_TEMP_DATA, + .calibrate = sun50i_h6_ths_calibrate, + .init = sun50i_h6_thermal_init, + .irq_ack = sun50i_h6_irq_ack, + .calc_temp = sun8i_ths_calc_temp, +}; + static const struct ths_thermal_chip sun50i_h5_ths = { .sensor_num = 2, .has_mod_clk = true, @@ -619,6 +632,7 @@ static const struct of_device_id of_ths_match[] = { { .compatible = "allwinner,sun8i-h3-ths", .data = &sun8i_h3_ths }, { .compatible = "allwinner,sun8i-r40-ths", .data = &sun8i_r40_ths }, { .compatible = "allwinner,sun50i-a64-ths", .data = &sun50i_a64_ths }, + { .compatible = "allwinner,sun50i-a100-ths", .data = &sun50i_a100_ths }, { .compatible = "allwinner,sun50i-h5-ths", .data = &sun50i_h5_ths }, { .compatible = "allwinner,sun50i-h6-ths", .data = &sun50i_h6_ths }, { /* sentinel */ }, diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index a6616e530a84..c6d74bc1c90b 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -603,7 +603,6 @@ static void thermal_zone_device_check(struct work_struct *work) /** * power_actor_get_max_power() - get the maximum power that a cdev can consume * @cdev: pointer to &thermal_cooling_device - * @tz: a valid thermal zone device pointer * @max_power: pointer in which to store the maximum power * * Calculate the maximum power consumption in milliwats that the @@ -613,18 +612,17 @@ static void thermal_zone_device_check(struct work_struct *work) * power_actor API or -E* on other error. */ int power_actor_get_max_power(struct thermal_cooling_device *cdev, - struct thermal_zone_device *tz, u32 *max_power) + u32 *max_power) { if (!cdev_is_power_actor(cdev)) return -EINVAL; - return cdev->ops->state2power(cdev, tz, 0, max_power); + return cdev->ops->state2power(cdev, 0, max_power); } /** * power_actor_get_min_power() - get the mainimum power that a cdev can consume * @cdev: pointer to &thermal_cooling_device - * @tz: a valid thermal zone device pointer * @min_power: pointer in which to store the minimum power * * Calculate the minimum power consumption in milliwatts that the @@ -634,7 +632,7 @@ int power_actor_get_max_power(struct thermal_cooling_device *cdev, * power_actor API or -E* on other error. */ int power_actor_get_min_power(struct thermal_cooling_device *cdev, - struct thermal_zone_device *tz, u32 *min_power) + u32 *min_power) { unsigned long max_state; int ret; @@ -646,7 +644,7 @@ int power_actor_get_min_power(struct thermal_cooling_device *cdev, if (ret) return ret; - return cdev->ops->state2power(cdev, tz, max_state, min_power); + return cdev->ops->state2power(cdev, max_state, min_power); } /** @@ -670,7 +668,7 @@ int power_actor_set_power(struct thermal_cooling_device *cdev, if (!cdev_is_power_actor(cdev)) return -EINVAL; - ret = cdev->ops->power2state(cdev, instance->tz, power, &state); + ret = cdev->ops->power2state(cdev, power, &state); if (ret) return ret; @@ -1652,7 +1650,6 @@ static int __init thermal_init(void) if (result) goto error; - mutex_init(&poweroff_lock); result = thermal_register_governors(); if (result) goto error; diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h index e00fc5585ea8..764c2de31771 100644 --- a/drivers/thermal/thermal_core.h +++ b/drivers/thermal/thermal_core.h @@ -66,9 +66,9 @@ static inline bool cdev_is_power_actor(struct thermal_cooling_device *cdev) } int power_actor_get_max_power(struct thermal_cooling_device *cdev, - struct thermal_zone_device *tz, u32 *max_power); + u32 *max_power); int power_actor_get_min_power(struct thermal_cooling_device *cdev, - struct thermal_zone_device *tz, u32 *min_power); + u32 *min_power); int power_actor_set_power(struct thermal_cooling_device *cdev, struct thermal_instance *ti, u32 power); /** diff --git a/drivers/thermal/thermal_netlink.c b/drivers/thermal/thermal_netlink.c index da2891fcac89..1234dbe95895 100644 --- a/drivers/thermal/thermal_netlink.c +++ b/drivers/thermal/thermal_netlink.c @@ -78,7 +78,7 @@ int thermal_genl_sampling_temp(int id, int temp) hdr = genlmsg_put(skb, 0, 0, &thermal_gnl_family, 0, THERMAL_GENL_SAMPLING_TEMP); if (!hdr) - return -EMSGSIZE; + goto out_free; if (nla_put_u32(skb, THERMAL_GENL_ATTR_TZ_ID, id)) goto out_cancel; @@ -93,6 +93,7 @@ int thermal_genl_sampling_temp(int id, int temp) return 0; out_cancel: genlmsg_cancel(skb, hdr); +out_free: nlmsg_free(skb); return -EMSGSIZE; diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c index 8c231219e15d..a6f371fc9af2 100644 --- a/drivers/thermal/thermal_sysfs.c +++ b/drivers/thermal/thermal_sysfs.c @@ -448,7 +448,7 @@ static umode_t thermal_zone_passive_is_visible(struct kobject *kobj, struct attribute *attr, int attrno) { - struct device *dev = container_of(kobj, struct device, kobj); + struct device *dev = kobj_to_dev(kobj); struct thermal_zone_device *tz; enum thermal_trip_type trip_type; int count, passive = 0; diff --git a/drivers/thermal/ti-soc-thermal/ti-bandgap.c b/drivers/thermal/ti-soc-thermal/ti-bandgap.c index ab19ceff6e2a..5e596168ba73 100644 --- a/drivers/thermal/ti-soc-thermal/ti-bandgap.c +++ b/drivers/thermal/ti-soc-thermal/ti-bandgap.c @@ -25,10 +25,20 @@ #include <linux/of_platform.h> #include <linux/of_irq.h> #include <linux/io.h> +#include <linux/cpu_pm.h> +#include <linux/device.h> +#include <linux/pm_runtime.h> +#include <linux/pm.h> +#include <linux/of.h> +#include <linux/of_device.h> #include "ti-bandgap.h" static int ti_bandgap_force_single_read(struct ti_bandgap *bgp, int id); +#ifdef CONFIG_PM_SLEEP +static int bandgap_omap_cpu_notifier(struct notifier_block *nb, + unsigned long cmd, void *v); +#endif /*** Helper functions to access registers and their bitfields ***/ @@ -1008,6 +1018,11 @@ int ti_bandgap_probe(struct platform_device *pdev) } } +#ifdef CONFIG_PM_SLEEP + bgp->nb.notifier_call = bandgap_omap_cpu_notifier; + cpu_pm_register_notifier(&bgp->nb); +#endif + return 0; remove_last_cooling: @@ -1041,7 +1056,9 @@ int ti_bandgap_remove(struct platform_device *pdev) struct ti_bandgap *bgp = platform_get_drvdata(pdev); int i; - /* First thing is to remove sensor interfaces */ + cpu_pm_unregister_notifier(&bgp->nb); + + /* Remove sensor interfaces */ for (i = 0; i < bgp->conf->sensor_count; i++) { if (bgp->conf->sensors[i].unregister_cooling) bgp->conf->sensors[i].unregister_cooling(bgp, i); @@ -1150,9 +1167,43 @@ static int ti_bandgap_suspend(struct device *dev) if (TI_BANDGAP_HAS(bgp, CLK_CTRL)) clk_disable_unprepare(bgp->fclock); + bgp->is_suspended = true; + return err; } +static int bandgap_omap_cpu_notifier(struct notifier_block *nb, + unsigned long cmd, void *v) +{ + struct ti_bandgap *bgp; + + bgp = container_of(nb, struct ti_bandgap, nb); + + spin_lock(&bgp->lock); + switch (cmd) { + case CPU_CLUSTER_PM_ENTER: + if (bgp->is_suspended) + break; + ti_bandgap_save_ctxt(bgp); + ti_bandgap_power(bgp, false); + if (TI_BANDGAP_HAS(bgp, CLK_CTRL)) + clk_disable(bgp->fclock); + break; + case CPU_CLUSTER_PM_ENTER_FAILED: + case CPU_CLUSTER_PM_EXIT: + if (bgp->is_suspended) + break; + if (TI_BANDGAP_HAS(bgp, CLK_CTRL)) + clk_enable(bgp->fclock); + ti_bandgap_power(bgp, true); + ti_bandgap_restore_ctxt(bgp); + break; + } + spin_unlock(&bgp->lock); + + return NOTIFY_OK; +} + static int ti_bandgap_resume(struct device *dev) { struct ti_bandgap *bgp = dev_get_drvdata(dev); @@ -1161,6 +1212,7 @@ static int ti_bandgap_resume(struct device *dev) clk_prepare_enable(bgp->fclock); ti_bandgap_power(bgp, true); + bgp->is_suspended = false; return ti_bandgap_restore_ctxt(bgp); } diff --git a/drivers/thermal/ti-soc-thermal/ti-bandgap.h b/drivers/thermal/ti-soc-thermal/ti-bandgap.h index fce4657e9486..ed0ea4b17b25 100644 --- a/drivers/thermal/ti-soc-thermal/ti-bandgap.h +++ b/drivers/thermal/ti-soc-thermal/ti-bandgap.h @@ -12,6 +12,10 @@ #include <linux/spinlock.h> #include <linux/types.h> #include <linux/err.h> +#include <linux/cpu_pm.h> +#include <linux/device.h> +#include <linux/pm_runtime.h> +#include <linux/pm.h> struct gpio_desc; @@ -203,6 +207,8 @@ struct ti_bandgap { int irq; struct gpio_desc *tshut_gpiod; u32 clk_rate; + struct notifier_block nb; + unsigned int is_suspended:1; }; /** diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig index 20b98a3ba046..1044fc387691 100644 --- a/drivers/tty/serial/Kconfig +++ b/drivers/tty/serial/Kconfig @@ -236,7 +236,7 @@ config SERIAL_CLPS711X_CONSOLE config SERIAL_SAMSUNG tristate "Samsung SoC serial support" - depends on PLAT_SAMSUNG || ARCH_EXYNOS || COMPILE_TEST + depends on PLAT_SAMSUNG || ARCH_S5PV210 || ARCH_EXYNOS || COMPILE_TEST select SERIAL_CORE help Support for the on-chip UARTs on the Samsung S3C24XX series CPUs, diff --git a/drivers/tty/serial/sb1250-duart.c b/drivers/tty/serial/sb1250-duart.c index bd5e7e9938ce..22c7bc90b104 100644 --- a/drivers/tty/serial/sb1250-duart.c +++ b/drivers/tty/serial/sb1250-duart.c @@ -35,7 +35,6 @@ #include <linux/refcount.h> #include <asm/io.h> -#include <asm/war.h> #include <asm/sibyte/sb1250.h> #include <asm/sibyte/sb1250_uart.h> @@ -157,7 +156,7 @@ static unsigned char read_sbdchn(struct sbd_port *sport, int reg) unsigned char retval; retval = __read_sbdchn(sport, reg); - if (SIBYTE_1956_WAR) + if (IS_ENABLED(CONFIG_SB1_PASS_2_WORKAROUNDS)) __war_sbd1956(sport); return retval; } @@ -167,7 +166,7 @@ static unsigned char read_sbdshr(struct sbd_port *sport, int reg) unsigned char retval; retval = __read_sbdshr(sport, reg); - if (SIBYTE_1956_WAR) + if (IS_ENABLED(CONFIG_SB1_PASS_2_WORKAROUNDS)) __war_sbd1956(sport); return retval; } @@ -175,14 +174,14 @@ static unsigned char read_sbdshr(struct sbd_port *sport, int reg) static void write_sbdchn(struct sbd_port *sport, int reg, unsigned int value) { __write_sbdchn(sport, reg, value); - if (SIBYTE_1956_WAR) + if (IS_ENABLED(CONFIG_SB1_PASS_2_WORKAROUNDS)) __war_sbd1956(sport); } static void write_sbdshr(struct sbd_port *sport, int reg, unsigned int value) { __write_sbdshr(sport, reg, value); - if (SIBYTE_1956_WAR) + if (IS_ENABLED(CONFIG_SB1_PASS_2_WORKAROUNDS)) __war_sbd1956(sport); } diff --git a/drivers/usb/gadget/udc/s3c-hsudc.c b/drivers/usb/gadget/udc/s3c-hsudc.c index aaca1b0a2f59..7bd5182ce3ef 100644 --- a/drivers/usb/gadget/udc/s3c-hsudc.c +++ b/drivers/usb/gadget/udc/s3c-hsudc.c @@ -30,8 +30,6 @@ #include <linux/regulator/consumer.h> #include <linux/pm_runtime.h> -#include <mach/regs-s3c2443-clock.h> - #define S3C_HSUDC_REG(x) (x) /* Non-Indexed Registers */ @@ -186,53 +184,6 @@ static inline void __orr32(void __iomem *ptr, u32 val) writel(readl(ptr) | val, ptr); } -static void s3c_hsudc_init_phy(void) -{ - u32 cfg; - - cfg = readl(S3C2443_PWRCFG) | S3C2443_PWRCFG_USBPHY; - writel(cfg, S3C2443_PWRCFG); - - cfg = readl(S3C2443_URSTCON); - cfg |= (S3C2443_URSTCON_FUNCRST | S3C2443_URSTCON_PHYRST); - writel(cfg, S3C2443_URSTCON); - mdelay(1); - - cfg = readl(S3C2443_URSTCON); - cfg &= ~(S3C2443_URSTCON_FUNCRST | S3C2443_URSTCON_PHYRST); - writel(cfg, S3C2443_URSTCON); - - cfg = readl(S3C2443_PHYCTRL); - cfg &= ~(S3C2443_PHYCTRL_CLKSEL | S3C2443_PHYCTRL_DSPORT); - cfg |= (S3C2443_PHYCTRL_EXTCLK | S3C2443_PHYCTRL_PLLSEL); - writel(cfg, S3C2443_PHYCTRL); - - cfg = readl(S3C2443_PHYPWR); - cfg &= ~(S3C2443_PHYPWR_FSUSPEND | S3C2443_PHYPWR_PLL_PWRDN | - S3C2443_PHYPWR_XO_ON | S3C2443_PHYPWR_PLL_REFCLK | - S3C2443_PHYPWR_ANALOG_PD); - cfg |= S3C2443_PHYPWR_COMMON_ON; - writel(cfg, S3C2443_PHYPWR); - - cfg = readl(S3C2443_UCLKCON); - cfg |= (S3C2443_UCLKCON_DETECT_VBUS | S3C2443_UCLKCON_FUNC_CLKEN | - S3C2443_UCLKCON_TCLKEN); - writel(cfg, S3C2443_UCLKCON); -} - -static void s3c_hsudc_uninit_phy(void) -{ - u32 cfg; - - cfg = readl(S3C2443_PWRCFG) & ~S3C2443_PWRCFG_USBPHY; - writel(cfg, S3C2443_PWRCFG); - - writel(S3C2443_PHYPWR_FSUSPEND, S3C2443_PHYPWR); - - cfg = readl(S3C2443_UCLKCON) & ~S3C2443_UCLKCON_FUNC_CLKEN; - writel(cfg, S3C2443_UCLKCON); -} - /** * s3c_hsudc_complete_request - Complete a transfer request. * @hsep: Endpoint to which the request belongs. @@ -1188,7 +1139,8 @@ static int s3c_hsudc_start(struct usb_gadget *gadget, pm_runtime_get_sync(hsudc->dev); - s3c_hsudc_init_phy(); + if (hsudc->pd->phy_init) + hsudc->pd->phy_init(); if (hsudc->pd->gpio_init) hsudc->pd->gpio_init(); @@ -1210,7 +1162,8 @@ static int s3c_hsudc_stop(struct usb_gadget *gadget) spin_lock_irqsave(&hsudc->lock, flags); hsudc->gadget.speed = USB_SPEED_UNKNOWN; - s3c_hsudc_uninit_phy(); + if (hsudc->pd->phy_uninit) + hsudc->pd->phy_uninit(); pm_runtime_put(hsudc->dev); diff --git a/drivers/usb/gadget/udc/s3c2410_udc.c b/drivers/usb/gadget/udc/s3c2410_udc.c index e875a0b967c0..f1ea51476add 100644 --- a/drivers/usb/gadget/udc/s3c2410_udc.c +++ b/drivers/usb/gadget/udc/s3c2410_udc.c @@ -36,15 +36,11 @@ #include <asm/byteorder.h> #include <asm/irq.h> #include <asm/unaligned.h> -#include <mach/irqs.h> -#include <mach/hardware.h> - -#include <plat/regs-udc.h> #include <linux/platform_data/usb-s3c2410_udc.h> - #include "s3c2410_udc.h" +#include "s3c2410_udc_regs.h" #define DRIVER_DESC "S3C2410 USB Device Controller Gadget" #define DRIVER_AUTHOR "Herbert Pötzl <herbert@13thfloor.at>, " \ @@ -57,6 +53,7 @@ static struct s3c2410_udc *the_controller; static struct clk *udc_clock; static struct clk *usb_bus_clock; static void __iomem *base_addr; +static int irq_usbd; static u64 rsrc_start; static u64 rsrc_len; static struct dentry *s3c2410_udc_debugfs_root; @@ -835,8 +832,6 @@ static void s3c2410_udc_handle_ep(struct s3c2410_ep *ep) } } -#include <mach/regs-irq.h> - /* * s3c2410_udc_irq - interrupt handler */ @@ -977,7 +972,7 @@ static irqreturn_t s3c2410_udc_irq(int dummy, void *_dev) } } - dprintk(DEBUG_VERBOSE, "irq: %d s3c2410_udc_done.\n", IRQ_USBD); + dprintk(DEBUG_VERBOSE, "irq: %d s3c2410_udc_done.\n", irq_usbd); /* Restore old index */ udc_write(idx, S3C2410_UDC_INDEX_REG); @@ -1777,13 +1772,7 @@ static int s3c2410_udc_probe(struct platform_device *pdev) spin_lock_init(&udc->lock); udc_info = dev_get_platdata(&pdev->dev); - rsrc_start = S3C2410_PA_USBDEV; - rsrc_len = S3C24XX_SZ_USBDEV; - - if (!request_mem_region(rsrc_start, rsrc_len, gadget_name)) - return -EBUSY; - - base_addr = ioremap(rsrc_start, rsrc_len); + base_addr = devm_platform_ioremap_resource(pdev, 0); if (!base_addr) { retval = -ENOMEM; goto err_mem; @@ -1795,17 +1784,19 @@ static int s3c2410_udc_probe(struct platform_device *pdev) s3c2410_udc_disable(udc); s3c2410_udc_reinit(udc); + irq_usbd = platform_get_irq(pdev, 0); + /* irq setup after old hardware state is cleaned up */ - retval = request_irq(IRQ_USBD, s3c2410_udc_irq, + retval = request_irq(irq_usbd, s3c2410_udc_irq, 0, gadget_name, udc); if (retval != 0) { - dev_err(dev, "cannot get irq %i, err %d\n", IRQ_USBD, retval); + dev_err(dev, "cannot get irq %i, err %d\n", irq_usbd, retval); retval = -EBUSY; goto err_map; } - dev_dbg(dev, "got irq %i\n", IRQ_USBD); + dev_dbg(dev, "got irq %i\n", irq_usbd); if (udc_info && udc_info->vbus_pin > 0) { retval = gpio_request(udc_info->vbus_pin, "udc vbus"); @@ -1872,7 +1863,7 @@ err_gpio_claim: if (udc_info && udc_info->vbus_pin > 0) gpio_free(udc_info->vbus_pin); err_int: - free_irq(IRQ_USBD, udc); + free_irq(irq_usbd, udc); err_map: iounmap(base_addr); err_mem: @@ -1906,7 +1897,7 @@ static int s3c2410_udc_remove(struct platform_device *pdev) free_irq(irq, udc); } - free_irq(IRQ_USBD, udc); + free_irq(irq_usbd, udc); iounmap(base_addr); release_mem_region(rsrc_start, rsrc_len); diff --git a/drivers/usb/gadget/udc/s3c2410_udc.h b/drivers/usb/gadget/udc/s3c2410_udc.h index bdcaa8dd300f..68bdf3e5aac2 100644 --- a/drivers/usb/gadget/udc/s3c2410_udc.h +++ b/drivers/usb/gadget/udc/s3c2410_udc.h @@ -90,6 +90,7 @@ struct s3c2410_udc { unsigned req_pending : 1; u8 vbus; struct dentry *regs_info; + int irq; }; #define to_s3c2410(g) (container_of((g), struct s3c2410_udc, gadget)) diff --git a/drivers/usb/gadget/udc/s3c2410_udc_regs.h b/drivers/usb/gadget/udc/s3c2410_udc_regs.h new file mode 100644 index 000000000000..d8d2eeaca088 --- /dev/null +++ b/drivers/usb/gadget/udc/s3c2410_udc_regs.h @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (C) 2004 Herbert Poetzl <herbert@13thfloor.at> + */ + +#ifndef __ASM_ARCH_REGS_UDC_H +#define __ASM_ARCH_REGS_UDC_H + +#define S3C2410_USBDREG(x) (x) + +#define S3C2410_UDC_FUNC_ADDR_REG S3C2410_USBDREG(0x0140) +#define S3C2410_UDC_PWR_REG S3C2410_USBDREG(0x0144) +#define S3C2410_UDC_EP_INT_REG S3C2410_USBDREG(0x0148) + +#define S3C2410_UDC_USB_INT_REG S3C2410_USBDREG(0x0158) +#define S3C2410_UDC_EP_INT_EN_REG S3C2410_USBDREG(0x015c) + +#define S3C2410_UDC_USB_INT_EN_REG S3C2410_USBDREG(0x016c) + +#define S3C2410_UDC_FRAME_NUM1_REG S3C2410_USBDREG(0x0170) +#define S3C2410_UDC_FRAME_NUM2_REG S3C2410_USBDREG(0x0174) + +#define S3C2410_UDC_EP0_FIFO_REG S3C2410_USBDREG(0x01c0) +#define S3C2410_UDC_EP1_FIFO_REG S3C2410_USBDREG(0x01c4) +#define S3C2410_UDC_EP2_FIFO_REG S3C2410_USBDREG(0x01c8) +#define S3C2410_UDC_EP3_FIFO_REG S3C2410_USBDREG(0x01cc) +#define S3C2410_UDC_EP4_FIFO_REG S3C2410_USBDREG(0x01d0) + +#define S3C2410_UDC_EP1_DMA_CON S3C2410_USBDREG(0x0200) +#define S3C2410_UDC_EP1_DMA_UNIT S3C2410_USBDREG(0x0204) +#define S3C2410_UDC_EP1_DMA_FIFO S3C2410_USBDREG(0x0208) +#define S3C2410_UDC_EP1_DMA_TTC_L S3C2410_USBDREG(0x020c) +#define S3C2410_UDC_EP1_DMA_TTC_M S3C2410_USBDREG(0x0210) +#define S3C2410_UDC_EP1_DMA_TTC_H S3C2410_USBDREG(0x0214) + +#define S3C2410_UDC_EP2_DMA_CON S3C2410_USBDREG(0x0218) +#define S3C2410_UDC_EP2_DMA_UNIT S3C2410_USBDREG(0x021c) +#define S3C2410_UDC_EP2_DMA_FIFO S3C2410_USBDREG(0x0220) +#define S3C2410_UDC_EP2_DMA_TTC_L S3C2410_USBDREG(0x0224) +#define S3C2410_UDC_EP2_DMA_TTC_M S3C2410_USBDREG(0x0228) +#define S3C2410_UDC_EP2_DMA_TTC_H S3C2410_USBDREG(0x022c) + +#define S3C2410_UDC_EP3_DMA_CON S3C2410_USBDREG(0x0240) +#define S3C2410_UDC_EP3_DMA_UNIT S3C2410_USBDREG(0x0244) +#define S3C2410_UDC_EP3_DMA_FIFO S3C2410_USBDREG(0x0248) +#define S3C2410_UDC_EP3_DMA_TTC_L S3C2410_USBDREG(0x024c) +#define S3C2410_UDC_EP3_DMA_TTC_M S3C2410_USBDREG(0x0250) +#define S3C2410_UDC_EP3_DMA_TTC_H S3C2410_USBDREG(0x0254) + +#define S3C2410_UDC_EP4_DMA_CON S3C2410_USBDREG(0x0258) +#define S3C2410_UDC_EP4_DMA_UNIT S3C2410_USBDREG(0x025c) +#define S3C2410_UDC_EP4_DMA_FIFO S3C2410_USBDREG(0x0260) +#define S3C2410_UDC_EP4_DMA_TTC_L S3C2410_USBDREG(0x0264) +#define S3C2410_UDC_EP4_DMA_TTC_M S3C2410_USBDREG(0x0268) +#define S3C2410_UDC_EP4_DMA_TTC_H S3C2410_USBDREG(0x026c) + +#define S3C2410_UDC_INDEX_REG S3C2410_USBDREG(0x0178) + +/* indexed registers */ + +#define S3C2410_UDC_MAXP_REG S3C2410_USBDREG(0x0180) + +#define S3C2410_UDC_EP0_CSR_REG S3C2410_USBDREG(0x0184) + +#define S3C2410_UDC_IN_CSR1_REG S3C2410_USBDREG(0x0184) +#define S3C2410_UDC_IN_CSR2_REG S3C2410_USBDREG(0x0188) + +#define S3C2410_UDC_OUT_CSR1_REG S3C2410_USBDREG(0x0190) +#define S3C2410_UDC_OUT_CSR2_REG S3C2410_USBDREG(0x0194) +#define S3C2410_UDC_OUT_FIFO_CNT1_REG S3C2410_USBDREG(0x0198) +#define S3C2410_UDC_OUT_FIFO_CNT2_REG S3C2410_USBDREG(0x019c) + +#define S3C2410_UDC_FUNCADDR_UPDATE (1 << 7) + +#define S3C2410_UDC_PWR_ISOUP (1 << 7) /* R/W */ +#define S3C2410_UDC_PWR_RESET (1 << 3) /* R */ +#define S3C2410_UDC_PWR_RESUME (1 << 2) /* R/W */ +#define S3C2410_UDC_PWR_SUSPEND (1 << 1) /* R */ +#define S3C2410_UDC_PWR_ENSUSPEND (1 << 0) /* R/W */ + +#define S3C2410_UDC_PWR_DEFAULT (0x00) + +#define S3C2410_UDC_INT_EP4 (1 << 4) /* R/W (clear only) */ +#define S3C2410_UDC_INT_EP3 (1 << 3) /* R/W (clear only) */ +#define S3C2410_UDC_INT_EP2 (1 << 2) /* R/W (clear only) */ +#define S3C2410_UDC_INT_EP1 (1 << 1) /* R/W (clear only) */ +#define S3C2410_UDC_INT_EP0 (1 << 0) /* R/W (clear only) */ + +#define S3C2410_UDC_USBINT_RESET (1 << 2) /* R/W (clear only) */ +#define S3C2410_UDC_USBINT_RESUME (1 << 1) /* R/W (clear only) */ +#define S3C2410_UDC_USBINT_SUSPEND (1 << 0) /* R/W (clear only) */ + +#define S3C2410_UDC_INTE_EP4 (1 << 4) /* R/W */ +#define S3C2410_UDC_INTE_EP3 (1 << 3) /* R/W */ +#define S3C2410_UDC_INTE_EP2 (1 << 2) /* R/W */ +#define S3C2410_UDC_INTE_EP1 (1 << 1) /* R/W */ +#define S3C2410_UDC_INTE_EP0 (1 << 0) /* R/W */ + +#define S3C2410_UDC_USBINTE_RESET (1 << 2) /* R/W */ +#define S3C2410_UDC_USBINTE_SUSPEND (1 << 0) /* R/W */ + +#define S3C2410_UDC_INDEX_EP0 (0x00) +#define S3C2410_UDC_INDEX_EP1 (0x01) +#define S3C2410_UDC_INDEX_EP2 (0x02) +#define S3C2410_UDC_INDEX_EP3 (0x03) +#define S3C2410_UDC_INDEX_EP4 (0x04) + +#define S3C2410_UDC_ICSR1_CLRDT (1 << 6) /* R/W */ +#define S3C2410_UDC_ICSR1_SENTSTL (1 << 5) /* R/W (clear only) */ +#define S3C2410_UDC_ICSR1_SENDSTL (1 << 4) /* R/W */ +#define S3C2410_UDC_ICSR1_FFLUSH (1 << 3) /* W (set only) */ +#define S3C2410_UDC_ICSR1_UNDRUN (1 << 2) /* R/W (clear only) */ +#define S3C2410_UDC_ICSR1_PKTRDY (1 << 0) /* R/W (set only) */ + +#define S3C2410_UDC_ICSR2_AUTOSET (1 << 7) /* R/W */ +#define S3C2410_UDC_ICSR2_ISO (1 << 6) /* R/W */ +#define S3C2410_UDC_ICSR2_MODEIN (1 << 5) /* R/W */ +#define S3C2410_UDC_ICSR2_DMAIEN (1 << 4) /* R/W */ + +#define S3C2410_UDC_OCSR1_CLRDT (1 << 7) /* R/W */ +#define S3C2410_UDC_OCSR1_SENTSTL (1 << 6) /* R/W (clear only) */ +#define S3C2410_UDC_OCSR1_SENDSTL (1 << 5) /* R/W */ +#define S3C2410_UDC_OCSR1_FFLUSH (1 << 4) /* R/W */ +#define S3C2410_UDC_OCSR1_DERROR (1 << 3) /* R */ +#define S3C2410_UDC_OCSR1_OVRRUN (1 << 2) /* R/W (clear only) */ +#define S3C2410_UDC_OCSR1_PKTRDY (1 << 0) /* R/W (clear only) */ + +#define S3C2410_UDC_OCSR2_AUTOCLR (1 << 7) /* R/W */ +#define S3C2410_UDC_OCSR2_ISO (1 << 6) /* R/W */ +#define S3C2410_UDC_OCSR2_DMAIEN (1 << 5) /* R/W */ + +#define S3C2410_UDC_EP0_CSR_OPKRDY (1 << 0) +#define S3C2410_UDC_EP0_CSR_IPKRDY (1 << 1) +#define S3C2410_UDC_EP0_CSR_SENTSTL (1 << 2) +#define S3C2410_UDC_EP0_CSR_DE (1 << 3) +#define S3C2410_UDC_EP0_CSR_SE (1 << 4) +#define S3C2410_UDC_EP0_CSR_SENDSTL (1 << 5) +#define S3C2410_UDC_EP0_CSR_SOPKTRDY (1 << 6) +#define S3C2410_UDC_EP0_CSR_SSE (1 << 7) + +#define S3C2410_UDC_MAXP_8 (1 << 0) +#define S3C2410_UDC_MAXP_16 (1 << 1) +#define S3C2410_UDC_MAXP_32 (1 << 2) +#define S3C2410_UDC_MAXP_64 (1 << 3) + +#endif diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index 74264e590695..1fa6fcac8299 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -1522,6 +1522,11 @@ static inline bool mlx5_vdpa_is_little_endian(struct mlx5_vdpa_dev *mvdev) (mvdev->actual_features & (1ULL << VIRTIO_F_VERSION_1)); } +static __virtio16 cpu_to_mlx5vdpa16(struct mlx5_vdpa_dev *mvdev, u16 val) +{ + return __cpu_to_virtio16(mlx5_vdpa_is_little_endian(mvdev), val); +} + static int mlx5_vdpa_set_features(struct vdpa_device *vdev, u64 features) { struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); @@ -1535,8 +1540,8 @@ static int mlx5_vdpa_set_features(struct vdpa_device *vdev, u64 features) return err; ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features; - ndev->config.mtu = __cpu_to_virtio16(mlx5_vdpa_is_little_endian(mvdev), - ndev->mtu); + ndev->config.mtu = cpu_to_mlx5vdpa16(mvdev, ndev->mtu); + ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); return err; } @@ -1653,6 +1658,9 @@ static int mlx5_vdpa_change_map(struct mlx5_vdpa_net *ndev, struct vhost_iotlb * if (err) goto err_mr; + if (!(ndev->mvdev.status & VIRTIO_CONFIG_S_DRIVER_OK)) + return 0; + restore_channels_info(ndev); err = setup_driver(ndev); if (err) diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index fd17db9b432f..5533df91b257 100644 --- a/drivers/vfio/Kconfig +++ b/drivers/vfio/Kconfig @@ -47,4 +47,5 @@ menuconfig VFIO_NOIOMMU source "drivers/vfio/pci/Kconfig" source "drivers/vfio/platform/Kconfig" source "drivers/vfio/mdev/Kconfig" +source "drivers/vfio/fsl-mc/Kconfig" source "virt/lib/Kconfig" diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile index de67c4725cce..fee73f3d9480 100644 --- a/drivers/vfio/Makefile +++ b/drivers/vfio/Makefile @@ -9,3 +9,4 @@ obj-$(CONFIG_VFIO_SPAPR_EEH) += vfio_spapr_eeh.o obj-$(CONFIG_VFIO_PCI) += pci/ obj-$(CONFIG_VFIO_PLATFORM) += platform/ obj-$(CONFIG_VFIO_MDEV) += mdev/ +obj-$(CONFIG_VFIO_FSL_MC) += fsl-mc/ diff --git a/drivers/vfio/fsl-mc/Kconfig b/drivers/vfio/fsl-mc/Kconfig new file mode 100644 index 000000000000..b1a527d6b6f2 --- /dev/null +++ b/drivers/vfio/fsl-mc/Kconfig @@ -0,0 +1,9 @@ +config VFIO_FSL_MC + tristate "VFIO support for QorIQ DPAA2 fsl-mc bus devices" + depends on VFIO && FSL_MC_BUS && EVENTFD + help + Driver to enable support for the VFIO QorIQ DPAA2 fsl-mc + (Management Complex) devices. This is required to passthrough + fsl-mc bus devices using the VFIO framework. + + If you don't know what to do here, say N. diff --git a/drivers/vfio/fsl-mc/Makefile b/drivers/vfio/fsl-mc/Makefile new file mode 100644 index 000000000000..cad6dbf0b735 --- /dev/null +++ b/drivers/vfio/fsl-mc/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) + +vfio-fsl-mc-y := vfio_fsl_mc.o vfio_fsl_mc_intr.o +obj-$(CONFIG_VFIO_FSL_MC) += vfio-fsl-mc.o diff --git a/drivers/vfio/fsl-mc/vfio_fsl_mc.c b/drivers/vfio/fsl-mc/vfio_fsl_mc.c new file mode 100644 index 000000000000..0113a980f974 --- /dev/null +++ b/drivers/vfio/fsl-mc/vfio_fsl_mc.c @@ -0,0 +1,683 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016-2017,2019-2020 NXP + */ + +#include <linux/device.h> +#include <linux/iommu.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/slab.h> +#include <linux/types.h> +#include <linux/vfio.h> +#include <linux/fsl/mc.h> +#include <linux/delay.h> +#include <linux/io-64-nonatomic-hi-lo.h> + +#include "vfio_fsl_mc_private.h" + +static struct fsl_mc_driver vfio_fsl_mc_driver; + +static DEFINE_MUTEX(reflck_lock); + +static void vfio_fsl_mc_reflck_get(struct vfio_fsl_mc_reflck *reflck) +{ + kref_get(&reflck->kref); +} + +static void vfio_fsl_mc_reflck_release(struct kref *kref) +{ + struct vfio_fsl_mc_reflck *reflck = container_of(kref, + struct vfio_fsl_mc_reflck, + kref); + + mutex_destroy(&reflck->lock); + kfree(reflck); + mutex_unlock(&reflck_lock); +} + +static void vfio_fsl_mc_reflck_put(struct vfio_fsl_mc_reflck *reflck) +{ + kref_put_mutex(&reflck->kref, vfio_fsl_mc_reflck_release, &reflck_lock); +} + +static struct vfio_fsl_mc_reflck *vfio_fsl_mc_reflck_alloc(void) +{ + struct vfio_fsl_mc_reflck *reflck; + + reflck = kzalloc(sizeof(*reflck), GFP_KERNEL); + if (!reflck) + return ERR_PTR(-ENOMEM); + + kref_init(&reflck->kref); + mutex_init(&reflck->lock); + + return reflck; +} + +static int vfio_fsl_mc_reflck_attach(struct vfio_fsl_mc_device *vdev) +{ + int ret = 0; + + mutex_lock(&reflck_lock); + if (is_fsl_mc_bus_dprc(vdev->mc_dev)) { + vdev->reflck = vfio_fsl_mc_reflck_alloc(); + ret = PTR_ERR_OR_ZERO(vdev->reflck); + } else { + struct device *mc_cont_dev = vdev->mc_dev->dev.parent; + struct vfio_device *device; + struct vfio_fsl_mc_device *cont_vdev; + + device = vfio_device_get_from_dev(mc_cont_dev); + if (!device) { + ret = -ENODEV; + goto unlock; + } + + cont_vdev = vfio_device_data(device); + if (!cont_vdev || !cont_vdev->reflck) { + vfio_device_put(device); + ret = -ENODEV; + goto unlock; + } + vfio_fsl_mc_reflck_get(cont_vdev->reflck); + vdev->reflck = cont_vdev->reflck; + vfio_device_put(device); + } + +unlock: + mutex_unlock(&reflck_lock); + return ret; +} + +static int vfio_fsl_mc_regions_init(struct vfio_fsl_mc_device *vdev) +{ + struct fsl_mc_device *mc_dev = vdev->mc_dev; + int count = mc_dev->obj_desc.region_count; + int i; + + vdev->regions = kcalloc(count, sizeof(struct vfio_fsl_mc_region), + GFP_KERNEL); + if (!vdev->regions) + return -ENOMEM; + + for (i = 0; i < count; i++) { + struct resource *res = &mc_dev->regions[i]; + int no_mmap = is_fsl_mc_bus_dprc(mc_dev); + + vdev->regions[i].addr = res->start; + vdev->regions[i].size = resource_size(res); + vdev->regions[i].type = mc_dev->regions[i].flags & IORESOURCE_BITS; + /* + * Only regions addressed with PAGE granularity may be + * MMAPed securely. + */ + if (!no_mmap && !(vdev->regions[i].addr & ~PAGE_MASK) && + !(vdev->regions[i].size & ~PAGE_MASK)) + vdev->regions[i].flags |= + VFIO_REGION_INFO_FLAG_MMAP; + vdev->regions[i].flags |= VFIO_REGION_INFO_FLAG_READ; + if (!(mc_dev->regions[i].flags & IORESOURCE_READONLY)) + vdev->regions[i].flags |= VFIO_REGION_INFO_FLAG_WRITE; + } + + return 0; +} + +static void vfio_fsl_mc_regions_cleanup(struct vfio_fsl_mc_device *vdev) +{ + struct fsl_mc_device *mc_dev = vdev->mc_dev; + int i; + + for (i = 0; i < mc_dev->obj_desc.region_count; i++) + iounmap(vdev->regions[i].ioaddr); + kfree(vdev->regions); +} + +static int vfio_fsl_mc_open(void *device_data) +{ + struct vfio_fsl_mc_device *vdev = device_data; + int ret; + + if (!try_module_get(THIS_MODULE)) + return -ENODEV; + + mutex_lock(&vdev->reflck->lock); + if (!vdev->refcnt) { + ret = vfio_fsl_mc_regions_init(vdev); + if (ret) + goto err_reg_init; + } + vdev->refcnt++; + + mutex_unlock(&vdev->reflck->lock); + + return 0; + +err_reg_init: + mutex_unlock(&vdev->reflck->lock); + module_put(THIS_MODULE); + return ret; +} + +static void vfio_fsl_mc_release(void *device_data) +{ + struct vfio_fsl_mc_device *vdev = device_data; + int ret; + + mutex_lock(&vdev->reflck->lock); + + if (!(--vdev->refcnt)) { + struct fsl_mc_device *mc_dev = vdev->mc_dev; + struct device *cont_dev = fsl_mc_cont_dev(&mc_dev->dev); + struct fsl_mc_device *mc_cont = to_fsl_mc_device(cont_dev); + + vfio_fsl_mc_regions_cleanup(vdev); + + /* reset the device before cleaning up the interrupts */ + ret = dprc_reset_container(mc_cont->mc_io, 0, + mc_cont->mc_handle, + mc_cont->obj_desc.id, + DPRC_RESET_OPTION_NON_RECURSIVE); + + if (ret) { + dev_warn(&mc_cont->dev, "VFIO_FLS_MC: reset device has failed (%d)\n", + ret); + WARN_ON(1); + } + + vfio_fsl_mc_irqs_cleanup(vdev); + + fsl_mc_cleanup_irq_pool(mc_cont); + } + + mutex_unlock(&vdev->reflck->lock); + + module_put(THIS_MODULE); +} + +static long vfio_fsl_mc_ioctl(void *device_data, unsigned int cmd, + unsigned long arg) +{ + unsigned long minsz; + struct vfio_fsl_mc_device *vdev = device_data; + struct fsl_mc_device *mc_dev = vdev->mc_dev; + + switch (cmd) { + case VFIO_DEVICE_GET_INFO: + { + struct vfio_device_info info; + + minsz = offsetofend(struct vfio_device_info, num_irqs); + + if (copy_from_user(&info, (void __user *)arg, minsz)) + return -EFAULT; + + if (info.argsz < minsz) + return -EINVAL; + + info.flags = VFIO_DEVICE_FLAGS_FSL_MC; + + if (is_fsl_mc_bus_dprc(mc_dev)) + info.flags |= VFIO_DEVICE_FLAGS_RESET; + + info.num_regions = mc_dev->obj_desc.region_count; + info.num_irqs = mc_dev->obj_desc.irq_count; + + return copy_to_user((void __user *)arg, &info, minsz) ? + -EFAULT : 0; + } + case VFIO_DEVICE_GET_REGION_INFO: + { + struct vfio_region_info info; + + minsz = offsetofend(struct vfio_region_info, offset); + + if (copy_from_user(&info, (void __user *)arg, minsz)) + return -EFAULT; + + if (info.argsz < minsz) + return -EINVAL; + + if (info.index >= mc_dev->obj_desc.region_count) + return -EINVAL; + + /* map offset to the physical address */ + info.offset = VFIO_FSL_MC_INDEX_TO_OFFSET(info.index); + info.size = vdev->regions[info.index].size; + info.flags = vdev->regions[info.index].flags; + + return copy_to_user((void __user *)arg, &info, minsz); + } + case VFIO_DEVICE_GET_IRQ_INFO: + { + struct vfio_irq_info info; + + minsz = offsetofend(struct vfio_irq_info, count); + if (copy_from_user(&info, (void __user *)arg, minsz)) + return -EFAULT; + + if (info.argsz < minsz) + return -EINVAL; + + if (info.index >= mc_dev->obj_desc.irq_count) + return -EINVAL; + + info.flags = VFIO_IRQ_INFO_EVENTFD; + info.count = 1; + + return copy_to_user((void __user *)arg, &info, minsz); + } + case VFIO_DEVICE_SET_IRQS: + { + struct vfio_irq_set hdr; + u8 *data = NULL; + int ret = 0; + size_t data_size = 0; + + minsz = offsetofend(struct vfio_irq_set, count); + + if (copy_from_user(&hdr, (void __user *)arg, minsz)) + return -EFAULT; + + ret = vfio_set_irqs_validate_and_prepare(&hdr, mc_dev->obj_desc.irq_count, + mc_dev->obj_desc.irq_count, &data_size); + if (ret) + return ret; + + if (data_size) { + data = memdup_user((void __user *)(arg + minsz), + data_size); + if (IS_ERR(data)) + return PTR_ERR(data); + } + + mutex_lock(&vdev->igate); + ret = vfio_fsl_mc_set_irqs_ioctl(vdev, hdr.flags, + hdr.index, hdr.start, + hdr.count, data); + mutex_unlock(&vdev->igate); + kfree(data); + + return ret; + } + case VFIO_DEVICE_RESET: + { + int ret; + struct fsl_mc_device *mc_dev = vdev->mc_dev; + + /* reset is supported only for the DPRC */ + if (!is_fsl_mc_bus_dprc(mc_dev)) + return -ENOTTY; + + ret = dprc_reset_container(mc_dev->mc_io, 0, + mc_dev->mc_handle, + mc_dev->obj_desc.id, + DPRC_RESET_OPTION_NON_RECURSIVE); + return ret; + + } + default: + return -ENOTTY; + } +} + +static ssize_t vfio_fsl_mc_read(void *device_data, char __user *buf, + size_t count, loff_t *ppos) +{ + struct vfio_fsl_mc_device *vdev = device_data; + unsigned int index = VFIO_FSL_MC_OFFSET_TO_INDEX(*ppos); + loff_t off = *ppos & VFIO_FSL_MC_OFFSET_MASK; + struct fsl_mc_device *mc_dev = vdev->mc_dev; + struct vfio_fsl_mc_region *region; + u64 data[8]; + int i; + + if (index >= mc_dev->obj_desc.region_count) + return -EINVAL; + + region = &vdev->regions[index]; + + if (!(region->flags & VFIO_REGION_INFO_FLAG_READ)) + return -EINVAL; + + if (!region->ioaddr) { + region->ioaddr = ioremap(region->addr, region->size); + if (!region->ioaddr) + return -ENOMEM; + } + + if (count != 64 || off != 0) + return -EINVAL; + + for (i = 7; i >= 0; i--) + data[i] = readq(region->ioaddr + i * sizeof(uint64_t)); + + if (copy_to_user(buf, data, 64)) + return -EFAULT; + + return count; +} + +#define MC_CMD_COMPLETION_TIMEOUT_MS 5000 +#define MC_CMD_COMPLETION_POLLING_MAX_SLEEP_USECS 500 + +static int vfio_fsl_mc_send_command(void __iomem *ioaddr, uint64_t *cmd_data) +{ + int i; + enum mc_cmd_status status; + unsigned long timeout_usecs = MC_CMD_COMPLETION_TIMEOUT_MS * 1000; + + /* Write at command parameter into portal */ + for (i = 7; i >= 1; i--) + writeq_relaxed(cmd_data[i], ioaddr + i * sizeof(uint64_t)); + + /* Write command header in the end */ + writeq(cmd_data[0], ioaddr); + + /* Wait for response before returning to user-space + * This can be optimized in future to even prepare response + * before returning to user-space and avoid read ioctl. + */ + for (;;) { + u64 header; + struct mc_cmd_header *resp_hdr; + + header = cpu_to_le64(readq_relaxed(ioaddr)); + + resp_hdr = (struct mc_cmd_header *)&header; + status = (enum mc_cmd_status)resp_hdr->status; + if (status != MC_CMD_STATUS_READY) + break; + + udelay(MC_CMD_COMPLETION_POLLING_MAX_SLEEP_USECS); + timeout_usecs -= MC_CMD_COMPLETION_POLLING_MAX_SLEEP_USECS; + if (timeout_usecs == 0) + return -ETIMEDOUT; + } + + return 0; +} + +static ssize_t vfio_fsl_mc_write(void *device_data, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct vfio_fsl_mc_device *vdev = device_data; + unsigned int index = VFIO_FSL_MC_OFFSET_TO_INDEX(*ppos); + loff_t off = *ppos & VFIO_FSL_MC_OFFSET_MASK; + struct fsl_mc_device *mc_dev = vdev->mc_dev; + struct vfio_fsl_mc_region *region; + u64 data[8]; + int ret; + + if (index >= mc_dev->obj_desc.region_count) + return -EINVAL; + + region = &vdev->regions[index]; + + if (!(region->flags & VFIO_REGION_INFO_FLAG_WRITE)) + return -EINVAL; + + if (!region->ioaddr) { + region->ioaddr = ioremap(region->addr, region->size); + if (!region->ioaddr) + return -ENOMEM; + } + + if (count != 64 || off != 0) + return -EINVAL; + + if (copy_from_user(&data, buf, 64)) + return -EFAULT; + + ret = vfio_fsl_mc_send_command(region->ioaddr, data); + if (ret) + return ret; + + return count; + +} + +static int vfio_fsl_mc_mmap_mmio(struct vfio_fsl_mc_region region, + struct vm_area_struct *vma) +{ + u64 size = vma->vm_end - vma->vm_start; + u64 pgoff, base; + u8 region_cacheable; + + pgoff = vma->vm_pgoff & + ((1U << (VFIO_FSL_MC_OFFSET_SHIFT - PAGE_SHIFT)) - 1); + base = pgoff << PAGE_SHIFT; + + if (region.size < PAGE_SIZE || base + size > region.size) + return -EINVAL; + + region_cacheable = (region.type & FSL_MC_REGION_CACHEABLE) && + (region.type & FSL_MC_REGION_SHAREABLE); + if (!region_cacheable) + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff; + + return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, + size, vma->vm_page_prot); +} + +static int vfio_fsl_mc_mmap(void *device_data, struct vm_area_struct *vma) +{ + struct vfio_fsl_mc_device *vdev = device_data; + struct fsl_mc_device *mc_dev = vdev->mc_dev; + int index; + + index = vma->vm_pgoff >> (VFIO_FSL_MC_OFFSET_SHIFT - PAGE_SHIFT); + + if (vma->vm_end < vma->vm_start) + return -EINVAL; + if (vma->vm_start & ~PAGE_MASK) + return -EINVAL; + if (vma->vm_end & ~PAGE_MASK) + return -EINVAL; + if (!(vma->vm_flags & VM_SHARED)) + return -EINVAL; + if (index >= mc_dev->obj_desc.region_count) + return -EINVAL; + + if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_MMAP)) + return -EINVAL; + + if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_READ) + && (vma->vm_flags & VM_READ)) + return -EINVAL; + + if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE) + && (vma->vm_flags & VM_WRITE)) + return -EINVAL; + + vma->vm_private_data = mc_dev; + + return vfio_fsl_mc_mmap_mmio(vdev->regions[index], vma); +} + +static const struct vfio_device_ops vfio_fsl_mc_ops = { + .name = "vfio-fsl-mc", + .open = vfio_fsl_mc_open, + .release = vfio_fsl_mc_release, + .ioctl = vfio_fsl_mc_ioctl, + .read = vfio_fsl_mc_read, + .write = vfio_fsl_mc_write, + .mmap = vfio_fsl_mc_mmap, +}; + +static int vfio_fsl_mc_bus_notifier(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct vfio_fsl_mc_device *vdev = container_of(nb, + struct vfio_fsl_mc_device, nb); + struct device *dev = data; + struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev); + struct fsl_mc_device *mc_cont = to_fsl_mc_device(mc_dev->dev.parent); + + if (action == BUS_NOTIFY_ADD_DEVICE && + vdev->mc_dev == mc_cont) { + mc_dev->driver_override = kasprintf(GFP_KERNEL, "%s", + vfio_fsl_mc_ops.name); + if (!mc_dev->driver_override) + dev_warn(dev, "VFIO_FSL_MC: Setting driver override for device in dprc %s failed\n", + dev_name(&mc_cont->dev)); + else + dev_info(dev, "VFIO_FSL_MC: Setting driver override for device in dprc %s\n", + dev_name(&mc_cont->dev)); + } else if (action == BUS_NOTIFY_BOUND_DRIVER && + vdev->mc_dev == mc_cont) { + struct fsl_mc_driver *mc_drv = to_fsl_mc_driver(dev->driver); + + if (mc_drv && mc_drv != &vfio_fsl_mc_driver) + dev_warn(dev, "VFIO_FSL_MC: Object %s bound to driver %s while DPRC bound to vfio-fsl-mc\n", + dev_name(dev), mc_drv->driver.name); + } + + return 0; +} + +static int vfio_fsl_mc_init_device(struct vfio_fsl_mc_device *vdev) +{ + struct fsl_mc_device *mc_dev = vdev->mc_dev; + int ret; + + /* Non-dprc devices share mc_io from parent */ + if (!is_fsl_mc_bus_dprc(mc_dev)) { + struct fsl_mc_device *mc_cont = to_fsl_mc_device(mc_dev->dev.parent); + + mc_dev->mc_io = mc_cont->mc_io; + return 0; + } + + vdev->nb.notifier_call = vfio_fsl_mc_bus_notifier; + ret = bus_register_notifier(&fsl_mc_bus_type, &vdev->nb); + if (ret) + return ret; + + /* open DPRC, allocate a MC portal */ + ret = dprc_setup(mc_dev); + if (ret) { + dev_err(&mc_dev->dev, "VFIO_FSL_MC: Failed to setup DPRC (%d)\n", ret); + goto out_nc_unreg; + } + + ret = dprc_scan_container(mc_dev, false); + if (ret) { + dev_err(&mc_dev->dev, "VFIO_FSL_MC: Container scanning failed (%d)\n", ret); + goto out_dprc_cleanup; + } + + return 0; + +out_dprc_cleanup: + dprc_remove_devices(mc_dev, NULL, 0); + dprc_cleanup(mc_dev); +out_nc_unreg: + bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb); + vdev->nb.notifier_call = NULL; + + return ret; +} + +static int vfio_fsl_mc_probe(struct fsl_mc_device *mc_dev) +{ + struct iommu_group *group; + struct vfio_fsl_mc_device *vdev; + struct device *dev = &mc_dev->dev; + int ret; + + group = vfio_iommu_group_get(dev); + if (!group) { + dev_err(dev, "VFIO_FSL_MC: No IOMMU group\n"); + return -EINVAL; + } + + vdev = devm_kzalloc(dev, sizeof(*vdev), GFP_KERNEL); + if (!vdev) { + ret = -ENOMEM; + goto out_group_put; + } + + vdev->mc_dev = mc_dev; + + ret = vfio_add_group_dev(dev, &vfio_fsl_mc_ops, vdev); + if (ret) { + dev_err(dev, "VFIO_FSL_MC: Failed to add to vfio group\n"); + goto out_group_put; + } + + ret = vfio_fsl_mc_reflck_attach(vdev); + if (ret) + goto out_group_dev; + + ret = vfio_fsl_mc_init_device(vdev); + if (ret) + goto out_reflck; + + mutex_init(&vdev->igate); + + return 0; + +out_reflck: + vfio_fsl_mc_reflck_put(vdev->reflck); +out_group_dev: + vfio_del_group_dev(dev); +out_group_put: + vfio_iommu_group_put(group, dev); + return ret; +} + +static int vfio_fsl_mc_remove(struct fsl_mc_device *mc_dev) +{ + struct vfio_fsl_mc_device *vdev; + struct device *dev = &mc_dev->dev; + + vdev = vfio_del_group_dev(dev); + if (!vdev) + return -EINVAL; + + mutex_destroy(&vdev->igate); + + vfio_fsl_mc_reflck_put(vdev->reflck); + + if (is_fsl_mc_bus_dprc(mc_dev)) { + dprc_remove_devices(mc_dev, NULL, 0); + dprc_cleanup(mc_dev); + } + + if (vdev->nb.notifier_call) + bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb); + + vfio_iommu_group_put(mc_dev->dev.iommu_group, dev); + + return 0; +} + +static struct fsl_mc_driver vfio_fsl_mc_driver = { + .probe = vfio_fsl_mc_probe, + .remove = vfio_fsl_mc_remove, + .driver = { + .name = "vfio-fsl-mc", + .owner = THIS_MODULE, + }, +}; + +static int __init vfio_fsl_mc_driver_init(void) +{ + return fsl_mc_driver_register(&vfio_fsl_mc_driver); +} + +static void __exit vfio_fsl_mc_driver_exit(void) +{ + fsl_mc_driver_unregister(&vfio_fsl_mc_driver); +} + +module_init(vfio_fsl_mc_driver_init); +module_exit(vfio_fsl_mc_driver_exit); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("VFIO for FSL-MC devices - User Level meta-driver"); diff --git a/drivers/vfio/fsl-mc/vfio_fsl_mc_intr.c b/drivers/vfio/fsl-mc/vfio_fsl_mc_intr.c new file mode 100644 index 000000000000..c80dceb46f79 --- /dev/null +++ b/drivers/vfio/fsl-mc/vfio_fsl_mc_intr.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2019 NXP + */ + +#include <linux/vfio.h> +#include <linux/slab.h> +#include <linux/types.h> +#include <linux/eventfd.h> +#include <linux/msi.h> + +#include "linux/fsl/mc.h" +#include "vfio_fsl_mc_private.h" + +int vfio_fsl_mc_irqs_allocate(struct vfio_fsl_mc_device *vdev) +{ + struct fsl_mc_device *mc_dev = vdev->mc_dev; + struct vfio_fsl_mc_irq *mc_irq; + int irq_count; + int ret, i; + + /* Device does not support any interrupt */ + if (mc_dev->obj_desc.irq_count == 0) + return 0; + + /* interrupts were already allocated for this device */ + if (vdev->mc_irqs) + return 0; + + irq_count = mc_dev->obj_desc.irq_count; + + mc_irq = kcalloc(irq_count, sizeof(*mc_irq), GFP_KERNEL); + if (!mc_irq) + return -ENOMEM; + + /* Allocate IRQs */ + ret = fsl_mc_allocate_irqs(mc_dev); + if (ret) { + kfree(mc_irq); + return ret; + } + + for (i = 0; i < irq_count; i++) { + mc_irq[i].count = 1; + mc_irq[i].flags = VFIO_IRQ_INFO_EVENTFD; + } + + vdev->mc_irqs = mc_irq; + + return 0; +} + +static irqreturn_t vfio_fsl_mc_irq_handler(int irq_num, void *arg) +{ + struct vfio_fsl_mc_irq *mc_irq = (struct vfio_fsl_mc_irq *)arg; + + eventfd_signal(mc_irq->trigger, 1); + return IRQ_HANDLED; +} + +static int vfio_set_trigger(struct vfio_fsl_mc_device *vdev, + int index, int fd) +{ + struct vfio_fsl_mc_irq *irq = &vdev->mc_irqs[index]; + struct eventfd_ctx *trigger; + int hwirq; + int ret; + + hwirq = vdev->mc_dev->irqs[index]->msi_desc->irq; + if (irq->trigger) { + free_irq(hwirq, irq); + kfree(irq->name); + eventfd_ctx_put(irq->trigger); + irq->trigger = NULL; + } + + if (fd < 0) /* Disable only */ + return 0; + + irq->name = kasprintf(GFP_KERNEL, "vfio-irq[%d](%s)", + hwirq, dev_name(&vdev->mc_dev->dev)); + if (!irq->name) + return -ENOMEM; + + trigger = eventfd_ctx_fdget(fd); + if (IS_ERR(trigger)) { + kfree(irq->name); + return PTR_ERR(trigger); + } + + irq->trigger = trigger; + + ret = request_irq(hwirq, vfio_fsl_mc_irq_handler, 0, + irq->name, irq); + if (ret) { + kfree(irq->name); + eventfd_ctx_put(trigger); + irq->trigger = NULL; + return ret; + } + + return 0; +} + +static int vfio_fsl_mc_set_irq_trigger(struct vfio_fsl_mc_device *vdev, + unsigned int index, unsigned int start, + unsigned int count, u32 flags, + void *data) +{ + struct fsl_mc_device *mc_dev = vdev->mc_dev; + int ret, hwirq; + struct vfio_fsl_mc_irq *irq; + struct device *cont_dev = fsl_mc_cont_dev(&mc_dev->dev); + struct fsl_mc_device *mc_cont = to_fsl_mc_device(cont_dev); + + if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) + return vfio_set_trigger(vdev, index, -1); + + if (start != 0 || count != 1) + return -EINVAL; + + mutex_lock(&vdev->reflck->lock); + ret = fsl_mc_populate_irq_pool(mc_cont, + FSL_MC_IRQ_POOL_MAX_TOTAL_IRQS); + if (ret) + goto unlock; + + ret = vfio_fsl_mc_irqs_allocate(vdev); + if (ret) + goto unlock; + mutex_unlock(&vdev->reflck->lock); + + if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { + s32 fd = *(s32 *)data; + + return vfio_set_trigger(vdev, index, fd); + } + + hwirq = vdev->mc_dev->irqs[index]->msi_desc->irq; + + irq = &vdev->mc_irqs[index]; + + if (flags & VFIO_IRQ_SET_DATA_NONE) { + vfio_fsl_mc_irq_handler(hwirq, irq); + + } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { + u8 trigger = *(u8 *)data; + + if (trigger) + vfio_fsl_mc_irq_handler(hwirq, irq); + } + + return 0; + +unlock: + mutex_unlock(&vdev->reflck->lock); + return ret; + +} + +int vfio_fsl_mc_set_irqs_ioctl(struct vfio_fsl_mc_device *vdev, + u32 flags, unsigned int index, + unsigned int start, unsigned int count, + void *data) +{ + if (flags & VFIO_IRQ_SET_ACTION_TRIGGER) + return vfio_fsl_mc_set_irq_trigger(vdev, index, start, + count, flags, data); + else + return -EINVAL; +} + +/* Free All IRQs for the given MC object */ +void vfio_fsl_mc_irqs_cleanup(struct vfio_fsl_mc_device *vdev) +{ + struct fsl_mc_device *mc_dev = vdev->mc_dev; + int irq_count = mc_dev->obj_desc.irq_count; + int i; + + /* + * Device does not support any interrupt or the interrupts + * were not configured + */ + if (!vdev->mc_irqs) + return; + + for (i = 0; i < irq_count; i++) + vfio_set_trigger(vdev, i, -1); + + fsl_mc_free_irqs(mc_dev); + kfree(vdev->mc_irqs); + vdev->mc_irqs = NULL; +} diff --git a/drivers/vfio/fsl-mc/vfio_fsl_mc_private.h b/drivers/vfio/fsl-mc/vfio_fsl_mc_private.h new file mode 100644 index 000000000000..a97ee691ed47 --- /dev/null +++ b/drivers/vfio/fsl-mc/vfio_fsl_mc_private.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016,2019-2020 NXP + */ + +#ifndef VFIO_FSL_MC_PRIVATE_H +#define VFIO_FSL_MC_PRIVATE_H + +#define VFIO_FSL_MC_OFFSET_SHIFT 40 +#define VFIO_FSL_MC_OFFSET_MASK (((u64)(1) << VFIO_FSL_MC_OFFSET_SHIFT) - 1) + +#define VFIO_FSL_MC_OFFSET_TO_INDEX(off) ((off) >> VFIO_FSL_MC_OFFSET_SHIFT) + +#define VFIO_FSL_MC_INDEX_TO_OFFSET(index) \ + ((u64)(index) << VFIO_FSL_MC_OFFSET_SHIFT) + +struct vfio_fsl_mc_irq { + u32 flags; + u32 count; + struct eventfd_ctx *trigger; + char *name; +}; + +struct vfio_fsl_mc_reflck { + struct kref kref; + struct mutex lock; +}; + +struct vfio_fsl_mc_region { + u32 flags; + u32 type; + u64 addr; + resource_size_t size; + void __iomem *ioaddr; +}; + +struct vfio_fsl_mc_device { + struct fsl_mc_device *mc_dev; + struct notifier_block nb; + int refcnt; + struct vfio_fsl_mc_region *regions; + struct vfio_fsl_mc_reflck *reflck; + struct mutex igate; + struct vfio_fsl_mc_irq *mc_irqs; +}; + +extern int vfio_fsl_mc_set_irqs_ioctl(struct vfio_fsl_mc_device *vdev, + u32 flags, unsigned int index, + unsigned int start, unsigned int count, + void *data); + +void vfio_fsl_mc_irqs_cleanup(struct vfio_fsl_mc_device *vdev); + +#endif /* VFIO_FSL_MC_PRIVATE_H */ diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig index ac3c1dd3edef..40a223381ab6 100644 --- a/drivers/vfio/pci/Kconfig +++ b/drivers/vfio/pci/Kconfig @@ -45,3 +45,15 @@ config VFIO_PCI_NVLINK2 depends on VFIO_PCI && PPC_POWERNV help VFIO PCI support for P9 Witherspoon machine with NVIDIA V100 GPUs + +config VFIO_PCI_ZDEV + bool "VFIO PCI ZPCI device CLP support" + depends on VFIO_PCI && S390 + default y + help + Enabling this option exposes VFIO capabilities containing hardware + configuration for zPCI devices. This enables userspace (e.g. QEMU) + to supply proper configuration values instead of hard-coded defaults + for zPCI devices passed through via VFIO on s390. + + Say Y here. diff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile index f027f8a0e89c..781e0809d6ee 100644 --- a/drivers/vfio/pci/Makefile +++ b/drivers/vfio/pci/Makefile @@ -3,5 +3,6 @@ vfio-pci-y := vfio_pci.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o vfio-pci-$(CONFIG_VFIO_PCI_IGD) += vfio_pci_igd.o vfio-pci-$(CONFIG_VFIO_PCI_NVLINK2) += vfio_pci_nvlink2.o +vfio-pci-$(CONFIG_VFIO_PCI_ZDEV) += vfio_pci_zdev.o obj-$(CONFIG_VFIO_PCI) += vfio-pci.o diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index 1ab1f5cda4ac..fbd2b3404184 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -807,15 +807,25 @@ static long vfio_pci_ioctl(void *device_data, if (cmd == VFIO_DEVICE_GET_INFO) { struct vfio_device_info info; + struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; + unsigned long capsz; minsz = offsetofend(struct vfio_device_info, num_irqs); + /* For backward compatibility, cannot require this */ + capsz = offsetofend(struct vfio_iommu_type1_info, cap_offset); + if (copy_from_user(&info, (void __user *)arg, minsz)) return -EFAULT; if (info.argsz < minsz) return -EINVAL; + if (info.argsz >= capsz) { + minsz = capsz; + info.cap_offset = 0; + } + info.flags = VFIO_DEVICE_FLAGS_PCI; if (vdev->reset_works) @@ -824,6 +834,33 @@ static long vfio_pci_ioctl(void *device_data, info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; info.num_irqs = VFIO_PCI_NUM_IRQS; + if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV)) { + int ret = vfio_pci_info_zdev_add_caps(vdev, &caps); + + if (ret && ret != -ENODEV) { + pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n"); + return ret; + } + } + + if (caps.size) { + info.flags |= VFIO_DEVICE_FLAGS_CAPS; + if (info.argsz < sizeof(info) + caps.size) { + info.argsz = sizeof(info) + caps.size; + } else { + vfio_info_cap_shift(&caps, sizeof(info)); + if (copy_to_user((void __user *)arg + + sizeof(info), caps.buf, + caps.size)) { + kfree(caps.buf); + return -EFAULT; + } + info.cap_offset = sizeof(info); + } + + kfree(caps.buf); + } + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; @@ -1480,31 +1517,29 @@ static int vfio_pci_zap_and_vma_lock(struct vfio_pci_device *vdev, bool try) } else { mmap_read_lock(mm); } - if (mmget_still_valid(mm)) { - if (try) { - if (!mutex_trylock(&vdev->vma_lock)) { - mmap_read_unlock(mm); - mmput(mm); - return 0; - } - } else { - mutex_lock(&vdev->vma_lock); + if (try) { + if (!mutex_trylock(&vdev->vma_lock)) { + mmap_read_unlock(mm); + mmput(mm); + return 0; } - list_for_each_entry_safe(mmap_vma, tmp, - &vdev->vma_list, vma_next) { - struct vm_area_struct *vma = mmap_vma->vma; + } else { + mutex_lock(&vdev->vma_lock); + } + list_for_each_entry_safe(mmap_vma, tmp, + &vdev->vma_list, vma_next) { + struct vm_area_struct *vma = mmap_vma->vma; - if (vma->vm_mm != mm) - continue; + if (vma->vm_mm != mm) + continue; - list_del(&mmap_vma->vma_next); - kfree(mmap_vma); + list_del(&mmap_vma->vma_next); + kfree(mmap_vma); - zap_vma_ptes(vma, vma->vm_start, - vma->vm_end - vma->vm_start); - } - mutex_unlock(&vdev->vma_lock); + zap_vma_ptes(vma, vma->vm_start, + vma->vm_end - vma->vm_start); } + mutex_unlock(&vdev->vma_lock); mmap_read_unlock(mm); mmput(mm); } @@ -1862,7 +1897,6 @@ static const struct vfio_device_ops vfio_pci_ops = { static int vfio_pci_reflck_attach(struct vfio_pci_device *vdev); static void vfio_pci_reflck_put(struct vfio_pci_reflck *reflck); -static struct pci_driver vfio_pci_driver; static int vfio_pci_bus_notifier(struct notifier_block *nb, unsigned long action, void *data) diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c index d98843feddce..a402adee8a21 100644 --- a/drivers/vfio/pci/vfio_pci_config.c +++ b/drivers/vfio/pci/vfio_pci_config.c @@ -406,7 +406,7 @@ bool __vfio_pci_memory_enabled(struct vfio_pci_device *vdev) * PF SR-IOV capability, there's therefore no need to trigger * faults based on the virtual value. */ - return pdev->is_virtfn || (cmd & PCI_COMMAND_MEMORY); + return pdev->no_command_memory || (cmd & PCI_COMMAND_MEMORY); } /* @@ -467,6 +467,9 @@ static void vfio_bar_fixup(struct vfio_pci_device *vdev) __le32 *vbar; u64 mask; + if (!vdev->bardirty) + return; + vbar = (__le32 *)&vdev->vconfig[PCI_BASE_ADDRESS_0]; for (i = 0; i < PCI_STD_NUM_BARS; i++, vbar++) { @@ -520,8 +523,8 @@ static int vfio_basic_config_read(struct vfio_pci_device *vdev, int pos, count = vfio_default_config_read(vdev, pos, count, perm, offset, val); - /* Mask in virtual memory enable for SR-IOV devices */ - if (offset == PCI_COMMAND && vdev->pdev->is_virtfn) { + /* Mask in virtual memory enable */ + if (offset == PCI_COMMAND && vdev->pdev->no_command_memory) { u16 cmd = le16_to_cpu(*(__le16 *)&vdev->vconfig[PCI_COMMAND]); u32 tmp_val = le32_to_cpu(*val); @@ -589,9 +592,11 @@ static int vfio_basic_config_write(struct vfio_pci_device *vdev, int pos, * shows it disabled (phys_mem/io, then the device has * undergone some kind of backdoor reset and needs to be * restored before we allow it to enable the bars. - * SR-IOV devices will trigger this, but we catch them later + * SR-IOV devices will trigger this - for mem enable let's + * catch this now and for io enable it will be caught later */ - if ((new_mem && virt_mem && !phys_mem) || + if ((new_mem && virt_mem && !phys_mem && + !pdev->no_command_memory) || (new_io && virt_io && !phys_io) || vfio_need_bar_restore(vdev)) vfio_bar_restore(vdev); @@ -1734,12 +1739,14 @@ int vfio_config_init(struct vfio_pci_device *vdev) vconfig[PCI_INTERRUPT_PIN]); vconfig[PCI_INTERRUPT_PIN] = 0; /* Gratuitous for good VFs */ - + } + if (pdev->no_command_memory) { /* - * VFs do no implement the memory enable bit of the COMMAND - * register therefore we'll not have it set in our initial - * copy of config space after pci_enable_device(). For - * consistency with PFs, set the virtual enable bit here. + * VFs and devices that set pdev->no_command_memory do not + * implement the memory enable bit of the COMMAND register + * therefore we'll not have it set in our initial copy of + * config space after pci_enable_device(). For consistency + * with PFs, set the virtual enable bit here. */ *(__le16 *)&vconfig[PCI_COMMAND] |= cpu_to_le16(PCI_COMMAND_MEMORY); diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c index 1d9fb2592945..869dce5f134d 100644 --- a/drivers/vfio/pci/vfio_pci_intrs.c +++ b/drivers/vfio/pci/vfio_pci_intrs.c @@ -352,11 +352,13 @@ static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, vdev->ctx[vector].producer.token = trigger; vdev->ctx[vector].producer.irq = irq; ret = irq_bypass_register_producer(&vdev->ctx[vector].producer); - if (unlikely(ret)) + if (unlikely(ret)) { dev_info(&pdev->dev, "irq bypass producer (token %p) registration fails: %d\n", vdev->ctx[vector].producer.token, ret); + vdev->ctx[vector].producer.token = NULL; + } vdev->ctx[vector].trigger = trigger; return 0; diff --git a/drivers/vfio/pci/vfio_pci_private.h b/drivers/vfio/pci/vfio_pci_private.h index 61ca8ab165dc..5c90e560c5c7 100644 --- a/drivers/vfio/pci/vfio_pci_private.h +++ b/drivers/vfio/pci/vfio_pci_private.h @@ -213,4 +213,16 @@ static inline int vfio_pci_ibm_npu2_init(struct vfio_pci_device *vdev) return -ENODEV; } #endif + +#ifdef CONFIG_VFIO_PCI_ZDEV +extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev, + struct vfio_info_cap *caps); +#else +static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev, + struct vfio_info_cap *caps) +{ + return -ENODEV; +} +#endif + #endif /* VFIO_PCI_PRIVATE_H */ diff --git a/drivers/vfio/pci/vfio_pci_zdev.c b/drivers/vfio/pci/vfio_pci_zdev.c new file mode 100644 index 000000000000..229685634031 --- /dev/null +++ b/drivers/vfio/pci/vfio_pci_zdev.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * VFIO ZPCI devices support + * + * Copyright (C) IBM Corp. 2020. All rights reserved. + * Author(s): Pierre Morel <pmorel@linux.ibm.com> + * Matthew Rosato <mjrosato@linux.ibm.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ +#include <linux/io.h> +#include <linux/pci.h> +#include <linux/uaccess.h> +#include <linux/vfio.h> +#include <linux/vfio_zdev.h> +#include <asm/pci_clp.h> +#include <asm/pci_io.h> + +#include "vfio_pci_private.h" + +/* + * Add the Base PCI Function information to the device info region. + */ +static int zpci_base_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, + struct vfio_info_cap *caps) +{ + struct vfio_device_info_cap_zpci_base cap = { + .header.id = VFIO_DEVICE_INFO_CAP_ZPCI_BASE, + .header.version = 1, + .start_dma = zdev->start_dma, + .end_dma = zdev->end_dma, + .pchid = zdev->pchid, + .vfn = zdev->vfn, + .fmb_length = zdev->fmb_length, + .pft = zdev->pft, + .gid = zdev->pfgid + }; + + return vfio_info_add_capability(caps, &cap.header, sizeof(cap)); +} + +/* + * Add the Base PCI Function Group information to the device info region. + */ +static int zpci_group_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, + struct vfio_info_cap *caps) +{ + struct vfio_device_info_cap_zpci_group cap = { + .header.id = VFIO_DEVICE_INFO_CAP_ZPCI_GROUP, + .header.version = 1, + .dasm = zdev->dma_mask, + .msi_addr = zdev->msi_addr, + .flags = VFIO_DEVICE_INFO_ZPCI_FLAG_REFRESH, + .mui = zdev->fmb_update, + .noi = zdev->max_msi, + .maxstbl = ZPCI_MAX_WRITE_SIZE, + .version = zdev->version + }; + + return vfio_info_add_capability(caps, &cap.header, sizeof(cap)); +} + +/* + * Add the device utility string to the device info region. + */ +static int zpci_util_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, + struct vfio_info_cap *caps) +{ + struct vfio_device_info_cap_zpci_util *cap; + int cap_size = sizeof(*cap) + CLP_UTIL_STR_LEN; + int ret; + + cap = kmalloc(cap_size, GFP_KERNEL); + + cap->header.id = VFIO_DEVICE_INFO_CAP_ZPCI_UTIL; + cap->header.version = 1; + cap->size = CLP_UTIL_STR_LEN; + memcpy(cap->util_str, zdev->util_str, cap->size); + + ret = vfio_info_add_capability(caps, &cap->header, cap_size); + + kfree(cap); + + return ret; +} + +/* + * Add the function path string to the device info region. + */ +static int zpci_pfip_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, + struct vfio_info_cap *caps) +{ + struct vfio_device_info_cap_zpci_pfip *cap; + int cap_size = sizeof(*cap) + CLP_PFIP_NR_SEGMENTS; + int ret; + + cap = kmalloc(cap_size, GFP_KERNEL); + + cap->header.id = VFIO_DEVICE_INFO_CAP_ZPCI_PFIP; + cap->header.version = 1; + cap->size = CLP_PFIP_NR_SEGMENTS; + memcpy(cap->pfip, zdev->pfip, cap->size); + + ret = vfio_info_add_capability(caps, &cap->header, cap_size); + + kfree(cap); + + return ret; +} + +/* + * Add all supported capabilities to the VFIO_DEVICE_GET_INFO capability chain. + */ +int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev, + struct vfio_info_cap *caps) +{ + struct zpci_dev *zdev = to_zpci(vdev->pdev); + int ret; + + if (!zdev) + return -ENODEV; + + ret = zpci_base_cap(zdev, vdev, caps); + if (ret) + return ret; + + ret = zpci_group_cap(zdev, vdev, caps); + if (ret) + return ret; + + if (zdev->util_str_avail) { + ret = zpci_util_cap(zdev, vdev, caps); + if (ret) + return ret; + } + + ret = zpci_pfip_cap(zdev, vdev, caps); + + return ret; +} diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index 262ab0efd06c..2151bc7f87ab 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -1949,8 +1949,10 @@ int vfio_pin_pages(struct device *dev, unsigned long *user_pfn, int npage, if (!group) return -ENODEV; - if (group->dev_counter > 1) - return -EINVAL; + if (group->dev_counter > 1) { + ret = -EINVAL; + goto err_pin_pages; + } ret = vfio_group_add_container_user(group); if (ret) @@ -2051,6 +2053,9 @@ int vfio_group_pin_pages(struct vfio_group *group, if (!group || !user_iova_pfn || !phys_pfn || !npage) return -EINVAL; + if (group->dev_counter > 1) + return -EINVAL; + if (npage > VFIO_PIN_PAGES_MAX_ENTRIES) return -E2BIG; diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index c255a6683f31..bb2684cc245e 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -693,7 +693,8 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, ret = vfio_add_to_pfn_list(dma, iova, phys_pfn[i]); if (ret) { - vfio_unpin_page_external(dma, iova, do_accounting); + if (put_pfn(phys_pfn[i], dma->prot) && do_accounting) + vfio_lock_acct(dma, -1, true); goto pin_unwind; } @@ -2609,6 +2610,20 @@ static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu, return vfio_info_add_capability(caps, &cap_mig.header, sizeof(cap_mig)); } +static int vfio_iommu_dma_avail_build_caps(struct vfio_iommu *iommu, + struct vfio_info_cap *caps) +{ + struct vfio_iommu_type1_info_dma_avail cap_dma_avail; + + cap_dma_avail.header.id = VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL; + cap_dma_avail.header.version = 1; + + cap_dma_avail.avail = iommu->dma_avail; + + return vfio_info_add_capability(caps, &cap_dma_avail.header, + sizeof(cap_dma_avail)); +} + static int vfio_iommu_type1_get_info(struct vfio_iommu *iommu, unsigned long arg) { @@ -2642,6 +2657,9 @@ static int vfio_iommu_type1_get_info(struct vfio_iommu *iommu, ret = vfio_iommu_migration_build_caps(iommu, &caps); if (!ret) + ret = vfio_iommu_dma_avail_build_caps(iommu, &caps); + + if (!ret) ret = vfio_iommu_iova_build_caps(iommu, &caps); mutex_unlock(&iommu->lock); @@ -2933,7 +2951,8 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, * size */ bitmap_set(dma->bitmap, offset >> pgshift, - *copied >> pgshift); + ((offset + *copied - 1) >> pgshift) - + (offset >> pgshift) + 1); } } else *copied = copy_from_user(data, (void __user *)vaddr, diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 62a9bb0efc55..a2dbc85e0b0d 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -22,7 +22,6 @@ #include <linux/nospec.h> #include <linux/vhost.h> #include <linux/virtio_net.h> -#include <linux/kernel.h> #include "vhost.h" @@ -97,26 +96,20 @@ static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid) return; irq = ops->get_vq_irq(vdpa, qid); - spin_lock(&vq->call_ctx.ctx_lock); irq_bypass_unregister_producer(&vq->call_ctx.producer); - if (!vq->call_ctx.ctx || irq < 0) { - spin_unlock(&vq->call_ctx.ctx_lock); + if (!vq->call_ctx.ctx || irq < 0) return; - } vq->call_ctx.producer.token = vq->call_ctx.ctx; vq->call_ctx.producer.irq = irq; ret = irq_bypass_register_producer(&vq->call_ctx.producer); - spin_unlock(&vq->call_ctx.ctx_lock); } static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid) { struct vhost_virtqueue *vq = &v->vqs[qid]; - spin_lock(&vq->call_ctx.ctx_lock); irq_bypass_unregister_producer(&vq->call_ctx.producer); - spin_unlock(&vq->call_ctx.ctx_lock); } static void vhost_vdpa_reset(struct vhost_vdpa *v) diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 9ad45e1d27f0..5c835a292783 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -302,7 +302,6 @@ static void vhost_vring_call_reset(struct vhost_vring_call *call_ctx) { call_ctx->ctx = NULL; memset(&call_ctx->producer, 0x0, sizeof(struct irq_bypass_producer)); - spin_lock_init(&call_ctx->ctx_lock); } static void vhost_vq_reset(struct vhost_dev *dev, @@ -1650,9 +1649,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg break; } - spin_lock(&vq->call_ctx.ctx_lock); swap(ctx, vq->call_ctx.ctx); - spin_unlock(&vq->call_ctx.ctx_lock); break; case VHOST_SET_VRING_ERR: if (copy_from_user(&f, argp, sizeof f)) { @@ -1897,7 +1894,7 @@ static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len) static int log_used(struct vhost_virtqueue *vq, u64 used_offset, u64 len) { - struct iovec iov[64]; + struct iovec *iov = vq->log_iov; int i, ret; if (!vq->iotlb) diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index 9032d3c2a9f4..e016cd3fa02f 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -64,7 +64,6 @@ enum vhost_uaddr_type { struct vhost_vring_call { struct eventfd_ctx *ctx; struct irq_bypass_producer producer; - spinlock_t ctx_lock; }; /* The virtqueue structure describes a queue attached to a device. */ @@ -123,6 +122,7 @@ struct vhost_virtqueue { /* Log write descriptors */ void __user *log_base; struct vhost_log *log; + struct iovec log_iov[64]; /* Ring endianness. Defaults to legacy native endianness. * Set to true when starting a modern virtio device. */ diff --git a/drivers/vhost/vringh.c b/drivers/vhost/vringh.c index e059a9a47cdf..8bd8b403f087 100644 --- a/drivers/vhost/vringh.c +++ b/drivers/vhost/vringh.c @@ -284,13 +284,14 @@ __vringh_iov(struct vringh *vrh, u16 i, desc_max = vrh->vring.num; up_next = -1; + /* You must want something! */ + if (WARN_ON(!riov && !wiov)) + return -EINVAL; + if (riov) riov->i = riov->used = 0; - else if (wiov) + if (wiov) wiov->i = wiov->used = 0; - else - /* You must want something! */ - BUG(); for (;;) { void *addr; diff --git a/drivers/video/fbdev/s3c2410fb-regs-lcd.h b/drivers/video/fbdev/s3c2410fb-regs-lcd.h new file mode 100644 index 000000000000..1e46f7a788e5 --- /dev/null +++ b/drivers/video/fbdev/s3c2410fb-regs-lcd.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2003 Simtec Electronics <linux@simtec.co.uk> + * http://www.simtec.co.uk/products/SWLINUX/ + */ + +#ifndef ___ASM_ARCH_REGS_LCD_H +#define ___ASM_ARCH_REGS_LCD_H + +/* + * a couple of values are used as platform data in + * include/linux/platform_data/fb-s3c2410.h and not + * duplicated here. + */ +#include <linux/platform_data/fb-s3c2410.h> + +#define S3C2410_LCDREG(x) (x) + +/* LCD control registers */ +#define S3C2410_LCDCON1 S3C2410_LCDREG(0x00) +#define S3C2410_LCDCON2 S3C2410_LCDREG(0x04) +#define S3C2410_LCDCON3 S3C2410_LCDREG(0x08) +#define S3C2410_LCDCON4 S3C2410_LCDREG(0x0C) +#define S3C2410_LCDCON5 S3C2410_LCDREG(0x10) + +#define S3C2410_LCDCON1_CLKVAL(x) ((x) << 8) +#define S3C2410_LCDCON1_MMODE (1<<7) +#define S3C2410_LCDCON1_DSCAN4 (0<<5) +#define S3C2410_LCDCON1_STN4 (1<<5) +#define S3C2410_LCDCON1_STN8 (2<<5) +#define S3C2410_LCDCON1_TFT (3<<5) + +#define S3C2410_LCDCON1_STN1BPP (0<<1) +#define S3C2410_LCDCON1_STN2GREY (1<<1) +#define S3C2410_LCDCON1_STN4GREY (2<<1) +#define S3C2410_LCDCON1_STN8BPP (3<<1) +#define S3C2410_LCDCON1_STN12BPP (4<<1) + +#define S3C2410_LCDCON1_ENVID (1) + +#define S3C2410_LCDCON1_MODEMASK 0x1E + +#define S3C2410_LCDCON2_VBPD(x) ((x) << 24) +#define S3C2410_LCDCON2_LINEVAL(x) ((x) << 14) +#define S3C2410_LCDCON2_VFPD(x) ((x) << 6) +#define S3C2410_LCDCON2_VSPW(x) ((x) << 0) + +#define S3C2410_LCDCON2_GET_VBPD(x) ( ((x) >> 24) & 0xFF) +#define S3C2410_LCDCON2_GET_VFPD(x) ( ((x) >> 6) & 0xFF) +#define S3C2410_LCDCON2_GET_VSPW(x) ( ((x) >> 0) & 0x3F) + +#define S3C2410_LCDCON3_HBPD(x) ((x) << 19) +#define S3C2410_LCDCON3_WDLY(x) ((x) << 19) +#define S3C2410_LCDCON3_HOZVAL(x) ((x) << 8) +#define S3C2410_LCDCON3_HFPD(x) ((x) << 0) +#define S3C2410_LCDCON3_LINEBLANK(x)((x) << 0) + +#define S3C2410_LCDCON3_GET_HBPD(x) ( ((x) >> 19) & 0x7F) +#define S3C2410_LCDCON3_GET_HFPD(x) ( ((x) >> 0) & 0xFF) + +/* LDCCON4 changes for STN mode on the S3C2412 */ + +#define S3C2410_LCDCON4_MVAL(x) ((x) << 8) +#define S3C2410_LCDCON4_HSPW(x) ((x) << 0) +#define S3C2410_LCDCON4_WLH(x) ((x) << 0) + +#define S3C2410_LCDCON4_GET_HSPW(x) ( ((x) >> 0) & 0xFF) + +/* framebuffer start addressed */ +#define S3C2410_LCDSADDR1 S3C2410_LCDREG(0x14) +#define S3C2410_LCDSADDR2 S3C2410_LCDREG(0x18) +#define S3C2410_LCDSADDR3 S3C2410_LCDREG(0x1C) + +#define S3C2410_LCDBANK(x) ((x) << 21) +#define S3C2410_LCDBASEU(x) (x) + +#define S3C2410_OFFSIZE(x) ((x) << 11) +#define S3C2410_PAGEWIDTH(x) (x) + +/* colour lookup and miscellaneous controls */ + +#define S3C2410_REDLUT S3C2410_LCDREG(0x20) +#define S3C2410_GREENLUT S3C2410_LCDREG(0x24) +#define S3C2410_BLUELUT S3C2410_LCDREG(0x28) + +#define S3C2410_DITHMODE S3C2410_LCDREG(0x4C) +#define S3C2410_TPAL S3C2410_LCDREG(0x50) + +#define S3C2410_TPAL_EN (1<<24) + +/* interrupt info */ +#define S3C2410_LCDINTPND S3C2410_LCDREG(0x54) +#define S3C2410_LCDSRCPND S3C2410_LCDREG(0x58) +#define S3C2410_LCDINTMSK S3C2410_LCDREG(0x5C) +#define S3C2410_LCDINT_FIWSEL (1<<2) +#define S3C2410_LCDINT_FRSYNC (1<<1) +#define S3C2410_LCDINT_FICNT (1<<0) + +/* s3c2442 extra stn registers */ + +#define S3C2442_REDLUT S3C2410_LCDREG(0x20) +#define S3C2442_GREENLUT S3C2410_LCDREG(0x24) +#define S3C2442_BLUELUT S3C2410_LCDREG(0x28) +#define S3C2442_DITHMODE S3C2410_LCDREG(0x20) + +#define S3C2410_LPCSEL S3C2410_LCDREG(0x60) + +#define S3C2410_TFTPAL(x) S3C2410_LCDREG((0x400 + (x)*4)) + +/* S3C2412 registers */ + +#define S3C2412_TPAL S3C2410_LCDREG(0x20) + +#define S3C2412_LCDINTPND S3C2410_LCDREG(0x24) +#define S3C2412_LCDSRCPND S3C2410_LCDREG(0x28) +#define S3C2412_LCDINTMSK S3C2410_LCDREG(0x2C) + +#define S3C2412_TCONSEL S3C2410_LCDREG(0x30) + +#define S3C2412_LCDCON6 S3C2410_LCDREG(0x34) +#define S3C2412_LCDCON7 S3C2410_LCDREG(0x38) +#define S3C2412_LCDCON8 S3C2410_LCDREG(0x3C) +#define S3C2412_LCDCON9 S3C2410_LCDREG(0x40) + +#define S3C2412_REDLUT(x) S3C2410_LCDREG(0x44 + ((x)*4)) +#define S3C2412_GREENLUT(x) S3C2410_LCDREG(0x60 + ((x)*4)) +#define S3C2412_BLUELUT(x) S3C2410_LCDREG(0x98 + ((x)*4)) + +#define S3C2412_FRCPAT(x) S3C2410_LCDREG(0xB4 + ((x)*4)) + +/* general registers */ + +/* base of the LCD registers, where INTPND, INTSRC and then INTMSK + * are available. */ + +#define S3C2410_LCDINTBASE S3C2410_LCDREG(0x54) +#define S3C2412_LCDINTBASE S3C2410_LCDREG(0x24) + +#define S3C24XX_LCDINTPND (0x00) +#define S3C24XX_LCDSRCPND (0x04) +#define S3C24XX_LCDINTMSK (0x08) + +#endif /* ___ASM_ARCH_REGS_LCD_H */ diff --git a/drivers/video/fbdev/s3c2410fb.c b/drivers/video/fbdev/s3c2410fb.c index 6f8fa501583f..d8ae5258de46 100644 --- a/drivers/video/fbdev/s3c2410fb.c +++ b/drivers/video/fbdev/s3c2410fb.c @@ -29,19 +29,18 @@ #include <linux/clk.h> #include <linux/cpufreq.h> #include <linux/io.h> +#include <linux/platform_data/fb-s3c2410.h> #include <asm/div64.h> #include <asm/mach/map.h> -#include <mach/regs-lcd.h> -#include <mach/regs-gpio.h> -#include <mach/fb.h> #ifdef CONFIG_PM #include <linux/pm.h> #endif #include "s3c2410fb.h" +#include "s3c2410fb-regs-lcd.h" /* Debugging stuff */ static int debug = IS_BUILTIN(CONFIG_FB_S3C2410_DEBUG); @@ -672,6 +671,9 @@ static inline void modify_gpio(void __iomem *reg, { unsigned long tmp; + if (!reg) + return; + tmp = readl(reg) & ~mask; writel(tmp | set, reg); } @@ -702,10 +704,10 @@ static int s3c2410fb_init_registers(struct fb_info *info) /* modify the gpio(s) with interrupts set (bjd) */ - modify_gpio(S3C2410_GPCUP, mach_info->gpcup, mach_info->gpcup_mask); - modify_gpio(S3C2410_GPCCON, mach_info->gpccon, mach_info->gpccon_mask); - modify_gpio(S3C2410_GPDUP, mach_info->gpdup, mach_info->gpdup_mask); - modify_gpio(S3C2410_GPDCON, mach_info->gpdcon, mach_info->gpdcon_mask); + modify_gpio(mach_info->gpcup_reg, mach_info->gpcup, mach_info->gpcup_mask); + modify_gpio(mach_info->gpccon_reg, mach_info->gpccon, mach_info->gpccon_mask); + modify_gpio(mach_info->gpdup_reg, mach_info->gpdup, mach_info->gpdup_mask); + modify_gpio(mach_info->gpdcon_reg, mach_info->gpdcon, mach_info->gpdcon_mask); local_irq_restore(flags); diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index e76e9b9ba93c..7b41130d3f35 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -6,6 +6,12 @@ config VIRTIO bus, such as CONFIG_VIRTIO_PCI, CONFIG_VIRTIO_MMIO, CONFIG_RPMSG or CONFIG_S390_GUEST. +config ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS + bool + help + This option is selected if the architecture may need to enforce + VIRTIO_F_ACCESS_PLATFORM + menuconfig VIRTIO_MENU bool "Virtio drivers" default y diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index 5d46f0ded92d..42e09cc1b8ac 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -176,6 +176,21 @@ int virtio_finalize_features(struct virtio_device *dev) if (ret) return ret; + ret = arch_has_restricted_virtio_memory_access(); + if (ret) { + if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1)) { + dev_warn(&dev->dev, + "device must provide VIRTIO_F_VERSION_1\n"); + return -ENODEV; + } + + if (!virtio_has_feature(dev, VIRTIO_F_ACCESS_PLATFORM)) { + dev_warn(&dev->dev, + "device must provide VIRTIO_F_ACCESS_PLATFORM\n"); + return -ENODEV; + } + } + if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1)) return 0; diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 31cc97f2f515..481611c09dae 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -128,7 +128,7 @@ struct virtio_balloon { struct page_reporting_dev_info pr_dev_info; }; -static struct virtio_device_id id_table[] = { +static const struct virtio_device_id id_table[] = { { VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID }, { 0 }, }; diff --git a/drivers/virtio/virtio_input.c b/drivers/virtio/virtio_input.c index 877b2ea3ed05..f1f6208edcf5 100644 --- a/drivers/virtio/virtio_input.c +++ b/drivers/virtio/virtio_input.c @@ -363,7 +363,7 @@ static int virtinput_restore(struct virtio_device *vdev) static unsigned int features[] = { /* none */ }; -static struct virtio_device_id id_table[] = { +static const struct virtio_device_id id_table[] = { { VIRTIO_ID_INPUT, VIRTIO_DEV_ANY_ID }, { 0 }, }; diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 834b7c13ef3d..181e2f18beae 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -424,7 +424,8 @@ static int virtio_mem_mb_add(struct virtio_mem *vm, unsigned long mb_id) dev_dbg(&vm->vdev->dev, "adding memory block: %lu\n", mb_id); return add_memory_driver_managed(nid, addr, memory_block_size_bytes(), - vm->resource_name); + vm->resource_name, + MEMHP_MERGE_RESOURCE); } /* @@ -1926,7 +1927,7 @@ static unsigned int virtio_mem_features[] = { #endif }; -static struct virtio_device_id virtio_mem_id_table[] = { +static const struct virtio_device_id virtio_mem_id_table[] = { { VIRTIO_ID_MEM, VIRTIO_DEV_ANY_ID }, { 0 }, }; diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig index ba698340ed6f..fd7968635e6d 100644 --- a/drivers/watchdog/Kconfig +++ b/drivers/watchdog/Kconfig @@ -489,16 +489,10 @@ config IXP4XX_WATCHDOG Say N if you are unsure. -config HAVE_S3C2410_WATCHDOG - bool - help - This will include watchdog timer support for Samsung SoCs. If - you want to include watchdog support for any machine, kindly - select this in the respective mach-XXXX/Kconfig file. - config S3C2410_WATCHDOG tristate "S3C2410 Watchdog" - depends on HAVE_S3C2410_WATCHDOG || COMPILE_TEST + depends on ARCH_S3C24XX || ARCH_S3C64XX || ARCH_S5PV210 || ARCH_EXYNOS || \ + COMPILE_TEST select WATCHDOG_CORE select MFD_SYSCON if ARCH_EXYNOS help @@ -1015,6 +1009,14 @@ config PM8916_WATCHDOG Say Y here to include support watchdog timer embedded into the pm8916 module. +config VISCONTI_WATCHDOG + tristate "Toshiba Visconti series watchdog support" + depends on ARCH_VISCONTI || COMPILE_TEST + select WATCHDOG_CORE + help + Say Y here to include support for the watchdog timer in Toshiba + Visconti SoCs. + # X86 (i386 + ia64 + x86_64) Architecture config ACQUIRE_WDT diff --git a/drivers/watchdog/Makefile b/drivers/watchdog/Makefile index aa6e41126901..071a2e50be98 100644 --- a/drivers/watchdog/Makefile +++ b/drivers/watchdog/Makefile @@ -95,6 +95,7 @@ obj-$(CONFIG_RTD119X_WATCHDOG) += rtd119x_wdt.o obj-$(CONFIG_SPRD_WATCHDOG) += sprd_wdt.o obj-$(CONFIG_PM8916_WATCHDOG) += pm8916_wdt.o obj-$(CONFIG_ARM_SMC_WATCHDOG) += arm_smc_wdt.o +obj-$(CONFIG_VISCONTI_WATCHDOG) += visconti_wdt.o # X86 (i386 + ia64 + x86_64) Architecture obj-$(CONFIG_ACQUIRE_WDT) += acquirewdt.o diff --git a/drivers/watchdog/cadence_wdt.c b/drivers/watchdog/cadence_wdt.c index 672b184da875..bc99e9164930 100644 --- a/drivers/watchdog/cadence_wdt.c +++ b/drivers/watchdog/cadence_wdt.c @@ -334,12 +334,9 @@ static int cdns_wdt_probe(struct platform_device *pdev) watchdog_set_drvdata(cdns_wdt_device, wdt); wdt->clk = devm_clk_get(dev, NULL); - if (IS_ERR(wdt->clk)) { - ret = PTR_ERR(wdt->clk); - if (ret != -EPROBE_DEFER) - dev_err(dev, "input clock not found\n"); - return ret; - } + if (IS_ERR(wdt->clk)) + return dev_err_probe(dev, PTR_ERR(wdt->clk), + "input clock not found\n"); ret = clk_prepare_enable(wdt->clk); if (ret) { diff --git a/drivers/watchdog/davinci_wdt.c b/drivers/watchdog/davinci_wdt.c index 2b3f3cd382ef..e6eaba6bae5b 100644 --- a/drivers/watchdog/davinci_wdt.c +++ b/drivers/watchdog/davinci_wdt.c @@ -206,12 +206,9 @@ static int davinci_wdt_probe(struct platform_device *pdev) return -ENOMEM; davinci_wdt->clk = devm_clk_get(dev, NULL); - - if (IS_ERR(davinci_wdt->clk)) { - if (PTR_ERR(davinci_wdt->clk) != -EPROBE_DEFER) - dev_err(dev, "failed to get clock node\n"); - return PTR_ERR(davinci_wdt->clk); - } + if (IS_ERR(davinci_wdt->clk)) + return dev_err_probe(dev, PTR_ERR(davinci_wdt->clk), + "failed to get clock node\n"); ret = clk_prepare_enable(davinci_wdt->clk); if (ret) { diff --git a/drivers/watchdog/imx7ulp_wdt.c b/drivers/watchdog/imx7ulp_wdt.c index 7993c8c41b3a..922b60374295 100644 --- a/drivers/watchdog/imx7ulp_wdt.c +++ b/drivers/watchdog/imx7ulp_wdt.c @@ -5,6 +5,7 @@ #include <linux/clk.h> #include <linux/io.h> +#include <linux/iopoll.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/of.h> @@ -21,6 +22,8 @@ #define WDOG_CS_CLK (LPO_CLK << LPO_CLK_SHIFT) #define WDOG_CS_EN BIT(7) #define WDOG_CS_UPDATE BIT(5) +#define WDOG_CS_WAIT BIT(1) +#define WDOG_CS_STOP BIT(0) #define WDOG_CNT 0x4 #define WDOG_TOVAL 0x8 @@ -36,6 +39,7 @@ #define DEFAULT_TIMEOUT 60 #define MAX_TIMEOUT 128 #define WDOG_CLOCK_RATE 1000 +#define WDOG_WAIT_TIMEOUT 20 static bool nowayout = WATCHDOG_NOWAYOUT; module_param(nowayout, bool, 0000); @@ -48,17 +52,40 @@ struct imx7ulp_wdt_device { struct clk *clk; }; -static void imx7ulp_wdt_enable(struct watchdog_device *wdog, bool enable) +static int imx7ulp_wdt_wait(void __iomem *base, u32 mask) +{ + u32 val = readl(base + WDOG_CS); + + if (!(val & mask) && readl_poll_timeout_atomic(base + WDOG_CS, val, + val & mask, 0, + WDOG_WAIT_TIMEOUT)) + return -ETIMEDOUT; + + return 0; +} + +static int imx7ulp_wdt_enable(struct watchdog_device *wdog, bool enable) { struct imx7ulp_wdt_device *wdt = watchdog_get_drvdata(wdog); u32 val = readl(wdt->base + WDOG_CS); + int ret; + local_irq_disable(); writel(UNLOCK, wdt->base + WDOG_CNT); + ret = imx7ulp_wdt_wait(wdt->base, WDOG_CS_ULK); + if (ret) + goto enable_out; if (enable) writel(val | WDOG_CS_EN, wdt->base + WDOG_CS); else writel(val & ~WDOG_CS_EN, wdt->base + WDOG_CS); + imx7ulp_wdt_wait(wdt->base, WDOG_CS_RCS); + +enable_out: + local_irq_enable(); + + return ret; } static bool imx7ulp_wdt_is_enabled(void __iomem *base) @@ -79,17 +106,12 @@ static int imx7ulp_wdt_ping(struct watchdog_device *wdog) static int imx7ulp_wdt_start(struct watchdog_device *wdog) { - - imx7ulp_wdt_enable(wdog, true); - - return 0; + return imx7ulp_wdt_enable(wdog, true); } static int imx7ulp_wdt_stop(struct watchdog_device *wdog) { - imx7ulp_wdt_enable(wdog, false); - - return 0; + return imx7ulp_wdt_enable(wdog, false); } static int imx7ulp_wdt_set_timeout(struct watchdog_device *wdog, @@ -97,22 +119,37 @@ static int imx7ulp_wdt_set_timeout(struct watchdog_device *wdog, { struct imx7ulp_wdt_device *wdt = watchdog_get_drvdata(wdog); u32 val = WDOG_CLOCK_RATE * timeout; + int ret; + local_irq_disable(); writel(UNLOCK, wdt->base + WDOG_CNT); + ret = imx7ulp_wdt_wait(wdt->base, WDOG_CS_ULK); + if (ret) + goto timeout_out; writel(val, wdt->base + WDOG_TOVAL); + imx7ulp_wdt_wait(wdt->base, WDOG_CS_RCS); wdog->timeout = timeout; - return 0; +timeout_out: + local_irq_enable(); + + return ret; } static int imx7ulp_wdt_restart(struct watchdog_device *wdog, unsigned long action, void *data) { struct imx7ulp_wdt_device *wdt = watchdog_get_drvdata(wdog); + int ret; + + ret = imx7ulp_wdt_enable(wdog, true); + if (ret) + return ret; - imx7ulp_wdt_enable(wdog, true); - imx7ulp_wdt_set_timeout(&wdt->wdd, 1); + ret = imx7ulp_wdt_set_timeout(&wdt->wdd, 1); + if (ret) + return ret; /* wait for wdog to fire */ while (true) @@ -136,19 +173,31 @@ static const struct watchdog_info imx7ulp_wdt_info = { WDIOF_MAGICCLOSE, }; -static void imx7ulp_wdt_init(void __iomem *base, unsigned int timeout) +static int imx7ulp_wdt_init(void __iomem *base, unsigned int timeout) { u32 val; + int ret; + local_irq_disable(); /* unlock the wdog for reconfiguration */ writel_relaxed(UNLOCK_SEQ0, base + WDOG_CNT); writel_relaxed(UNLOCK_SEQ1, base + WDOG_CNT); + ret = imx7ulp_wdt_wait(base, WDOG_CS_ULK); + if (ret) + goto init_out; /* set an initial timeout value in TOVAL */ writel(timeout, base + WDOG_TOVAL); /* enable 32bit command sequence and reconfigure */ - val = WDOG_CS_CMD32EN | WDOG_CS_CLK | WDOG_CS_UPDATE; + val = WDOG_CS_CMD32EN | WDOG_CS_CLK | WDOG_CS_UPDATE | + WDOG_CS_WAIT | WDOG_CS_STOP; writel(val, base + WDOG_CS); + imx7ulp_wdt_wait(base, WDOG_CS_RCS); + +init_out: + local_irq_enable(); + + return ret; } static void imx7ulp_wdt_action(void *data) @@ -199,7 +248,9 @@ static int imx7ulp_wdt_probe(struct platform_device *pdev) watchdog_stop_on_reboot(wdog); watchdog_stop_on_unregister(wdog); watchdog_set_drvdata(wdog, imx7ulp_wdt); - imx7ulp_wdt_init(imx7ulp_wdt->base, wdog->timeout * WDOG_CLOCK_RATE); + ret = imx7ulp_wdt_init(imx7ulp_wdt->base, wdog->timeout * WDOG_CLOCK_RATE); + if (ret) + return ret; return devm_watchdog_register_device(dev, wdog); } diff --git a/drivers/watchdog/it87_wdt.c b/drivers/watchdog/it87_wdt.c index f3bf3ea50e39..2b4831842162 100644 --- a/drivers/watchdog/it87_wdt.c +++ b/drivers/watchdog/it87_wdt.c @@ -15,7 +15,7 @@ * Support of the watchdog timers, which are available on * IT8607, IT8620, IT8622, IT8625, IT8628, IT8655, IT8665, IT8686, * IT8702, IT8712, IT8716, IT8718, IT8720, IT8721, IT8726, IT8728, - * and IT8783. + * IT8772, IT8783 and IT8784. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -66,7 +66,9 @@ #define IT8721_ID 0x8721 #define IT8726_ID 0x8726 /* the data sheet suggest wrongly 0x8716 */ #define IT8728_ID 0x8728 +#define IT8772_ID 0x8772 #define IT8783_ID 0x8783 +#define IT8784_ID 0x8784 #define IT8786_ID 0x8786 /* GPIO Configuration Registers LDN=0x07 */ @@ -294,7 +296,9 @@ static int __init it87_wdt_init(void) case IT8720_ID: case IT8721_ID: case IT8728_ID: + case IT8772_ID: case IT8783_ID: + case IT8784_ID: case IT8786_ID: max_units = 65535; break; diff --git a/drivers/watchdog/pcwd_usb.c b/drivers/watchdog/pcwd_usb.c index 41a928eb91ed..1bdaf17c1d38 100644 --- a/drivers/watchdog/pcwd_usb.c +++ b/drivers/watchdog/pcwd_usb.c @@ -656,7 +656,7 @@ static int usb_pcwd_probe(struct usb_interface *interface, /* set up the memory buffer's */ usb_pcwd->intr_buffer = usb_alloc_coherent(udev, usb_pcwd->intr_size, - GFP_ATOMIC, &usb_pcwd->intr_dma); + GFP_KERNEL, &usb_pcwd->intr_dma); if (!usb_pcwd->intr_buffer) { pr_err("Out of memory\n"); goto error; diff --git a/drivers/watchdog/rdc321x_wdt.c b/drivers/watchdog/rdc321x_wdt.c index 57187efeb86f..f0c94ea51c3e 100644 --- a/drivers/watchdog/rdc321x_wdt.c +++ b/drivers/watchdog/rdc321x_wdt.c @@ -231,6 +231,8 @@ static int rdc321x_wdt_probe(struct platform_device *pdev) rdc321x_wdt_device.sb_pdev = pdata->sb_pdev; rdc321x_wdt_device.base_reg = r->start; + rdc321x_wdt_device.queue = 0; + rdc321x_wdt_device.default_ticks = ticks; err = misc_register(&rdc321x_wdt_misc); if (err < 0) { @@ -245,14 +247,11 @@ static int rdc321x_wdt_probe(struct platform_device *pdev) rdc321x_wdt_device.base_reg, RDC_WDT_RST); init_completion(&rdc321x_wdt_device.stop); - rdc321x_wdt_device.queue = 0; clear_bit(0, &rdc321x_wdt_device.inuse); timer_setup(&rdc321x_wdt_device.timer, rdc321x_wdt_trigger, 0); - rdc321x_wdt_device.default_ticks = ticks; - dev_info(&pdev->dev, "watchdog init success\n"); return 0; diff --git a/drivers/watchdog/renesas_wdt.c b/drivers/watchdog/renesas_wdt.c index 00662a8e039c..47fce4de0110 100644 --- a/drivers/watchdog/renesas_wdt.c +++ b/drivers/watchdog/renesas_wdt.c @@ -194,6 +194,7 @@ static int rwdt_probe(struct platform_device *pdev) struct clk *clk; unsigned long clks_per_sec; int ret, i; + u8 csra; if (rwdt_blacklisted(dev)) return -ENODEV; @@ -213,8 +214,8 @@ static int rwdt_probe(struct platform_device *pdev) pm_runtime_enable(dev); pm_runtime_get_sync(dev); priv->clk_rate = clk_get_rate(clk); - priv->wdev.bootstatus = (readb_relaxed(priv->base + RWTCSRA) & - RWTCSRA_WOVF) ? WDIOF_CARDRESET : 0; + csra = readb_relaxed(priv->base + RWTCSRA); + priv->wdev.bootstatus = csra & RWTCSRA_WOVF ? WDIOF_CARDRESET : 0; pm_runtime_put(dev); if (!priv->clk_rate) { @@ -252,6 +253,13 @@ static int rwdt_probe(struct platform_device *pdev) /* This overrides the default timeout only if DT configuration was found */ watchdog_init_timeout(&priv->wdev, 0, dev); + /* Check if FW enabled the watchdog */ + if (csra & RWTCSRA_TME) { + /* Ensure properly initialized dividers */ + rwdt_start(&priv->wdev); + set_bit(WDOG_HW_RUNNING, &priv->wdev.status); + } + ret = watchdog_register_device(&priv->wdev); if (ret < 0) goto out_pm_disable; diff --git a/drivers/watchdog/rti_wdt.c b/drivers/watchdog/rti_wdt.c index 705e8f7523e8..836319cbaca9 100644 --- a/drivers/watchdog/rti_wdt.c +++ b/drivers/watchdog/rti_wdt.c @@ -205,11 +205,8 @@ static int rti_wdt_probe(struct platform_device *pdev) return -ENOMEM; clk = clk_get(dev, NULL); - if (IS_ERR(clk)) { - if (PTR_ERR(clk) != -EPROBE_DEFER) - dev_err(dev, "failed to get clock\n"); - return PTR_ERR(clk); - } + if (IS_ERR(clk)) + return dev_err_probe(dev, PTR_ERR(clk), "failed to get clock\n"); wdt->freq = clk_get_rate(clk); @@ -230,11 +227,8 @@ static int rti_wdt_probe(struct platform_device *pdev) pm_runtime_enable(dev); ret = pm_runtime_get_sync(dev); - if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "runtime pm failed\n"); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "runtime pm failed\n"); platform_set_drvdata(pdev, wdt); diff --git a/drivers/watchdog/sp5100_tco.c b/drivers/watchdog/sp5100_tco.c index 85e9664318c9..a730ecbf78cd 100644 --- a/drivers/watchdog/sp5100_tco.c +++ b/drivers/watchdog/sp5100_tco.c @@ -17,6 +17,12 @@ * AMD Publication 51192 "AMD Bolton FCH Register Reference Guide" * AMD Publication 52740 "BIOS and Kernel Developer’s Guide (BKDG) * for AMD Family 16h Models 30h-3Fh Processors" + * AMD Publication 55570-B1-PUB "Processor Programming Reference (PPR) + * for AMD Family 17h Model 18h, Revision B1 + * Processors (PUB) + * AMD Publication 55772-A1-PUB "Processor Programming Reference (PPR) + * for AMD Family 17h Model 20h, Revision A1 + * Processors (PUB) */ /* @@ -241,6 +247,18 @@ static int sp5100_tco_setupdevice(struct device *dev, break; case efch: dev_name = SB800_DEVNAME; + /* + * On Family 17h devices, the EFCH_PM_DECODEEN_WDT_TMREN bit of + * EFCH_PM_DECODEEN not only enables the EFCH_PM_WDT_ADDR memory + * region, it also enables the watchdog itself. + */ + if (boot_cpu_data.x86 == 0x17) { + val = sp5100_tco_read_pm_reg8(EFCH_PM_DECODEEN); + if (!(val & EFCH_PM_DECODEEN_WDT_TMREN)) { + sp5100_tco_update_pm_reg8(EFCH_PM_DECODEEN, 0xff, + EFCH_PM_DECODEEN_WDT_TMREN); + } + } val = sp5100_tco_read_pm_reg8(EFCH_PM_DECODEEN); if (val & EFCH_PM_DECODEEN_WDT_TMREN) mmio_addr = EFCH_PM_WDT_ADDR; diff --git a/drivers/watchdog/sp5100_tco.h b/drivers/watchdog/sp5100_tco.h index 87eaf357ae01..adf015aa4126 100644 --- a/drivers/watchdog/sp5100_tco.h +++ b/drivers/watchdog/sp5100_tco.h @@ -70,7 +70,7 @@ #define EFCH_PM_DECODEEN_WDT_TMREN BIT(7) -#define EFCH_PM_DECODEEN3 0x00 +#define EFCH_PM_DECODEEN3 0x03 #define EFCH_PM_DECODEEN_SECOND_RES GENMASK(1, 0) #define EFCH_PM_WATCHDOG_DISABLE ((u8)GENMASK(3, 2)) diff --git a/drivers/watchdog/visconti_wdt.c b/drivers/watchdog/visconti_wdt.c new file mode 100644 index 000000000000..83ef55e66ca8 --- /dev/null +++ b/drivers/watchdog/visconti_wdt.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 TOSHIBA CORPORATION + * Copyright (c) 2020 Toshiba Electronic Devices & Storage Corporation + * Copyright (c) 2020 Nobuhiro Iwamatsu <nobuhiro1.iwamatsu@toshiba.co.jp> + */ + +#include <linux/clk.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/watchdog.h> + +#define WDT_CNT 0x00 +#define WDT_MIN 0x04 +#define WDT_MAX 0x08 +#define WDT_CTL 0x0c +#define WDT_CMD 0x10 +#define WDT_CMD_CLEAR 0x4352 +#define WDT_CMD_START_STOP 0x5354 +#define WDT_DIV 0x30 + +#define VISCONTI_WDT_FREQ 2000000 /* 2MHz */ +#define WDT_DEFAULT_TIMEOUT 10U /* in seconds */ + +static bool nowayout = WATCHDOG_NOWAYOUT; +module_param(nowayout, bool, 0); +MODULE_PARM_DESC( + nowayout, + "Watchdog cannot be stopped once started (default=" __MODULE_STRING(WATCHDOG_NOWAYOUT)")"); + +struct visconti_wdt_priv { + struct watchdog_device wdev; + void __iomem *base; + u32 div; +}; + +static int visconti_wdt_start(struct watchdog_device *wdev) +{ + struct visconti_wdt_priv *priv = watchdog_get_drvdata(wdev); + u32 timeout = wdev->timeout * VISCONTI_WDT_FREQ; + + writel(priv->div, priv->base + WDT_DIV); + writel(0, priv->base + WDT_MIN); + writel(timeout, priv->base + WDT_MAX); + writel(0, priv->base + WDT_CTL); + writel(WDT_CMD_START_STOP, priv->base + WDT_CMD); + + return 0; +} + +static int visconti_wdt_stop(struct watchdog_device *wdev) +{ + struct visconti_wdt_priv *priv = watchdog_get_drvdata(wdev); + + writel(1, priv->base + WDT_CTL); + writel(WDT_CMD_START_STOP, priv->base + WDT_CMD); + + return 0; +} + +static int visconti_wdt_ping(struct watchdog_device *wdd) +{ + struct visconti_wdt_priv *priv = watchdog_get_drvdata(wdd); + + writel(WDT_CMD_CLEAR, priv->base + WDT_CMD); + + return 0; +} + +static unsigned int visconti_wdt_get_timeleft(struct watchdog_device *wdev) +{ + struct visconti_wdt_priv *priv = watchdog_get_drvdata(wdev); + u32 timeout = wdev->timeout * VISCONTI_WDT_FREQ; + u32 cnt = readl(priv->base + WDT_CNT); + + if (timeout <= cnt) + return 0; + timeout -= cnt; + + return timeout / VISCONTI_WDT_FREQ; +} + +static int visconti_wdt_set_timeout(struct watchdog_device *wdev, unsigned int timeout) +{ + u32 val; + struct visconti_wdt_priv *priv = watchdog_get_drvdata(wdev); + + wdev->timeout = timeout; + val = wdev->timeout * VISCONTI_WDT_FREQ; + + /* Clear counter before setting timeout because WDT expires */ + writel(WDT_CMD_CLEAR, priv->base + WDT_CMD); + writel(val, priv->base + WDT_MAX); + + return 0; +} + +static const struct watchdog_info visconti_wdt_info = { + .options = WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE | WDIOF_KEEPALIVEPING, + .identity = "Visconti Watchdog", +}; + +static const struct watchdog_ops visconti_wdt_ops = { + .owner = THIS_MODULE, + .start = visconti_wdt_start, + .stop = visconti_wdt_stop, + .ping = visconti_wdt_ping, + .get_timeleft = visconti_wdt_get_timeleft, + .set_timeout = visconti_wdt_set_timeout, +}; + +static void visconti_clk_disable_unprepare(void *data) +{ + clk_disable_unprepare(data); +} + +static int visconti_wdt_probe(struct platform_device *pdev) +{ + struct watchdog_device *wdev; + struct visconti_wdt_priv *priv; + struct device *dev = &pdev->dev; + struct clk *clk; + int ret; + unsigned long clk_freq; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->base)) + return PTR_ERR(priv->base); + + clk = devm_clk_get(dev, NULL); + if (IS_ERR(clk)) + return dev_err_probe(dev, PTR_ERR(clk), "Could not get clock\n"); + + ret = clk_prepare_enable(clk); + if (ret) { + dev_err(dev, "Could not enable clock\n"); + return ret; + } + + ret = devm_add_action_or_reset(dev, visconti_clk_disable_unprepare, clk); + if (ret) + return ret; + + clk_freq = clk_get_rate(clk); + if (!clk_freq) + return -EINVAL; + + priv->div = clk_freq / VISCONTI_WDT_FREQ; + + /* Initialize struct watchdog_device. */ + wdev = &priv->wdev; + wdev->info = &visconti_wdt_info; + wdev->ops = &visconti_wdt_ops; + wdev->parent = dev; + wdev->min_timeout = 1; + wdev->max_timeout = 0xffffffff / VISCONTI_WDT_FREQ; + wdev->timeout = min(wdev->max_timeout, WDT_DEFAULT_TIMEOUT); + + watchdog_set_drvdata(wdev, priv); + watchdog_set_nowayout(wdev, nowayout); + watchdog_stop_on_unregister(wdev); + + /* This overrides the default timeout only if DT configuration was found */ + ret = watchdog_init_timeout(wdev, 0, dev); + if (ret) + dev_warn(dev, "Specified timeout value invalid, using default\n"); + + return devm_watchdog_register_device(dev, wdev); +} + +static const struct of_device_id visconti_wdt_of_match[] = { + { .compatible = "toshiba,visconti-wdt", }, + {} +}; +MODULE_DEVICE_TABLE(of, visconti_wdt_of_match); + +static struct platform_driver visconti_wdt_driver = { + .driver = { + .name = "visconti_wdt", + .of_match_table = visconti_wdt_of_match, + }, + .probe = visconti_wdt_probe, +}; +module_platform_driver(visconti_wdt_driver); + +MODULE_DESCRIPTION("TOSHIBA Visconti Watchdog Driver"); +MODULE_AUTHOR("Nobuhiro Iwamatsu <nobuhiro1.iwamatsu@toshiba.co.jp"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/watchdog/watchdog_dev.c b/drivers/watchdog/watchdog_dev.c index 6798addabd5a..2946f3a63110 100644 --- a/drivers/watchdog/watchdog_dev.c +++ b/drivers/watchdog/watchdog_dev.c @@ -43,8 +43,6 @@ #include <linux/watchdog.h> /* For watchdog specific items */ #include <linux/uaccess.h> /* For copy_to_user/put_user/... */ -#include <uapi/linux/sched/types.h> /* For struct sched_param */ - #include "watchdog_core.h" #include "watchdog_pretimeout.h" @@ -994,8 +992,10 @@ static int watchdog_cdev_register(struct watchdog_device *wdd) wd_data->wdd = wdd; wdd->wd_data = wd_data; - if (IS_ERR_OR_NULL(watchdog_kworker)) + if (IS_ERR_OR_NULL(watchdog_kworker)) { + kfree(wd_data); return -ENODEV; + } device_initialize(&wd_data->dev); wd_data->dev.devt = MKDEV(MAJOR(watchdog_devt), wdd->id); @@ -1021,7 +1021,7 @@ static int watchdog_cdev_register(struct watchdog_device *wdd) pr_err("%s: a legacy watchdog module is probably present.\n", wdd->info->identity); old_wd_data = NULL; - kfree(wd_data); + put_device(&wd_data->dev); return err; } } diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index 51427c752b37..b57b2067ecbf 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -331,7 +331,7 @@ static enum bp_state reserve_additional_memory(void) mutex_unlock(&balloon_mutex); /* add_memory_resource() requires the device_hotplug lock */ lock_device_hotplug(); - rc = add_memory_resource(nid, resource); + rc = add_memory_resource(nid, resource, MEMHP_MERGE_RESOURCE); unlock_device_hotplug(); mutex_lock(&balloon_mutex); diff --git a/drivers/xen/events/events_2l.c b/drivers/xen/events/events_2l.c index 64df919a2111..fe5ad0e89cd8 100644 --- a/drivers/xen/events/events_2l.c +++ b/drivers/xen/events/events_2l.c @@ -91,6 +91,8 @@ static void evtchn_2l_unmask(evtchn_port_t port) BUG_ON(!irqs_disabled()); + smp_wmb(); /* All writes before unmask must be visible. */ + if (unlikely((cpu != cpu_from_evtchn(port)))) do_hypercall = 1; else { @@ -159,7 +161,7 @@ static inline xen_ulong_t active_evtchns(unsigned int cpu, * a bitset of words which contain pending event bits. The second * level is a bitset of pending events themselves. */ -static void evtchn_2l_handle_events(unsigned cpu) +static void evtchn_2l_handle_events(unsigned cpu, struct evtchn_loop_ctrl *ctrl) { int irq; xen_ulong_t pending_words; @@ -240,10 +242,7 @@ static void evtchn_2l_handle_events(unsigned cpu) /* Process port. */ port = (word_idx * BITS_PER_EVTCHN_WORD) + bit_idx; - irq = get_evtchn_to_irq(port); - - if (irq != -1) - generic_handle_irq(irq); + handle_irq_for_port(port, ctrl); bit_idx = (bit_idx + 1) % BITS_PER_EVTCHN_WORD; diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c index 6f02c18fa65c..cc317739e786 100644 --- a/drivers/xen/events/events_base.c +++ b/drivers/xen/events/events_base.c @@ -33,6 +33,10 @@ #include <linux/slab.h> #include <linux/irqnr.h> #include <linux/pci.h> +#include <linux/spinlock.h> +#include <linux/cpuhotplug.h> +#include <linux/atomic.h> +#include <linux/ktime.h> #ifdef CONFIG_X86 #include <asm/desc.h> @@ -63,6 +67,15 @@ #include "events_internal.h" +#undef MODULE_PARAM_PREFIX +#define MODULE_PARAM_PREFIX "xen." + +static uint __read_mostly event_loop_timeout = 2; +module_param(event_loop_timeout, uint, 0644); + +static uint __read_mostly event_eoi_delay = 10; +module_param(event_eoi_delay, uint, 0644); + const struct evtchn_ops *evtchn_ops; /* @@ -71,6 +84,24 @@ const struct evtchn_ops *evtchn_ops; */ static DEFINE_MUTEX(irq_mapping_update_lock); +/* + * Lock protecting event handling loop against removing event channels. + * Adding of event channels is no issue as the associated IRQ becomes active + * only after everything is setup (before request_[threaded_]irq() the handler + * can't be entered for an event, as the event channel will be unmasked only + * then). + */ +static DEFINE_RWLOCK(evtchn_rwlock); + +/* + * Lock hierarchy: + * + * irq_mapping_update_lock + * evtchn_rwlock + * IRQ-desc lock + * percpu eoi_list_lock + */ + static LIST_HEAD(xen_irq_list_head); /* IRQ <-> VIRQ mapping. */ @@ -95,17 +126,20 @@ static bool (*pirq_needs_eoi)(unsigned irq); static struct irq_info *legacy_info_ptrs[NR_IRQS_LEGACY]; static struct irq_chip xen_dynamic_chip; +static struct irq_chip xen_lateeoi_chip; static struct irq_chip xen_percpu_chip; static struct irq_chip xen_pirq_chip; static void enable_dynirq(struct irq_data *data); static void disable_dynirq(struct irq_data *data); +static DEFINE_PER_CPU(unsigned int, irq_epoch); + static void clear_evtchn_to_irq_row(unsigned row) { unsigned col; for (col = 0; col < EVTCHN_PER_ROW; col++) - evtchn_to_irq[row][col] = -1; + WRITE_ONCE(evtchn_to_irq[row][col], -1); } static void clear_evtchn_to_irq_all(void) @@ -142,7 +176,7 @@ static int set_evtchn_to_irq(evtchn_port_t evtchn, unsigned int irq) clear_evtchn_to_irq_row(row); } - evtchn_to_irq[row][col] = irq; + WRITE_ONCE(evtchn_to_irq[row][col], irq); return 0; } @@ -152,7 +186,7 @@ int get_evtchn_to_irq(evtchn_port_t evtchn) return -1; if (evtchn_to_irq[EVTCHN_ROW(evtchn)] == NULL) return -1; - return evtchn_to_irq[EVTCHN_ROW(evtchn)][EVTCHN_COL(evtchn)]; + return READ_ONCE(evtchn_to_irq[EVTCHN_ROW(evtchn)][EVTCHN_COL(evtchn)]); } /* Get info for IRQ */ @@ -261,10 +295,14 @@ static void xen_irq_info_cleanup(struct irq_info *info) */ evtchn_port_t evtchn_from_irq(unsigned irq) { - if (WARN(irq >= nr_irqs, "Invalid irq %d!\n", irq)) + const struct irq_info *info = NULL; + + if (likely(irq < nr_irqs)) + info = info_for_irq(irq); + if (!info) return 0; - return info_for_irq(irq)->evtchn; + return info->evtchn; } unsigned int irq_from_evtchn(evtchn_port_t evtchn) @@ -375,9 +413,157 @@ void notify_remote_via_irq(int irq) } EXPORT_SYMBOL_GPL(notify_remote_via_irq); +struct lateeoi_work { + struct delayed_work delayed; + spinlock_t eoi_list_lock; + struct list_head eoi_list; +}; + +static DEFINE_PER_CPU(struct lateeoi_work, lateeoi); + +static void lateeoi_list_del(struct irq_info *info) +{ + struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu); + unsigned long flags; + + spin_lock_irqsave(&eoi->eoi_list_lock, flags); + list_del_init(&info->eoi_list); + spin_unlock_irqrestore(&eoi->eoi_list_lock, flags); +} + +static void lateeoi_list_add(struct irq_info *info) +{ + struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu); + struct irq_info *elem; + u64 now = get_jiffies_64(); + unsigned long delay; + unsigned long flags; + + if (now < info->eoi_time) + delay = info->eoi_time - now; + else + delay = 1; + + spin_lock_irqsave(&eoi->eoi_list_lock, flags); + + if (list_empty(&eoi->eoi_list)) { + list_add(&info->eoi_list, &eoi->eoi_list); + mod_delayed_work_on(info->eoi_cpu, system_wq, + &eoi->delayed, delay); + } else { + list_for_each_entry_reverse(elem, &eoi->eoi_list, eoi_list) { + if (elem->eoi_time <= info->eoi_time) + break; + } + list_add(&info->eoi_list, &elem->eoi_list); + } + + spin_unlock_irqrestore(&eoi->eoi_list_lock, flags); +} + +static void xen_irq_lateeoi_locked(struct irq_info *info, bool spurious) +{ + evtchn_port_t evtchn; + unsigned int cpu; + unsigned int delay = 0; + + evtchn = info->evtchn; + if (!VALID_EVTCHN(evtchn) || !list_empty(&info->eoi_list)) + return; + + if (spurious) { + if ((1 << info->spurious_cnt) < (HZ << 2)) + info->spurious_cnt++; + if (info->spurious_cnt > 1) { + delay = 1 << (info->spurious_cnt - 2); + if (delay > HZ) + delay = HZ; + if (!info->eoi_time) + info->eoi_cpu = smp_processor_id(); + info->eoi_time = get_jiffies_64() + delay; + } + } else { + info->spurious_cnt = 0; + } + + cpu = info->eoi_cpu; + if (info->eoi_time && + (info->irq_epoch == per_cpu(irq_epoch, cpu) || delay)) { + lateeoi_list_add(info); + return; + } + + info->eoi_time = 0; + unmask_evtchn(evtchn); +} + +static void xen_irq_lateeoi_worker(struct work_struct *work) +{ + struct lateeoi_work *eoi; + struct irq_info *info; + u64 now = get_jiffies_64(); + unsigned long flags; + + eoi = container_of(to_delayed_work(work), struct lateeoi_work, delayed); + + read_lock_irqsave(&evtchn_rwlock, flags); + + while (true) { + spin_lock(&eoi->eoi_list_lock); + + info = list_first_entry_or_null(&eoi->eoi_list, struct irq_info, + eoi_list); + + if (info == NULL || now < info->eoi_time) { + spin_unlock(&eoi->eoi_list_lock); + break; + } + + list_del_init(&info->eoi_list); + + spin_unlock(&eoi->eoi_list_lock); + + info->eoi_time = 0; + + xen_irq_lateeoi_locked(info, false); + } + + if (info) + mod_delayed_work_on(info->eoi_cpu, system_wq, + &eoi->delayed, info->eoi_time - now); + + read_unlock_irqrestore(&evtchn_rwlock, flags); +} + +static void xen_cpu_init_eoi(unsigned int cpu) +{ + struct lateeoi_work *eoi = &per_cpu(lateeoi, cpu); + + INIT_DELAYED_WORK(&eoi->delayed, xen_irq_lateeoi_worker); + spin_lock_init(&eoi->eoi_list_lock); + INIT_LIST_HEAD(&eoi->eoi_list); +} + +void xen_irq_lateeoi(unsigned int irq, unsigned int eoi_flags) +{ + struct irq_info *info; + unsigned long flags; + + read_lock_irqsave(&evtchn_rwlock, flags); + + info = info_for_irq(irq); + + if (info) + xen_irq_lateeoi_locked(info, eoi_flags & XEN_EOI_FLAG_SPURIOUS); + + read_unlock_irqrestore(&evtchn_rwlock, flags); +} +EXPORT_SYMBOL_GPL(xen_irq_lateeoi); + static void xen_irq_init(unsigned irq) { struct irq_info *info; + #ifdef CONFIG_SMP /* By default all event channels notify CPU#0. */ cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(0)); @@ -392,6 +578,7 @@ static void xen_irq_init(unsigned irq) set_info_for_irq(irq, info); + INIT_LIST_HEAD(&info->eoi_list); list_add_tail(&info->list, &xen_irq_list_head); } @@ -440,16 +627,24 @@ static int __must_check xen_allocate_irq_gsi(unsigned gsi) static void xen_free_irq(unsigned irq) { struct irq_info *info = info_for_irq(irq); + unsigned long flags; if (WARN_ON(!info)) return; + write_lock_irqsave(&evtchn_rwlock, flags); + + if (!list_empty(&info->eoi_list)) + lateeoi_list_del(info); + list_del(&info->list); set_info_for_irq(irq, NULL); WARN_ON(info->refcnt > 0); + write_unlock_irqrestore(&evtchn_rwlock, flags); + kfree(info); /* Legacy IRQ descriptors are managed by the arch. */ @@ -841,7 +1036,7 @@ int xen_pirq_from_irq(unsigned irq) } EXPORT_SYMBOL_GPL(xen_pirq_from_irq); -int bind_evtchn_to_irq(evtchn_port_t evtchn) +static int bind_evtchn_to_irq_chip(evtchn_port_t evtchn, struct irq_chip *chip) { int irq; int ret; @@ -858,7 +1053,7 @@ int bind_evtchn_to_irq(evtchn_port_t evtchn) if (irq < 0) goto out; - irq_set_chip_and_handler_name(irq, &xen_dynamic_chip, + irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "event"); ret = xen_irq_info_evtchn_setup(irq, evtchn); @@ -879,8 +1074,19 @@ out: return irq; } + +int bind_evtchn_to_irq(evtchn_port_t evtchn) +{ + return bind_evtchn_to_irq_chip(evtchn, &xen_dynamic_chip); +} EXPORT_SYMBOL_GPL(bind_evtchn_to_irq); +int bind_evtchn_to_irq_lateeoi(evtchn_port_t evtchn) +{ + return bind_evtchn_to_irq_chip(evtchn, &xen_lateeoi_chip); +} +EXPORT_SYMBOL_GPL(bind_evtchn_to_irq_lateeoi); + static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu) { struct evtchn_bind_ipi bind_ipi; @@ -922,8 +1128,9 @@ static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu) return irq; } -int bind_interdomain_evtchn_to_irq(unsigned int remote_domain, - evtchn_port_t remote_port) +static int bind_interdomain_evtchn_to_irq_chip(unsigned int remote_domain, + evtchn_port_t remote_port, + struct irq_chip *chip) { struct evtchn_bind_interdomain bind_interdomain; int err; @@ -934,10 +1141,26 @@ int bind_interdomain_evtchn_to_irq(unsigned int remote_domain, err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, &bind_interdomain); - return err ? : bind_evtchn_to_irq(bind_interdomain.local_port); + return err ? : bind_evtchn_to_irq_chip(bind_interdomain.local_port, + chip); +} + +int bind_interdomain_evtchn_to_irq(unsigned int remote_domain, + evtchn_port_t remote_port) +{ + return bind_interdomain_evtchn_to_irq_chip(remote_domain, remote_port, + &xen_dynamic_chip); } EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq); +int bind_interdomain_evtchn_to_irq_lateeoi(unsigned int remote_domain, + evtchn_port_t remote_port) +{ + return bind_interdomain_evtchn_to_irq_chip(remote_domain, remote_port, + &xen_lateeoi_chip); +} +EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq_lateeoi); + static int find_virq(unsigned int virq, unsigned int cpu, evtchn_port_t *evtchn) { struct evtchn_status status; @@ -1034,14 +1257,15 @@ static void unbind_from_irq(unsigned int irq) mutex_unlock(&irq_mapping_update_lock); } -int bind_evtchn_to_irqhandler(evtchn_port_t evtchn, - irq_handler_t handler, - unsigned long irqflags, - const char *devname, void *dev_id) +static int bind_evtchn_to_irqhandler_chip(evtchn_port_t evtchn, + irq_handler_t handler, + unsigned long irqflags, + const char *devname, void *dev_id, + struct irq_chip *chip) { int irq, retval; - irq = bind_evtchn_to_irq(evtchn); + irq = bind_evtchn_to_irq_chip(evtchn, chip); if (irq < 0) return irq; retval = request_irq(irq, handler, irqflags, devname, dev_id); @@ -1052,18 +1276,38 @@ int bind_evtchn_to_irqhandler(evtchn_port_t evtchn, return irq; } + +int bind_evtchn_to_irqhandler(evtchn_port_t evtchn, + irq_handler_t handler, + unsigned long irqflags, + const char *devname, void *dev_id) +{ + return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags, + devname, dev_id, + &xen_dynamic_chip); +} EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler); -int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain, - evtchn_port_t remote_port, - irq_handler_t handler, - unsigned long irqflags, - const char *devname, - void *dev_id) +int bind_evtchn_to_irqhandler_lateeoi(evtchn_port_t evtchn, + irq_handler_t handler, + unsigned long irqflags, + const char *devname, void *dev_id) +{ + return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags, + devname, dev_id, + &xen_lateeoi_chip); +} +EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler_lateeoi); + +static int bind_interdomain_evtchn_to_irqhandler_chip( + unsigned int remote_domain, evtchn_port_t remote_port, + irq_handler_t handler, unsigned long irqflags, + const char *devname, void *dev_id, struct irq_chip *chip) { int irq, retval; - irq = bind_interdomain_evtchn_to_irq(remote_domain, remote_port); + irq = bind_interdomain_evtchn_to_irq_chip(remote_domain, remote_port, + chip); if (irq < 0) return irq; @@ -1075,8 +1319,33 @@ int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain, return irq; } + +int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain, + evtchn_port_t remote_port, + irq_handler_t handler, + unsigned long irqflags, + const char *devname, + void *dev_id) +{ + return bind_interdomain_evtchn_to_irqhandler_chip(remote_domain, + remote_port, handler, irqflags, devname, + dev_id, &xen_dynamic_chip); +} EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler); +int bind_interdomain_evtchn_to_irqhandler_lateeoi(unsigned int remote_domain, + evtchn_port_t remote_port, + irq_handler_t handler, + unsigned long irqflags, + const char *devname, + void *dev_id) +{ + return bind_interdomain_evtchn_to_irqhandler_chip(remote_domain, + remote_port, handler, irqflags, devname, + dev_id, &xen_lateeoi_chip); +} +EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler_lateeoi); + int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu, irq_handler_t handler, unsigned long irqflags, const char *devname, void *dev_id) @@ -1189,7 +1458,7 @@ int evtchn_get(evtchn_port_t evtchn) goto done; err = -EINVAL; - if (info->refcnt <= 0) + if (info->refcnt <= 0 || info->refcnt == SHRT_MAX) goto done; info->refcnt++; @@ -1228,21 +1497,81 @@ void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector) notify_remote_via_irq(irq); } +struct evtchn_loop_ctrl { + ktime_t timeout; + unsigned count; + bool defer_eoi; +}; + +void handle_irq_for_port(evtchn_port_t port, struct evtchn_loop_ctrl *ctrl) +{ + int irq; + struct irq_info *info; + + irq = get_evtchn_to_irq(port); + if (irq == -1) + return; + + /* + * Check for timeout every 256 events. + * We are setting the timeout value only after the first 256 + * events in order to not hurt the common case of few loop + * iterations. The 256 is basically an arbitrary value. + * + * In case we are hitting the timeout we need to defer all further + * EOIs in order to ensure to leave the event handling loop rather + * sooner than later. + */ + if (!ctrl->defer_eoi && !(++ctrl->count & 0xff)) { + ktime_t kt = ktime_get(); + + if (!ctrl->timeout) { + kt = ktime_add_ms(kt, + jiffies_to_msecs(event_loop_timeout)); + ctrl->timeout = kt; + } else if (kt > ctrl->timeout) { + ctrl->defer_eoi = true; + } + } + + info = info_for_irq(irq); + + if (ctrl->defer_eoi) { + info->eoi_cpu = smp_processor_id(); + info->irq_epoch = __this_cpu_read(irq_epoch); + info->eoi_time = get_jiffies_64() + event_eoi_delay; + } + + generic_handle_irq(irq); +} + static void __xen_evtchn_do_upcall(void) { struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu); int cpu = smp_processor_id(); + struct evtchn_loop_ctrl ctrl = { 0 }; + + read_lock(&evtchn_rwlock); do { vcpu_info->evtchn_upcall_pending = 0; - xen_evtchn_handle_events(cpu); + xen_evtchn_handle_events(cpu, &ctrl); BUG_ON(!irqs_disabled()); virt_rmb(); /* Hypervisor can set upcall pending. */ } while (vcpu_info->evtchn_upcall_pending); + + read_unlock(&evtchn_rwlock); + + /* + * Increment irq_epoch only now to defer EOIs only for + * xen_irq_lateeoi() invocations occurring from inside the loop + * above. + */ + __this_cpu_inc(irq_epoch); } void xen_evtchn_do_upcall(struct pt_regs *regs) @@ -1606,6 +1935,21 @@ static struct irq_chip xen_dynamic_chip __read_mostly = { .irq_retrigger = retrigger_dynirq, }; +static struct irq_chip xen_lateeoi_chip __read_mostly = { + /* The chip name needs to contain "xen-dyn" for irqbalance to work. */ + .name = "xen-dyn-lateeoi", + + .irq_disable = disable_dynirq, + .irq_mask = disable_dynirq, + .irq_unmask = enable_dynirq, + + .irq_ack = mask_ack_dynirq, + .irq_mask_ack = mask_ack_dynirq, + + .irq_set_affinity = set_affinity_irq, + .irq_retrigger = retrigger_dynirq, +}; + static struct irq_chip xen_pirq_chip __read_mostly = { .name = "xen-pirq", @@ -1676,12 +2020,31 @@ void xen_setup_callback_vector(void) {} static inline void xen_alloc_callback_vector(void) {} #endif -#undef MODULE_PARAM_PREFIX -#define MODULE_PARAM_PREFIX "xen." - static bool fifo_events = true; module_param(fifo_events, bool, 0); +static int xen_evtchn_cpu_prepare(unsigned int cpu) +{ + int ret = 0; + + xen_cpu_init_eoi(cpu); + + if (evtchn_ops->percpu_init) + ret = evtchn_ops->percpu_init(cpu); + + return ret; +} + +static int xen_evtchn_cpu_dead(unsigned int cpu) +{ + int ret = 0; + + if (evtchn_ops->percpu_deinit) + ret = evtchn_ops->percpu_deinit(cpu); + + return ret; +} + void __init xen_init_IRQ(void) { int ret = -EINVAL; @@ -1692,6 +2055,12 @@ void __init xen_init_IRQ(void) if (ret < 0) xen_evtchn_2l_init(); + xen_cpu_init_eoi(smp_processor_id()); + + cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE, + "xen/evtchn:prepare", + xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead); + evtchn_to_irq = kcalloc(EVTCHN_ROW(xen_evtchn_max_channels()), sizeof(*evtchn_to_irq), GFP_KERNEL); BUG_ON(!evtchn_to_irq); diff --git a/drivers/xen/events/events_fifo.c b/drivers/xen/events/events_fifo.c index c60ee0450173..6085a808da95 100644 --- a/drivers/xen/events/events_fifo.c +++ b/drivers/xen/events/events_fifo.c @@ -227,19 +227,25 @@ static bool evtchn_fifo_is_masked(evtchn_port_t port) return sync_test_bit(EVTCHN_FIFO_BIT(MASKED, word), BM(word)); } /* - * Clear MASKED, spinning if BUSY is set. + * Clear MASKED if not PENDING, spinning if BUSY is set. + * Return true if mask was cleared. */ -static void clear_masked(volatile event_word_t *word) +static bool clear_masked_cond(volatile event_word_t *word) { event_word_t new, old, w; w = *word; do { + if (w & (1 << EVTCHN_FIFO_PENDING)) + return false; + old = w & ~(1 << EVTCHN_FIFO_BUSY); new = old & ~(1 << EVTCHN_FIFO_MASKED); w = sync_cmpxchg(word, old, new); } while (w != old); + + return true; } static void evtchn_fifo_unmask(evtchn_port_t port) @@ -248,8 +254,7 @@ static void evtchn_fifo_unmask(evtchn_port_t port) BUG_ON(!irqs_disabled()); - clear_masked(word); - if (evtchn_fifo_is_pending(port)) { + if (!clear_masked_cond(word)) { struct evtchn_unmask unmask = { .port = port }; (void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask); } @@ -270,19 +275,9 @@ static uint32_t clear_linked(volatile event_word_t *word) return w & EVTCHN_FIFO_LINK_MASK; } -static void handle_irq_for_port(evtchn_port_t port) -{ - int irq; - - irq = get_evtchn_to_irq(port); - if (irq != -1) - generic_handle_irq(irq); -} - -static void consume_one_event(unsigned cpu, +static void consume_one_event(unsigned cpu, struct evtchn_loop_ctrl *ctrl, struct evtchn_fifo_control_block *control_block, - unsigned priority, unsigned long *ready, - bool drop) + unsigned priority, unsigned long *ready) { struct evtchn_fifo_queue *q = &per_cpu(cpu_queue, cpu); uint32_t head; @@ -315,16 +310,17 @@ static void consume_one_event(unsigned cpu, clear_bit(priority, ready); if (evtchn_fifo_is_pending(port) && !evtchn_fifo_is_masked(port)) { - if (unlikely(drop)) + if (unlikely(!ctrl)) pr_warn("Dropping pending event for port %u\n", port); else - handle_irq_for_port(port); + handle_irq_for_port(port, ctrl); } q->head[priority] = head; } -static void __evtchn_fifo_handle_events(unsigned cpu, bool drop) +static void __evtchn_fifo_handle_events(unsigned cpu, + struct evtchn_loop_ctrl *ctrl) { struct evtchn_fifo_control_block *control_block; unsigned long ready; @@ -336,14 +332,15 @@ static void __evtchn_fifo_handle_events(unsigned cpu, bool drop) while (ready) { q = find_first_bit(&ready, EVTCHN_FIFO_MAX_QUEUES); - consume_one_event(cpu, control_block, q, &ready, drop); + consume_one_event(cpu, ctrl, control_block, q, &ready); ready |= xchg(&control_block->ready, 0); } } -static void evtchn_fifo_handle_events(unsigned cpu) +static void evtchn_fifo_handle_events(unsigned cpu, + struct evtchn_loop_ctrl *ctrl) { - __evtchn_fifo_handle_events(cpu, false); + __evtchn_fifo_handle_events(cpu, ctrl); } static void evtchn_fifo_resume(void) @@ -380,21 +377,6 @@ static void evtchn_fifo_resume(void) event_array_pages = 0; } -static const struct evtchn_ops evtchn_ops_fifo = { - .max_channels = evtchn_fifo_max_channels, - .nr_channels = evtchn_fifo_nr_channels, - .setup = evtchn_fifo_setup, - .bind_to_cpu = evtchn_fifo_bind_to_cpu, - .clear_pending = evtchn_fifo_clear_pending, - .set_pending = evtchn_fifo_set_pending, - .is_pending = evtchn_fifo_is_pending, - .test_and_set_mask = evtchn_fifo_test_and_set_mask, - .mask = evtchn_fifo_mask, - .unmask = evtchn_fifo_unmask, - .handle_events = evtchn_fifo_handle_events, - .resume = evtchn_fifo_resume, -}; - static int evtchn_fifo_alloc_control_block(unsigned cpu) { void *control_block = NULL; @@ -417,19 +399,36 @@ static int evtchn_fifo_alloc_control_block(unsigned cpu) return ret; } -static int xen_evtchn_cpu_prepare(unsigned int cpu) +static int evtchn_fifo_percpu_init(unsigned int cpu) { if (!per_cpu(cpu_control_block, cpu)) return evtchn_fifo_alloc_control_block(cpu); return 0; } -static int xen_evtchn_cpu_dead(unsigned int cpu) +static int evtchn_fifo_percpu_deinit(unsigned int cpu) { - __evtchn_fifo_handle_events(cpu, true); + __evtchn_fifo_handle_events(cpu, NULL); return 0; } +static const struct evtchn_ops evtchn_ops_fifo = { + .max_channels = evtchn_fifo_max_channels, + .nr_channels = evtchn_fifo_nr_channels, + .setup = evtchn_fifo_setup, + .bind_to_cpu = evtchn_fifo_bind_to_cpu, + .clear_pending = evtchn_fifo_clear_pending, + .set_pending = evtchn_fifo_set_pending, + .is_pending = evtchn_fifo_is_pending, + .test_and_set_mask = evtchn_fifo_test_and_set_mask, + .mask = evtchn_fifo_mask, + .unmask = evtchn_fifo_unmask, + .handle_events = evtchn_fifo_handle_events, + .resume = evtchn_fifo_resume, + .percpu_init = evtchn_fifo_percpu_init, + .percpu_deinit = evtchn_fifo_percpu_deinit, +}; + int __init xen_evtchn_fifo_init(void) { int cpu = smp_processor_id(); @@ -443,9 +442,5 @@ int __init xen_evtchn_fifo_init(void) evtchn_ops = &evtchn_ops_fifo; - cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE, - "xen/evtchn:prepare", - xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead); - return ret; } diff --git a/drivers/xen/events/events_internal.h b/drivers/xen/events/events_internal.h index 10684feb094e..82937d90d7d7 100644 --- a/drivers/xen/events/events_internal.h +++ b/drivers/xen/events/events_internal.h @@ -30,11 +30,16 @@ enum xen_irq_type { */ struct irq_info { struct list_head list; - int refcnt; + struct list_head eoi_list; + short refcnt; + short spurious_cnt; enum xen_irq_type type; /* type */ unsigned irq; evtchn_port_t evtchn; /* event channel */ unsigned short cpu; /* cpu bound */ + unsigned short eoi_cpu; /* EOI must happen on this cpu */ + unsigned int irq_epoch; /* If eoi_cpu valid: irq_epoch of event */ + u64 eoi_time; /* Time in jiffies when to EOI. */ union { unsigned short virq; @@ -53,6 +58,8 @@ struct irq_info { #define PIRQ_SHAREABLE (1 << 1) #define PIRQ_MSI_GROUP (1 << 2) +struct evtchn_loop_ctrl; + struct evtchn_ops { unsigned (*max_channels)(void); unsigned (*nr_channels)(void); @@ -67,14 +74,18 @@ struct evtchn_ops { void (*mask)(evtchn_port_t port); void (*unmask)(evtchn_port_t port); - void (*handle_events)(unsigned cpu); + void (*handle_events)(unsigned cpu, struct evtchn_loop_ctrl *ctrl); void (*resume)(void); + + int (*percpu_init)(unsigned int cpu); + int (*percpu_deinit)(unsigned int cpu); }; extern const struct evtchn_ops *evtchn_ops; extern int **evtchn_to_irq; int get_evtchn_to_irq(evtchn_port_t evtchn); +void handle_irq_for_port(evtchn_port_t port, struct evtchn_loop_ctrl *ctrl); struct irq_info *info_for_irq(unsigned irq); unsigned cpu_from_irq(unsigned irq); @@ -132,9 +143,10 @@ static inline void unmask_evtchn(evtchn_port_t port) return evtchn_ops->unmask(port); } -static inline void xen_evtchn_handle_events(unsigned cpu) +static inline void xen_evtchn_handle_events(unsigned cpu, + struct evtchn_loop_ctrl *ctrl) { - return evtchn_ops->handle_events(cpu); + return evtchn_ops->handle_events(cpu, ctrl); } static inline void xen_evtchn_resume(void) diff --git a/drivers/xen/evtchn.c b/drivers/xen/evtchn.c index 6e0b1dd5573c..5dc016d68f83 100644 --- a/drivers/xen/evtchn.c +++ b/drivers/xen/evtchn.c @@ -167,7 +167,6 @@ static irqreturn_t evtchn_interrupt(int irq, void *data) "Interrupt for port %u, but apparently not enabled; per-user %p\n", evtchn->port, u); - disable_irq_nosync(irq); evtchn->enabled = false; spin_lock(&u->ring_prod_lock); @@ -293,7 +292,7 @@ static ssize_t evtchn_write(struct file *file, const char __user *buf, evtchn = find_evtchn(u, port); if (evtchn && !evtchn->enabled) { evtchn->enabled = true; - enable_irq(irq_from_evtchn(port)); + xen_irq_lateeoi(irq_from_evtchn(port), 0); } } @@ -393,8 +392,8 @@ static int evtchn_bind_to_user(struct per_user_data *u, evtchn_port_t port) if (rc < 0) goto err; - rc = bind_evtchn_to_irqhandler(port, evtchn_interrupt, 0, - u->name, evtchn); + rc = bind_evtchn_to_irqhandler_lateeoi(port, evtchn_interrupt, 0, + u->name, evtchn); if (rc < 0) goto err; diff --git a/drivers/xen/pvcalls-back.c b/drivers/xen/pvcalls-back.c index 9eae1fceec1e..a7d293fa8d14 100644 --- a/drivers/xen/pvcalls-back.c +++ b/drivers/xen/pvcalls-back.c @@ -66,6 +66,7 @@ struct sock_mapping { atomic_t write; atomic_t io; atomic_t release; + atomic_t eoi; void (*saved_data_ready)(struct sock *sk); struct pvcalls_ioworker ioworker; }; @@ -87,7 +88,7 @@ static int pvcalls_back_release_active(struct xenbus_device *dev, struct pvcalls_fedata *fedata, struct sock_mapping *map); -static void pvcalls_conn_back_read(void *opaque) +static bool pvcalls_conn_back_read(void *opaque) { struct sock_mapping *map = (struct sock_mapping *)opaque; struct msghdr msg; @@ -107,17 +108,17 @@ static void pvcalls_conn_back_read(void *opaque) virt_mb(); if (error) - return; + return false; size = pvcalls_queued(prod, cons, array_size); if (size >= array_size) - return; + return false; spin_lock_irqsave(&map->sock->sk->sk_receive_queue.lock, flags); if (skb_queue_empty(&map->sock->sk->sk_receive_queue)) { atomic_set(&map->read, 0); spin_unlock_irqrestore(&map->sock->sk->sk_receive_queue.lock, flags); - return; + return true; } spin_unlock_irqrestore(&map->sock->sk->sk_receive_queue.lock, flags); wanted = array_size - size; @@ -141,7 +142,7 @@ static void pvcalls_conn_back_read(void *opaque) ret = inet_recvmsg(map->sock, &msg, wanted, MSG_DONTWAIT); WARN_ON(ret > wanted); if (ret == -EAGAIN) /* shouldn't happen */ - return; + return true; if (!ret) ret = -ENOTCONN; spin_lock_irqsave(&map->sock->sk->sk_receive_queue.lock, flags); @@ -160,10 +161,10 @@ static void pvcalls_conn_back_read(void *opaque) virt_wmb(); notify_remote_via_irq(map->irq); - return; + return true; } -static void pvcalls_conn_back_write(struct sock_mapping *map) +static bool pvcalls_conn_back_write(struct sock_mapping *map) { struct pvcalls_data_intf *intf = map->ring; struct pvcalls_data *data = &map->data; @@ -180,7 +181,7 @@ static void pvcalls_conn_back_write(struct sock_mapping *map) array_size = XEN_FLEX_RING_SIZE(map->ring_order); size = pvcalls_queued(prod, cons, array_size); if (size == 0) - return; + return false; memset(&msg, 0, sizeof(msg)); msg.msg_flags |= MSG_DONTWAIT; @@ -198,12 +199,11 @@ static void pvcalls_conn_back_write(struct sock_mapping *map) atomic_set(&map->write, 0); ret = inet_sendmsg(map->sock, &msg, size); - if (ret == -EAGAIN || (ret >= 0 && ret < size)) { + if (ret == -EAGAIN) { atomic_inc(&map->write); atomic_inc(&map->io); + return true; } - if (ret == -EAGAIN) - return; /* write the data, then update the indexes */ virt_wmb(); @@ -216,9 +216,13 @@ static void pvcalls_conn_back_write(struct sock_mapping *map) } /* update the indexes, then notify the other end */ virt_wmb(); - if (prod != cons + ret) + if (prod != cons + ret) { atomic_inc(&map->write); + atomic_inc(&map->io); + } notify_remote_via_irq(map->irq); + + return true; } static void pvcalls_back_ioworker(struct work_struct *work) @@ -227,6 +231,7 @@ static void pvcalls_back_ioworker(struct work_struct *work) struct pvcalls_ioworker, register_work); struct sock_mapping *map = container_of(ioworker, struct sock_mapping, ioworker); + unsigned int eoi_flags = XEN_EOI_FLAG_SPURIOUS; while (atomic_read(&map->io) > 0) { if (atomic_read(&map->release) > 0) { @@ -234,10 +239,18 @@ static void pvcalls_back_ioworker(struct work_struct *work) return; } - if (atomic_read(&map->read) > 0) - pvcalls_conn_back_read(map); - if (atomic_read(&map->write) > 0) - pvcalls_conn_back_write(map); + if (atomic_read(&map->read) > 0 && + pvcalls_conn_back_read(map)) + eoi_flags = 0; + if (atomic_read(&map->write) > 0 && + pvcalls_conn_back_write(map)) + eoi_flags = 0; + + if (atomic_read(&map->eoi) > 0 && !atomic_read(&map->write)) { + atomic_set(&map->eoi, 0); + xen_irq_lateeoi(map->irq, eoi_flags); + eoi_flags = XEN_EOI_FLAG_SPURIOUS; + } atomic_dec(&map->io); } @@ -334,12 +347,9 @@ static struct sock_mapping *pvcalls_new_active_socket( goto out; map->bytes = page; - ret = bind_interdomain_evtchn_to_irqhandler(fedata->dev->otherend_id, - evtchn, - pvcalls_back_conn_event, - 0, - "pvcalls-backend", - map); + ret = bind_interdomain_evtchn_to_irqhandler_lateeoi( + fedata->dev->otherend_id, evtchn, + pvcalls_back_conn_event, 0, "pvcalls-backend", map); if (ret < 0) goto out; map->irq = ret; @@ -873,15 +883,18 @@ static irqreturn_t pvcalls_back_event(int irq, void *dev_id) { struct xenbus_device *dev = dev_id; struct pvcalls_fedata *fedata = NULL; + unsigned int eoi_flags = XEN_EOI_FLAG_SPURIOUS; - if (dev == NULL) - return IRQ_HANDLED; + if (dev) { + fedata = dev_get_drvdata(&dev->dev); + if (fedata) { + pvcalls_back_work(fedata); + eoi_flags = 0; + } + } - fedata = dev_get_drvdata(&dev->dev); - if (fedata == NULL) - return IRQ_HANDLED; + xen_irq_lateeoi(irq, eoi_flags); - pvcalls_back_work(fedata); return IRQ_HANDLED; } @@ -891,12 +904,15 @@ static irqreturn_t pvcalls_back_conn_event(int irq, void *sock_map) struct pvcalls_ioworker *iow; if (map == NULL || map->sock == NULL || map->sock->sk == NULL || - map->sock->sk->sk_user_data != map) + map->sock->sk->sk_user_data != map) { + xen_irq_lateeoi(irq, 0); return IRQ_HANDLED; + } iow = &map->ioworker; atomic_inc(&map->write); + atomic_inc(&map->eoi); atomic_inc(&map->io); queue_work(iow->wq, &iow->register_work); @@ -932,7 +948,7 @@ static int backend_connect(struct xenbus_device *dev) goto error; } - err = bind_interdomain_evtchn_to_irq(dev->otherend_id, evtchn); + err = bind_interdomain_evtchn_to_irq_lateeoi(dev->otherend_id, evtchn); if (err < 0) goto error; fedata->irq = err; diff --git a/drivers/xen/xen-pciback/pci_stub.c b/drivers/xen/xen-pciback/pci_stub.c index e876c3d6dad1..cb904ac83006 100644 --- a/drivers/xen/xen-pciback/pci_stub.c +++ b/drivers/xen/xen-pciback/pci_stub.c @@ -734,10 +734,17 @@ static pci_ers_result_t common_process(struct pcistub_device *psdev, wmb(); notify_remote_via_irq(pdev->evtchn_irq); + /* Enable IRQ to signal "request done". */ + xen_pcibk_lateeoi(pdev, 0); + ret = wait_event_timeout(xen_pcibk_aer_wait_queue, !(test_bit(_XEN_PCIB_active, (unsigned long *) &sh_info->flags)), 300*HZ); + /* Enable IRQ for pcifront request if not already active. */ + if (!test_bit(_PDEVF_op_active, &pdev->flags)) + xen_pcibk_lateeoi(pdev, 0); + if (!ret) { if (test_bit(_XEN_PCIB_active, (unsigned long *)&sh_info->flags)) { @@ -751,12 +758,6 @@ static pci_ers_result_t common_process(struct pcistub_device *psdev, } clear_bit(_PCIB_op_pending, (unsigned long *)&pdev->flags); - if (test_bit(_XEN_PCIF_active, - (unsigned long *)&sh_info->flags)) { - dev_dbg(&psdev->dev->dev, "schedule pci_conf service\n"); - xen_pcibk_test_and_schedule_op(psdev->pdev); - } - res = (pci_ers_result_t)aer_op->err; return res; } diff --git a/drivers/xen/xen-pciback/pciback.h b/drivers/xen/xen-pciback/pciback.h index f1ed2dbf685c..95e28ee48d52 100644 --- a/drivers/xen/xen-pciback/pciback.h +++ b/drivers/xen/xen-pciback/pciback.h @@ -14,6 +14,7 @@ #include <linux/spinlock.h> #include <linux/workqueue.h> #include <linux/atomic.h> +#include <xen/events.h> #include <xen/interface/io/pciif.h> #define DRV_NAME "xen-pciback" @@ -27,6 +28,8 @@ struct pci_dev_entry { #define PDEVF_op_active (1<<(_PDEVF_op_active)) #define _PCIB_op_pending (1) #define PCIB_op_pending (1<<(_PCIB_op_pending)) +#define _EOI_pending (2) +#define EOI_pending (1<<(_EOI_pending)) struct xen_pcibk_device { void *pci_dev_data; @@ -183,10 +186,15 @@ static inline void xen_pcibk_release_devices(struct xen_pcibk_device *pdev) irqreturn_t xen_pcibk_handle_event(int irq, void *dev_id); void xen_pcibk_do_op(struct work_struct *data); +static inline void xen_pcibk_lateeoi(struct xen_pcibk_device *pdev, + unsigned int eoi_flag) +{ + if (test_and_clear_bit(_EOI_pending, &pdev->flags)) + xen_irq_lateeoi(pdev->evtchn_irq, eoi_flag); +} + int xen_pcibk_xenbus_register(void); void xen_pcibk_xenbus_unregister(void); - -void xen_pcibk_test_and_schedule_op(struct xen_pcibk_device *pdev); #endif /* Handles shared IRQs that can to device domain and control domain. */ diff --git a/drivers/xen/xen-pciback/pciback_ops.c b/drivers/xen/xen-pciback/pciback_ops.c index e11a7438e1a2..3fbc21466a93 100644 --- a/drivers/xen/xen-pciback/pciback_ops.c +++ b/drivers/xen/xen-pciback/pciback_ops.c @@ -276,26 +276,41 @@ int xen_pcibk_disable_msix(struct xen_pcibk_device *pdev, return 0; } #endif + +static inline bool xen_pcibk_test_op_pending(struct xen_pcibk_device *pdev) +{ + return test_bit(_XEN_PCIF_active, + (unsigned long *)&pdev->sh_info->flags) && + !test_and_set_bit(_PDEVF_op_active, &pdev->flags); +} + /* * Now the same evtchn is used for both pcifront conf_read_write request * as well as pcie aer front end ack. We use a new work_queue to schedule * xen_pcibk conf_read_write service for avoiding confict with aer_core * do_recovery job which also use the system default work_queue */ -void xen_pcibk_test_and_schedule_op(struct xen_pcibk_device *pdev) +static void xen_pcibk_test_and_schedule_op(struct xen_pcibk_device *pdev) { + bool eoi = true; + /* Check that frontend is requesting an operation and that we are not * already processing a request */ - if (test_bit(_XEN_PCIF_active, (unsigned long *)&pdev->sh_info->flags) - && !test_and_set_bit(_PDEVF_op_active, &pdev->flags)) { + if (xen_pcibk_test_op_pending(pdev)) { schedule_work(&pdev->op_work); + eoi = false; } /*_XEN_PCIB_active should have been cleared by pcifront. And also make sure xen_pcibk is waiting for ack by checking _PCIB_op_pending*/ if (!test_bit(_XEN_PCIB_active, (unsigned long *)&pdev->sh_info->flags) && test_bit(_PCIB_op_pending, &pdev->flags)) { wake_up(&xen_pcibk_aer_wait_queue); + eoi = false; } + + /* EOI if there was nothing to do. */ + if (eoi) + xen_pcibk_lateeoi(pdev, XEN_EOI_FLAG_SPURIOUS); } /* Performing the configuration space reads/writes must not be done in atomic @@ -303,10 +318,8 @@ void xen_pcibk_test_and_schedule_op(struct xen_pcibk_device *pdev) * use of semaphores). This function is intended to be called from a work * queue in process context taking a struct xen_pcibk_device as a parameter */ -void xen_pcibk_do_op(struct work_struct *data) +static void xen_pcibk_do_one_op(struct xen_pcibk_device *pdev) { - struct xen_pcibk_device *pdev = - container_of(data, struct xen_pcibk_device, op_work); struct pci_dev *dev; struct xen_pcibk_dev_data *dev_data = NULL; struct xen_pci_op *op = &pdev->op; @@ -379,16 +392,31 @@ void xen_pcibk_do_op(struct work_struct *data) smp_mb__before_atomic(); /* /after/ clearing PCIF_active */ clear_bit(_PDEVF_op_active, &pdev->flags); smp_mb__after_atomic(); /* /before/ final check for work */ +} - /* Check to see if the driver domain tried to start another request in - * between clearing _XEN_PCIF_active and clearing _PDEVF_op_active. - */ - xen_pcibk_test_and_schedule_op(pdev); +void xen_pcibk_do_op(struct work_struct *data) +{ + struct xen_pcibk_device *pdev = + container_of(data, struct xen_pcibk_device, op_work); + + do { + xen_pcibk_do_one_op(pdev); + } while (xen_pcibk_test_op_pending(pdev)); + + xen_pcibk_lateeoi(pdev, 0); } irqreturn_t xen_pcibk_handle_event(int irq, void *dev_id) { struct xen_pcibk_device *pdev = dev_id; + bool eoi; + + /* IRQs might come in before pdev->evtchn_irq is written. */ + if (unlikely(pdev->evtchn_irq != irq)) + pdev->evtchn_irq = irq; + + eoi = test_and_set_bit(_EOI_pending, &pdev->flags); + WARN(eoi, "IRQ while EOI pending\n"); xen_pcibk_test_and_schedule_op(pdev); diff --git a/drivers/xen/xen-pciback/xenbus.c b/drivers/xen/xen-pciback/xenbus.c index b500466a6c37..4b99ec3dec58 100644 --- a/drivers/xen/xen-pciback/xenbus.c +++ b/drivers/xen/xen-pciback/xenbus.c @@ -123,7 +123,7 @@ static int xen_pcibk_do_attach(struct xen_pcibk_device *pdev, int gnt_ref, pdev->sh_info = vaddr; - err = bind_interdomain_evtchn_to_irqhandler( + err = bind_interdomain_evtchn_to_irqhandler_lateeoi( pdev->xdev->otherend_id, remote_evtchn, xen_pcibk_handle_event, 0, DRV_NAME, pdev); if (err < 0) { diff --git a/drivers/xen/xen-scsiback.c b/drivers/xen/xen-scsiback.c index 1e8cfd80a4e6..4acc4e899600 100644 --- a/drivers/xen/xen-scsiback.c +++ b/drivers/xen/xen-scsiback.c @@ -91,7 +91,6 @@ struct vscsibk_info { unsigned int irq; struct vscsiif_back_ring ring; - int ring_error; spinlock_t ring_lock; atomic_t nr_unreplied_reqs; @@ -722,7 +721,8 @@ static struct vscsibk_pend *prepare_pending_reqs(struct vscsibk_info *info, return pending_req; } -static int scsiback_do_cmd_fn(struct vscsibk_info *info) +static int scsiback_do_cmd_fn(struct vscsibk_info *info, + unsigned int *eoi_flags) { struct vscsiif_back_ring *ring = &info->ring; struct vscsiif_request ring_req; @@ -739,11 +739,12 @@ static int scsiback_do_cmd_fn(struct vscsibk_info *info) rc = ring->rsp_prod_pvt; pr_warn("Dom%d provided bogus ring requests (%#x - %#x = %u). Halting ring processing\n", info->domid, rp, rc, rp - rc); - info->ring_error = 1; - return 0; + return -EINVAL; } while ((rc != rp)) { + *eoi_flags &= ~XEN_EOI_FLAG_SPURIOUS; + if (RING_REQUEST_CONS_OVERFLOW(ring, rc)) break; @@ -802,13 +803,16 @@ static int scsiback_do_cmd_fn(struct vscsibk_info *info) static irqreturn_t scsiback_irq_fn(int irq, void *dev_id) { struct vscsibk_info *info = dev_id; + int rc; + unsigned int eoi_flags = XEN_EOI_FLAG_SPURIOUS; - if (info->ring_error) - return IRQ_HANDLED; - - while (scsiback_do_cmd_fn(info)) + while ((rc = scsiback_do_cmd_fn(info, &eoi_flags)) > 0) cond_resched(); + /* In case of a ring error we keep the event channel masked. */ + if (!rc) + xen_irq_lateeoi(irq, eoi_flags); + return IRQ_HANDLED; } @@ -829,7 +833,7 @@ static int scsiback_init_sring(struct vscsibk_info *info, grant_ref_t ring_ref, sring = (struct vscsiif_sring *)area; BACK_RING_INIT(&info->ring, sring, PAGE_SIZE); - err = bind_interdomain_evtchn_to_irq(info->domid, evtchn); + err = bind_interdomain_evtchn_to_irq_lateeoi(info->domid, evtchn); if (err < 0) goto unmap_page; @@ -1253,7 +1257,6 @@ static int scsiback_probe(struct xenbus_device *dev, info->domid = dev->otherend_id; spin_lock_init(&info->ring_lock); - info->ring_error = 0; atomic_set(&info->nr_unreplied_reqs, 0); init_waitqueue_head(&info->waiting_to_free); info->dev = dev; diff --git a/drivers/xen/xenbus/xenbus_client.c b/drivers/xen/xenbus/xenbus_client.c index 2690318ad50f..fd80e318b99c 100644 --- a/drivers/xen/xenbus/xenbus_client.c +++ b/drivers/xen/xenbus/xenbus_client.c @@ -73,16 +73,13 @@ struct map_ring_valloc { struct xenbus_map_node *node; /* Why do we need two arrays? See comment of __xenbus_map_ring */ - union { - unsigned long addrs[XENBUS_MAX_RING_GRANTS]; - pte_t *ptes[XENBUS_MAX_RING_GRANTS]; - }; + unsigned long addrs[XENBUS_MAX_RING_GRANTS]; phys_addr_t phys_addrs[XENBUS_MAX_RING_GRANTS]; struct gnttab_map_grant_ref map[XENBUS_MAX_RING_GRANTS]; struct gnttab_unmap_grant_ref unmap[XENBUS_MAX_RING_GRANTS]; - unsigned int idx; /* HVM only. */ + unsigned int idx; }; static DEFINE_SPINLOCK(xenbus_valloc_lock); @@ -686,6 +683,14 @@ int xenbus_unmap_ring_vfree(struct xenbus_device *dev, void *vaddr) EXPORT_SYMBOL_GPL(xenbus_unmap_ring_vfree); #ifdef CONFIG_XEN_PV +static int map_ring_apply(pte_t *pte, unsigned long addr, void *data) +{ + struct map_ring_valloc *info = data; + + info->phys_addrs[info->idx++] = arbitrary_virt_to_machine(pte).maddr; + return 0; +} + static int xenbus_map_ring_pv(struct xenbus_device *dev, struct map_ring_valloc *info, grant_ref_t *gnt_refs, @@ -694,18 +699,15 @@ static int xenbus_map_ring_pv(struct xenbus_device *dev, { struct xenbus_map_node *node = info->node; struct vm_struct *area; - int err = GNTST_okay; - int i; - bool leaked; + bool leaked = false; + int err = -ENOMEM; - area = alloc_vm_area(XEN_PAGE_SIZE * nr_grefs, info->ptes); + area = get_vm_area(XEN_PAGE_SIZE * nr_grefs, VM_IOREMAP); if (!area) return -ENOMEM; - - for (i = 0; i < nr_grefs; i++) - info->phys_addrs[i] = - arbitrary_virt_to_machine(info->ptes[i]).maddr; - + if (apply_to_page_range(&init_mm, (unsigned long)area->addr, + XEN_PAGE_SIZE * nr_grefs, map_ring_apply, info)) + goto failed; err = __xenbus_map_ring(dev, gnt_refs, nr_grefs, node->handles, info, GNTMAP_host_map | GNTMAP_contains_pte, &leaked); |