diff options
Diffstat (limited to 'drivers/hv/hv.c')
| -rw-r--r-- | drivers/hv/hv.c | 832 |
1 files changed, 532 insertions, 300 deletions
diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c index 88f4096fa078..c100f04b3581 100644 --- a/drivers/hv/hv.c +++ b/drivers/hv/hv.c @@ -1,128 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2009, Microsoft Corporation. * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 Temple - * Place - Suite 330, Boston, MA 02111-1307 USA. - * * Authors: * Haiyang Zhang <haiyangz@microsoft.com> * Hank Janssen <hjanssen@microsoft.com> - * */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include <linux/io.h> #include <linux/kernel.h> #include <linux/mm.h> #include <linux/slab.h> #include <linux/vmalloc.h> #include <linux/hyperv.h> -#include <linux/version.h> +#include <linux/random.h> +#include <linux/clockchips.h> +#include <linux/delay.h> #include <linux/interrupt.h> -#include <asm/hyperv.h> +#include <linux/export.h> +#include <clocksource/hyperv_timer.h> +#include <asm/mshyperv.h> +#include <linux/set_memory.h> #include "hyperv_vmbus.h" /* The one and only */ -struct hv_context hv_context = { - .synic_initialized = false, - .hypercall_page = NULL, -}; - -/* - * query_hypervisor_info - Get version info of the windows hypervisor - */ -unsigned int host_info_eax; -unsigned int host_info_ebx; -unsigned int host_info_ecx; -unsigned int host_info_edx; - -static int query_hypervisor_info(void) -{ - unsigned int eax; - unsigned int ebx; - unsigned int ecx; - unsigned int edx; - unsigned int max_leaf; - unsigned int op; - - /* - * Its assumed that this is called after confirming that Viridian - * is present. Query id and revision. - */ - eax = 0; - ebx = 0; - ecx = 0; - edx = 0; - op = HVCPUID_VENDOR_MAXFUNCTION; - cpuid(op, &eax, &ebx, &ecx, &edx); - - max_leaf = eax; - - if (max_leaf >= HVCPUID_VERSION) { - eax = 0; - ebx = 0; - ecx = 0; - edx = 0; - op = HVCPUID_VERSION; - cpuid(op, &eax, &ebx, &ecx, &edx); - host_info_eax = eax; - host_info_ebx = ebx; - host_info_ecx = ecx; - host_info_edx = edx; - } - return max_leaf; -} - -/* - * do_hypercall- Invoke the specified hypercall - */ -static u64 do_hypercall(u64 control, void *input, void *output) -{ -#ifdef CONFIG_X86_64 - u64 hv_status = 0; - u64 input_address = (input) ? virt_to_phys(input) : 0; - u64 output_address = (output) ? virt_to_phys(output) : 0; - void *hypercall_page = hv_context.hypercall_page; - - __asm__ __volatile__("mov %0, %%r8" : : "r" (output_address) : "r8"); - __asm__ __volatile__("call *%3" : "=a" (hv_status) : - "c" (control), "d" (input_address), - "m" (hypercall_page)); - - return hv_status; - -#else - - u32 control_hi = control >> 32; - u32 control_lo = control & 0xFFFFFFFF; - u32 hv_status_hi = 1; - u32 hv_status_lo = 1; - u64 input_address = (input) ? virt_to_phys(input) : 0; - u32 input_address_hi = input_address >> 32; - u32 input_address_lo = input_address & 0xFFFFFFFF; - u64 output_address = (output) ? virt_to_phys(output) : 0; - u32 output_address_hi = output_address >> 32; - u32 output_address_lo = output_address & 0xFFFFFFFF; - void *hypercall_page = hv_context.hypercall_page; - - __asm__ __volatile__ ("call *%8" : "=d"(hv_status_hi), - "=a"(hv_status_lo) : "d" (control_hi), - "a" (control_lo), "b" (input_address_hi), - "c" (input_address_lo), "D"(output_address_hi), - "S"(output_address_lo), "m" (hypercall_page)); - - return hv_status_lo | ((u64)hv_status_hi << 32); -#endif /* !x86_64 */ -} +struct hv_context hv_context; +EXPORT_SYMBOL_FOR_MODULES(hv_context, "mshv_vtl"); /* * hv_init - Main initialization routine. @@ -131,81 +35,10 @@ static u64 do_hypercall(u64 control, void *input, void *output) */ int hv_init(void) { - int max_leaf; - union hv_x64_msr_hypercall_contents hypercall_msr; - void *virtaddr = NULL; - - memset(hv_context.synic_event_page, 0, sizeof(void *) * NR_CPUS); - memset(hv_context.synic_message_page, 0, - sizeof(void *) * NR_CPUS); - memset(hv_context.vp_index, 0, - sizeof(int) * NR_CPUS); - memset(hv_context.event_dpc, 0, - sizeof(void *) * NR_CPUS); - - max_leaf = query_hypervisor_info(); - - /* - * Write our OS ID. - */ - hv_context.guestid = generate_guest_id(0, LINUX_VERSION_CODE, 0); - wrmsrl(HV_X64_MSR_GUEST_OS_ID, hv_context.guestid); - - /* See if the hypercall page is already set */ - rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); - - virtaddr = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL_EXEC); - - if (!virtaddr) - goto cleanup; - - hypercall_msr.enable = 1; - - hypercall_msr.guest_physical_address = vmalloc_to_pfn(virtaddr); - wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); - - /* Confirm that hypercall page did get setup. */ - hypercall_msr.as_uint64 = 0; - rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); - - if (!hypercall_msr.enable) - goto cleanup; - - hv_context.hypercall_page = virtaddr; - + hv_context.cpu_context = alloc_percpu(struct hv_per_cpu_context); + if (!hv_context.cpu_context) + return -ENOMEM; return 0; - -cleanup: - if (virtaddr) { - if (hypercall_msr.enable) { - hypercall_msr.as_uint64 = 0; - wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); - } - - vfree(virtaddr); - } - - return -ENOTSUPP; -} - -/* - * hv_cleanup - Cleanup routine. - * - * This routine is called normally during driver unloading or exiting. - */ -void hv_cleanup(void) -{ - union hv_x64_msr_hypercall_contents hypercall_msr; - - /* Reset our OS id */ - wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); - - if (hv_context.hypercall_page) { - hypercall_msr.as_uint64 = 0; - wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); - vfree(hv_context.hypercall_page); - hv_context.hypercall_page = NULL; - } } /* @@ -214,211 +47,610 @@ void hv_cleanup(void) * This involves a hypercall. */ int hv_post_message(union hv_connection_id connection_id, - enum hv_message_type message_type, - void *payload, size_t payload_size) + enum hv_message_type message_type, + void *payload, size_t payload_size) { - struct aligned_input { - u64 alignment8; - struct hv_input_post_message msg; - }; - struct hv_input_post_message *aligned_msg; - u16 status; - unsigned long addr; + unsigned long flags; + u64 status; if (payload_size > HV_MESSAGE_PAYLOAD_BYTE_COUNT) return -EMSGSIZE; - addr = (unsigned long)kmalloc(sizeof(struct aligned_input), GFP_ATOMIC); - if (!addr) - return -ENOMEM; + local_irq_save(flags); - aligned_msg = (struct hv_input_post_message *) - (ALIGN(addr, HV_HYPERCALL_PARAM_ALIGN)); + /* + * A TDX VM with the paravisor must use the decrypted post_msg_page: see + * the comment in struct hv_per_cpu_context. A SNP VM with the paravisor + * can use the encrypted hyperv_pcpu_input_arg because it copies the + * input into the GHCB page, which has been decrypted by the paravisor. + */ + if (hv_isolation_type_tdx() && ms_hyperv.paravisor_present) + aligned_msg = this_cpu_ptr(hv_context.cpu_context)->post_msg_page; + else + aligned_msg = *this_cpu_ptr(hyperv_pcpu_input_arg); aligned_msg->connectionid = connection_id; + aligned_msg->reserved = 0; aligned_msg->message_type = message_type; aligned_msg->payload_size = payload_size; memcpy((void *)aligned_msg->payload, payload, payload_size); - status = do_hypercall(HVCALL_POST_MESSAGE, aligned_msg, NULL) - & 0xFFFF; + if (ms_hyperv.paravisor_present && !vmbus_is_confidential()) { + /* + * If the VMBus isn't confidential, use the CoCo-specific + * mechanism to communicate with the hypervisor. + */ + if (hv_isolation_type_tdx()) + status = hv_tdx_hypercall(HVCALL_POST_MESSAGE, + virt_to_phys(aligned_msg), 0); + else if (hv_isolation_type_snp()) + status = hv_ghcb_hypercall(HVCALL_POST_MESSAGE, + aligned_msg, NULL, + sizeof(*aligned_msg)); + else + status = HV_STATUS_INVALID_PARAMETER; + } else { + u64 control = HVCALL_POST_MESSAGE; + + control |= hv_nested ? HV_HYPERCALL_NESTED : 0; + /* + * If there is no paravisor, this will go to the hypervisor. + * In the Confidential VMBus case, there is the paravisor + * to which this will trap. + */ + status = hv_do_hypercall(control, aligned_msg, NULL); + } - kfree((void *)addr); + local_irq_restore(flags); - return status; + return hv_result(status); } +EXPORT_SYMBOL_FOR_MODULES(hv_post_message, "mshv_vtl"); - -/* - * hv_signal_event - - * Signal an event on the specified connection using the hypervisor event IPC. - * - * This involves a hypercall. - */ -u16 hv_signal_event(void *con_id) +static int hv_alloc_page(void **page, bool decrypt, const char *note) { - u16 status; + int ret = 0; - status = (do_hypercall(HVCALL_SIGNAL_EVENT, con_id, NULL) & 0xFFFF); + /* + * After the page changes its encryption status, its contents might + * appear scrambled on some hardware. Thus `get_zeroed_page` would + * zero the page out in vain, so do that explicitly exactly once. + * + * By default, the page is allocated encrypted in a CoCo VM. + */ + *page = (void *)__get_free_page(GFP_KERNEL); + if (!*page) + return -ENOMEM; - return status; + if (decrypt) + ret = set_memory_decrypted((unsigned long)*page, 1); + if (ret) + goto failed; + + memset(*page, 0, PAGE_SIZE); + return 0; + +failed: + /* + * Report the failure but don't put the page back on the free list as + * its encryption status is unknown. + */ + pr_err("allocation failed for %s page, error %d, decrypted %d\n", + note, ret, decrypt); + *page = NULL; + return ret; } +static int hv_free_page(void **page, bool encrypt, const char *note) +{ + int ret = 0; + + if (!*page) + return 0; + + if (encrypt) + ret = set_memory_encrypted((unsigned long)*page, 1); + + /* + * In the case of the failure, the page is leaked. Something is wrong, + * prefer to lose the page with the unknown encryption status and stay afloat. + */ + if (ret) + pr_err("deallocation failed for %s page, error %d, encrypt %d\n", + note, ret, encrypt); + else + free_page((unsigned long)*page); + + *page = NULL; + + return ret; +} int hv_synic_alloc(void) { - size_t size = sizeof(struct tasklet_struct); - int cpu; + int cpu, ret = -ENOMEM; + struct hv_per_cpu_context *hv_cpu; + const bool decrypt = !vmbus_is_confidential(); - for_each_online_cpu(cpu) { - hv_context.event_dpc[cpu] = kmalloc(size, GFP_ATOMIC); - if (hv_context.event_dpc[cpu] == NULL) { - pr_err("Unable to allocate event dpc\n"); - goto err; - } - tasklet_init(hv_context.event_dpc[cpu], vmbus_on_event, cpu); + /* + * First, zero all per-cpu memory areas so hv_synic_free() can + * detect what memory has been allocated and cleanup properly + * after any failures. + */ + for_each_present_cpu(cpu) { + hv_cpu = per_cpu_ptr(hv_context.cpu_context, cpu); + memset(hv_cpu, 0, sizeof(*hv_cpu)); + } - hv_context.synic_message_page[cpu] = - (void *)get_zeroed_page(GFP_ATOMIC); + hv_context.hv_numa_map = kcalloc(nr_node_ids, sizeof(struct cpumask), + GFP_KERNEL); + if (!hv_context.hv_numa_map) { + pr_err("Unable to allocate NUMA map\n"); + goto err; + } + + for_each_present_cpu(cpu) { + hv_cpu = per_cpu_ptr(hv_context.cpu_context, cpu); - if (hv_context.synic_message_page[cpu] == NULL) { - pr_err("Unable to allocate SYNIC message page\n"); - goto err; + tasklet_init(&hv_cpu->msg_dpc, + vmbus_on_msg_dpc, (unsigned long)hv_cpu); + + if (ms_hyperv.paravisor_present && hv_isolation_type_tdx()) { + ret = hv_alloc_page(&hv_cpu->post_msg_page, + decrypt, "post msg"); + if (ret) + goto err; } - hv_context.synic_event_page[cpu] = - (void *)get_zeroed_page(GFP_ATOMIC); + /* + * If these SynIC pages are not allocated, SIEF and SIM pages + * are configured using what the root partition or the paravisor + * provides upon reading the SIEFP and SIMP registers. + */ + if (!ms_hyperv.paravisor_present && !hv_root_partition()) { + ret = hv_alloc_page(&hv_cpu->hyp_synic_message_page, + decrypt, "hypervisor SynIC msg"); + if (ret) + goto err; + ret = hv_alloc_page(&hv_cpu->hyp_synic_event_page, + decrypt, "hypervisor SynIC event"); + if (ret) + goto err; + } - if (hv_context.synic_event_page[cpu] == NULL) { - pr_err("Unable to allocate SYNIC event page\n"); - goto err; + if (vmbus_is_confidential()) { + ret = hv_alloc_page(&hv_cpu->para_synic_message_page, + false, "paravisor SynIC msg"); + if (ret) + goto err; + ret = hv_alloc_page(&hv_cpu->para_synic_event_page, + false, "paravisor SynIC event"); + if (ret) + goto err; } } return 0; -err: - return -ENOMEM; -} -void hv_synic_free_cpu(int cpu) -{ - kfree(hv_context.event_dpc[cpu]); - if (hv_context.synic_message_page[cpu]) - free_page((unsigned long)hv_context.synic_event_page[cpu]); - if (hv_context.synic_message_page[cpu]) - free_page((unsigned long)hv_context.synic_message_page[cpu]); +err: + /* + * Any memory allocations that succeeded will be freed when + * the caller cleans up by calling hv_synic_free() + */ + return ret; } void hv_synic_free(void) { int cpu; + const bool encrypt = !vmbus_is_confidential(); + + for_each_present_cpu(cpu) { + struct hv_per_cpu_context *hv_cpu = + per_cpu_ptr(hv_context.cpu_context, cpu); + + if (ms_hyperv.paravisor_present && hv_isolation_type_tdx()) + hv_free_page(&hv_cpu->post_msg_page, + encrypt, "post msg"); + if (!ms_hyperv.paravisor_present && !hv_root_partition()) { + hv_free_page(&hv_cpu->hyp_synic_event_page, + encrypt, "hypervisor SynIC event"); + hv_free_page(&hv_cpu->hyp_synic_message_page, + encrypt, "hypervisor SynIC msg"); + } + if (vmbus_is_confidential()) { + hv_free_page(&hv_cpu->para_synic_event_page, + false, "paravisor SynIC event"); + hv_free_page(&hv_cpu->para_synic_message_page, + false, "paravisor SynIC msg"); + } + } - for_each_online_cpu(cpu) - hv_synic_free_cpu(cpu); + kfree(hv_context.hv_numa_map); } /* - * hv_synic_init - Initialize the Synthethic Interrupt Controller. - * - * If it is already initialized by another entity (ie x2v shim), we need to - * retrieve the initialized message and event pages. Otherwise, we create and - * initialize the message and event pages. + * hv_hyp_synic_enable_regs - Initialize the Synthetic Interrupt Controller + * with the hypervisor. */ -void hv_synic_init(void *arg) +void hv_hyp_synic_enable_regs(unsigned int cpu) { - u64 version; + struct hv_per_cpu_context *hv_cpu = + per_cpu_ptr(hv_context.cpu_context, cpu); union hv_synic_simp simp; union hv_synic_siefp siefp; union hv_synic_sint shared_sint; - union hv_synic_scontrol sctrl; - u64 vp_index; - - int cpu = smp_processor_id(); - - if (!hv_context.hypercall_page) - return; - /* Check the version */ - rdmsrl(HV_X64_MSR_SVERSION, version); - - /* Setup the Synic's message page */ - rdmsrl(HV_X64_MSR_SIMP, simp.as_uint64); + /* Setup the Synic's message page with the hypervisor. */ + simp.as_uint64 = hv_get_msr(HV_MSR_SIMP); simp.simp_enabled = 1; - simp.base_simp_gpa = virt_to_phys(hv_context.synic_message_page[cpu]) - >> PAGE_SHIFT; - wrmsrl(HV_X64_MSR_SIMP, simp.as_uint64); + if (ms_hyperv.paravisor_present || hv_root_partition()) { + /* Mask out vTOM bit. ioremap_cache() maps decrypted */ + u64 base = (simp.base_simp_gpa << HV_HYP_PAGE_SHIFT) & + ~ms_hyperv.shared_gpa_boundary; + hv_cpu->hyp_synic_message_page = + (void *)ioremap_cache(base, HV_HYP_PAGE_SIZE); + if (!hv_cpu->hyp_synic_message_page) + pr_err("Fail to map synic message page.\n"); + } else { + simp.base_simp_gpa = virt_to_phys(hv_cpu->hyp_synic_message_page) + >> HV_HYP_PAGE_SHIFT; + } + + hv_set_msr(HV_MSR_SIMP, simp.as_uint64); - /* Setup the Synic's event page */ - rdmsrl(HV_X64_MSR_SIEFP, siefp.as_uint64); + /* Setup the Synic's event page with the hypervisor. */ + siefp.as_uint64 = hv_get_msr(HV_MSR_SIEFP); siefp.siefp_enabled = 1; - siefp.base_siefp_gpa = virt_to_phys(hv_context.synic_event_page[cpu]) - >> PAGE_SHIFT; - wrmsrl(HV_X64_MSR_SIEFP, siefp.as_uint64); + if (ms_hyperv.paravisor_present || hv_root_partition()) { + /* Mask out vTOM bit. ioremap_cache() maps decrypted */ + u64 base = (siefp.base_siefp_gpa << HV_HYP_PAGE_SHIFT) & + ~ms_hyperv.shared_gpa_boundary; + hv_cpu->hyp_synic_event_page = + (void *)ioremap_cache(base, HV_HYP_PAGE_SIZE); + if (!hv_cpu->hyp_synic_event_page) + pr_err("Fail to map synic event page.\n"); + } else { + siefp.base_siefp_gpa = virt_to_phys(hv_cpu->hyp_synic_event_page) + >> HV_HYP_PAGE_SHIFT; + } + + hv_set_msr(HV_MSR_SIEFP, siefp.as_uint64); + hv_enable_coco_interrupt(cpu, vmbus_interrupt, true); /* Setup the shared SINT. */ - rdmsrl(HV_X64_MSR_SINT0 + VMBUS_MESSAGE_SINT, shared_sint.as_uint64); + if (vmbus_irq != -1) + enable_percpu_irq(vmbus_irq, 0); + shared_sint.as_uint64 = hv_get_msr(HV_MSR_SINT0 + VMBUS_MESSAGE_SINT); - shared_sint.as_uint64 = 0; - shared_sint.vector = HYPERVISOR_CALLBACK_VECTOR; + shared_sint.vector = vmbus_interrupt; shared_sint.masked = false; - shared_sint.auto_eoi = true; + shared_sint.auto_eoi = hv_recommend_using_aeoi(); + hv_set_msr(HV_MSR_SINT0 + VMBUS_MESSAGE_SINT, shared_sint.as_uint64); +} - wrmsrl(HV_X64_MSR_SINT0 + VMBUS_MESSAGE_SINT, shared_sint.as_uint64); +static void hv_hyp_synic_enable_interrupts(void) +{ + union hv_synic_scontrol sctrl; /* Enable the global synic bit */ - rdmsrl(HV_X64_MSR_SCONTROL, sctrl.as_uint64); + sctrl.as_uint64 = hv_get_msr(HV_MSR_SCONTROL); sctrl.enable = 1; - wrmsrl(HV_X64_MSR_SCONTROL, sctrl.as_uint64); + hv_set_msr(HV_MSR_SCONTROL, sctrl.as_uint64); +} - hv_context.synic_initialized = true; +static void hv_para_synic_enable_regs(unsigned int cpu) +{ + union hv_synic_simp simp; + union hv_synic_siefp siefp; + struct hv_per_cpu_context *hv_cpu + = per_cpu_ptr(hv_context.cpu_context, cpu); + + /* Setup the Synic's message page with the paravisor. */ + simp.as_uint64 = hv_para_get_synic_register(HV_MSR_SIMP); + simp.simp_enabled = 1; + simp.base_simp_gpa = virt_to_phys(hv_cpu->para_synic_message_page) + >> HV_HYP_PAGE_SHIFT; + hv_para_set_synic_register(HV_MSR_SIMP, simp.as_uint64); + + /* Setup the Synic's event page with the paravisor. */ + siefp.as_uint64 = hv_para_get_synic_register(HV_MSR_SIEFP); + siefp.siefp_enabled = 1; + siefp.base_siefp_gpa = virt_to_phys(hv_cpu->para_synic_event_page) + >> HV_HYP_PAGE_SHIFT; + hv_para_set_synic_register(HV_MSR_SIEFP, siefp.as_uint64); +} + +static void hv_para_synic_enable_interrupts(void) +{ + union hv_synic_scontrol sctrl; + + /* Enable the global synic bit */ + sctrl.as_uint64 = hv_para_get_synic_register(HV_MSR_SCONTROL); + sctrl.enable = 1; + hv_para_set_synic_register(HV_MSR_SCONTROL, sctrl.as_uint64); +} + +int hv_synic_init(unsigned int cpu) +{ + if (vmbus_is_confidential()) + hv_para_synic_enable_regs(cpu); /* - * Setup the mapping between Hyper-V's notion - * of cpuid and Linux' notion of cpuid. - * This array will be indexed using Linux cpuid. + * The SINT is set in hv_hyp_synic_enable_regs() by calling + * hv_set_msr(). hv_set_msr() in turn has special case code for the + * SINT MSRs that write to the hypervisor version of the MSR *and* + * the paravisor version of the MSR (but *without* the proxy bit when + * VMBus is confidential). + * + * Then enable interrupts via the paravisor if VMBus is confidential, + * and otherwise via the hypervisor. */ - rdmsrl(HV_X64_MSR_VP_INDEX, vp_index); - hv_context.vp_index[cpu] = (u32)vp_index; - return; + + hv_hyp_synic_enable_regs(cpu); + if (vmbus_is_confidential()) + hv_para_synic_enable_interrupts(); + else + hv_hyp_synic_enable_interrupts(); + + hv_stimer_legacy_init(cpu, VMBUS_MESSAGE_SINT); + + return 0; } -/* - * hv_synic_cleanup - Cleanup routine for hv_synic_init(). - */ -void hv_synic_cleanup(void *arg) +void hv_hyp_synic_disable_regs(unsigned int cpu) { + struct hv_per_cpu_context *hv_cpu = + per_cpu_ptr(hv_context.cpu_context, cpu); union hv_synic_sint shared_sint; union hv_synic_simp simp; union hv_synic_siefp siefp; - int cpu = smp_processor_id(); - - if (!hv_context.synic_initialized) - return; - rdmsrl(HV_X64_MSR_SINT0 + VMBUS_MESSAGE_SINT, shared_sint.as_uint64); + shared_sint.as_uint64 = hv_get_msr(HV_MSR_SINT0 + VMBUS_MESSAGE_SINT); shared_sint.masked = 1; /* Need to correctly cleanup in the case of SMP!!! */ /* Disable the interrupt */ - wrmsrl(HV_X64_MSR_SINT0 + VMBUS_MESSAGE_SINT, shared_sint.as_uint64); + hv_set_msr(HV_MSR_SINT0 + VMBUS_MESSAGE_SINT, shared_sint.as_uint64); + hv_enable_coco_interrupt(cpu, vmbus_interrupt, false); - rdmsrl(HV_X64_MSR_SIMP, simp.as_uint64); + simp.as_uint64 = hv_get_msr(HV_MSR_SIMP); + /* + * In Isolation VM, simp and sief pages are allocated by + * paravisor. These pages also will be used by kdump + * kernel. So just reset enable bit here and keep page + * addresses. + */ simp.simp_enabled = 0; - simp.base_simp_gpa = 0; + if (ms_hyperv.paravisor_present || hv_root_partition()) { + if (hv_cpu->hyp_synic_message_page) { + iounmap(hv_cpu->hyp_synic_message_page); + hv_cpu->hyp_synic_message_page = NULL; + } + } else { + simp.base_simp_gpa = 0; + } + + hv_set_msr(HV_MSR_SIMP, simp.as_uint64); + + siefp.as_uint64 = hv_get_msr(HV_MSR_SIEFP); + siefp.siefp_enabled = 0; + + if (ms_hyperv.paravisor_present || hv_root_partition()) { + if (hv_cpu->hyp_synic_event_page) { + iounmap(hv_cpu->hyp_synic_event_page); + hv_cpu->hyp_synic_event_page = NULL; + } + } else { + siefp.base_siefp_gpa = 0; + } + + hv_set_msr(HV_MSR_SIEFP, siefp.as_uint64); +} + +static void hv_hyp_synic_disable_interrupts(void) +{ + union hv_synic_scontrol sctrl; - wrmsrl(HV_X64_MSR_SIMP, simp.as_uint64); + /* Disable the global synic bit */ + sctrl.as_uint64 = hv_get_msr(HV_MSR_SCONTROL); + sctrl.enable = 0; + hv_set_msr(HV_MSR_SCONTROL, sctrl.as_uint64); +} - rdmsrl(HV_X64_MSR_SIEFP, siefp.as_uint64); +static void hv_para_synic_disable_regs(unsigned int cpu) +{ + union hv_synic_simp simp; + union hv_synic_siefp siefp; + + /* Disable SynIC's message page in the paravisor. */ + simp.as_uint64 = hv_para_get_synic_register(HV_MSR_SIMP); + simp.simp_enabled = 0; + hv_para_set_synic_register(HV_MSR_SIMP, simp.as_uint64); + + /* Disable SynIC's event page in the paravisor. */ + siefp.as_uint64 = hv_para_get_synic_register(HV_MSR_SIEFP); siefp.siefp_enabled = 0; - siefp.base_siefp_gpa = 0; + hv_para_set_synic_register(HV_MSR_SIEFP, siefp.as_uint64); +} + +static void hv_para_synic_disable_interrupts(void) +{ + union hv_synic_scontrol sctrl; + + /* Disable the global synic bit */ + sctrl.as_uint64 = hv_para_get_synic_register(HV_MSR_SCONTROL); + sctrl.enable = 0; + hv_para_set_synic_register(HV_MSR_SCONTROL, sctrl.as_uint64); +} + +#define HV_MAX_TRIES 3 +/* + * Scan the event flags page of 'this' CPU looking for any bit that is set. If we find one + * bit set, then wait for a few milliseconds. Repeat these steps for a maximum of 3 times. + * Return 'true', if there is still any set bit after this operation; 'false', otherwise. + * + * If a bit is set, that means there is a pending channel interrupt. The expectation is + * that the normal interrupt handling mechanism will find and process the channel interrupt + * "very soon", and in the process clear the bit. + */ +static bool __hv_synic_event_pending(union hv_synic_event_flags *event, int sint) +{ + unsigned long *recv_int_page; + bool pending; + u32 relid; + int tries = 0; + + if (!event) + return false; + + event += sint; + recv_int_page = event->flags; /* assumes VMBus version >= VERSION_WIN8 */ +retry: + pending = false; + for_each_set_bit(relid, recv_int_page, HV_EVENT_FLAGS_COUNT) { + /* Special case - VMBus channel protocol messages */ + if (relid == 0) + continue; + pending = true; + break; + } + if (pending && tries++ < HV_MAX_TRIES) { + usleep_range(10000, 20000); + goto retry; + } + return pending; +} + +static bool hv_synic_event_pending(void) +{ + struct hv_per_cpu_context *hv_cpu = this_cpu_ptr(hv_context.cpu_context); + union hv_synic_event_flags *hyp_synic_event_page = hv_cpu->hyp_synic_event_page; + union hv_synic_event_flags *para_synic_event_page = hv_cpu->para_synic_event_page; + + return + __hv_synic_event_pending(hyp_synic_event_page, VMBUS_MESSAGE_SINT) || + __hv_synic_event_pending(para_synic_event_page, VMBUS_MESSAGE_SINT); +} + +static int hv_pick_new_cpu(struct vmbus_channel *channel) +{ + int ret = -EBUSY; + int start; + int cpu; + + lockdep_assert_cpus_held(); + lockdep_assert_held(&vmbus_connection.channel_mutex); + + /* + * We can't assume that the relevant interrupts will be sent before + * the cpu is offlined on older versions of hyperv. + */ + if (vmbus_proto_version < VERSION_WIN10_V5_3) + return -EBUSY; + + start = get_random_u32_below(nr_cpu_ids); + + for_each_cpu_wrap(cpu, cpu_online_mask, start) { + if (channel->target_cpu == cpu || + channel->target_cpu == VMBUS_CONNECT_CPU) + continue; + + ret = vmbus_channel_set_cpu(channel, cpu); + if (!ret) + break; + } + + if (ret) + ret = vmbus_channel_set_cpu(channel, VMBUS_CONNECT_CPU); + + return ret; +} + +/* + * hv_synic_cleanup - Cleanup routine for hv_synic_init(). + */ +int hv_synic_cleanup(unsigned int cpu) +{ + struct vmbus_channel *channel, *sc; + int ret = 0; + + if (vmbus_connection.conn_state != CONNECTED) + goto always_cleanup; + + /* + * Hyper-V does not provide a way to change the connect CPU once + * it is set; we must prevent the connect CPU from going offline + * while the VM is running normally. But in the panic or kexec() + * path where the vmbus is already disconnected, the CPU must be + * allowed to shut down. + */ + if (cpu == VMBUS_CONNECT_CPU) + return -EBUSY; + + /* + * Search for channels which are bound to the CPU we're about to + * cleanup. + */ + mutex_lock(&vmbus_connection.channel_mutex); + list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { + if (channel->target_cpu == cpu) { + ret = hv_pick_new_cpu(channel); + if (ret) { + mutex_unlock(&vmbus_connection.channel_mutex); + return ret; + } + } + list_for_each_entry(sc, &channel->sc_list, sc_list) { + if (sc->target_cpu == cpu) { + ret = hv_pick_new_cpu(sc); + if (ret) { + mutex_unlock(&vmbus_connection.channel_mutex); + return ret; + } + } + } + } + mutex_unlock(&vmbus_connection.channel_mutex); + + /* + * Scan the event flags page looking for bits that are set and waiting + * with a timeout for vmbus_chan_sched() to process such bits. If bits + * are still set after this operation and VMBus is connected, fail the + * CPU offlining operation. + */ + if (vmbus_proto_version >= VERSION_WIN10_V4_1 && hv_synic_event_pending()) + return -EBUSY; + +always_cleanup: + hv_stimer_legacy_cleanup(cpu); - wrmsrl(HV_X64_MSR_SIEFP, siefp.as_uint64); + /* + * First, disable the event and message pages + * used for communicating with the host, and then + * disable the host interrupts if VMBus is not + * confidential. + */ + hv_hyp_synic_disable_regs(cpu); + if (!vmbus_is_confidential()) + hv_hyp_synic_disable_interrupts(); + + /* + * Perform the same steps for the Confidential VMBus. + * The sequencing provides the guarantee that no data + * may be posted for processing before disabling interrupts. + */ + if (vmbus_is_confidential()) { + hv_para_synic_disable_regs(cpu); + hv_para_synic_disable_interrupts(); + } + if (vmbus_irq != -1) + disable_percpu_irq(vmbus_irq); - free_page((unsigned long)hv_context.synic_message_page[cpu]); - free_page((unsigned long)hv_context.synic_event_page[cpu]); + return ret; } |