// SPDX-License-Identifier: GPL-2.0 /* * Copyright IBM Corp. 1999, 2023 */ #include #include #include #include #include #include #include #include #define __SUBCODE_MASK 0x0600 #define __PF_RES_FIELD 0x8000000000000000UL /* * 'pfault' pseudo page faults routines. */ static int pfault_disable; static int __init nopfault(char *str) { pfault_disable = 1; return 1; } early_param("nopfault", nopfault); struct pfault_refbk { u16 refdiagc; u16 reffcode; u16 refdwlen; u16 refversn; u64 refgaddr; u64 refselmk; u64 refcmpmk; u64 reserved; }; static struct pfault_refbk pfault_init_refbk = { .refdiagc = 0x258, .reffcode = 0, .refdwlen = 5, .refversn = 2, .refgaddr = __LC_LPP, .refselmk = 1UL << 48, .refcmpmk = 1UL << 48, .reserved = __PF_RES_FIELD }; int __pfault_init(void) { int rc = -EOPNOTSUPP; if (pfault_disable) return rc; diag_stat_inc(DIAG_STAT_X258); asm volatile( " diag %[refbk],%[rc],0x258\n" "0: nopr %%r7\n" EX_TABLE(0b, 0b) : [rc] "+d" (rc) : [refbk] "a" (&pfault_init_refbk), "m" (pfault_init_refbk) : "cc"); return rc; } static struct pfault_refbk pfault_fini_refbk = { .refdiagc = 0x258, .reffcode = 1, .refdwlen = 5, .refversn = 2, }; void __pfault_fini(void) { if (pfault_disable) return; diag_stat_inc(DIAG_STAT_X258); asm volatile( " diag %[refbk],0,0x258\n" "0: nopr %%r7\n" EX_TABLE(0b, 0b) : : [refbk] "a" (&pfault_fini_refbk), "m" (pfault_fini_refbk) : "cc"); } static DEFINE_SPINLOCK(pfault_lock); static LIST_HEAD(pfault_list); #define PF_COMPLETE 0x0080 /* * The mechanism of our pfault code: if Linux is running as guest, runs a user * space process and the user space process accesses a page that the host has * paged out we get a pfault interrupt. * * This allows us, within the guest, to schedule a different process. Without * this mechanism the host would have to suspend the whole virtual cpu until * the page has been paged in. * * So when we get such an interrupt then we set the state of the current task * to uninterruptible and also set the need_resched flag. Both happens within * interrupt context(!). If we later on want to return to user space we * recognize the need_resched flag and then call schedule(). It's not very * obvious how this works... * * Of course we have a lot of additional fun with the completion interrupt (-> * host signals that a page of a process has been paged in and the process can * continue to run). This interrupt can arrive on any cpu and, since we have * virtual cpus, actually appear before the interrupt that signals that a page * is missing. */ static void pfault_interrupt(struct ext_code ext_code, unsigned int param32, unsigned long param64) { struct task_struct *tsk; __u16 subcode; pid_t pid; /* * Get the external interruption subcode & pfault initial/completion * signal bit. VM stores this in the 'cpu address' field associated * with the external interrupt. */ subcode = ext_code.subcode; if ((subcode & 0xff00) != __SUBCODE_MASK) return; inc_irq_stat(IRQEXT_PFL); /* Get the token (= pid of the affected task). */ pid = param64 & LPP_PID_MASK; rcu_read_lock(); tsk = find_task_by_pid_ns(pid, &init_pid_ns); if (tsk) get_task_struct(tsk); rcu_read_unlock(); if (!tsk) return; spin_lock(&pfault_lock); if (subcode & PF_COMPLETE) { /* signal bit is set -> a page has been swapped in by VM */ if (tsk->thread.pfault_wait == 1) { /* * Initial interrupt was faster than the completion * interrupt. pfault_wait is valid. Set pfault_wait * back to zero and wake up the process. This can * safely be done because the task is still sleeping * and can't produce new pfaults. */ tsk->thread.pfault_wait = 0; list_del(&tsk->thread.list); wake_up_process(tsk); put_task_struct(tsk); } else { /* * Completion interrupt was faster than initial * interrupt. Set pfault_wait to -1 so the initial * interrupt doesn't put the task to sleep. * If the task is not running, ignore the completion * interrupt since it must be a leftover of a PFAULT * CANCEL operation which didn't remove all pending * completion interrupts. */ if (task_is_running(tsk)) tsk->thread.pfault_wait = -1; } } else { /* signal bit not set -> a real page is missing. */ if (WARN_ON_ONCE(tsk != current)) goto out; if (tsk->thread.pfault_wait == 1) { /* Already on the list with a reference: put to sleep */ goto block; } else if (tsk->thread.pfault_wait == -1) { /* * Completion interrupt was faster than the initial * interrupt (pfault_wait == -1). Set pfault_wait * back to zero and exit. */ tsk->thread.pfault_wait = 0; } else { /* * Initial interrupt arrived before completion * interrupt. Let the task sleep. * An extra task reference is needed since a different * cpu may set the task state to TASK_RUNNING again * before the scheduler is reached. */ get_task_struct(tsk); tsk->thread.pfault_wait = 1; list_add(&tsk->thread.list, &pfault_list); block: /* * Since this must be a userspace fault, there * is no kernel task state to trample. Rely on the * return to userspace schedule() to block. */ __set_current_state(TASK_UNINTERRUPTIBLE); set_tsk_need_resched(tsk); set_preempt_need_resched(); } } out: spin_unlock(&pfault_lock); put_task_struct(tsk); } static int pfault_cpu_dead(unsigned int cpu) { struct thread_struct *thread, *next; struct task_struct *tsk; spin_lock_irq(&pfault_lock); list_for_each_entry_safe(thread, next, &pfault_list, list) { thread->pfault_wait = 0; list_del(&thread->list); tsk = container_of(thread, struct task_struct, thread); wake_up_process(tsk); put_task_struct(tsk); } spin_unlock_irq(&pfault_lock); return 0; } static int __init pfault_irq_init(void) { int rc; rc = register_external_irq(EXT_IRQ_CP_SERVICE, pfault_interrupt); if (rc) goto out_extint; rc = pfault_init() == 0 ? 0 : -EOPNOTSUPP; if (rc) goto out_pfault; irq_subclass_register(IRQ_SUBCLASS_SERVICE_SIGNAL); cpuhp_setup_state_nocalls(CPUHP_S390_PFAULT_DEAD, "s390/pfault:dead", NULL, pfault_cpu_dead); return 0; out_pfault: unregister_external_irq(EXT_IRQ_CP_SERVICE, pfault_interrupt); out_extint: pfault_disable = 1; return rc; } early_initcall(pfault_irq_init);