summaryrefslogtreecommitdiff
path: root/arch/mips/include/asm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/mips/include/asm')
-rw-r--r--arch/mips/include/asm/checksum.h4
-rw-r--r--arch/mips/include/asm/cpu-features.h10
-rw-r--r--arch/mips/include/asm/cpu-info.h2
-rw-r--r--arch/mips/include/asm/cpu.h1
-rw-r--r--arch/mips/include/asm/kvm_host.h468
-rw-r--r--arch/mips/include/asm/maar.h10
-rw-r--r--arch/mips/include/asm/mipsregs.h62
-rw-r--r--arch/mips/include/asm/octeon/cvmx-helper-rgmii.h2
-rw-r--r--arch/mips/include/asm/pci.h5
-rw-r--r--arch/mips/include/asm/r4kcache.h4
-rw-r--r--arch/mips/include/asm/tlb.h6
-rw-r--r--arch/mips/include/asm/uaccess.h449
12 files changed, 524 insertions, 499 deletions
diff --git a/arch/mips/include/asm/checksum.h b/arch/mips/include/asm/checksum.h
index c8b574f7e0cc..77cad232a1c6 100644
--- a/arch/mips/include/asm/checksum.h
+++ b/arch/mips/include/asm/checksum.h
@@ -50,7 +50,7 @@ __wsum csum_partial_copy_from_user(const void __user *src, void *dst, int len,
__wsum sum, int *err_ptr)
{
might_fault();
- if (segment_eq(get_fs(), get_ds()))
+ if (uaccess_kernel())
return __csum_partial_copy_kernel((__force void *)src, dst,
len, sum, err_ptr);
else
@@ -82,7 +82,7 @@ __wsum csum_and_copy_to_user(const void *src, void __user *dst, int len,
{
might_fault();
if (access_ok(VERIFY_WRITE, dst, len)) {
- if (segment_eq(get_fs(), get_ds()))
+ if (uaccess_kernel())
return __csum_partial_copy_kernel(src,
(__force void *)dst,
len, sum, err_ptr);
diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h
index e961c8a7ea66..494d38274142 100644
--- a/arch/mips/include/asm/cpu-features.h
+++ b/arch/mips/include/asm/cpu-features.h
@@ -444,6 +444,10 @@
# define cpu_has_msa 0
#endif
+#ifndef cpu_has_ufr
+# define cpu_has_ufr (cpu_data[0].options & MIPS_CPU_UFR)
+#endif
+
#ifndef cpu_has_fre
# define cpu_has_fre (cpu_data[0].options & MIPS_CPU_FRE)
#endif
@@ -528,6 +532,9 @@
#ifndef cpu_guest_has_htw
#define cpu_guest_has_htw (cpu_data[0].guest.options & MIPS_CPU_HTW)
#endif
+#ifndef cpu_guest_has_mvh
+#define cpu_guest_has_mvh (cpu_data[0].guest.options & MIPS_CPU_MVH)
+#endif
#ifndef cpu_guest_has_msa
#define cpu_guest_has_msa (cpu_data[0].guest.ases & MIPS_ASE_MSA)
#endif
@@ -543,6 +550,9 @@
#ifndef cpu_guest_has_maar
#define cpu_guest_has_maar (cpu_data[0].guest.options & MIPS_CPU_MAAR)
#endif
+#ifndef cpu_guest_has_userlocal
+#define cpu_guest_has_userlocal (cpu_data[0].guest.options & MIPS_CPU_ULRI)
+#endif
/*
* Guest dynamic capabilities
diff --git a/arch/mips/include/asm/cpu-info.h b/arch/mips/include/asm/cpu-info.h
index d43b3045109e..cd6efb07c980 100644
--- a/arch/mips/include/asm/cpu-info.h
+++ b/arch/mips/include/asm/cpu-info.h
@@ -32,6 +32,7 @@ struct guest_info {
unsigned long ases_dyn;
unsigned long long options;
unsigned long long options_dyn;
+ int tlbsize;
u8 conf;
u8 kscratch_mask;
};
@@ -108,6 +109,7 @@ struct cpuinfo_mips {
struct guest_info guest;
unsigned int gtoffset_mask;
unsigned int guestid_mask;
+ unsigned int guestid_cache;
} __attribute__((aligned(SMP_CACHE_BYTES)));
extern struct cpuinfo_mips cpu_data[];
diff --git a/arch/mips/include/asm/cpu.h b/arch/mips/include/asm/cpu.h
index 9a8372484edc..98f59307e6a3 100644
--- a/arch/mips/include/asm/cpu.h
+++ b/arch/mips/include/asm/cpu.h
@@ -415,6 +415,7 @@ enum cpu_type_enum {
#define MIPS_CPU_GUESTCTL2 MBIT_ULL(50) /* CPU has VZ GuestCtl2 register */
#define MIPS_CPU_GUESTID MBIT_ULL(51) /* CPU uses VZ ASE GuestID feature */
#define MIPS_CPU_DRG MBIT_ULL(52) /* CPU has VZ Direct Root to Guest (DRG) */
+#define MIPS_CPU_UFR MBIT_ULL(53) /* CPU supports User mode FR switching */
/*
* CPU ASE encodings
diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h
index 05e785fc061d..2998479fd4e8 100644
--- a/arch/mips/include/asm/kvm_host.h
+++ b/arch/mips/include/asm/kvm_host.h
@@ -10,6 +10,7 @@
#ifndef __MIPS_KVM_HOST_H__
#define __MIPS_KVM_HOST_H__
+#include <linux/cpumask.h>
#include <linux/mutex.h>
#include <linux/hrtimer.h>
#include <linux/interrupt.h>
@@ -33,12 +34,23 @@
#define KVM_REG_MIPS_CP0_ENTRYLO0 MIPS_CP0_64(2, 0)
#define KVM_REG_MIPS_CP0_ENTRYLO1 MIPS_CP0_64(3, 0)
#define KVM_REG_MIPS_CP0_CONTEXT MIPS_CP0_64(4, 0)
+#define KVM_REG_MIPS_CP0_CONTEXTCONFIG MIPS_CP0_32(4, 1)
#define KVM_REG_MIPS_CP0_USERLOCAL MIPS_CP0_64(4, 2)
+#define KVM_REG_MIPS_CP0_XCONTEXTCONFIG MIPS_CP0_64(4, 3)
#define KVM_REG_MIPS_CP0_PAGEMASK MIPS_CP0_32(5, 0)
#define KVM_REG_MIPS_CP0_PAGEGRAIN MIPS_CP0_32(5, 1)
+#define KVM_REG_MIPS_CP0_SEGCTL0 MIPS_CP0_64(5, 2)
+#define KVM_REG_MIPS_CP0_SEGCTL1 MIPS_CP0_64(5, 3)
+#define KVM_REG_MIPS_CP0_SEGCTL2 MIPS_CP0_64(5, 4)
+#define KVM_REG_MIPS_CP0_PWBASE MIPS_CP0_64(5, 5)
+#define KVM_REG_MIPS_CP0_PWFIELD MIPS_CP0_64(5, 6)
+#define KVM_REG_MIPS_CP0_PWSIZE MIPS_CP0_64(5, 7)
#define KVM_REG_MIPS_CP0_WIRED MIPS_CP0_32(6, 0)
+#define KVM_REG_MIPS_CP0_PWCTL MIPS_CP0_32(6, 6)
#define KVM_REG_MIPS_CP0_HWRENA MIPS_CP0_32(7, 0)
#define KVM_REG_MIPS_CP0_BADVADDR MIPS_CP0_64(8, 0)
+#define KVM_REG_MIPS_CP0_BADINSTR MIPS_CP0_32(8, 1)
+#define KVM_REG_MIPS_CP0_BADINSTRP MIPS_CP0_32(8, 2)
#define KVM_REG_MIPS_CP0_COUNT MIPS_CP0_32(9, 0)
#define KVM_REG_MIPS_CP0_ENTRYHI MIPS_CP0_64(10, 0)
#define KVM_REG_MIPS_CP0_COMPARE MIPS_CP0_32(11, 0)
@@ -55,6 +67,7 @@
#define KVM_REG_MIPS_CP0_CONFIG4 MIPS_CP0_32(16, 4)
#define KVM_REG_MIPS_CP0_CONFIG5 MIPS_CP0_32(16, 5)
#define KVM_REG_MIPS_CP0_CONFIG7 MIPS_CP0_32(16, 7)
+#define KVM_REG_MIPS_CP0_MAARI MIPS_CP0_64(17, 2)
#define KVM_REG_MIPS_CP0_XCONTEXT MIPS_CP0_64(20, 0)
#define KVM_REG_MIPS_CP0_ERROREPC MIPS_CP0_64(30, 0)
#define KVM_REG_MIPS_CP0_KSCRATCH1 MIPS_CP0_64(31, 2)
@@ -70,9 +83,13 @@
/* memory slots that does not exposed to userspace */
#define KVM_PRIVATE_MEM_SLOTS 0
-#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
#define KVM_HALT_POLL_NS_DEFAULT 500000
+#ifdef CONFIG_KVM_MIPS_VZ
+extern unsigned long GUESTID_MASK;
+extern unsigned long GUESTID_FIRST_VERSION;
+extern unsigned long GUESTID_VERSION_MASK;
+#endif
/*
@@ -145,6 +162,16 @@ struct kvm_vcpu_stat {
u64 fpe_exits;
u64 msa_disabled_exits;
u64 flush_dcache_exits;
+#ifdef CONFIG_KVM_MIPS_VZ
+ u64 vz_gpsi_exits;
+ u64 vz_gsfc_exits;
+ u64 vz_hc_exits;
+ u64 vz_grr_exits;
+ u64 vz_gva_exits;
+ u64 vz_ghfc_exits;
+ u64 vz_gpa_exits;
+ u64 vz_resvd_exits;
+#endif
u64 halt_successful_poll;
u64 halt_attempted_poll;
u64 halt_poll_invalid;
@@ -157,6 +184,8 @@ struct kvm_arch_memory_slot {
struct kvm_arch {
/* Guest physical mm */
struct mm_struct gpa_mm;
+ /* Mask of CPUs needing GPA ASID flush */
+ cpumask_t asid_flush_mask;
};
#define N_MIPS_COPROC_REGS 32
@@ -214,6 +243,11 @@ struct mips_coproc {
#define MIPS_CP0_CONFIG4_SEL 4
#define MIPS_CP0_CONFIG5_SEL 5
+#define MIPS_CP0_GUESTCTL2 10
+#define MIPS_CP0_GUESTCTL2_SEL 5
+#define MIPS_CP0_GTOFFSET 12
+#define MIPS_CP0_GTOFFSET_SEL 7
+
/* Resume Flags */
#define RESUME_FLAG_DR (1<<0) /* Reload guest nonvolatile state? */
#define RESUME_FLAG_HOST (1<<1) /* Resume host? */
@@ -229,6 +263,7 @@ enum emulation_result {
EMULATE_WAIT, /* WAIT instruction */
EMULATE_PRIV_FAIL,
EMULATE_EXCEPT, /* A guest exception has been generated */
+ EMULATE_HYPERCALL, /* HYPCALL instruction */
};
#define mips3_paddr_to_tlbpfn(x) \
@@ -276,13 +311,18 @@ struct kvm_mmu_memory_cache {
struct kvm_vcpu_arch {
void *guest_ebase;
int (*vcpu_run)(struct kvm_run *run, struct kvm_vcpu *vcpu);
+
+ /* Host registers preserved across guest mode execution */
unsigned long host_stack;
unsigned long host_gp;
+ unsigned long host_pgd;
+ unsigned long host_entryhi;
/* Host CP0 registers used when handling exits from guest */
unsigned long host_cp0_badvaddr;
unsigned long host_cp0_epc;
u32 host_cp0_cause;
+ u32 host_cp0_guestctl0;
u32 host_cp0_badinstr;
u32 host_cp0_badinstrp;
@@ -340,7 +380,23 @@ struct kvm_vcpu_arch {
/* Cache some mmu pages needed inside spinlock regions */
struct kvm_mmu_memory_cache mmu_page_cache;
+#ifdef CONFIG_KVM_MIPS_VZ
+ /* vcpu's vzguestid is different on each host cpu in an smp system */
+ u32 vzguestid[NR_CPUS];
+
+ /* wired guest TLB entries */
+ struct kvm_mips_tlb *wired_tlb;
+ unsigned int wired_tlb_limit;
+ unsigned int wired_tlb_used;
+
+ /* emulated guest MAAR registers */
+ unsigned long maar[6];
+#endif
+
+ /* Last CPU the VCPU state was loaded on */
int last_sched_cpu;
+ /* Last CPU the VCPU actually executed guest code on */
+ int last_exec_cpu;
/* WAIT executed */
int wait;
@@ -349,78 +405,6 @@ struct kvm_vcpu_arch {
u8 msa_enabled;
};
-
-#define kvm_read_c0_guest_index(cop0) (cop0->reg[MIPS_CP0_TLB_INDEX][0])
-#define kvm_write_c0_guest_index(cop0, val) (cop0->reg[MIPS_CP0_TLB_INDEX][0] = val)
-#define kvm_read_c0_guest_entrylo0(cop0) (cop0->reg[MIPS_CP0_TLB_LO0][0])
-#define kvm_write_c0_guest_entrylo0(cop0, val) (cop0->reg[MIPS_CP0_TLB_LO0][0] = (val))
-#define kvm_read_c0_guest_entrylo1(cop0) (cop0->reg[MIPS_CP0_TLB_LO1][0])
-#define kvm_write_c0_guest_entrylo1(cop0, val) (cop0->reg[MIPS_CP0_TLB_LO1][0] = (val))
-#define kvm_read_c0_guest_context(cop0) (cop0->reg[MIPS_CP0_TLB_CONTEXT][0])
-#define kvm_write_c0_guest_context(cop0, val) (cop0->reg[MIPS_CP0_TLB_CONTEXT][0] = (val))
-#define kvm_read_c0_guest_userlocal(cop0) (cop0->reg[MIPS_CP0_TLB_CONTEXT][2])
-#define kvm_write_c0_guest_userlocal(cop0, val) (cop0->reg[MIPS_CP0_TLB_CONTEXT][2] = (val))
-#define kvm_read_c0_guest_pagemask(cop0) (cop0->reg[MIPS_CP0_TLB_PG_MASK][0])
-#define kvm_write_c0_guest_pagemask(cop0, val) (cop0->reg[MIPS_CP0_TLB_PG_MASK][0] = (val))
-#define kvm_read_c0_guest_wired(cop0) (cop0->reg[MIPS_CP0_TLB_WIRED][0])
-#define kvm_write_c0_guest_wired(cop0, val) (cop0->reg[MIPS_CP0_TLB_WIRED][0] = (val))
-#define kvm_read_c0_guest_hwrena(cop0) (cop0->reg[MIPS_CP0_HWRENA][0])
-#define kvm_write_c0_guest_hwrena(cop0, val) (cop0->reg[MIPS_CP0_HWRENA][0] = (val))
-#define kvm_read_c0_guest_badvaddr(cop0) (cop0->reg[MIPS_CP0_BAD_VADDR][0])
-#define kvm_write_c0_guest_badvaddr(cop0, val) (cop0->reg[MIPS_CP0_BAD_VADDR][0] = (val))
-#define kvm_read_c0_guest_count(cop0) (cop0->reg[MIPS_CP0_COUNT][0])
-#define kvm_write_c0_guest_count(cop0, val) (cop0->reg[MIPS_CP0_COUNT][0] = (val))
-#define kvm_read_c0_guest_entryhi(cop0) (cop0->reg[MIPS_CP0_TLB_HI][0])
-#define kvm_write_c0_guest_entryhi(cop0, val) (cop0->reg[MIPS_CP0_TLB_HI][0] = (val))
-#define kvm_read_c0_guest_compare(cop0) (cop0->reg[MIPS_CP0_COMPARE][0])
-#define kvm_write_c0_guest_compare(cop0, val) (cop0->reg[MIPS_CP0_COMPARE][0] = (val))
-#define kvm_read_c0_guest_status(cop0) (cop0->reg[MIPS_CP0_STATUS][0])
-#define kvm_write_c0_guest_status(cop0, val) (cop0->reg[MIPS_CP0_STATUS][0] = (val))
-#define kvm_read_c0_guest_intctl(cop0) (cop0->reg[MIPS_CP0_STATUS][1])
-#define kvm_write_c0_guest_intctl(cop0, val) (cop0->reg[MIPS_CP0_STATUS][1] = (val))
-#define kvm_read_c0_guest_cause(cop0) (cop0->reg[MIPS_CP0_CAUSE][0])
-#define kvm_write_c0_guest_cause(cop0, val) (cop0->reg[MIPS_CP0_CAUSE][0] = (val))
-#define kvm_read_c0_guest_epc(cop0) (cop0->reg[MIPS_CP0_EXC_PC][0])
-#define kvm_write_c0_guest_epc(cop0, val) (cop0->reg[MIPS_CP0_EXC_PC][0] = (val))
-#define kvm_read_c0_guest_prid(cop0) (cop0->reg[MIPS_CP0_PRID][0])
-#define kvm_write_c0_guest_prid(cop0, val) (cop0->reg[MIPS_CP0_PRID][0] = (val))
-#define kvm_read_c0_guest_ebase(cop0) (cop0->reg[MIPS_CP0_PRID][1])
-#define kvm_write_c0_guest_ebase(cop0, val) (cop0->reg[MIPS_CP0_PRID][1] = (val))
-#define kvm_read_c0_guest_config(cop0) (cop0->reg[MIPS_CP0_CONFIG][0])
-#define kvm_read_c0_guest_config1(cop0) (cop0->reg[MIPS_CP0_CONFIG][1])
-#define kvm_read_c0_guest_config2(cop0) (cop0->reg[MIPS_CP0_CONFIG][2])
-#define kvm_read_c0_guest_config3(cop0) (cop0->reg[MIPS_CP0_CONFIG][3])
-#define kvm_read_c0_guest_config4(cop0) (cop0->reg[MIPS_CP0_CONFIG][4])
-#define kvm_read_c0_guest_config5(cop0) (cop0->reg[MIPS_CP0_CONFIG][5])
-#define kvm_read_c0_guest_config7(cop0) (cop0->reg[MIPS_CP0_CONFIG][7])
-#define kvm_write_c0_guest_config(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][0] = (val))
-#define kvm_write_c0_guest_config1(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][1] = (val))
-#define kvm_write_c0_guest_config2(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][2] = (val))
-#define kvm_write_c0_guest_config3(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][3] = (val))
-#define kvm_write_c0_guest_config4(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][4] = (val))
-#define kvm_write_c0_guest_config5(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][5] = (val))
-#define kvm_write_c0_guest_config7(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][7] = (val))
-#define kvm_read_c0_guest_errorepc(cop0) (cop0->reg[MIPS_CP0_ERROR_PC][0])
-#define kvm_write_c0_guest_errorepc(cop0, val) (cop0->reg[MIPS_CP0_ERROR_PC][0] = (val))
-#define kvm_read_c0_guest_kscratch1(cop0) (cop0->reg[MIPS_CP0_DESAVE][2])
-#define kvm_read_c0_guest_kscratch2(cop0) (cop0->reg[MIPS_CP0_DESAVE][3])
-#define kvm_read_c0_guest_kscratch3(cop0) (cop0->reg[MIPS_CP0_DESAVE][4])
-#define kvm_read_c0_guest_kscratch4(cop0) (cop0->reg[MIPS_CP0_DESAVE][5])
-#define kvm_read_c0_guest_kscratch5(cop0) (cop0->reg[MIPS_CP0_DESAVE][6])
-#define kvm_read_c0_guest_kscratch6(cop0) (cop0->reg[MIPS_CP0_DESAVE][7])
-#define kvm_write_c0_guest_kscratch1(cop0, val) (cop0->reg[MIPS_CP0_DESAVE][2] = (val))
-#define kvm_write_c0_guest_kscratch2(cop0, val) (cop0->reg[MIPS_CP0_DESAVE][3] = (val))
-#define kvm_write_c0_guest_kscratch3(cop0, val) (cop0->reg[MIPS_CP0_DESAVE][4] = (val))
-#define kvm_write_c0_guest_kscratch4(cop0, val) (cop0->reg[MIPS_CP0_DESAVE][5] = (val))
-#define kvm_write_c0_guest_kscratch5(cop0, val) (cop0->reg[MIPS_CP0_DESAVE][6] = (val))
-#define kvm_write_c0_guest_kscratch6(cop0, val) (cop0->reg[MIPS_CP0_DESAVE][7] = (val))
-
-/*
- * Some of the guest registers may be modified asynchronously (e.g. from a
- * hrtimer callback in hard irq context) and therefore need stronger atomicity
- * guarantees than other registers.
- */
-
static inline void _kvm_atomic_set_c0_guest_reg(unsigned long *reg,
unsigned long val)
{
@@ -471,26 +455,286 @@ static inline void _kvm_atomic_change_c0_guest_reg(unsigned long *reg,
} while (unlikely(!temp));
}
-#define kvm_set_c0_guest_status(cop0, val) (cop0->reg[MIPS_CP0_STATUS][0] |= (val))
-#define kvm_clear_c0_guest_status(cop0, val) (cop0->reg[MIPS_CP0_STATUS][0] &= ~(val))
+/* Guest register types, used in accessor build below */
+#define __KVMT32 u32
+#define __KVMTl unsigned long
-/* Cause can be modified asynchronously from hardirq hrtimer callback */
-#define kvm_set_c0_guest_cause(cop0, val) \
- _kvm_atomic_set_c0_guest_reg(&cop0->reg[MIPS_CP0_CAUSE][0], val)
-#define kvm_clear_c0_guest_cause(cop0, val) \
- _kvm_atomic_clear_c0_guest_reg(&cop0->reg[MIPS_CP0_CAUSE][0], val)
-#define kvm_change_c0_guest_cause(cop0, change, val) \
- _kvm_atomic_change_c0_guest_reg(&cop0->reg[MIPS_CP0_CAUSE][0], \
- change, val)
-
-#define kvm_set_c0_guest_ebase(cop0, val) (cop0->reg[MIPS_CP0_PRID][1] |= (val))
-#define kvm_clear_c0_guest_ebase(cop0, val) (cop0->reg[MIPS_CP0_PRID][1] &= ~(val))
-#define kvm_change_c0_guest_ebase(cop0, change, val) \
+/*
+ * __BUILD_KVM_$ops_SAVED(): kvm_$op_sw_gc0_$reg()
+ * These operate on the saved guest C0 state in RAM.
+ */
+
+/* Generate saved context simple accessors */
+#define __BUILD_KVM_RW_SAVED(name, type, _reg, sel) \
+static inline __KVMT##type kvm_read_sw_gc0_##name(struct mips_coproc *cop0) \
+{ \
+ return cop0->reg[(_reg)][(sel)]; \
+} \
+static inline void kvm_write_sw_gc0_##name(struct mips_coproc *cop0, \
+ __KVMT##type val) \
+{ \
+ cop0->reg[(_reg)][(sel)] = val; \
+}
+
+/* Generate saved context bitwise modifiers */
+#define __BUILD_KVM_SET_SAVED(name, type, _reg, sel) \
+static inline void kvm_set_sw_gc0_##name(struct mips_coproc *cop0, \
+ __KVMT##type val) \
+{ \
+ cop0->reg[(_reg)][(sel)] |= val; \
+} \
+static inline void kvm_clear_sw_gc0_##name(struct mips_coproc *cop0, \
+ __KVMT##type val) \
+{ \
+ cop0->reg[(_reg)][(sel)] &= ~val; \
+} \
+static inline void kvm_change_sw_gc0_##name(struct mips_coproc *cop0, \
+ __KVMT##type mask, \
+ __KVMT##type val) \
+{ \
+ unsigned long _mask = mask; \
+ cop0->reg[(_reg)][(sel)] &= ~_mask; \
+ cop0->reg[(_reg)][(sel)] |= val & _mask; \
+}
+
+/* Generate saved context atomic bitwise modifiers */
+#define __BUILD_KVM_ATOMIC_SAVED(name, type, _reg, sel) \
+static inline void kvm_set_sw_gc0_##name(struct mips_coproc *cop0, \
+ __KVMT##type val) \
+{ \
+ _kvm_atomic_set_c0_guest_reg(&cop0->reg[(_reg)][(sel)], val); \
+} \
+static inline void kvm_clear_sw_gc0_##name(struct mips_coproc *cop0, \
+ __KVMT##type val) \
+{ \
+ _kvm_atomic_clear_c0_guest_reg(&cop0->reg[(_reg)][(sel)], val); \
+} \
+static inline void kvm_change_sw_gc0_##name(struct mips_coproc *cop0, \
+ __KVMT##type mask, \
+ __KVMT##type val) \
+{ \
+ _kvm_atomic_change_c0_guest_reg(&cop0->reg[(_reg)][(sel)], mask, \
+ val); \
+}
+
+/*
+ * __BUILD_KVM_$ops_VZ(): kvm_$op_vz_gc0_$reg()
+ * These operate on the VZ guest C0 context in hardware.
+ */
+
+/* Generate VZ guest context simple accessors */
+#define __BUILD_KVM_RW_VZ(name, type, _reg, sel) \
+static inline __KVMT##type kvm_read_vz_gc0_##name(struct mips_coproc *cop0) \
+{ \
+ return read_gc0_##name(); \
+} \
+static inline void kvm_write_vz_gc0_##name(struct mips_coproc *cop0, \
+ __KVMT##type val) \
+{ \
+ write_gc0_##name(val); \
+}
+
+/* Generate VZ guest context bitwise modifiers */
+#define __BUILD_KVM_SET_VZ(name, type, _reg, sel) \
+static inline void kvm_set_vz_gc0_##name(struct mips_coproc *cop0, \
+ __KVMT##type val) \
+{ \
+ set_gc0_##name(val); \
+} \
+static inline void kvm_clear_vz_gc0_##name(struct mips_coproc *cop0, \
+ __KVMT##type val) \
+{ \
+ clear_gc0_##name(val); \
+} \
+static inline void kvm_change_vz_gc0_##name(struct mips_coproc *cop0, \
+ __KVMT##type mask, \
+ __KVMT##type val) \
+{ \
+ change_gc0_##name(mask, val); \
+}
+
+/* Generate VZ guest context save/restore to/from saved context */
+#define __BUILD_KVM_SAVE_VZ(name, _reg, sel) \
+static inline void kvm_restore_gc0_##name(struct mips_coproc *cop0) \
+{ \
+ write_gc0_##name(cop0->reg[(_reg)][(sel)]); \
+} \
+static inline void kvm_save_gc0_##name(struct mips_coproc *cop0) \
+{ \
+ cop0->reg[(_reg)][(sel)] = read_gc0_##name(); \
+}
+
+/*
+ * __BUILD_KVM_$ops_WRAP(): kvm_$op_$name1() -> kvm_$op_$name2()
+ * These wrap a set of operations to provide them with a different name.
+ */
+
+/* Generate simple accessor wrapper */
+#define __BUILD_KVM_RW_WRAP(name1, name2, type) \
+static inline __KVMT##type kvm_read_##name1(struct mips_coproc *cop0) \
+{ \
+ return kvm_read_##name2(cop0); \
+} \
+static inline void kvm_write_##name1(struct mips_coproc *cop0, \
+ __KVMT##type val) \
+{ \
+ kvm_write_##name2(cop0, val); \
+}
+
+/* Generate bitwise modifier wrapper */
+#define __BUILD_KVM_SET_WRAP(name1, name2, type) \
+static inline void kvm_set_##name1(struct mips_coproc *cop0, \
+ __KVMT##type val) \
{ \
- kvm_clear_c0_guest_ebase(cop0, change); \
- kvm_set_c0_guest_ebase(cop0, ((val) & (change))); \
+ kvm_set_##name2(cop0, val); \
+} \
+static inline void kvm_clear_##name1(struct mips_coproc *cop0, \
+ __KVMT##type val) \
+{ \
+ kvm_clear_##name2(cop0, val); \
+} \
+static inline void kvm_change_##name1(struct mips_coproc *cop0, \
+ __KVMT##type mask, \
+ __KVMT##type val) \
+{ \
+ kvm_change_##name2(cop0, mask, val); \
}
+/*
+ * __BUILD_KVM_$ops_SW(): kvm_$op_c0_guest_$reg() -> kvm_$op_sw_gc0_$reg()
+ * These generate accessors operating on the saved context in RAM, and wrap them
+ * with the common guest C0 accessors (for use by common emulation code).
+ */
+
+#define __BUILD_KVM_RW_SW(name, type, _reg, sel) \
+ __BUILD_KVM_RW_SAVED(name, type, _reg, sel) \
+ __BUILD_KVM_RW_WRAP(c0_guest_##name, sw_gc0_##name, type)
+
+#define __BUILD_KVM_SET_SW(name, type, _reg, sel) \
+ __BUILD_KVM_SET_SAVED(name, type, _reg, sel) \
+ __BUILD_KVM_SET_WRAP(c0_guest_##name, sw_gc0_##name, type)
+
+#define __BUILD_KVM_ATOMIC_SW(name, type, _reg, sel) \
+ __BUILD_KVM_ATOMIC_SAVED(name, type, _reg, sel) \
+ __BUILD_KVM_SET_WRAP(c0_guest_##name, sw_gc0_##name, type)
+
+#ifndef CONFIG_KVM_MIPS_VZ
+
+/*
+ * T&E (trap & emulate software based virtualisation)
+ * We generate the common accessors operating exclusively on the saved context
+ * in RAM.
+ */
+
+#define __BUILD_KVM_RW_HW __BUILD_KVM_RW_SW
+#define __BUILD_KVM_SET_HW __BUILD_KVM_SET_SW
+#define __BUILD_KVM_ATOMIC_HW __BUILD_KVM_ATOMIC_SW
+
+#else
+
+/*
+ * VZ (hardware assisted virtualisation)
+ * These macros use the active guest state in VZ mode (hardware registers),
+ */
+
+/*
+ * __BUILD_KVM_$ops_HW(): kvm_$op_c0_guest_$reg() -> kvm_$op_vz_gc0_$reg()
+ * These generate accessors operating on the VZ guest context in hardware, and
+ * wrap them with the common guest C0 accessors (for use by common emulation
+ * code).
+ *
+ * Accessors operating on the saved context in RAM are also generated to allow
+ * convenient explicit saving and restoring of the state.
+ */
+
+#define __BUILD_KVM_RW_HW(name, type, _reg, sel) \
+ __BUILD_KVM_RW_SAVED(name, type, _reg, sel) \
+ __BUILD_KVM_RW_VZ(name, type, _reg, sel) \
+ __BUILD_KVM_RW_WRAP(c0_guest_##name, vz_gc0_##name, type) \
+ __BUILD_KVM_SAVE_VZ(name, _reg, sel)
+
+#define __BUILD_KVM_SET_HW(name, type, _reg, sel) \
+ __BUILD_KVM_SET_SAVED(name, type, _reg, sel) \
+ __BUILD_KVM_SET_VZ(name, type, _reg, sel) \
+ __BUILD_KVM_SET_WRAP(c0_guest_##name, vz_gc0_##name, type)
+
+/*
+ * We can't do atomic modifications of COP0 state if hardware can modify it.
+ * Races must be handled explicitly.
+ */
+#define __BUILD_KVM_ATOMIC_HW __BUILD_KVM_SET_HW
+
+#endif
+
+/*
+ * Define accessors for CP0 registers that are accessible to the guest. These
+ * are primarily used by common emulation code, which may need to access the
+ * registers differently depending on the implementation.
+ *
+ * fns_hw/sw name type reg num select
+ */
+__BUILD_KVM_RW_HW(index, 32, MIPS_CP0_TLB_INDEX, 0)
+__BUILD_KVM_RW_HW(entrylo0, l, MIPS_CP0_TLB_LO0, 0)
+__BUILD_KVM_RW_HW(entrylo1, l, MIPS_CP0_TLB_LO1, 0)
+__BUILD_KVM_RW_HW(context, l, MIPS_CP0_TLB_CONTEXT, 0)
+__BUILD_KVM_RW_HW(contextconfig, 32, MIPS_CP0_TLB_CONTEXT, 1)
+__BUILD_KVM_RW_HW(userlocal, l, MIPS_CP0_TLB_CONTEXT, 2)
+__BUILD_KVM_RW_HW(xcontextconfig, l, MIPS_CP0_TLB_CONTEXT, 3)
+__BUILD_KVM_RW_HW(pagemask, l, MIPS_CP0_TLB_PG_MASK, 0)
+__BUILD_KVM_RW_HW(pagegrain, 32, MIPS_CP0_TLB_PG_MASK, 1)
+__BUILD_KVM_RW_HW(segctl0, l, MIPS_CP0_TLB_PG_MASK, 2)
+__BUILD_KVM_RW_HW(segctl1, l, MIPS_CP0_TLB_PG_MASK, 3)
+__BUILD_KVM_RW_HW(segctl2, l, MIPS_CP0_TLB_PG_MASK, 4)
+__BUILD_KVM_RW_HW(pwbase, l, MIPS_CP0_TLB_PG_MASK, 5)
+__BUILD_KVM_RW_HW(pwfield, l, MIPS_CP0_TLB_PG_MASK, 6)
+__BUILD_KVM_RW_HW(pwsize, l, MIPS_CP0_TLB_PG_MASK, 7)
+__BUILD_KVM_RW_HW(wired, 32, MIPS_CP0_TLB_WIRED, 0)
+__BUILD_KVM_RW_HW(pwctl, 32, MIPS_CP0_TLB_WIRED, 6)
+__BUILD_KVM_RW_HW(hwrena, 32, MIPS_CP0_HWRENA, 0)
+__BUILD_KVM_RW_HW(badvaddr, l, MIPS_CP0_BAD_VADDR, 0)
+__BUILD_KVM_RW_HW(badinstr, 32, MIPS_CP0_BAD_VADDR, 1)
+__BUILD_KVM_RW_HW(badinstrp, 32, MIPS_CP0_BAD_VADDR, 2)
+__BUILD_KVM_RW_SW(count, 32, MIPS_CP0_COUNT, 0)
+__BUILD_KVM_RW_HW(entryhi, l, MIPS_CP0_TLB_HI, 0)
+__BUILD_KVM_RW_HW(compare, 32, MIPS_CP0_COMPARE, 0)
+__BUILD_KVM_RW_HW(status, 32, MIPS_CP0_STATUS, 0)
+__BUILD_KVM_RW_HW(intctl, 32, MIPS_CP0_STATUS, 1)
+__BUILD_KVM_RW_HW(cause, 32, MIPS_CP0_CAUSE, 0)
+__BUILD_KVM_RW_HW(epc, l, MIPS_CP0_EXC_PC, 0)
+__BUILD_KVM_RW_SW(prid, 32, MIPS_CP0_PRID, 0)
+__BUILD_KVM_RW_HW(ebase, l, MIPS_CP0_PRID, 1)
+__BUILD_KVM_RW_HW(config, 32, MIPS_CP0_CONFIG, 0)
+__BUILD_KVM_RW_HW(config1, 32, MIPS_CP0_CONFIG, 1)
+__BUILD_KVM_RW_HW(config2, 32, MIPS_CP0_CONFIG, 2)
+__BUILD_KVM_RW_HW(config3, 32, MIPS_CP0_CONFIG, 3)
+__BUILD_KVM_RW_HW(config4, 32, MIPS_CP0_CONFIG, 4)
+__BUILD_KVM_RW_HW(config5, 32, MIPS_CP0_CONFIG, 5)
+__BUILD_KVM_RW_HW(config6, 32, MIPS_CP0_CONFIG, 6)
+__BUILD_KVM_RW_HW(config7, 32, MIPS_CP0_CONFIG, 7)
+__BUILD_KVM_RW_SW(maari, l, MIPS_CP0_LLADDR, 2)
+__BUILD_KVM_RW_HW(xcontext, l, MIPS_CP0_TLB_XCONTEXT, 0)
+__BUILD_KVM_RW_HW(errorepc, l, MIPS_CP0_ERROR_PC, 0)
+__BUILD_KVM_RW_HW(kscratch1, l, MIPS_CP0_DESAVE, 2)
+__BUILD_KVM_RW_HW(kscratch2, l, MIPS_CP0_DESAVE, 3)
+__BUILD_KVM_RW_HW(kscratch3, l, MIPS_CP0_DESAVE, 4)
+__BUILD_KVM_RW_HW(kscratch4, l, MIPS_CP0_DESAVE, 5)
+__BUILD_KVM_RW_HW(kscratch5, l, MIPS_CP0_DESAVE, 6)
+__BUILD_KVM_RW_HW(kscratch6, l, MIPS_CP0_DESAVE, 7)
+
+/* Bitwise operations (on HW state) */
+__BUILD_KVM_SET_HW(status, 32, MIPS_CP0_STATUS, 0)
+/* Cause can be modified asynchronously from hardirq hrtimer callback */
+__BUILD_KVM_ATOMIC_HW(cause, 32, MIPS_CP0_CAUSE, 0)
+__BUILD_KVM_SET_HW(ebase, l, MIPS_CP0_PRID, 1)
+
+/* Bitwise operations (on saved state) */
+__BUILD_KVM_SET_SAVED(config, 32, MIPS_CP0_CONFIG, 0)
+__BUILD_KVM_SET_SAVED(config1, 32, MIPS_CP0_CONFIG, 1)
+__BUILD_KVM_SET_SAVED(config2, 32, MIPS_CP0_CONFIG, 2)
+__BUILD_KVM_SET_SAVED(config3, 32, MIPS_CP0_CONFIG, 3)
+__BUILD_KVM_SET_SAVED(config4, 32, MIPS_CP0_CONFIG, 4)
+__BUILD_KVM_SET_SAVED(config5, 32, MIPS_CP0_CONFIG, 5)
+
/* Helpers */
static inline bool kvm_mips_guest_can_have_fpu(struct kvm_vcpu_arch *vcpu)
@@ -531,6 +775,10 @@ struct kvm_mips_callbacks {
int (*handle_msa_fpe)(struct kvm_vcpu *vcpu);
int (*handle_fpe)(struct kvm_vcpu *vcpu);
int (*handle_msa_disabled)(struct kvm_vcpu *vcpu);
+ int (*handle_guest_exit)(struct kvm_vcpu *vcpu);
+ int (*hardware_enable)(void);
+ void (*hardware_disable)(void);
+ int (*check_extension)(struct kvm *kvm, long ext);
int (*vcpu_init)(struct kvm_vcpu *vcpu);
void (*vcpu_uninit)(struct kvm_vcpu *vcpu);
int (*vcpu_setup)(struct kvm_vcpu *vcpu);
@@ -599,6 +847,10 @@ u32 kvm_get_user_asid(struct kvm_vcpu *vcpu);
u32 kvm_get_commpage_asid (struct kvm_vcpu *vcpu);
+#ifdef CONFIG_KVM_MIPS_VZ
+int kvm_mips_handle_vz_root_tlb_fault(unsigned long badvaddr,
+ struct kvm_vcpu *vcpu, bool write_fault);
+#endif
extern int kvm_mips_handle_kseg0_tlb_fault(unsigned long badbaddr,
struct kvm_vcpu *vcpu,
bool write_fault);
@@ -625,6 +877,18 @@ extern int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long entryhi,
extern int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu,
unsigned long entryhi);
+#ifdef CONFIG_KVM_MIPS_VZ
+int kvm_vz_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long entryhi);
+int kvm_vz_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long gva,
+ unsigned long *gpa);
+void kvm_vz_local_flush_roottlb_all_guests(void);
+void kvm_vz_local_flush_guesttlb_all(void);
+void kvm_vz_save_guesttlb(struct kvm_mips_tlb *buf, unsigned int index,
+ unsigned int count);
+void kvm_vz_load_guesttlb(const struct kvm_mips_tlb *buf, unsigned int index,
+ unsigned int count);
+#endif
+
void kvm_mips_suspend_mm(int cpu);
void kvm_mips_resume_mm(int cpu);
@@ -795,7 +1059,7 @@ extern enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu,
u32 kvm_mips_read_count(struct kvm_vcpu *vcpu);
void kvm_mips_write_count(struct kvm_vcpu *vcpu, u32 count);
void kvm_mips_write_compare(struct kvm_vcpu *vcpu, u32 compare, bool ack);
-void kvm_mips_init_count(struct kvm_vcpu *vcpu);
+void kvm_mips_init_count(struct kvm_vcpu *vcpu, unsigned long count_hz);
int kvm_mips_set_count_ctl(struct kvm_vcpu *vcpu, s64 count_ctl);
int kvm_mips_set_count_resume(struct kvm_vcpu *vcpu, s64 count_resume);
int kvm_mips_set_count_hz(struct kvm_vcpu *vcpu, s64 count_hz);
@@ -803,6 +1067,20 @@ void kvm_mips_count_enable_cause(struct kvm_vcpu *vcpu);
void kvm_mips_count_disable_cause(struct kvm_vcpu *vcpu);
enum hrtimer_restart kvm_mips_count_timeout(struct kvm_vcpu *vcpu);
+/* fairly internal functions requiring some care to use */
+int kvm_mips_count_disabled(struct kvm_vcpu *vcpu);
+ktime_t kvm_mips_freeze_hrtimer(struct kvm_vcpu *vcpu, u32 *count);
+int kvm_mips_restore_hrtimer(struct kvm_vcpu *vcpu, ktime_t before,
+ u32 count, int min_drift);
+
+#ifdef CONFIG_KVM_MIPS_VZ
+void kvm_vz_acquire_htimer(struct kvm_vcpu *vcpu);
+void kvm_vz_lose_htimer(struct kvm_vcpu *vcpu);
+#else
+static inline void kvm_vz_acquire_htimer(struct kvm_vcpu *vcpu) {}
+static inline void kvm_vz_lose_htimer(struct kvm_vcpu *vcpu) {}
+#endif
+
enum emulation_result kvm_mips_check_privilege(u32 cause,
u32 *opc,
struct kvm_run *run,
@@ -827,11 +1105,20 @@ enum emulation_result kvm_mips_emulate_load(union mips_instruction inst,
struct kvm_run *run,
struct kvm_vcpu *vcpu);
+/* COP0 */
+enum emulation_result kvm_mips_emul_wait(struct kvm_vcpu *vcpu);
+
unsigned int kvm_mips_config1_wrmask(struct kvm_vcpu *vcpu);
unsigned int kvm_mips_config3_wrmask(struct kvm_vcpu *vcpu);
unsigned int kvm_mips_config4_wrmask(struct kvm_vcpu *vcpu);
unsigned int kvm_mips_config5_wrmask(struct kvm_vcpu *vcpu);
+/* Hypercalls (hypcall.c) */
+
+enum emulation_result kvm_mips_emul_hypcall(struct kvm_vcpu *vcpu,
+ union mips_instruction inst);
+int kvm_mips_handle_hypcall(struct kvm_vcpu *vcpu);
+
/* Dynamic binary translation */
extern int kvm_mips_trans_cache_index(union mips_instruction inst,
u32 *opc, struct kvm_vcpu *vcpu);
@@ -846,7 +1133,6 @@ extern int kvm_mips_trans_mtc0(union mips_instruction inst, u32 *opc,
extern void kvm_mips_dump_stats(struct kvm_vcpu *vcpu);
extern unsigned long kvm_mips_get_ramsize(struct kvm *kvm);
-static inline void kvm_arch_hardware_disable(void) {}
static inline void kvm_arch_hardware_unsetup(void) {}
static inline void kvm_arch_sync_events(struct kvm *kvm) {}
static inline void kvm_arch_free_memslot(struct kvm *kvm,
diff --git a/arch/mips/include/asm/maar.h b/arch/mips/include/asm/maar.h
index 21d9607c80d7..e10f78befbd9 100644
--- a/arch/mips/include/asm/maar.h
+++ b/arch/mips/include/asm/maar.h
@@ -36,7 +36,7 @@ unsigned platform_maar_init(unsigned num_pairs);
* @upper: The highest address that the MAAR pair will affect. Must be
* aligned to one byte before a 2^16 byte boundary.
* @attrs: The accessibility attributes to program, eg. MIPS_MAAR_S. The
- * MIPS_MAAR_V attribute will automatically be set.
+ * MIPS_MAAR_VL attribute will automatically be set.
*
* Program the pair of MAAR registers specified by idx to apply the attributes
* specified by attrs to the range of addresses from lower to higher.
@@ -49,10 +49,10 @@ static inline void write_maar_pair(unsigned idx, phys_addr_t lower,
BUG_ON(((upper & 0xffff) != 0xffff)
|| ((upper & ~0xffffull) & ~(MIPS_MAAR_ADDR << 4)));
- /* Automatically set MIPS_MAAR_V */
- attrs |= MIPS_MAAR_V;
+ /* Automatically set MIPS_MAAR_VL */
+ attrs |= MIPS_MAAR_VL;
- /* Write the upper address & attributes (only MIPS_MAAR_V matters) */
+ /* Write the upper address & attributes (only MIPS_MAAR_VL matters) */
write_c0_maari(idx << 1);
back_to_back_c0_hazard();
write_c0_maar(((upper >> 4) & MIPS_MAAR_ADDR) | attrs);
@@ -81,7 +81,7 @@ extern void maar_init(void);
* @upper: The highest address that the MAAR pair will affect. Must be
* aligned to one byte before a 2^16 byte boundary.
* @attrs: The accessibility attributes to program, eg. MIPS_MAAR_S. The
- * MIPS_MAAR_V attribute will automatically be set.
+ * MIPS_MAAR_VL attribute will automatically be set.
*
* Describes the configuration of a pair of Memory Accessibility Attribute
* Registers - applying attributes from attrs to the range of physical
diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h
index f8d1d2f1d80d..6875b69f59f7 100644
--- a/arch/mips/include/asm/mipsregs.h
+++ b/arch/mips/include/asm/mipsregs.h
@@ -34,8 +34,10 @@
*/
#ifdef __ASSEMBLY__
#define _ULCAST_
+#define _U64CAST_
#else
#define _ULCAST_ (unsigned long)
+#define _U64CAST_ (u64)
#endif
/*
@@ -217,8 +219,10 @@
/*
* Wired register bits
*/
-#define MIPSR6_WIRED_LIMIT (_ULCAST_(0xffff) << 16)
-#define MIPSR6_WIRED_WIRED (_ULCAST_(0xffff) << 0)
+#define MIPSR6_WIRED_LIMIT_SHIFT 16
+#define MIPSR6_WIRED_LIMIT (_ULCAST_(0xffff) << MIPSR6_WIRED_LIMIT_SHIFT)
+#define MIPSR6_WIRED_WIRED_SHIFT 0
+#define MIPSR6_WIRED_WIRED (_ULCAST_(0xffff) << MIPSR6_WIRED_WIRED_SHIFT)
/*
* Values used for computation of new tlb entries
@@ -645,6 +649,7 @@
#define MIPS_CONF5_LLB (_ULCAST_(1) << 4)
#define MIPS_CONF5_MVH (_ULCAST_(1) << 5)
#define MIPS_CONF5_VP (_ULCAST_(1) << 7)
+#define MIPS_CONF5_SBRI (_ULCAST_(1) << 6)
#define MIPS_CONF5_FRE (_ULCAST_(1) << 8)
#define MIPS_CONF5_UFE (_ULCAST_(1) << 9)
#define MIPS_CONF5_MSAEN (_ULCAST_(1) << 27)
@@ -719,10 +724,14 @@
#define XLR_PERFCTRL_ALLTHREADS (_ULCAST_(1) << 13)
/* MAAR bit definitions */
+#define MIPS_MAAR_VH (_U64CAST_(1) << 63)
#define MIPS_MAAR_ADDR ((BIT_ULL(BITS_PER_LONG - 12) - 1) << 12)
#define MIPS_MAAR_ADDR_SHIFT 12
#define MIPS_MAAR_S (_ULCAST_(1) << 1)
-#define MIPS_MAAR_V (_ULCAST_(1) << 0)
+#define MIPS_MAAR_VL (_ULCAST_(1) << 0)
+
+/* MAARI bit definitions */
+#define MIPS_MAARI_INDEX (_ULCAST_(0x3f) << 0)
/* EBase bit definitions */
#define MIPS_EBASE_CPUNUM_SHIFT 0
@@ -736,6 +745,10 @@
#define MIPS_CMGCRB_BASE 11
#define MIPS_CMGCRF_BASE (~_ULCAST_((1 << MIPS_CMGCRB_BASE) - 1))
+/* LLAddr bit definitions */
+#define MIPS_LLADDR_LLB_SHIFT 0
+#define MIPS_LLADDR_LLB (_ULCAST_(1) << MIPS_LLADDR_LLB_SHIFT)
+
/*
* Bits in the MIPS32 Memory Segmentation registers.
*/
@@ -961,6 +974,22 @@
/* Flush FTLB */
#define LOONGSON_DIAG_FTLB (_ULCAST_(1) << 13)
+/* CvmCtl register field definitions */
+#define CVMCTL_IPPCI_SHIFT 7
+#define CVMCTL_IPPCI (_U64CAST_(0x7) << CVMCTL_IPPCI_SHIFT)
+#define CVMCTL_IPTI_SHIFT 4
+#define CVMCTL_IPTI (_U64CAST_(0x7) << CVMCTL_IPTI_SHIFT)
+
+/* CvmMemCtl2 register field definitions */
+#define CVMMEMCTL2_INHIBITTS (_U64CAST_(1) << 17)
+
+/* CvmVMConfig register field definitions */
+#define CVMVMCONF_DGHT (_U64CAST_(1) << 60)
+#define CVMVMCONF_MMUSIZEM1_S 12
+#define CVMVMCONF_MMUSIZEM1 (_U64CAST_(0xff) << CVMVMCONF_MMUSIZEM1_S)
+#define CVMVMCONF_RMMUSIZEM1_S 0
+#define CVMVMCONF_RMMUSIZEM1 (_U64CAST_(0xff) << CVMVMCONF_RMMUSIZEM1_S)
+
/*
* Coprocessor 1 (FPU) register names
*/
@@ -1720,6 +1749,13 @@ do { \
#define read_c0_cvmmemctl() __read_64bit_c0_register($11, 7)
#define write_c0_cvmmemctl(val) __write_64bit_c0_register($11, 7, val)
+
+#define read_c0_cvmmemctl2() __read_64bit_c0_register($16, 6)
+#define write_c0_cvmmemctl2(val) __write_64bit_c0_register($16, 6, val)
+
+#define read_c0_cvmvmconfig() __read_64bit_c0_register($16, 7)
+#define write_c0_cvmvmconfig(val) __write_64bit_c0_register($16, 7, val)
+
/*
* The cacheerr registers are not standardized. On OCTEON, they are
* 64 bits wide.
@@ -1989,6 +2025,8 @@ do { \
#define read_gc0_epc() __read_ulong_gc0_register(14, 0)
#define write_gc0_epc(val) __write_ulong_gc0_register(14, 0, val)
+#define read_gc0_prid() __read_32bit_gc0_register(15, 0)
+
#define read_gc0_ebase() __read_32bit_gc0_register(15, 1)
#define write_gc0_ebase(val) __write_32bit_gc0_register(15, 1, val)
@@ -2012,6 +2050,9 @@ do { \
#define write_gc0_config6(val) __write_32bit_gc0_register(16, 6, val)
#define write_gc0_config7(val) __write_32bit_gc0_register(16, 7, val)
+#define read_gc0_lladdr() __read_ulong_gc0_register(17, 0)
+#define write_gc0_lladdr(val) __write_ulong_gc0_register(17, 0, val)
+
#define read_gc0_watchlo0() __read_ulong_gc0_register(18, 0)
#define read_gc0_watchlo1() __read_ulong_gc0_register(18, 1)
#define read_gc0_watchlo2() __read_ulong_gc0_register(18, 2)
@@ -2090,6 +2131,19 @@ do { \
#define write_gc0_kscratch5(val) __write_ulong_gc0_register(31, 6, val)
#define write_gc0_kscratch6(val) __write_ulong_gc0_register(31, 7, val)
+/* Cavium OCTEON (cnMIPS) */
+#define read_gc0_cvmcount() __read_ulong_gc0_register(9, 6)
+#define write_gc0_cvmcount(val) __write_ulong_gc0_register(9, 6, val)
+
+#define read_gc0_cvmctl() __read_64bit_gc0_register(9, 7)
+#define write_gc0_cvmctl(val) __write_64bit_gc0_register(9, 7, val)
+
+#define read_gc0_cvmmemctl() __read_64bit_gc0_register(11, 7)
+#define write_gc0_cvmmemctl(val) __write_64bit_gc0_register(11, 7, val)
+
+#define read_gc0_cvmmemctl2() __read_64bit_gc0_register(16, 6)
+#define write_gc0_cvmmemctl2(val) __write_64bit_gc0_register(16, 6, val)
+
/*
* Macros to access the floating point coprocessor control registers
*/
@@ -2696,9 +2750,11 @@ __BUILD_SET_C0(brcm_mode)
*/
#define __BUILD_SET_GC0(name) __BUILD_SET_COMMON(gc0_##name)
+__BUILD_SET_GC0(wired)
__BUILD_SET_GC0(status)
__BUILD_SET_GC0(cause)
__BUILD_SET_GC0(ebase)
+__BUILD_SET_GC0(config1)
/*
* Return low 10 bits of ebase.
diff --git a/arch/mips/include/asm/octeon/cvmx-helper-rgmii.h b/arch/mips/include/asm/octeon/cvmx-helper-rgmii.h
index f89775be7654..f7a95d7de140 100644
--- a/arch/mips/include/asm/octeon/cvmx-helper-rgmii.h
+++ b/arch/mips/include/asm/octeon/cvmx-helper-rgmii.h
@@ -55,7 +55,7 @@ extern int __cvmx_helper_rgmii_probe(int interface);
extern void cvmx_helper_rgmii_internal_loopback(int port);
/**
- * Configure all of the ASX, GMX, and PKO regsiters required
+ * Configure all of the ASX, GMX, and PKO registers required
* to get RGMII to function on the supplied interface.
*
* @interface: PKO Interface to configure (0 or 1)
diff --git a/arch/mips/include/asm/pci.h b/arch/mips/include/asm/pci.h
index 30d1129d8624..1000c1b4c875 100644
--- a/arch/mips/include/asm/pci.h
+++ b/arch/mips/include/asm/pci.h
@@ -110,10 +110,7 @@ extern unsigned long PCIBIOS_MIN_MEM;
extern void pcibios_set_master(struct pci_dev *dev);
#define HAVE_PCI_MMAP
-
-extern int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma,
- enum pci_mmap_state mmap_state, int write_combine);
-
+#define ARCH_GENERIC_PCI_MMAP_RESOURCE
#define HAVE_ARCH_PCI_RESOURCE_TO_USER
/*
diff --git a/arch/mips/include/asm/r4kcache.h b/arch/mips/include/asm/r4kcache.h
index 55fd94e6cd0b..7f12d7e27c94 100644
--- a/arch/mips/include/asm/r4kcache.h
+++ b/arch/mips/include/asm/r4kcache.h
@@ -20,7 +20,7 @@
#include <asm/cpu-features.h>
#include <asm/cpu-type.h>
#include <asm/mipsmtregs.h>
-#include <linux/uaccess.h> /* for segment_eq() */
+#include <linux/uaccess.h> /* for uaccess_kernel() */
extern void (*r4k_blast_dcache)(void);
extern void (*r4k_blast_icache)(void);
@@ -714,7 +714,7 @@ static inline void protected_blast_##pfx##cache##_range(unsigned long start,\
\
__##pfx##flush_prologue \
\
- if (segment_eq(get_fs(), USER_DS)) { \
+ if (!uaccess_kernel()) { \
while (1) { \
protected_cachee_op(hitop, addr); \
if (addr == aend) \
diff --git a/arch/mips/include/asm/tlb.h b/arch/mips/include/asm/tlb.h
index dd179fd8acda..939734de4359 100644
--- a/arch/mips/include/asm/tlb.h
+++ b/arch/mips/include/asm/tlb.h
@@ -21,9 +21,11 @@
*/
#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
-#define UNIQUE_ENTRYHI(idx) \
- ((CKSEG0 + ((idx) << (PAGE_SHIFT + 1))) | \
+#define _UNIQUE_ENTRYHI(base, idx) \
+ (((base) + ((idx) << (PAGE_SHIFT + 1))) | \
(cpu_has_tlbinv ? MIPS_ENTRYHI_EHINV : 0))
+#define UNIQUE_ENTRYHI(idx) _UNIQUE_ENTRYHI(CKSEG0, idx)
+#define UNIQUE_GUEST_ENTRYHI(idx) _UNIQUE_ENTRYHI(CKSEG1, idx)
static inline unsigned int num_wired_entries(void)
{
diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h
index 5347cfe15af2..99e629a590a5 100644
--- a/arch/mips/include/asm/uaccess.h
+++ b/arch/mips/include/asm/uaccess.h
@@ -12,8 +12,6 @@
#define _ASM_UACCESS_H
#include <linux/kernel.h>
-#include <linux/errno.h>
-#include <linux/thread_info.h>
#include <linux/string.h>
#include <asm/asm-eva.h>
#include <asm/extable.h>
@@ -71,9 +69,6 @@ extern u64 __ua_limit;
#define USER_DS ((mm_segment_t) { __UA_LIMIT })
#endif
-#define VERIFY_READ 0
-#define VERIFY_WRITE 1
-
#define get_ds() (KERNEL_DS)
#define get_fs() (current_thread_info()->addr_limit)
#define set_fs(x) (current_thread_info()->addr_limit = (x))
@@ -93,7 +88,7 @@ static inline bool eva_kernel_access(void)
if (!IS_ENABLED(CONFIG_EVA))
return false;
- return segment_eq(get_fs(), get_ds());
+ return uaccess_kernel();
}
/*
@@ -133,23 +128,14 @@ static inline bool eva_kernel_access(void)
* this function, memory access functions may still return -EFAULT.
*/
-#define __access_mask get_fs().seg
-
-#define __access_ok(addr, size, mask) \
-({ \
- unsigned long __addr = (unsigned long) (addr); \
- unsigned long __size = size; \
- unsigned long __mask = mask; \
- unsigned long __ok; \
- \
- __chk_user_ptr(addr); \
- __ok = (signed long)(__mask & (__addr | (__addr + __size) | \
- __ua_size(__size))); \
- __ok == 0; \
-})
+static inline int __access_ok(const void __user *p, unsigned long size)
+{
+ unsigned long addr = (unsigned long)p;
+ return (get_fs().seg & (addr | (addr + size) | __ua_size(size))) == 0;
+}
#define access_ok(type, addr, size) \
- likely(__access_ok((addr), (size), __access_mask))
+ likely(__access_ok((addr), (size)))
/*
* put_user: - Write a simple value into user space.
@@ -811,157 +797,7 @@ extern void __put_user_unaligned_unknown(void);
extern size_t __copy_user(void *__to, const void *__from, size_t __n);
-#ifndef CONFIG_EVA
-#define __invoke_copy_to_user(to, from, n) \
-({ \
- register void __user *__cu_to_r __asm__("$4"); \
- register const void *__cu_from_r __asm__("$5"); \
- register long __cu_len_r __asm__("$6"); \
- \
- __cu_to_r = (to); \
- __cu_from_r = (from); \
- __cu_len_r = (n); \
- __asm__ __volatile__( \
- __MODULE_JAL(__copy_user) \
- : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
- : \
- : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \
- DADDI_SCRATCH, "memory"); \
- __cu_len_r; \
-})
-
-#define __invoke_copy_to_kernel(to, from, n) \
- __invoke_copy_to_user(to, from, n)
-
-#endif
-
-/*
- * __copy_to_user: - Copy a block of data into user space, with less checking.
- * @to: Destination address, in user space.
- * @from: Source address, in kernel space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from kernel space to user space. Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- */
-#define __copy_to_user(to, from, n) \
-({ \
- void __user *__cu_to; \
- const void *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- \
- check_object_size(__cu_from, __cu_len, true); \
- might_fault(); \
- \
- if (eva_kernel_access()) \
- __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from, \
- __cu_len); \
- else \
- __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, \
- __cu_len); \
- __cu_len; \
-})
-
-extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
-
-#define __copy_to_user_inatomic(to, from, n) \
-({ \
- void __user *__cu_to; \
- const void *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- \
- check_object_size(__cu_from, __cu_len, true); \
- \
- if (eva_kernel_access()) \
- __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from, \
- __cu_len); \
- else \
- __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, \
- __cu_len); \
- __cu_len; \
-})
-
-#define __copy_from_user_inatomic(to, from, n) \
-({ \
- void *__cu_to; \
- const void __user *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- \
- check_object_size(__cu_to, __cu_len, false); \
- \
- if (eva_kernel_access()) \
- __cu_len = __invoke_copy_from_kernel_inatomic(__cu_to, \
- __cu_from,\
- __cu_len);\
- else \
- __cu_len = __invoke_copy_from_user_inatomic(__cu_to, \
- __cu_from, \
- __cu_len); \
- __cu_len; \
-})
-
-/*
- * copy_to_user: - Copy a block of data into user space.
- * @to: Destination address, in user space.
- * @from: Source address, in kernel space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from kernel space to user space.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- */
-#define copy_to_user(to, from, n) \
-({ \
- void __user *__cu_to; \
- const void *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- \
- check_object_size(__cu_from, __cu_len, true); \
- \
- if (eva_kernel_access()) { \
- __cu_len = __invoke_copy_to_kernel(__cu_to, \
- __cu_from, \
- __cu_len); \
- } else { \
- if (access_ok(VERIFY_WRITE, __cu_to, __cu_len)) { \
- might_fault(); \
- __cu_len = __invoke_copy_to_user(__cu_to, \
- __cu_from, \
- __cu_len); \
- } \
- } \
- __cu_len; \
-})
-
-#ifndef CONFIG_EVA
-
-#define __invoke_copy_from_user(to, from, n) \
+#define __invoke_copy_from(func, to, from, n) \
({ \
register void *__cu_to_r __asm__("$4"); \
register const void __user *__cu_from_r __asm__("$5"); \
@@ -972,7 +808,7 @@ extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
__cu_len_r = (n); \
__asm__ __volatile__( \
".set\tnoreorder\n\t" \
- __MODULE_JAL(__copy_user) \
+ __MODULE_JAL(func) \
".set\tnoat\n\t" \
__UA_ADDU "\t$1, %1, %2\n\t" \
".set\tat\n\t" \
@@ -984,33 +820,17 @@ extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
__cu_len_r; \
})
-#define __invoke_copy_from_kernel(to, from, n) \
- __invoke_copy_from_user(to, from, n)
-
-/* For userland <-> userland operations */
-#define ___invoke_copy_in_user(to, from, n) \
- __invoke_copy_from_user(to, from, n)
-
-/* For kernel <-> kernel operations */
-#define ___invoke_copy_in_kernel(to, from, n) \
- __invoke_copy_from_user(to, from, n)
-
-#define __invoke_copy_from_user_inatomic(to, from, n) \
+#define __invoke_copy_to(func, to, from, n) \
({ \
- register void *__cu_to_r __asm__("$4"); \
- register const void __user *__cu_from_r __asm__("$5"); \
+ register void __user *__cu_to_r __asm__("$4"); \
+ register const void *__cu_from_r __asm__("$5"); \
register long __cu_len_r __asm__("$6"); \
\
__cu_to_r = (to); \
__cu_from_r = (from); \
__cu_len_r = (n); \
__asm__ __volatile__( \
- ".set\tnoreorder\n\t" \
- __MODULE_JAL(__copy_user_inatomic) \
- ".set\tnoat\n\t" \
- __UA_ADDU "\t$1, %1, %2\n\t" \
- ".set\tat\n\t" \
- ".set\treorder" \
+ __MODULE_JAL(func) \
: "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
: \
: "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \
@@ -1018,228 +838,79 @@ extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
__cu_len_r; \
})
-#define __invoke_copy_from_kernel_inatomic(to, from, n) \
- __invoke_copy_from_user_inatomic(to, from, n) \
+#define __invoke_copy_from_kernel(to, from, n) \
+ __invoke_copy_from(__copy_user, to, from, n)
+
+#define __invoke_copy_to_kernel(to, from, n) \
+ __invoke_copy_to(__copy_user, to, from, n)
+
+#define ___invoke_copy_in_kernel(to, from, n) \
+ __invoke_copy_from(__copy_user, to, from, n)
+
+#ifndef CONFIG_EVA
+#define __invoke_copy_from_user(to, from, n) \
+ __invoke_copy_from(__copy_user, to, from, n)
+
+#define __invoke_copy_to_user(to, from, n) \
+ __invoke_copy_to(__copy_user, to, from, n)
+
+#define ___invoke_copy_in_user(to, from, n) \
+ __invoke_copy_from(__copy_user, to, from, n)
#else
/* EVA specific functions */
-extern size_t __copy_user_inatomic_eva(void *__to, const void *__from,
- size_t __n);
extern size_t __copy_from_user_eva(void *__to, const void *__from,
size_t __n);
extern size_t __copy_to_user_eva(void *__to, const void *__from,
size_t __n);
extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n);
-#define __invoke_copy_from_user_eva_generic(to, from, n, func_ptr) \
-({ \
- register void *__cu_to_r __asm__("$4"); \
- register const void __user *__cu_from_r __asm__("$5"); \
- register long __cu_len_r __asm__("$6"); \
- \
- __cu_to_r = (to); \
- __cu_from_r = (from); \
- __cu_len_r = (n); \
- __asm__ __volatile__( \
- ".set\tnoreorder\n\t" \
- __MODULE_JAL(func_ptr) \
- ".set\tnoat\n\t" \
- __UA_ADDU "\t$1, %1, %2\n\t" \
- ".set\tat\n\t" \
- ".set\treorder" \
- : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
- : \
- : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \
- DADDI_SCRATCH, "memory"); \
- __cu_len_r; \
-})
-
-#define __invoke_copy_to_user_eva_generic(to, from, n, func_ptr) \
-({ \
- register void *__cu_to_r __asm__("$4"); \
- register const void __user *__cu_from_r __asm__("$5"); \
- register long __cu_len_r __asm__("$6"); \
- \
- __cu_to_r = (to); \
- __cu_from_r = (from); \
- __cu_len_r = (n); \
- __asm__ __volatile__( \
- __MODULE_JAL(func_ptr) \
- : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
- : \
- : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \
- DADDI_SCRATCH, "memory"); \
- __cu_len_r; \
-})
-
/*
* Source or destination address is in userland. We need to go through
* the TLB
*/
#define __invoke_copy_from_user(to, from, n) \
- __invoke_copy_from_user_eva_generic(to, from, n, __copy_from_user_eva)
-
-#define __invoke_copy_from_user_inatomic(to, from, n) \
- __invoke_copy_from_user_eva_generic(to, from, n, \
- __copy_user_inatomic_eva)
+ __invoke_copy_from(__copy_from_user_eva, to, from, n)
#define __invoke_copy_to_user(to, from, n) \
- __invoke_copy_to_user_eva_generic(to, from, n, __copy_to_user_eva)
+ __invoke_copy_to(__copy_to_user_eva, to, from, n)
#define ___invoke_copy_in_user(to, from, n) \
- __invoke_copy_from_user_eva_generic(to, from, n, __copy_in_user_eva)
-
-/*
- * Source or destination address in the kernel. We are not going through
- * the TLB
- */
-#define __invoke_copy_from_kernel(to, from, n) \
- __invoke_copy_from_user_eva_generic(to, from, n, __copy_user)
-
-#define __invoke_copy_from_kernel_inatomic(to, from, n) \
- __invoke_copy_from_user_eva_generic(to, from, n, __copy_user_inatomic)
-
-#define __invoke_copy_to_kernel(to, from, n) \
- __invoke_copy_to_user_eva_generic(to, from, n, __copy_user)
-
-#define ___invoke_copy_in_kernel(to, from, n) \
- __invoke_copy_from_user_eva_generic(to, from, n, __copy_user)
+ __invoke_copy_from(__copy_in_user_eva, to, from, n)
#endif /* CONFIG_EVA */
-/*
- * __copy_from_user: - Copy a block of data from user space, with less checking.
- * @to: Destination address, in kernel space.
- * @from: Source address, in user space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from user space to kernel space. Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- *
- * If some data could not be copied, this function will pad the copied
- * data to the requested size using zero bytes.
- */
-#define __copy_from_user(to, from, n) \
-({ \
- void *__cu_to; \
- const void __user *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- \
- check_object_size(__cu_to, __cu_len, false); \
- \
- if (eva_kernel_access()) { \
- __cu_len = __invoke_copy_from_kernel(__cu_to, \
- __cu_from, \
- __cu_len); \
- } else { \
- might_fault(); \
- __cu_len = __invoke_copy_from_user(__cu_to, __cu_from, \
- __cu_len); \
- } \
- __cu_len; \
-})
+static inline unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+ if (eva_kernel_access())
+ return __invoke_copy_to_kernel(to, from, n);
+ else
+ return __invoke_copy_to_user(to, from, n);
+}
-/*
- * copy_from_user: - Copy a block of data from user space.
- * @to: Destination address, in kernel space.
- * @from: Source address, in user space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from user space to kernel space.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- *
- * If some data could not be copied, this function will pad the copied
- * data to the requested size using zero bytes.
- */
-#define copy_from_user(to, from, n) \
-({ \
- void *__cu_to; \
- const void __user *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- \
- check_object_size(__cu_to, __cu_len, false); \
- \
- if (eva_kernel_access()) { \
- __cu_len = __invoke_copy_from_kernel(__cu_to, \
- __cu_from, \
- __cu_len); \
- } else { \
- if (access_ok(VERIFY_READ, __cu_from, __cu_len)) { \
- might_fault(); \
- __cu_len = __invoke_copy_from_user(__cu_to, \
- __cu_from, \
- __cu_len); \
- } else { \
- memset(__cu_to, 0, __cu_len); \
- } \
- } \
- __cu_len; \
-})
+static inline unsigned long
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+ if (eva_kernel_access())
+ return __invoke_copy_from_kernel(to, from, n);
+ else
+ return __invoke_copy_from_user(to, from, n);
+}
-#define __copy_in_user(to, from, n) \
-({ \
- void __user *__cu_to; \
- const void __user *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- if (eva_kernel_access()) { \
- __cu_len = ___invoke_copy_in_kernel(__cu_to, __cu_from, \
- __cu_len); \
- } else { \
- might_fault(); \
- __cu_len = ___invoke_copy_in_user(__cu_to, __cu_from, \
- __cu_len); \
- } \
- __cu_len; \
-})
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
-#define copy_in_user(to, from, n) \
-({ \
- void __user *__cu_to; \
- const void __user *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- if (eva_kernel_access()) { \
- __cu_len = ___invoke_copy_in_kernel(__cu_to,__cu_from, \
- __cu_len); \
- } else { \
- if (likely(access_ok(VERIFY_READ, __cu_from, __cu_len) &&\
- access_ok(VERIFY_WRITE, __cu_to, __cu_len))) {\
- might_fault(); \
- __cu_len = ___invoke_copy_in_user(__cu_to, \
- __cu_from, \
- __cu_len); \
- } \
- } \
- __cu_len; \
-})
+static inline unsigned long
+raw_copy_in_user(void __user*to, const void __user *from, unsigned long n)
+{
+ if (eva_kernel_access())
+ return ___invoke_copy_in_kernel(to, from, n);
+ else
+ return ___invoke_copy_in_user(to, from, n);
+}
extern __kernel_size_t __bzero_kernel(void __user *addr, __kernel_size_t size);
extern __kernel_size_t __bzero(void __user *addr, __kernel_size_t size);
generated by cgit (git 2.34.1) at 2025-04-17 12:09:29 +0000