summaryrefslogtreecommitdiff
path: root/arch/arm64/include/asm/vdso
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm64/include/asm/vdso')
-rw-r--r--arch/arm64/include/asm/vdso/clocksource.h11
-rw-r--r--arch/arm64/include/asm/vdso/compat_barrier.h7
-rw-r--r--arch/arm64/include/asm/vdso/compat_gettimeofday.h46
-rw-r--r--arch/arm64/include/asm/vdso/gettimeofday.h49
-rw-r--r--arch/arm64/include/asm/vdso/processor.h17
-rw-r--r--arch/arm64/include/asm/vdso/vsyscall.h9
6 files changed, 89 insertions, 50 deletions
diff --git a/arch/arm64/include/asm/vdso/clocksource.h b/arch/arm64/include/asm/vdso/clocksource.h
new file mode 100644
index 000000000000..b054d9febfb5
--- /dev/null
+++ b/arch/arm64/include/asm/vdso/clocksource.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_VDSOCLOCKSOURCE_H
+#define __ASM_VDSOCLOCKSOURCE_H
+
+#define VDSO_ARCH_CLOCKMODES \
+ /* vdso clocksource for both 32 and 64bit tasks */ \
+ VDSO_CLOCKMODE_ARCHTIMER, \
+ /* vdso clocksource for 64bit tasks only */ \
+ VDSO_CLOCKMODE_ARCHTIMER_NOCOMPAT
+
+#endif
diff --git a/arch/arm64/include/asm/vdso/compat_barrier.h b/arch/arm64/include/asm/vdso/compat_barrier.h
index 3fd8fd6d8fc2..3ac35f4a667c 100644
--- a/arch/arm64/include/asm/vdso/compat_barrier.h
+++ b/arch/arm64/include/asm/vdso/compat_barrier.h
@@ -20,16 +20,9 @@
#define dmb(option) __asm__ __volatile__ ("dmb " #option : : : "memory")
-#if __LINUX_ARM_ARCH__ >= 8 && defined(CONFIG_AS_DMB_ISHLD)
#define aarch32_smp_mb() dmb(ish)
#define aarch32_smp_rmb() dmb(ishld)
#define aarch32_smp_wmb() dmb(ishst)
-#else
-#define aarch32_smp_mb() dmb(ish)
-#define aarch32_smp_rmb() aarch32_smp_mb()
-#define aarch32_smp_wmb() dmb(ishst)
-#endif
-
#undef smp_mb
#undef smp_rmb
diff --git a/arch/arm64/include/asm/vdso/compat_gettimeofday.h b/arch/arm64/include/asm/vdso/compat_gettimeofday.h
index c50ee1b7d5cd..ecb6fd4c3c64 100644
--- a/arch/arm64/include/asm/vdso/compat_gettimeofday.h
+++ b/arch/arm64/include/asm/vdso/compat_gettimeofday.h
@@ -7,16 +7,15 @@
#ifndef __ASSEMBLY__
+#include <asm/barrier.h>
#include <asm/unistd.h>
-#include <uapi/linux/time.h>
+#include <asm/errno.h>
#include <asm/vdso/compat_barrier.h>
-#define __VDSO_USE_SYSCALL ULLONG_MAX
-
#define VDSO_HAS_CLOCK_GETRES 1
-#define VDSO_HAS_32BIT_FALLBACK 1
+#define BUILD_VDSO32 1
static __always_inline
int gettimeofday_fallback(struct __kernel_old_timeval *_tv,
@@ -78,10 +77,6 @@ int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts)
register long ret asm ("r0");
register long nr asm("r7") = __NR_compat_clock_getres_time64;
- /* The checks below are required for ABI consistency with arm */
- if ((_clkid >= MAX_CLOCKS) && (_ts == NULL))
- return -EINVAL;
-
asm volatile(
" swi #0\n"
: "=r" (ret)
@@ -99,10 +94,6 @@ int clock_getres32_fallback(clockid_t _clkid, struct old_timespec32 *_ts)
register long ret asm ("r0");
register long nr asm("r7") = __NR_compat_clock_getres;
- /* The checks below are required for ABI consistency with arm */
- if ((_clkid >= MAX_CLOCKS) && (_ts == NULL))
- return -EINVAL;
-
asm volatile(
" swi #0\n"
: "=r" (ret)
@@ -112,16 +103,18 @@ int clock_getres32_fallback(clockid_t _clkid, struct old_timespec32 *_ts)
return ret;
}
-static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)
+static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
+ const struct vdso_data *vd)
{
u64 res;
/*
- * clock_mode == 0 implies that vDSO are enabled otherwise
- * fallback on syscall.
+ * Core checks for mode already, so this raced against a concurrent
+ * update. Return something. Core will do another round and then
+ * see the mode change and fallback to the syscall.
*/
- if (clock_mode)
- return __VDSO_USE_SYSCALL;
+ if (clock_mode != VDSO_CLOCKMODE_ARCHTIMER)
+ return 0;
/*
* This isb() is required to prevent that the counter value
@@ -161,6 +154,25 @@ static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
return ret;
}
+#ifdef CONFIG_TIME_NS
+static __always_inline
+const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
+{
+ const struct vdso_data *ret;
+
+ /* See __arch_get_vdso_data(). */
+ asm volatile("mov %0, %1" : "=r"(ret) : "r"(_timens_data));
+
+ return ret;
+}
+#endif
+
+static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
+{
+ return vd->clock_mode == VDSO_CLOCKMODE_ARCHTIMER;
+}
+#define vdso_clocksource_ok vdso_clocksource_ok
+
#endif /* !__ASSEMBLY__ */
#endif /* __ASM_VDSO_GETTIMEOFDAY_H */
diff --git a/arch/arm64/include/asm/vdso/gettimeofday.h b/arch/arm64/include/asm/vdso/gettimeofday.h
index b08f476b72b4..764d13e2916c 100644
--- a/arch/arm64/include/asm/vdso/gettimeofday.h
+++ b/arch/arm64/include/asm/vdso/gettimeofday.h
@@ -7,10 +7,10 @@
#ifndef __ASSEMBLY__
+#include <asm/alternative.h>
+#include <asm/barrier.h>
#include <asm/unistd.h>
-#include <uapi/linux/time.h>
-
-#define __VDSO_USE_SYSCALL ULLONG_MAX
+#include <asm/sysreg.h>
#define VDSO_HAS_CLOCK_GETRES 1
@@ -66,28 +66,35 @@ int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts)
return ret;
}
-static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)
+static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
+ const struct vdso_data *vd)
{
u64 res;
/*
- * clock_mode == 0 implies that vDSO are enabled otherwise
- * fallback on syscall.
+ * Core checks for mode already, so this raced against a concurrent
+ * update. Return something. Core will do another round and then
+ * see the mode change and fallback to the syscall.
*/
- if (clock_mode)
- return __VDSO_USE_SYSCALL;
+ if (clock_mode == VDSO_CLOCKMODE_NONE)
+ return 0;
/*
- * This isb() is required to prevent that the counter value
+ * If FEAT_ECV is available, use the self-synchronizing counter.
+ * Otherwise the isb is required to prevent that the counter value
* is speculated.
- */
- isb();
- asm volatile("mrs %0, cntvct_el0" : "=r" (res) :: "memory");
- /*
- * This isb() is required to prevent that the seq lock is
- * speculated.#
- */
- isb();
+ */
+ asm volatile(
+ ALTERNATIVE("isb\n"
+ "mrs %0, cntvct_el0",
+ "nop\n"
+ __mrs_s("%0", SYS_CNTVCTSS_EL0),
+ ARM64_HAS_ECV)
+ : "=r" (res)
+ :
+ : "memory");
+
+ arch_counter_enforce_ordering(res);
return res;
}
@@ -98,6 +105,14 @@ const struct vdso_data *__arch_get_vdso_data(void)
return _vdso_data;
}
+#ifdef CONFIG_TIME_NS
+static __always_inline
+const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
+{
+ return _timens_data;
+}
+#endif
+
#endif /* !__ASSEMBLY__ */
#endif /* __ASM_VDSO_GETTIMEOFDAY_H */
diff --git a/arch/arm64/include/asm/vdso/processor.h b/arch/arm64/include/asm/vdso/processor.h
new file mode 100644
index 000000000000..ff830b766ad2
--- /dev/null
+++ b/arch/arm64/include/asm/vdso/processor.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 ARM Ltd.
+ */
+#ifndef __ASM_VDSO_PROCESSOR_H
+#define __ASM_VDSO_PROCESSOR_H
+
+#ifndef __ASSEMBLY__
+
+static inline void cpu_relax(void)
+{
+ asm volatile("yield" ::: "memory");
+}
+
+#endif /* __ASSEMBLY__ */
+
+#endif /* __ASM_VDSO_PROCESSOR_H */
diff --git a/arch/arm64/include/asm/vdso/vsyscall.h b/arch/arm64/include/asm/vdso/vsyscall.h
index 0c20a7c1bee5..f94b1457c117 100644
--- a/arch/arm64/include/asm/vdso/vsyscall.h
+++ b/arch/arm64/include/asm/vdso/vsyscall.h
@@ -22,15 +22,6 @@ struct vdso_data *__arm64_get_k_vdso_data(void)
#define __arch_get_k_vdso_data __arm64_get_k_vdso_data
static __always_inline
-int __arm64_get_clock_mode(struct timekeeper *tk)
-{
- u32 use_syscall = !tk->tkr_mono.clock->archdata.vdso_direct;
-
- return use_syscall;
-}
-#define __arch_get_clock_mode __arm64_get_clock_mode
-
-static __always_inline
void __arm64_update_vsyscall(struct vdso_data *vdata, struct timekeeper *tk)
{
vdata[CS_HRES_COARSE].mask = VDSO_PRECISION_MASK;
generated by cgit (git 2.34.1) at 2024-04-28 10:53:06 +0000