summaryrefslogtreecommitdiff
path: root/arch/x86/kernel/ftrace_32.S
blob: 58d9ed50fe6177957205b19f5888cf3c0ab05490 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
/* SPDX-License-Identifier: GPL-2.0 */
/*
 *  Copyright (C) 2017  Steven Rostedt, VMware Inc.
 */

#include <linux/export.h>
#include <linux/linkage.h>
#include <asm/page_types.h>
#include <asm/segment.h>
#include <asm/ftrace.h>
#include <asm/nospec-branch.h>
#include <asm/frame.h>
#include <asm/asm-offsets.h>

#ifdef CONFIG_FRAME_POINTER
# define MCOUNT_FRAME			1	/* using frame = true  */
#else
# define MCOUNT_FRAME			0	/* using frame = false */
#endif

SYM_FUNC_START(__fentry__)
	RET
SYM_FUNC_END(__fentry__)
EXPORT_SYMBOL(__fentry__)

SYM_CODE_START(ftrace_caller)

#ifdef CONFIG_FRAME_POINTER
	/*
	 * Frame pointers are of ip followed by bp.
	 * Since fentry is an immediate jump, we are left with
	 * parent-ip, function-ip. We need to add a frame with
	 * parent-ip followed by ebp.
	 */
	pushl	4(%esp)				/* parent ip */
	pushl	%ebp
	movl	%esp, %ebp
	pushl	2*4(%esp)			/* function ip */

	/* For mcount, the function ip is directly above */
	pushl	%ebp
	movl	%esp, %ebp
#endif
	pushl	%eax
	pushl	%ecx
	pushl	%edx
	pushl	$0				/* Pass NULL as regs pointer */

#ifdef CONFIG_FRAME_POINTER
	/* Load parent ebp into edx */
	movl	4*4(%esp), %edx
#else
	/* There's no frame pointer, load the appropriate stack addr instead */
	lea	4*4(%esp), %edx
#endif

	movl	(MCOUNT_FRAME+4)*4(%esp), %eax	/* load the rip */
	/* Get the parent ip */
	movl	4(%edx), %edx			/* edx has ebp */

	movl	function_trace_op, %ecx
	subl	$MCOUNT_INSN_SIZE, %eax

.globl ftrace_call
ftrace_call:
	call	ftrace_stub

	addl	$4, %esp			/* skip NULL pointer */
	popl	%edx
	popl	%ecx
	popl	%eax
#ifdef CONFIG_FRAME_POINTER
	popl	%ebp
	addl	$4,%esp				/* skip function ip */
	popl	%ebp				/* this is the orig bp */
	addl	$4, %esp			/* skip parent ip */
#endif
.Lftrace_ret:
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
.globl ftrace_graph_call
ftrace_graph_call:
	jmp	ftrace_stub
#endif

/* This is weak to keep gas from relaxing the jumps */
SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
	RET
SYM_CODE_END(ftrace_caller)

SYM_CODE_START(ftrace_regs_caller)
	/*
	 * We're here from an mcount/fentry CALL, and the stack frame looks like:
	 *
	 *  <previous context>
	 *  RET-IP
	 *
	 * The purpose of this function is to call out in an emulated INT3
	 * environment with a stack frame like:
	 *
	 *  <previous context>
	 *  gap / RET-IP
	 *  gap
	 *  gap
	 *  gap
	 *  pt_regs
	 *
	 * We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds
	 */
	subl	$3*4, %esp	# RET-IP + 3 gaps
	pushl	%ss		# ss
	pushl	%esp		# points at ss
	addl	$5*4, (%esp)	#   make it point at <previous context>
	pushfl			# flags
	pushl	$__KERNEL_CS	# cs
	pushl	7*4(%esp)	# ip <- RET-IP
	pushl	$0		# orig_eax

	pushl	%gs
	pushl	%fs
	pushl	%es
	pushl	%ds

	pushl	%eax
	pushl	%ebp
	pushl	%edi
	pushl	%esi
	pushl	%edx
	pushl	%ecx
	pushl	%ebx

	ENCODE_FRAME_POINTER

	movl	PT_EIP(%esp), %eax	# 1st argument: IP
	subl	$MCOUNT_INSN_SIZE, %eax
	movl	21*4(%esp), %edx	# 2nd argument: parent ip
	movl	function_trace_op, %ecx	# 3rd argument: ftrace_pos
	pushl	%esp			# 4th argument: pt_regs

SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)
	call	ftrace_stub

	addl	$4, %esp		# skip 4th argument

	/* place IP below the new SP */
	movl	PT_OLDESP(%esp), %eax
	movl	PT_EIP(%esp), %ecx
	movl	%ecx, -4(%eax)

	/* place EAX below that */
	movl	PT_EAX(%esp), %ecx
	movl	%ecx, -8(%eax)

	popl	%ebx
	popl	%ecx
	popl	%edx
	popl	%esi
	popl	%edi
	popl	%ebp

	lea	-8(%eax), %esp
	popl	%eax

	jmp	.Lftrace_ret
SYM_CODE_END(ftrace_regs_caller)

SYM_FUNC_START(ftrace_stub_direct_tramp)
	CALL_DEPTH_ACCOUNT
	RET
SYM_FUNC_END(ftrace_stub_direct_tramp)

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
SYM_CODE_START(ftrace_graph_caller)
	pushl	%eax
	pushl	%ecx
	pushl	%edx
	movl	3*4(%esp), %eax
	/* Even with frame pointers, fentry doesn't have one here */
	lea	4*4(%esp), %edx
	movl	$0, %ecx
	subl	$MCOUNT_INSN_SIZE, %eax
	call	prepare_ftrace_return
	popl	%edx
	popl	%ecx
	popl	%eax
	RET
SYM_CODE_END(ftrace_graph_caller)

.globl return_to_handler
return_to_handler:
	pushl	$0
	pushl	%edx
	pushl	%eax
	movl	%esp, %eax
	call	ftrace_return_to_handler
	movl	%eax, %ecx
	popl	%eax
	popl	%edx
	addl	$4, %esp		# skip ebp
	JMP_NOSPEC ecx
#endif