1//===-- xray_trampoline_x86.s -----------------------------------*- ASM -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of XRay, a dynamic runtime instrumentation system.
10//
11// This implements the X86-specific assembler for the trampolines.
12//
13//===----------------------------------------------------------------------===//
14
15#include "../builtins/assembly.h"
16#include "../sanitizer_common/sanitizer_asm.h"
17
18// XRay trampolines which are not produced by intrinsics are not System V AMD64
19// ABI compliant because they are called with a stack that is always misaligned
20// by 8 bytes with respect to a 16 bytes alignment. This is because they are
21// called immediately after the call to, or immediately before returning from,
22// the function being instrumented. This saves space in the patch point, but
23// misaligns the stack by 8 bytes.
24
25.macro ALIGN_STACK_16B
26#if defined(__APPLE__)
27 subq $$8, %rsp
28#else
29 subq $8, %rsp
30#endif
31 CFI_ADJUST_CFA_OFFSET(8)
32.endm
33
34.macro RESTORE_STACK_ALIGNMENT
35#if defined(__APPLE__)
36 addq $$8, %rsp
37#else
38 addq $8, %rsp
39#endif
40 CFI_ADJUST_CFA_OFFSET(-8)
41.endm
42
43// This macro should lower the stack pointer by an odd multiple of 8.
44.macro SAVE_REGISTERS
45 pushfq
46 CFI_ADJUST_CFA_OFFSET(8)
47 subq $240, %rsp
48 CFI_ADJUST_CFA_OFFSET(240)
49 movq %rbp, 232(%rsp)
50 movupd %xmm0, 216(%rsp)
51 movupd %xmm1, 200(%rsp)
52 movupd %xmm2, 184(%rsp)
53 movupd %xmm3, 168(%rsp)
54 movupd %xmm4, 152(%rsp)
55 movupd %xmm5, 136(%rsp)
56 movupd %xmm6, 120(%rsp)
57 movupd %xmm7, 104(%rsp)
58 movq %rdi, 96(%rsp)
59 movq %rax, 88(%rsp)
60 movq %rdx, 80(%rsp)
61 movq %rsi, 72(%rsp)
62 movq %rcx, 64(%rsp)
63 movq %r8, 56(%rsp)
64 movq %r9, 48(%rsp)
65 movq %r10, 40(%rsp)
66 movq %r11, 32(%rsp)
67 movq %r12, 24(%rsp)
68 movq %r13, 16(%rsp)
69 movq %r14, 8(%rsp)
70 movq %r15, 0(%rsp)
71.endm
72
73.macro RESTORE_REGISTERS
74 movq 232(%rsp), %rbp
75 movupd 216(%rsp), %xmm0
76 movupd 200(%rsp), %xmm1
77 movupd 184(%rsp), %xmm2
78 movupd 168(%rsp), %xmm3
79 movupd 152(%rsp), %xmm4
80 movupd 136(%rsp), %xmm5
81 movupd 120(%rsp) , %xmm6
82 movupd 104(%rsp) , %xmm7
83 movq 96(%rsp), %rdi
84 movq 88(%rsp), %rax
85 movq 80(%rsp), %rdx
86 movq 72(%rsp), %rsi
87 movq 64(%rsp), %rcx
88 movq 56(%rsp), %r8
89 movq 48(%rsp), %r9
90 movq 40(%rsp), %r10
91 movq 32(%rsp), %r11
92 movq 24(%rsp), %r12
93 movq 16(%rsp), %r13
94 movq 8(%rsp), %r14
95 movq 0(%rsp), %r15
96 addq $240, %rsp
97 CFI_ADJUST_CFA_OFFSET(-240)
98 popfq
99 CFI_ADJUST_CFA_OFFSET(-8)
100.endm
101
102 .text
103#if !defined(__APPLE__)
104 .section .text
105 .file "xray_trampoline_x86.S"
106#else
107 .section __TEXT,__text
108#endif
109
110//===----------------------------------------------------------------------===//
111
112 .globl ASM_SYMBOL(__xray_FunctionEntry)
113 ASM_HIDDEN(__xray_FunctionEntry)
114 .align 16, 0x90
115 ASM_TYPE_FUNCTION(__xray_FunctionEntry)
116# LLVM-MCA-BEGIN __xray_FunctionEntry
117ASM_SYMBOL(__xray_FunctionEntry):
118 CFI_STARTPROC
119 SAVE_REGISTERS
120 ALIGN_STACK_16B
121
122 // This load has to be atomic, it's concurrent with __xray_patch().
123 // On x86/amd64, a simple (type-aligned) MOV instruction is enough.
124 movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax
125 testq %rax, %rax
126 je LOCAL_LABEL(tmp0)
127
128 // The patched function prologue puts its xray_instr_map index into %r10d.
129 movl %r10d, %edi
130 xor %esi,%esi
131 callq *%rax
132
133LOCAL_LABEL(tmp0):
134 RESTORE_STACK_ALIGNMENT
135 RESTORE_REGISTERS
136 retq
137# LLVM-MCA-END
138 ASM_SIZE(__xray_FunctionEntry)
139 CFI_ENDPROC
140
141//===----------------------------------------------------------------------===//
142
143 .globl ASM_SYMBOL(__xray_FunctionExit)
144 ASM_HIDDEN(__xray_FunctionExit)
145 .align 16, 0x90
146 ASM_TYPE_FUNCTION(__xray_FunctionExit)
147# LLVM-MCA-BEGIN __xray_FunctionExit
148ASM_SYMBOL(__xray_FunctionExit):
149 CFI_STARTPROC
150 ALIGN_STACK_16B
151
152 // Save the important registers first. Since we're assuming that this
153 // function is only jumped into, we only preserve the registers for
154 // returning.
155 subq $64, %rsp
156 CFI_ADJUST_CFA_OFFSET(64)
157 movq %rbp, 48(%rsp)
158 movupd %xmm0, 32(%rsp)
159 movupd %xmm1, 16(%rsp)
160 movq %rax, 8(%rsp)
161 movq %rdx, 0(%rsp)
162 movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax
163 testq %rax,%rax
164 je LOCAL_LABEL(tmp2)
165
166 movl %r10d, %edi
167 movl $1, %esi
168 callq *%rax
169
170LOCAL_LABEL(tmp2):
171 // Restore the important registers.
172 movq 48(%rsp), %rbp
173 movupd 32(%rsp), %xmm0
174 movupd 16(%rsp), %xmm1
175 movq 8(%rsp), %rax
176 movq 0(%rsp), %rdx
177 addq $64, %rsp
178 CFI_ADJUST_CFA_OFFSET(-64)
179
180 RESTORE_STACK_ALIGNMENT
181 retq
182# LLVM-MCA-END
183 ASM_SIZE(__xray_FunctionExit)
184 CFI_ENDPROC
185
186//===----------------------------------------------------------------------===//
187
188 .globl ASM_SYMBOL(__xray_FunctionTailExit)
189 ASM_HIDDEN(__xray_FunctionTailExit)
190 .align 16, 0x90
191 ASM_TYPE_FUNCTION(__xray_FunctionTailExit)
192# LLVM-MCA-BEGIN __xray_FunctionTailExit
193ASM_SYMBOL(__xray_FunctionTailExit):
194 CFI_STARTPROC
195 SAVE_REGISTERS
196 ALIGN_STACK_16B
197
198 movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax
199 testq %rax,%rax
200 je LOCAL_LABEL(tmp4)
201
202 movl %r10d, %edi
203 movl $2, %esi
204 callq *%rax
205
206LOCAL_LABEL(tmp4):
207 RESTORE_STACK_ALIGNMENT
208 RESTORE_REGISTERS
209 retq
210# LLVM-MCA-END
211 ASM_SIZE(__xray_FunctionTailExit)
212 CFI_ENDPROC
213
214//===----------------------------------------------------------------------===//
215
216 .globl ASM_SYMBOL(__xray_ArgLoggerEntry)
217 ASM_HIDDEN(__xray_ArgLoggerEntry)
218 .align 16, 0x90
219 ASM_TYPE_FUNCTION(__xray_ArgLoggerEntry)
220# LLVM-MCA-BEGIN __xray_ArgLoggerEntry
221ASM_SYMBOL(__xray_ArgLoggerEntry):
222 CFI_STARTPROC
223 SAVE_REGISTERS
224 ALIGN_STACK_16B
225
226 // Again, these function pointer loads must be atomic; MOV is fine.
227 movq ASM_SYMBOL(_ZN6__xray13XRayArgLoggerE)(%rip), %rax
228 testq %rax, %rax
229 jne LOCAL_LABEL(arg1entryLog)
230
231 // If [arg1 logging handler] not set, defer to no-arg logging.
232 movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax
233 testq %rax, %rax
234 je LOCAL_LABEL(arg1entryFail)
235
236LOCAL_LABEL(arg1entryLog):
237
238 // First argument will become the third
239 movq %rdi, %rdx
240
241 // XRayEntryType::LOG_ARGS_ENTRY into the second
242 mov $0x3, %esi
243
244 // 32-bit function ID becomes the first
245 movl %r10d, %edi
246
247 callq *%rax
248
249LOCAL_LABEL(arg1entryFail):
250 RESTORE_STACK_ALIGNMENT
251 RESTORE_REGISTERS
252 retq
253# LLVM-MCA-END
254 ASM_SIZE(__xray_ArgLoggerEntry)
255 CFI_ENDPROC
256
257//===----------------------------------------------------------------------===//
258
259// __xray_*Event have default visibility so that they can be referenced by user
260// DSOs that do not link against the runtime.
261 .global ASM_SYMBOL(__xray_CustomEvent)
262 .align 16, 0x90
263 ASM_TYPE_FUNCTION(__xray_CustomEvent)
264# LLVM-MCA-BEGIN __xray_CustomEvent
265ASM_SYMBOL(__xray_CustomEvent):
266 CFI_STARTPROC
267 SAVE_REGISTERS
268
269 // We take two arguments to this trampoline, which should be in rdi and rsi
270 // already.
271 movq ASM_SYMBOL(_ZN6__xray22XRayPatchedCustomEventE)(%rip), %rax
272 testq %rax,%rax
273 je LOCAL_LABEL(customEventCleanup)
274
275 callq *%rax
276
277LOCAL_LABEL(customEventCleanup):
278 RESTORE_REGISTERS
279 retq
280# LLVM-MCA-END
281 ASM_SIZE(__xray_CustomEvent)
282 CFI_ENDPROC
283
284//===----------------------------------------------------------------------===//
285
286 .global ASM_SYMBOL(__xray_TypedEvent)
287 .align 16, 0x90
288 ASM_TYPE_FUNCTION(__xray_TypedEvent)
289# LLVM-MCA-BEGIN __xray_TypedEvent
290ASM_SYMBOL(__xray_TypedEvent):
291 CFI_STARTPROC
292 SAVE_REGISTERS
293
294 // We pass three arguments to this trampoline, which should be in rdi, rsi
295 // and rdx without our intervention.
296 movq ASM_SYMBOL(_ZN6__xray21XRayPatchedTypedEventE)(%rip), %rax
297 testq %rax,%rax
298 je LOCAL_LABEL(typedEventCleanup)
299
300 callq *%rax
301
302LOCAL_LABEL(typedEventCleanup):
303 RESTORE_REGISTERS
304 retq
305# LLVM-MCA-END
306 ASM_SIZE(__xray_TypedEvent)
307 CFI_ENDPROC
308
309//===----------------------------------------------------------------------===//
310
311NO_EXEC_STACK_DIRECTIVE
312