1//===-- X86RegisterInfo.cpp - X86 Register Information --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the X86 implementation of the TargetRegisterInfo class.
10// This file is responsible for the frame pointer elimination optimization
11// on X86.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86RegisterInfo.h"
16#include "X86FrameLowering.h"
17#include "X86MachineFunctionInfo.h"
18#include "X86Subtarget.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/SmallSet.h"
22#include "llvm/CodeGen/LiveRegMatrix.h"
23#include "llvm/CodeGen/MachineFrameInfo.h"
24#include "llvm/CodeGen/MachineRegisterInfo.h"
25#include "llvm/CodeGen/RegisterScavenging.h"
26#include "llvm/CodeGen/TargetFrameLowering.h"
27#include "llvm/CodeGen/TargetInstrInfo.h"
28#include "llvm/CodeGen/TileShapeInfo.h"
29#include "llvm/CodeGen/VirtRegMap.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/Type.h"
32#include "llvm/MC/MCContext.h"
33#include "llvm/Support/ErrorHandling.h"
34#include "llvm/Target/TargetMachine.h"
35
36using namespace llvm;
37
38#define GET_REGINFO_TARGET_DESC
39#include "X86GenRegisterInfo.inc"
40
41X86RegisterInfo::X86RegisterInfo(const Triple &TT)
42 : X86GenRegisterInfo((TT.isX86_64() ? X86::RIP : X86::EIP),
43 X86_MC::getDwarfRegFlavour(TT, isEH: false),
44 X86_MC::getDwarfRegFlavour(TT, isEH: true),
45 (TT.isX86_64() ? X86::RIP : X86::EIP)) {
46 X86_MC::initLLVMToSEHAndCVRegMapping(MRI: this);
47
48 // Cache some information.
49 Is64Bit = TT.isX86_64();
50 IsTarget64BitLP64 = Is64Bit && !TT.isX32();
51 IsWin64 = Is64Bit && TT.isOSWindows();
52 IsUEFI64 = Is64Bit && TT.isUEFI();
53
54 // Use a callee-saved register as the base pointer. These registers must
55 // not conflict with any ABI requirements. For example, in 32-bit mode PIC
56 // requires GOT in the EBX register before function calls via PLT GOT pointer.
57 if (Is64Bit) {
58 SlotSize = 8;
59 // This matches the simplified 32-bit pointer code in the data layout
60 // computation.
61 // FIXME: Should use the data layout?
62 bool Use64BitReg = !TT.isX32();
63 StackPtr = Use64BitReg ? X86::RSP : X86::ESP;
64 FramePtr = Use64BitReg ? X86::RBP : X86::EBP;
65 BasePtr = Use64BitReg ? X86::RBX : X86::EBX;
66 } else {
67 SlotSize = 4;
68 StackPtr = X86::ESP;
69 FramePtr = X86::EBP;
70 BasePtr = X86::ESI;
71 }
72}
73
74const TargetRegisterClass *
75X86RegisterInfo::getSubClassWithSubReg(const TargetRegisterClass *RC,
76 unsigned Idx) const {
77 // The sub_8bit sub-register index is more constrained in 32-bit mode.
78 // It behaves just like the sub_8bit_hi index.
79 if (!Is64Bit && Idx == X86::sub_8bit)
80 Idx = X86::sub_8bit_hi;
81
82 // Forward to TableGen's default version.
83 return X86GenRegisterInfo::getSubClassWithSubReg(RC, Idx);
84}
85
86const TargetRegisterClass *
87X86RegisterInfo::getMatchingSuperRegClass(const TargetRegisterClass *A,
88 const TargetRegisterClass *B,
89 unsigned SubIdx) const {
90 // The sub_8bit sub-register index is more constrained in 32-bit mode.
91 if (!Is64Bit && SubIdx == X86::sub_8bit) {
92 A = X86GenRegisterInfo::getSubClassWithSubReg(RC: A, Idx: X86::sub_8bit_hi);
93 if (!A)
94 return nullptr;
95 }
96 return X86GenRegisterInfo::getMatchingSuperRegClass(A, B, Idx: SubIdx);
97}
98
99const TargetRegisterClass *
100X86RegisterInfo::getLargestLegalSuperClass(const TargetRegisterClass *RC,
101 const MachineFunction &MF) const {
102 // Don't allow super-classes of GR8_NOREX. This class is only used after
103 // extracting sub_8bit_hi sub-registers. The H sub-registers cannot be copied
104 // to the full GR8 register class in 64-bit mode, so we cannot allow the
105 // reigster class inflation.
106 //
107 // The GR8_NOREX class is always used in a way that won't be constrained to a
108 // sub-class, so sub-classes like GR8_ABCD_L are allowed to expand to the
109 // full GR8 class.
110 if (RC == &X86::GR8_NOREXRegClass)
111 return RC;
112
113 // Keep using non-rex2 register class when APX feature (EGPR/NDD/NF) is not
114 // enabled for relocation.
115 const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
116 if (!Subtarget.getCLOpts().enable_apx_for_relocation && isNonRex2RegClass(RC))
117 return RC;
118
119 const TargetRegisterClass *Super = RC;
120 auto I = RC->superclasses().begin();
121 auto E = RC->superclasses().end();
122 do {
123 switch (Super->getID()) {
124 case X86::FR32RegClassID:
125 case X86::FR64RegClassID:
126 // If AVX-512 isn't supported we should only inflate to these classes.
127 if (!Subtarget.hasAVX512() &&
128 getRegSizeInBits(RC: *Super) == getRegSizeInBits(RC: *RC))
129 return Super;
130 break;
131 case X86::VR128RegClassID:
132 case X86::VR256RegClassID:
133 // If VLX isn't supported we should only inflate to these classes.
134 if (!Subtarget.hasVLX() &&
135 getRegSizeInBits(RC: *Super) == getRegSizeInBits(RC: *RC))
136 return Super;
137 break;
138 case X86::VR128XRegClassID:
139 case X86::VR256XRegClassID:
140 // If VLX isn't support we shouldn't inflate to these classes.
141 if (Subtarget.hasVLX() &&
142 getRegSizeInBits(RC: *Super) == getRegSizeInBits(RC: *RC))
143 return Super;
144 break;
145 case X86::FR32XRegClassID:
146 case X86::FR64XRegClassID:
147 // If AVX-512 isn't support we shouldn't inflate to these classes.
148 if (Subtarget.hasAVX512() &&
149 getRegSizeInBits(RC: *Super) == getRegSizeInBits(RC: *RC))
150 return Super;
151 break;
152 case X86::GR8RegClassID:
153 case X86::GR16RegClassID:
154 case X86::GR32RegClassID:
155 case X86::GR64RegClassID:
156 case X86::GR8_NOREX2RegClassID:
157 case X86::GR16_NOREX2RegClassID:
158 case X86::GR32_NOREX2RegClassID:
159 case X86::GR64_NOREX2RegClassID:
160 case X86::RFP32RegClassID:
161 case X86::RFP64RegClassID:
162 case X86::RFP80RegClassID:
163 case X86::VR512_0_15RegClassID:
164 case X86::VR512RegClassID:
165 // Don't return a super-class that would shrink the spill size.
166 // That can happen with the vector and float classes.
167 if (getRegSizeInBits(RC: *Super) == getRegSizeInBits(RC: *RC))
168 return Super;
169 }
170 if (I != E) {
171 Super = getRegClass(i: *I);
172 ++I;
173 } else {
174 Super = nullptr;
175 }
176 } while (Super);
177 return RC;
178}
179
180const TargetRegisterClass *
181X86RegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const {
182 if (RC == &X86::CCRRegClass) {
183 if (Is64Bit)
184 return &X86::GR64RegClass;
185 else
186 return &X86::GR32RegClass;
187 }
188 return RC;
189}
190
191unsigned
192X86RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
193 MachineFunction &MF) const {
194 const X86FrameLowering *TFI = getFrameLowering(MF);
195
196 unsigned FPDiff = TFI->hasFP(MF) ? 1 : 0;
197 switch (RC->getID()) {
198 default:
199 return 0;
200 case X86::GR32RegClassID:
201 return 4 - FPDiff;
202 case X86::GR64RegClassID:
203 return 12 - FPDiff;
204 case X86::VR128RegClassID:
205 return Is64Bit ? 10 : 4;
206 case X86::VR64RegClassID:
207 return 4;
208 }
209}
210
211const MCPhysReg *
212X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
213 assert(MF && "MachineFunction required");
214
215 const X86Subtarget &Subtarget = MF->getSubtarget<X86Subtarget>();
216 const Function &F = MF->getFunction();
217 bool HasSSE = Subtarget.hasSSE1();
218 bool HasAVX = Subtarget.hasAVX();
219 bool HasAVX512 = Subtarget.hasAVX512();
220 bool HasEGPR = Subtarget.hasEGPR();
221 bool CallsEHReturn = MF->callsEHReturn();
222
223 CallingConv::ID CC = F.getCallingConv();
224
225 // If attribute NoCallerSavedRegisters exists then we set X86_INTR calling
226 // convention because it has the CSR list.
227 if (MF->getFunction().hasFnAttribute(Kind: "no_caller_saved_registers"))
228 CC = CallingConv::X86_INTR;
229
230 // If atribute specified, override the CSRs normally specified by the
231 // calling convention and use the empty set instead.
232 if (MF->getFunction().hasFnAttribute(Kind: "no_callee_saved_registers"))
233 return CSR_NoRegs_SaveList;
234
235 switch (CC) {
236 case CallingConv::GHC:
237 case CallingConv::HiPE:
238 return CSR_NoRegs_SaveList;
239 case CallingConv::AnyReg:
240 if (HasAVX)
241 return HasEGPR ? CSR_64_APX_AllRegs_AVX_SaveList
242 : CSR_64_AllRegs_AVX_SaveList;
243 return HasEGPR ? CSR_64_APX_AllRegs_SaveList : CSR_64_AllRegs_SaveList;
244 case CallingConv::PreserveMost:
245 if (IsWin64)
246 return HasEGPR ? CSR_Win64_APX_RT_MostRegs_SaveList
247 : CSR_Win64_RT_MostRegs_SaveList;
248 return CSR_64_RT_MostRegs_SaveList;
249 case CallingConv::PreserveAll:
250 if (HasAVX)
251 return CSR_64_RT_AllRegs_AVX_SaveList;
252 return CSR_64_RT_AllRegs_SaveList;
253 case CallingConv::PreserveNone:
254 return CSR_64_NoneRegs_SaveList;
255 case CallingConv::CXX_FAST_TLS:
256 if (Is64Bit)
257 return MF->getInfo<X86MachineFunctionInfo>()->isSplitCSR() ?
258 CSR_64_CXX_TLS_Darwin_PE_SaveList : CSR_64_TLS_Darwin_SaveList;
259 break;
260 case CallingConv::Intel_OCL_BI: {
261 if (HasAVX512 && IsWin64)
262 return HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX512_SaveList
263 : CSR_Win64_Intel_OCL_BI_AVX512_SaveList;
264 if (HasAVX512 && Is64Bit)
265 return CSR_64_Intel_OCL_BI_AVX512_SaveList;
266 if (HasAVX && IsWin64)
267 return HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX_SaveList
268 : CSR_Win64_Intel_OCL_BI_AVX_SaveList;
269 if (HasAVX && Is64Bit)
270 return CSR_64_Intel_OCL_BI_AVX_SaveList;
271 if (!HasAVX && !IsWin64 && Is64Bit)
272 return CSR_64_Intel_OCL_BI_SaveList;
273 break;
274 }
275 case CallingConv::X86_RegCall:
276 if (Is64Bit) {
277 if (IsWin64) {
278 if (HasSSE)
279 return HasEGPR ? CSR_Win64_APX_RegCall_SaveList
280 : CSR_Win64_RegCall_SaveList;
281 return CSR_Win64_RegCall_NoSSE_SaveList;
282 }
283 return HasSSE ? CSR_SysV64_RegCall_SaveList
284 : CSR_SysV64_RegCall_NoSSE_SaveList;
285 }
286 return HasSSE ? CSR_32_RegCall_SaveList : CSR_32_RegCall_NoSSE_SaveList;
287 case CallingConv::CFGuard_Check:
288 assert(!Is64Bit && "CFGuard check mechanism only used on 32-bit X86");
289 return HasSSE ? CSR_Win32_CFGuard_Check_SaveList
290 : CSR_Win32_CFGuard_Check_NoSSE_SaveList;
291 case CallingConv::Cold:
292 if (Is64Bit)
293 return CSR_64_MostRegs_SaveList;
294 break;
295 case CallingConv::Win64:
296 if (HasSSE)
297 return HasEGPR ? CSR_Win64_APX_SaveList : CSR_Win64_SaveList;
298 return CSR_Win64_NoSSE_SaveList;
299 case CallingConv::SwiftTail:
300 if (!Is64Bit)
301 return CSR_32_SaveList;
302 if (IsWin64)
303 return HasEGPR ? CSR_Win64_APX_SwiftTail_SaveList
304 : CSR_Win64_SwiftTail_SaveList;
305 return CSR_64_SwiftTail_SaveList;
306 case CallingConv::X86_64_SysV:
307 if (CallsEHReturn)
308 return CSR_64EHRet_SaveList;
309 return CSR_64_SaveList;
310 case CallingConv::X86_INTR:
311 if (Is64Bit) {
312 if (HasAVX512)
313 return HasEGPR ? CSR_64_APX_AllRegs_AVX512_SaveList
314 : CSR_64_AllRegs_AVX512_SaveList;
315 if (HasAVX)
316 return HasEGPR ? CSR_64_APX_AllRegs_AVX_SaveList
317 : CSR_64_AllRegs_AVX_SaveList;
318 if (HasSSE)
319 return HasEGPR ? CSR_64_APX_AllRegs_SaveList : CSR_64_AllRegs_SaveList;
320 return HasEGPR ? CSR_64_APX_AllRegs_NoSSE_SaveList
321 : CSR_64_AllRegs_NoSSE_SaveList;
322 }
323 if (HasAVX512)
324 return CSR_32_AllRegs_AVX512_SaveList;
325 if (HasAVX)
326 return CSR_32_AllRegs_AVX_SaveList;
327 if (HasSSE)
328 return CSR_32_AllRegs_SSE_SaveList;
329 return CSR_32_AllRegs_SaveList;
330 default:
331 break;
332 }
333
334 if (Is64Bit) {
335 bool IsSwiftCC = Subtarget.getTargetLowering()->supportSwiftError() &&
336 F.getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError);
337 if (IsSwiftCC) {
338 if (IsWin64)
339 return HasEGPR ? CSR_Win64_APX_SwiftError_SaveList
340 : CSR_Win64_SwiftError_SaveList;
341 return CSR_64_SwiftError_SaveList;
342 }
343
344 if (IsWin64 || IsUEFI64) {
345 if (HasSSE)
346 return HasEGPR ? CSR_Win64_APX_SaveList : CSR_Win64_SaveList;
347 return CSR_Win64_NoSSE_SaveList;
348 }
349 if (CallsEHReturn)
350 return CSR_64EHRet_SaveList;
351 return CSR_64_SaveList;
352 }
353
354 return CallsEHReturn ? CSR_32EHRet_SaveList : CSR_32_SaveList;
355}
356
357const MCPhysReg *
358X86RegisterInfo::getIPRACSRegs(const MachineFunction *MF) const {
359 return Is64Bit ? CSR_IPRA_64_SaveList : CSR_IPRA_32_SaveList;
360}
361
362const MCPhysReg *X86RegisterInfo::getCalleeSavedRegsViaCopy(
363 const MachineFunction *MF) const {
364 assert(MF && "Invalid MachineFunction pointer.");
365 if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
366 MF->getInfo<X86MachineFunctionInfo>()->isSplitCSR())
367 return CSR_64_CXX_TLS_Darwin_ViaCopy_SaveList;
368 return nullptr;
369}
370
371const uint32_t *
372X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
373 CallingConv::ID CC) const {
374 const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
375 bool HasSSE = Subtarget.hasSSE1();
376 bool HasAVX = Subtarget.hasAVX();
377 bool HasAVX512 = Subtarget.hasAVX512();
378 bool HasEGPR = Subtarget.hasEGPR();
379
380 switch (CC) {
381 case CallingConv::GHC:
382 case CallingConv::HiPE:
383 return CSR_NoRegs_RegMask;
384 case CallingConv::AnyReg:
385 // Callers keep R16-R31 clobbered: callees built without EGPR or by older
386 // compilers may not save them (see CSR_64_APX_AllRegs).
387 if (HasAVX)
388 return CSR_64_AllRegs_AVX_RegMask;
389 return CSR_64_AllRegs_RegMask;
390 case CallingConv::PreserveMost:
391 if (IsWin64)
392 return HasEGPR ? CSR_Win64_APX_RT_MostRegs_RegMask
393 : CSR_Win64_RT_MostRegs_RegMask;
394 return CSR_64_RT_MostRegs_RegMask;
395 case CallingConv::PreserveAll:
396 if (HasAVX)
397 return CSR_64_RT_AllRegs_AVX_RegMask;
398 return CSR_64_RT_AllRegs_RegMask;
399 case CallingConv::PreserveNone:
400 return CSR_64_NoneRegs_RegMask;
401 case CallingConv::CXX_FAST_TLS:
402 if (Is64Bit)
403 return CSR_64_TLS_Darwin_RegMask;
404 break;
405 case CallingConv::Intel_OCL_BI: {
406 if (HasAVX512 && IsWin64)
407 return HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX512_RegMask
408 : CSR_Win64_Intel_OCL_BI_AVX512_RegMask;
409 if (HasAVX512 && Is64Bit)
410 return CSR_64_Intel_OCL_BI_AVX512_RegMask;
411 if (HasAVX && IsWin64)
412 return HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX_RegMask
413 : CSR_Win64_Intel_OCL_BI_AVX_RegMask;
414 if (HasAVX && Is64Bit)
415 return CSR_64_Intel_OCL_BI_AVX_RegMask;
416 if (!HasAVX && !IsWin64 && Is64Bit)
417 return CSR_64_Intel_OCL_BI_RegMask;
418 break;
419 }
420 case CallingConv::X86_RegCall:
421 if (Is64Bit) {
422 if (IsWin64) {
423 if (HasSSE)
424 return HasEGPR ? CSR_Win64_APX_RegCall_RegMask
425 : CSR_Win64_RegCall_RegMask;
426 return CSR_Win64_RegCall_NoSSE_RegMask;
427 }
428 return HasSSE ? CSR_SysV64_RegCall_RegMask
429 : CSR_SysV64_RegCall_NoSSE_RegMask;
430 }
431 return HasSSE ? CSR_32_RegCall_RegMask : CSR_32_RegCall_NoSSE_RegMask;
432 case CallingConv::CFGuard_Check:
433 if (Is64Bit) {
434 if (HasSSE)
435 return HasEGPR ? CSR_Win64_APX_CFGuard_Check_RegMask
436 : CSR_Win64_CFGuard_Check_RegMask;
437 return CSR_Win64_CFGuard_Check_NoSSE_RegMask;
438 }
439 return HasSSE ? CSR_Win32_CFGuard_Check_RegMask
440 : CSR_Win32_CFGuard_Check_NoSSE_RegMask;
441 case CallingConv::Cold:
442 if (Is64Bit)
443 return CSR_64_MostRegs_RegMask;
444 break;
445 case CallingConv::Win64:
446 return HasEGPR ? CSR_Win64_APX_RegMask : CSR_Win64_RegMask;
447 case CallingConv::SwiftTail:
448 if (!Is64Bit)
449 return CSR_32_RegMask;
450 if (IsWin64)
451 return HasEGPR ? CSR_Win64_APX_SwiftTail_RegMask
452 : CSR_Win64_SwiftTail_RegMask;
453 return CSR_64_SwiftTail_RegMask;
454 case CallingConv::X86_64_SysV:
455 return CSR_64_RegMask;
456 case CallingConv::X86_INTR:
457 // Callers keep R16-R31 clobbered: callees built without EGPR or by older
458 // compilers may not save them (see CSR_64_APX_AllRegs).
459 if (Is64Bit) {
460 if (HasAVX512)
461 return CSR_64_AllRegs_AVX512_RegMask;
462 if (HasAVX)
463 return CSR_64_AllRegs_AVX_RegMask;
464 if (HasSSE)
465 return CSR_64_AllRegs_RegMask;
466 return CSR_64_AllRegs_NoSSE_RegMask;
467 }
468 if (HasAVX512)
469 return CSR_32_AllRegs_AVX512_RegMask;
470 if (HasAVX)
471 return CSR_32_AllRegs_AVX_RegMask;
472 if (HasSSE)
473 return CSR_32_AllRegs_SSE_RegMask;
474 return CSR_32_AllRegs_RegMask;
475 default:
476 break;
477 }
478
479 // Unlike getCalleeSavedRegs(), we don't have MMI so we can't check
480 // callsEHReturn().
481 if (Is64Bit) {
482 const Function &F = MF.getFunction();
483 bool IsSwiftCC = Subtarget.getTargetLowering()->supportSwiftError() &&
484 F.getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError);
485 if (IsSwiftCC) {
486 if (IsWin64)
487 return HasEGPR ? CSR_Win64_APX_SwiftError_RegMask
488 : CSR_Win64_SwiftError_RegMask;
489 return CSR_64_SwiftError_RegMask;
490 }
491
492 if (IsWin64 || IsUEFI64)
493 return HasEGPR ? CSR_Win64_APX_RegMask : CSR_Win64_RegMask;
494 return CSR_64_RegMask;
495 }
496
497 return CSR_32_RegMask;
498}
499
500const uint32_t*
501X86RegisterInfo::getNoPreservedMask() const {
502 return CSR_NoRegs_RegMask;
503}
504
505const uint32_t *X86RegisterInfo::getDarwinTLSCallPreservedMask() const {
506 return CSR_64_TLS_Darwin_RegMask;
507}
508
509BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
510 BitVector Reserved(getNumRegs());
511 const X86FrameLowering *TFI = getFrameLowering(MF);
512
513 // Set the floating point control register as reserved.
514 Reserved.set(X86::FPCW);
515
516 // Set the floating point status register as reserved.
517 Reserved.set(X86::FPSW);
518
519 // Set the SIMD floating point control register as reserved.
520 Reserved.set(X86::MXCSR);
521
522 // Set the stack-pointer register and its aliases as reserved.
523 for (const MCPhysReg &SubReg : subregs_inclusive(Reg: X86::RSP))
524 Reserved.set(SubReg);
525
526 // Set the Shadow Stack Pointer as reserved.
527 Reserved.set(X86::SSP);
528
529 auto &ST = MF.getSubtarget<X86Subtarget>();
530 if (ST.hasUserReservedRegisters()) {
531 if (ST.is64Bit()) {
532 // Set r# as reserved register if user required.
533 for (unsigned Reg = X86::R8; Reg <= X86::R15; ++Reg)
534 if (ST.isRegisterReservedByUser(i: Reg))
535 for (const MCPhysReg &SubReg : subregs_inclusive(Reg))
536 Reserved.set(SubReg);
537 if (ST.hasEGPR())
538 for (unsigned Reg = X86::R16; Reg <= X86::R31; ++Reg)
539 if (ST.isRegisterReservedByUser(i: Reg))
540 for (const MCPhysReg &SubReg : subregs_inclusive(Reg))
541 Reserved.set(SubReg);
542 } else {
543 if (ST.isRegisterReservedByUser(i: X86::EDI))
544 for (const MCPhysReg &SubReg : sub_and_superregs_inclusive(Reg: X86::EDI))
545 Reserved.set(SubReg);
546 }
547 }
548
549 // Set the instruction pointer register and its aliases as reserved.
550 for (const MCPhysReg &SubReg : subregs_inclusive(Reg: X86::RIP))
551 Reserved.set(SubReg);
552
553 // Set the frame-pointer register and its aliases as reserved if needed.
554 if (TFI->hasFP(MF) || MF.framePointerIsReserved()) {
555 if (MF.getInfo<X86MachineFunctionInfo>()->getFPClobberedByInvoke())
556 MF.getContext().reportError(
557 L: SMLoc(),
558 Msg: "Frame pointer clobbered by function invoke is not supported.");
559
560 for (const MCPhysReg &SubReg : subregs_inclusive(Reg: X86::RBP))
561 Reserved.set(SubReg);
562 }
563
564 // Set the base-pointer register and its aliases as reserved if needed.
565 if (hasBasePointer(MF)) {
566 if (MF.getInfo<X86MachineFunctionInfo>()->getBPClobberedByInvoke())
567 MF.getContext().reportError(L: SMLoc(),
568 Msg: "Stack realignment in presence of dynamic "
569 "allocas is not supported with "
570 "this calling convention.");
571
572 Register BasePtr = getX86SubSuperRegister(Reg: getBaseRegister(), Size: 64);
573 for (const MCPhysReg &SubReg : subregs_inclusive(Reg: BasePtr))
574 Reserved.set(SubReg);
575 }
576
577 // Mark the segment registers as reserved.
578 Reserved.set(X86::CS);
579 Reserved.set(X86::SS);
580 Reserved.set(X86::DS);
581 Reserved.set(X86::ES);
582 Reserved.set(X86::FS);
583 Reserved.set(X86::GS);
584
585 // Mark the floating point stack registers as reserved.
586 for (unsigned n = 0; n != 8; ++n)
587 Reserved.set(X86::ST0 + n);
588
589 // Without usable x87 (soft float or -mno-x87), reserve the allocatable FPn
590 // pseudos (FP0-FP6; FP7 is already non-allocatable) so they aren't scrubbed.
591 if (ST.useSoftFloat() || !ST.hasX87())
592 for (unsigned n = 0; n != 7; ++n)
593 Reserved.set(X86::FP0 + n);
594
595 // Reserve the registers that only exist in 64-bit mode.
596 if (!Is64Bit) {
597 // These 8-bit registers are part of the x86-64 extension even though their
598 // super-registers are old 32-bits.
599 Reserved.set(X86::SIL);
600 Reserved.set(X86::DIL);
601 Reserved.set(X86::BPL);
602 Reserved.set(X86::SPL);
603 Reserved.set(X86::SIH);
604 Reserved.set(X86::DIH);
605 Reserved.set(X86::BPH);
606 Reserved.set(X86::SPH);
607
608 for (unsigned n = 0; n != 8; ++n) {
609 // R8, R9, ...
610 for (MCRegAliasIterator AI(X86::R8 + n, this, true); AI.isValid(); ++AI)
611 Reserved.set(*AI);
612
613 // XMM8, XMM9, ...
614 for (MCRegAliasIterator AI(X86::XMM8 + n, this, true); AI.isValid(); ++AI)
615 Reserved.set(*AI);
616 }
617 }
618 if (!Is64Bit || !MF.getSubtarget<X86Subtarget>().hasAVX512()) {
619 for (unsigned n = 0; n != 16; ++n) {
620 for (MCRegAliasIterator AI(X86::XMM16 + n, this, true); AI.isValid();
621 ++AI)
622 Reserved.set(*AI);
623 }
624 }
625
626 // Reserve the extended general purpose registers.
627 if (!Is64Bit || !MF.getSubtarget<X86Subtarget>().hasEGPR())
628 Reserved.set(I: X86::R16, E: X86::R31WH + 1);
629
630 // Due to specifics of setjmp unwinding in Win64 APX ABI, the unwinder
631 // cannot restore R30/R31. Reserve them to prevent register allocation.
632 // https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention#setjmplongjmp
633 if (MF.exposesReturnsTwice() && ST.isTargetWin64()) {
634 unsigned NumReservedCSRs = 0;
635 for (unsigned Reg = X86::R16; Reg <= X86::R31; ++Reg)
636 if (isCalleeSavedPhysReg(PhysReg: Reg, MF)) {
637 ++NumReservedCSRs;
638 for (const MCPhysReg &SubReg : subregs_inclusive(Reg))
639 Reserved.set(SubReg);
640 }
641 if (NumReservedCSRs &&
642 MF.size() > ST.getCLOpts().setjmp_csr_warning_threshold &&
643 !MF.getRegInfo().reservedRegsFrozen()) {
644 MF.getContext().reportWarning(
645 L: SMLoc(), Msg: Twine(NumReservedCSRs) +
646 " callee-saved register(s) reserved due to setjmp in '" +
647 MF.getName() +
648 "'; this may impact performance in large functions");
649 }
650 }
651
652 if (MF.getFunction().getCallingConv() == CallingConv::GRAAL) {
653 for (MCRegAliasIterator AI(X86::R14, this, true); AI.isValid(); ++AI)
654 Reserved.set(*AI);
655 for (MCRegAliasIterator AI(X86::R15, this, true); AI.isValid(); ++AI)
656 Reserved.set(*AI);
657 }
658
659 // Reserve registers for LFI sandboxing.
660 if (MF.getSubtarget<X86Subtarget>().isLFI()) {
661 for (MCRegAliasIterator AI(X86::R11, this, true); AI.isValid(); ++AI)
662 Reserved.set(*AI);
663 for (MCRegAliasIterator AI(X86::R14, this, true); AI.isValid(); ++AI)
664 Reserved.set(*AI);
665 for (MCRegAliasIterator AI(X86::R15, this, true); AI.isValid(); ++AI)
666 Reserved.set(*AI);
667 }
668
669 assert(checkAllSuperRegsMarked(Reserved,
670 {X86::SIL, X86::DIL, X86::BPL, X86::SPL,
671 X86::SIH, X86::DIH, X86::BPH, X86::SPH}));
672 return Reserved;
673}
674
675unsigned X86RegisterInfo::getNumSupportedRegs(const MachineFunction &MF) const {
676 // All existing Intel CPUs that support AMX support AVX512 and all existing
677 // Intel CPUs that support APX support AMX. AVX512 implies AVX.
678 //
679 // We enumerate the registers in X86GenRegisterInfo.inc in this order:
680 //
681 // Registers before AVX512,
682 // AVX512 registers (X/YMM16-31, ZMM0-31, K registers)
683 // AMX registers (TMM)
684 // APX registers (R16-R31)
685 //
686 // and try to return the minimum number of registers supported by the target.
687 static_assert((X86::R15WH + 1 == X86::YMM0) && (X86::YMM15 + 1 == X86::K0) &&
688 (X86::K6_K7 + 1 == X86::TMMCFG) &&
689 (X86::TMM7 + 1 == X86::R16) &&
690 (X86::R31WH + 1 == X86::NUM_TARGET_REGS),
691 "Register number may be incorrect");
692
693 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
694 if (ST.hasEGPR())
695 return X86::NUM_TARGET_REGS;
696 if (ST.hasAMXTILE())
697 return X86::TMM7 + 1;
698 if (ST.hasAVX512())
699 return X86::K6_K7 + 1;
700 if (ST.hasAVX())
701 return X86::YMM15 + 1;
702 return X86::R15WH + 1;
703}
704
705bool X86RegisterInfo::isArgumentRegister(const MachineFunction &MF,
706 MCRegister Reg) const {
707 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
708 const TargetRegisterInfo &TRI = *ST.getRegisterInfo();
709 auto IsSubReg = [&](MCRegister RegA, MCRegister RegB) {
710 return TRI.isSuperOrSubRegisterEq(RegA, RegB);
711 };
712
713 if (!ST.is64Bit())
714 return llvm::any_of(
715 Range: SmallVector<MCRegister>{X86::EAX, X86::ECX, X86::EDX},
716 P: [&](MCRegister &RegA) { return IsSubReg(RegA, Reg); }) ||
717 (ST.hasMMX() && X86::VR64RegClass.contains(Reg));
718
719 CallingConv::ID CC = MF.getFunction().getCallingConv();
720
721 if (CC == CallingConv::X86_64_SysV && IsSubReg(X86::RAX, Reg))
722 return true;
723
724 if (llvm::any_of(
725 Range: SmallVector<MCRegister>{X86::RDX, X86::RCX, X86::R8, X86::R9},
726 P: [&](MCRegister &RegA) { return IsSubReg(RegA, Reg); }))
727 return true;
728
729 if (CC != CallingConv::Win64 &&
730 llvm::any_of(Range: SmallVector<MCRegister>{X86::RDI, X86::RSI},
731 P: [&](MCRegister &RegA) { return IsSubReg(RegA, Reg); }))
732 return true;
733
734 if (ST.hasSSE1() &&
735 llvm::any_of(Range: SmallVector<MCRegister>{X86::XMM0, X86::XMM1, X86::XMM2,
736 X86::XMM3, X86::XMM4, X86::XMM5,
737 X86::XMM6, X86::XMM7},
738 P: [&](MCRegister &RegA) { return IsSubReg(RegA, Reg); }))
739 return true;
740
741 return X86GenRegisterInfo::isArgumentRegister(MF, PhysReg: Reg);
742}
743
744bool X86RegisterInfo::isFixedRegister(const MachineFunction &MF,
745 MCRegister PhysReg) const {
746 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
747 const TargetRegisterInfo &TRI = *ST.getRegisterInfo();
748
749 // Stack pointer.
750 if (TRI.isSuperOrSubRegisterEq(RegA: X86::RSP, RegB: PhysReg))
751 return true;
752
753 // Don't use the frame pointer if it's being used.
754 const X86FrameLowering &TFI = *getFrameLowering(MF);
755 if (TFI.hasFP(MF) && TRI.isSuperOrSubRegisterEq(RegA: X86::RBP, RegB: PhysReg))
756 return true;
757
758 return X86GenRegisterInfo::isFixedRegister(MF, PhysReg);
759}
760
761bool X86RegisterInfo::isTileRegisterClass(const TargetRegisterClass *RC) const {
762 return RC->getID() == X86::TILERegClassID;
763}
764
765void X86RegisterInfo::adjustStackMapLiveOutMask(uint32_t *Mask) const {
766 // Check if the EFLAGS register is marked as live-out. This shouldn't happen,
767 // because the calling convention defines the EFLAGS register as NOT
768 // preserved.
769 //
770 // Unfortunatelly the EFLAGS show up as live-out after branch folding. Adding
771 // an assert to track this and clear the register afterwards to avoid
772 // unnecessary crashes during release builds.
773 assert(!(Mask[X86::EFLAGS / 32] & (1U << (X86::EFLAGS % 32))) &&
774 "EFLAGS are not live-out from a patchpoint.");
775
776 // Also clean other registers that don't need preserving (IP).
777 for (auto Reg : {X86::EFLAGS, X86::RIP, X86::EIP, X86::IP})
778 Mask[Reg / 32] &= ~(1U << (Reg % 32));
779}
780
781//===----------------------------------------------------------------------===//
782// Stack Frame Processing methods
783//===----------------------------------------------------------------------===//
784
785static bool CantUseSP(const MachineFrameInfo &MFI) {
786 return MFI.hasVarSizedObjects() || MFI.hasOpaqueSPAdjustment();
787}
788
789bool X86RegisterInfo::hasBasePointer(const MachineFunction &MF) const {
790 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
791 // We have a virtual register to reference argument, and don't need base
792 // pointer.
793 if (X86FI->getStackPtrSaveMI() != nullptr)
794 return false;
795
796 if (X86FI->hasPreallocatedCall())
797 return true;
798
799 const MachineFrameInfo &MFI = MF.getFrameInfo();
800
801 if (!MF.getSubtarget<X86Subtarget>().getCLOpts().use_base_pointer)
802 return false;
803
804 // When we need stack realignment, we can't address the stack from the frame
805 // pointer. When we have dynamic allocas or stack-adjusting inline asm, we
806 // can't address variables from the stack pointer. MS inline asm can
807 // reference locals while also adjusting the stack pointer. When we can't
808 // use both the SP and the FP, we need a separate base pointer register.
809 bool CantUseFP = hasStackRealignment(MF);
810 return CantUseFP && CantUseSP(MFI);
811}
812
813bool X86RegisterInfo::canRealignStack(const MachineFunction &MF) const {
814 if (!TargetRegisterInfo::canRealignStack(MF))
815 return false;
816
817 const MachineFrameInfo &MFI = MF.getFrameInfo();
818 const MachineRegisterInfo *MRI = &MF.getRegInfo();
819
820 // Stack realignment requires a frame pointer. If we already started
821 // register allocation with frame pointer elimination, it is too late now.
822 if (!MRI->canReserveReg(PhysReg: FramePtr))
823 return false;
824
825 // If a base pointer is necessary. Check that it isn't too late to reserve
826 // it.
827 if (CantUseSP(MFI))
828 return MRI->canReserveReg(PhysReg: BasePtr);
829 return true;
830}
831
832bool X86RegisterInfo::shouldRealignStack(const MachineFunction &MF) const {
833 if (TargetRegisterInfo::shouldRealignStack(MF))
834 return true;
835
836 return !Is64Bit && MF.getFunction().getCallingConv() == CallingConv::X86_INTR;
837}
838
839// tryOptimizeLEAtoMOV - helper function that tries to replace a LEA instruction
840// of the form 'lea (%esp), %ebx' --> 'mov %esp, %ebx'.
841// TODO: In this case we should be really trying first to entirely eliminate
842// this instruction which is a plain copy.
843static bool tryOptimizeLEAtoMOV(MachineBasicBlock::iterator II) {
844 MachineInstr &MI = *II;
845 unsigned Opc = II->getOpcode();
846 // Check if this is a LEA of the form 'lea (%esp), %ebx'
847 if ((Opc != X86::LEA32r && Opc != X86::LEA64r && Opc != X86::LEA64_32r) ||
848 MI.getOperand(i: 2).getImm() != 1 ||
849 MI.getOperand(i: 3).getReg() != X86::NoRegister ||
850 MI.getOperand(i: 4).getImm() != 0 ||
851 MI.getOperand(i: 5).getReg() != X86::NoRegister)
852 return false;
853 Register BasePtr = MI.getOperand(i: 1).getReg();
854 // In X32 mode, ensure the base-pointer is a 32-bit operand, so the LEA will
855 // be replaced with a 32-bit operand MOV which will zero extend the upper
856 // 32-bits of the super register.
857 if (Opc == X86::LEA64_32r)
858 BasePtr = getX86SubSuperRegister(Reg: BasePtr, Size: 32);
859 Register NewDestReg = MI.getOperand(i: 0).getReg();
860 const X86InstrInfo *TII =
861 MI.getParent()->getParent()->getSubtarget<X86Subtarget>().getInstrInfo();
862 TII->copyPhysReg(MBB&: *MI.getParent(), MI: II, DL: MI.getDebugLoc(), DestReg: NewDestReg, SrcReg: BasePtr,
863 KillSrc: MI.getOperand(i: 1).isKill());
864 MI.eraseFromParent();
865 return true;
866}
867
868static bool isFuncletReturnInstr(MachineInstr &MI) {
869 switch (MI.getOpcode()) {
870 case X86::CATCHRET:
871 case X86::CLEANUPRET:
872 return true;
873 default:
874 return false;
875 }
876 llvm_unreachable("impossible");
877}
878
879void X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
880 unsigned FIOperandNum,
881 Register BaseReg,
882 int FIOffset) const {
883 MachineInstr &MI = *II;
884 unsigned Opc = MI.getOpcode();
885 if (Opc == TargetOpcode::LOCAL_ESCAPE) {
886 MachineOperand &FI = MI.getOperand(i: FIOperandNum);
887 FI.ChangeToImmediate(ImmVal: FIOffset);
888 return;
889 }
890
891 MI.getOperand(i: FIOperandNum).ChangeToRegister(Reg: BaseReg, isDef: false);
892
893 // The frame index format for stackmaps and patchpoints is different from the
894 // X86 format. It only has a FI and an offset.
895 if (Opc == TargetOpcode::STACKMAP || Opc == TargetOpcode::PATCHPOINT) {
896 assert(BasePtr == FramePtr && "Expected the FP as base register");
897 int64_t Offset = MI.getOperand(i: FIOperandNum + 1).getImm() + FIOffset;
898 MI.getOperand(i: FIOperandNum + 1).ChangeToImmediate(ImmVal: Offset);
899 return;
900 }
901
902 if (MI.getOperand(i: FIOperandNum + 3).isImm()) {
903 // Offset is a 32-bit integer.
904 int Imm = (int)(MI.getOperand(i: FIOperandNum + 3).getImm());
905 int Offset = FIOffset + Imm;
906 assert((!Is64Bit || isInt<32>((long long)FIOffset + Imm)) &&
907 "Requesting 64-bit offset in 32-bit immediate!");
908 if (Offset != 0)
909 MI.getOperand(i: FIOperandNum + 3).ChangeToImmediate(ImmVal: Offset);
910 } else {
911 // Offset is symbolic. This is extremely rare.
912 uint64_t Offset =
913 FIOffset + (uint64_t)MI.getOperand(i: FIOperandNum + 3).getOffset();
914 MI.getOperand(i: FIOperandNum + 3).setOffset(Offset);
915 }
916}
917
918bool
919X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
920 int SPAdj, unsigned FIOperandNum,
921 RegScavenger *RS) const {
922 MachineInstr &MI = *II;
923 MachineBasicBlock &MBB = *MI.getParent();
924 MachineFunction &MF = *MBB.getParent();
925 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
926 bool IsEHFuncletEpilogue = MBBI == MBB.end() ? false
927 : isFuncletReturnInstr(MI&: *MBBI);
928 const X86FrameLowering *TFI = getFrameLowering(MF);
929 int FrameIndex = MI.getOperand(i: FIOperandNum).getIndex();
930
931 // Determine base register and offset.
932 int64_t FIOffset;
933 Register BasePtr;
934 if (MI.isReturn()) {
935 assert((!hasStackRealignment(MF) ||
936 MF.getFrameInfo().isFixedObjectIndex(FrameIndex)) &&
937 "Return instruction can only reference SP relative frame objects");
938 FIOffset =
939 TFI->getFrameIndexReferenceSP(MF, FI: FrameIndex, SPReg&: BasePtr, Adjustment: 0).getFixed();
940 } else if (TFI->Is64Bit && (MBB.isEHFuncletEntry() || IsEHFuncletEpilogue)) {
941 FIOffset = TFI->getWin64EHFrameIndexRef(MF, FI: FrameIndex, SPReg&: BasePtr);
942 } else {
943 FIOffset = TFI->getFrameIndexReference(MF, FI: FrameIndex, FrameReg&: BasePtr).getFixed();
944 }
945
946 // LOCAL_ESCAPE uses a single offset, with no register. It only works in the
947 // simple FP case, and doesn't work with stack realignment. On 32-bit, the
948 // offset is from the traditional base pointer location. On 64-bit, the
949 // offset is from the SP at the end of the prologue, not the FP location. This
950 // matches the behavior of llvm.frameaddress.
951 unsigned Opc = MI.getOpcode();
952 if (Opc == TargetOpcode::LOCAL_ESCAPE) {
953 MachineOperand &FI = MI.getOperand(i: FIOperandNum);
954 FI.ChangeToImmediate(ImmVal: FIOffset);
955 return false;
956 }
957
958 // For LEA64_32r when BasePtr is 32-bits (X32) we can use full-size 64-bit
959 // register as source operand, semantic is the same and destination is
960 // 32-bits. It saves one byte per lea in code since 0x67 prefix is avoided.
961 // Don't change BasePtr since it is used later for stack adjustment.
962 Register MachineBasePtr = BasePtr;
963 if (Opc == X86::LEA64_32r && X86::GR32RegClass.contains(Reg: BasePtr))
964 MachineBasePtr = getX86SubSuperRegister(Reg: BasePtr, Size: 64);
965
966 // This must be part of a four operand memory reference. Replace the
967 // FrameIndex with base register. Add an offset to the offset.
968 MI.getOperand(i: FIOperandNum).ChangeToRegister(Reg: MachineBasePtr, isDef: false);
969
970 if (BasePtr == StackPtr)
971 FIOffset += SPAdj;
972
973 // The frame index format for stackmaps and patchpoints is different from the
974 // X86 format. It only has a FI and an offset.
975 if (Opc == TargetOpcode::STACKMAP || Opc == TargetOpcode::PATCHPOINT) {
976 assert(BasePtr == FramePtr && "Expected the FP as base register");
977 int64_t Offset = MI.getOperand(i: FIOperandNum + 1).getImm() + FIOffset;
978 MI.getOperand(i: FIOperandNum + 1).ChangeToImmediate(ImmVal: Offset);
979 return false;
980 }
981
982 if (MI.getOperand(i: FIOperandNum+3).isImm()) {
983 const X86InstrInfo *TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
984 const DebugLoc &DL = MI.getDebugLoc();
985 int64_t Imm = MI.getOperand(i: FIOperandNum + 3).getImm();
986 int64_t Offset = FIOffset + Imm;
987 bool FitsIn32Bits = isInt<32>(x: Offset);
988 // If the offset will not fit in a 32-bit displacement, then for 64-bit
989 // targets, scavenge a register to hold it. Otherwise...
990 if (Is64Bit && !FitsIn32Bits) {
991 assert(RS && "RegisterScavenger was NULL");
992
993 RS->enterBasicBlockEnd(MBB);
994 RS->backward(I: std::next(x: II));
995
996 Register ScratchReg = RS->scavengeRegisterBackwards(
997 RC: X86::GR64RegClass, To: II, /*RestoreAfter=*/false, /*SPAdj=*/0,
998 /*AllowSpill=*/true);
999 assert(ScratchReg != 0 && "scratch reg was 0");
1000 RS->setRegUsed(Reg: ScratchReg);
1001
1002 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: TII->get(Opcode: X86::MOV64ri), DestReg: ScratchReg).addImm(Val: Offset);
1003
1004 MI.getOperand(i: FIOperandNum + 3).setImm(0);
1005 if (MI.getOperand(i: FIOperandNum + 2).getReg() == X86::NoRegister) {
1006 MI.getOperand(i: FIOperandNum + 2).setReg(ScratchReg);
1007 } else {
1008 // The index register slot is already in use, fold the offset into
1009 // the base register instead. LEA does not clobber EFLAGS.
1010 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: TII->get(Opcode: X86::LEA64r), DestReg: ScratchReg)
1011 .addReg(RegNo: MachineBasePtr)
1012 .addImm(Val: 1)
1013 .addReg(RegNo: ScratchReg)
1014 .addImm(Val: 0)
1015 .addReg(RegNo: X86::NoRegister);
1016 MI.getOperand(i: FIOperandNum).setReg(ScratchReg);
1017 }
1018
1019 return false;
1020 }
1021
1022 // ... for 32-bit targets, this is a bug!
1023 if (!Is64Bit && !FitsIn32Bits) {
1024 MI.emitGenericError(ErrMsg: "64-bit offset calculated but target is 32-bit");
1025 // Trap so that the instruction verification pass does not fail if run.
1026 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::TRAP));
1027 return false;
1028 }
1029
1030 if (Offset != 0 || !tryOptimizeLEAtoMOV(II))
1031 MI.getOperand(i: FIOperandNum + 3).ChangeToImmediate(ImmVal: Offset);
1032 } else {
1033 // Offset is symbolic. This is extremely rare.
1034 uint64_t Offset = FIOffset +
1035 (uint64_t)MI.getOperand(i: FIOperandNum+3).getOffset();
1036 MI.getOperand(i: FIOperandNum + 3).setOffset(Offset);
1037 }
1038 return false;
1039}
1040
1041unsigned X86RegisterInfo::findDeadCallerSavedReg(
1042 MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI) const {
1043 const MachineFunction *MF = MBB.getParent();
1044 const MachineRegisterInfo &MRI = MF->getRegInfo();
1045 if (MF->callsEHReturn())
1046 return 0;
1047
1048 if (MBBI == MBB.end())
1049 return 0;
1050
1051 switch (MBBI->getOpcode()) {
1052 default:
1053 return 0;
1054 case TargetOpcode::PATCHABLE_RET:
1055 case X86::RET:
1056 case X86::RET32:
1057 case X86::RET64:
1058 case X86::RETI32:
1059 case X86::RETI64:
1060 case X86::TCRETURNdi:
1061 case X86::TCRETURNri:
1062 case X86::TCRETURN_WIN64ri:
1063 case X86::TCRETURN_HIPE32ri:
1064 case X86::TCRETURNmi:
1065 case X86::TCRETURNdi64:
1066 case X86::TCRETURNri64:
1067 case X86::TCRETURNri64_ImpCall:
1068 case X86::TCRETURNmi64:
1069 case X86::TCRETURN_WINmi64:
1070 case X86::EH_RETURN:
1071 case X86::EH_RETURN64: {
1072 LiveRegUnits LRU(*this);
1073 LRU.addLiveOuts(MBB);
1074 LRU.stepBackward(MI: *MBBI);
1075
1076 const TargetRegisterClass &RC =
1077 Is64Bit ? X86::GR64_NOSPRegClass : X86::GR32_NOSPRegClass;
1078 for (MCRegister Reg : RC) {
1079 if (LRU.available(Reg) && !MRI.isReserved(PhysReg: Reg))
1080 return Reg;
1081 }
1082 }
1083 }
1084
1085 return 0;
1086}
1087
1088Register X86RegisterInfo::getFrameRegister(const MachineFunction &MF) const {
1089 const X86FrameLowering *TFI = getFrameLowering(MF);
1090 return TFI->hasFP(MF) ? FramePtr : StackPtr;
1091}
1092
1093Register
1094X86RegisterInfo::getPtrSizedFrameRegister(const MachineFunction &MF) const {
1095 const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
1096 Register FrameReg = getFrameRegister(MF);
1097 if (Subtarget.isTarget64BitILP32())
1098 FrameReg = getX86SubSuperRegister(Reg: FrameReg, Size: 32);
1099 return FrameReg;
1100}
1101
1102Register
1103X86RegisterInfo::getPtrSizedStackRegister(const MachineFunction &MF) const {
1104 const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
1105 Register StackReg = getStackRegister();
1106 if (Subtarget.isTarget64BitILP32())
1107 StackReg = getX86SubSuperRegister(Reg: StackReg, Size: 32);
1108 return StackReg;
1109}
1110
1111static ShapeT getTileShape(Register VirtReg, VirtRegMap *VRM,
1112 const MachineRegisterInfo *MRI) {
1113 if (VRM->hasShape(virtReg: VirtReg))
1114 return VRM->getShape(virtReg: VirtReg);
1115
1116 const MachineOperand &Def = *MRI->def_begin(RegNo: VirtReg);
1117 MachineInstr *MI = const_cast<MachineInstr *>(Def.getParent());
1118 unsigned OpCode = MI->getOpcode();
1119 switch (OpCode) {
1120 default:
1121 llvm_unreachable("Unexpected machine instruction on tile register!");
1122 break;
1123 case X86::COPY: {
1124 Register SrcReg = MI->getOperand(i: 1).getReg();
1125 ShapeT Shape = getTileShape(VirtReg: SrcReg, VRM, MRI);
1126 VRM->assignVirt2Shape(virtReg: VirtReg, shape: Shape);
1127 return Shape;
1128 }
1129 // We only collect the tile shape that is defined.
1130 case X86::PTILELOADDV:
1131 case X86::PTILELOADDT1V:
1132 case X86::PTDPBSSDV:
1133 case X86::PTDPBSUDV:
1134 case X86::PTDPBUSDV:
1135 case X86::PTDPBUUDV:
1136 case X86::PTILEZEROV:
1137 case X86::PTDPBF16PSV:
1138 case X86::PTDPFP16PSV:
1139 case X86::PTCMMIMFP16PSV:
1140 case X86::PTCMMRLFP16PSV:
1141 case X86::PTILELOADDRSV:
1142 case X86::PTILELOADDRST1V:
1143 case X86::PTDPBF8PSV:
1144 case X86::PTDPBHF8PSV:
1145 case X86::PTDPHBF8PSV:
1146 case X86::PTDPHF8PSV: {
1147 MachineOperand &MO1 = MI->getOperand(i: 1);
1148 MachineOperand &MO2 = MI->getOperand(i: 2);
1149 ShapeT Shape(&MO1, &MO2, MRI);
1150 VRM->assignVirt2Shape(virtReg: VirtReg, shape: Shape);
1151 return Shape;
1152 }
1153 }
1154}
1155
1156bool X86RegisterInfo::getRegAllocationHints(
1157 Register VirtReg, ArrayRef<MCPhysReg> Order,
1158 SmallSetVector<MCPhysReg, 16> &Hints, const MachineFunction &MF,
1159 const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const {
1160 const MachineRegisterInfo *MRI = &MF.getRegInfo();
1161 const TargetRegisterClass &RC = *MRI->getRegClass(Reg: VirtReg);
1162 bool BaseImplRetVal = TargetRegisterInfo::getRegAllocationHints(
1163 VirtReg, Order, Hints, MF, VRM, Matrix);
1164 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
1165 const TargetRegisterInfo &TRI = *ST.getRegisterInfo();
1166
1167 unsigned ID = RC.getID();
1168
1169 if (!VRM)
1170 return BaseImplRetVal;
1171
1172 if (ID != X86::TILERegClassID) {
1173 if (ST.getCLOpts().disable_regalloc_hints_for_ndd || !ST.hasNDD() ||
1174 !TRI.isGeneralPurposeRegisterClass(RC: &RC))
1175 return BaseImplRetVal;
1176
1177 // Add any two address hints after any copy hints.
1178 SmallSet<unsigned, 4> TwoAddrHints;
1179
1180 auto TryAddNDDHint = [&](const MachineOperand &MO) {
1181 Register Reg = MO.getReg();
1182 Register PhysReg = Reg.isPhysical() ? Reg : Register(VRM->getPhys(virtReg: Reg));
1183 if (PhysReg && !MRI->isReserved(PhysReg) && !Hints.contains(key: PhysReg))
1184 TwoAddrHints.insert(V: PhysReg);
1185 };
1186
1187 // NDD instructions is compressible when Op0 is allocated to the same
1188 // physic register as Op1 (or Op2 if it's commutable).
1189 for (auto &MO : MRI->reg_nodbg_operands(Reg: VirtReg)) {
1190 const MachineInstr &MI = *MO.getParent();
1191 if (!X86::getNonNDVariant(Opc: MI.getOpcode()))
1192 continue;
1193 unsigned OpIdx = MI.getOperandNo(I: &MO);
1194 if (OpIdx == 0) {
1195 assert(MI.getOperand(1).isReg());
1196 TryAddNDDHint(MI.getOperand(i: 1));
1197 if (MI.isCommutable()) {
1198 assert(MI.getOperand(2).isReg());
1199 TryAddNDDHint(MI.getOperand(i: 2));
1200 }
1201 } else if (OpIdx == 1) {
1202 TryAddNDDHint(MI.getOperand(i: 0));
1203 } else if (MI.isCommutable() && OpIdx == 2) {
1204 TryAddNDDHint(MI.getOperand(i: 0));
1205 }
1206 }
1207
1208 for (MCPhysReg OrderReg : Order)
1209 if (TwoAddrHints.count(V: OrderReg))
1210 Hints.insert(X: OrderReg);
1211
1212 return BaseImplRetVal;
1213 }
1214
1215 ShapeT VirtShape = getTileShape(VirtReg, VRM: const_cast<VirtRegMap *>(VRM), MRI);
1216 auto AddHint = [&](MCPhysReg PhysReg) {
1217 Register VReg = Matrix->getOneVReg(PhysReg);
1218 if (VReg == MCRegister::NoRegister) { // Not allocated yet
1219 Hints.insert(X: PhysReg);
1220 return;
1221 }
1222 ShapeT PhysShape = getTileShape(VirtReg: VReg, VRM: const_cast<VirtRegMap *>(VRM), MRI);
1223 if (PhysShape == VirtShape)
1224 Hints.insert(X: PhysReg);
1225 };
1226
1227 SmallSetVector<MCPhysReg, 16> CopyHints(Hints);
1228 Hints.clear();
1229 for (auto Hint : CopyHints) {
1230 if (RC.contains(Reg: Hint) && !MRI->isReserved(PhysReg: Hint))
1231 AddHint(Hint);
1232 }
1233 for (MCPhysReg PhysReg : Order) {
1234 if (!CopyHints.contains(key: PhysReg) && RC.contains(Reg: PhysReg) &&
1235 !MRI->isReserved(PhysReg))
1236 AddHint(PhysReg);
1237 }
1238
1239#define DEBUG_TYPE "tile-hint"
1240 LLVM_DEBUG({
1241 dbgs() << "Hints for virtual register " << format_hex(VirtReg, 8) << "\n";
1242 for (auto Hint : Hints) {
1243 dbgs() << "tmm" << Hint << ",";
1244 }
1245 dbgs() << "\n";
1246 });
1247#undef DEBUG_TYPE
1248
1249 return true;
1250}
1251
1252const TargetRegisterClass *X86RegisterInfo::constrainRegClassToNonRex2(
1253 const TargetRegisterClass *RC) const {
1254 switch (RC->getID()) {
1255 default:
1256 return RC;
1257 case X86::GR8RegClassID:
1258 return &X86::GR8_NOREX2RegClass;
1259 case X86::GR16RegClassID:
1260 return &X86::GR16_NOREX2RegClass;
1261 case X86::GR32RegClassID:
1262 return &X86::GR32_NOREX2RegClass;
1263 case X86::GR64RegClassID:
1264 return &X86::GR64_NOREX2RegClass;
1265 case X86::GR32_NOSPRegClassID:
1266 return &X86::GR32_NOREX2_NOSPRegClass;
1267 case X86::GR64_NOSPRegClassID:
1268 return &X86::GR64_NOREX2_NOSPRegClass;
1269 }
1270}
1271
1272bool X86RegisterInfo::isNonRex2RegClass(const TargetRegisterClass *RC) const {
1273 switch (RC->getID()) {
1274 default:
1275 return false;
1276 case X86::GR8_NOREX2RegClassID:
1277 case X86::GR16_NOREX2RegClassID:
1278 case X86::GR32_NOREX2RegClassID:
1279 case X86::GR64_NOREX2RegClassID:
1280 case X86::GR32_NOREX2_NOSPRegClassID:
1281 case X86::GR64_NOREX2_NOSPRegClassID:
1282 case X86::GR64_with_sub_16bit_in_GR16_NOREX2RegClassID:
1283 return true;
1284 }
1285}
1286
1287unsigned X86RegisterInfo::getCSRFirstUseCost(const MachineFunction &MF) const {
1288 // PPX push/pop pairs don't access memory, but they still execute, so use a
1289 // non-zero cost to let RAGreedy keep CSR saves off hot paths.
1290 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
1291 if (ST.is64Bit() && ST.hasPPX())
1292 return 1;
1293
1294 // push + pop.
1295 return 2;
1296}
1297