1//===-- X86RegisterInfo.cpp - X86 Register Information --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the X86 implementation of the TargetRegisterInfo class.
10// This file is responsible for the frame pointer elimination optimization
11// on X86.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86RegisterInfo.h"
16#include "X86FrameLowering.h"
17#include "X86MachineFunctionInfo.h"
18#include "X86Subtarget.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/SmallSet.h"
22#include "llvm/CodeGen/LiveRegMatrix.h"
23#include "llvm/CodeGen/MachineFrameInfo.h"
24#include "llvm/CodeGen/MachineRegisterInfo.h"
25#include "llvm/CodeGen/RegisterScavenging.h"
26#include "llvm/CodeGen/TargetFrameLowering.h"
27#include "llvm/CodeGen/TargetInstrInfo.h"
28#include "llvm/CodeGen/TileShapeInfo.h"
29#include "llvm/CodeGen/VirtRegMap.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/Type.h"
32#include "llvm/MC/MCContext.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Target/TargetMachine.h"
36
37using namespace llvm;
38
39#define GET_REGINFO_TARGET_DESC
40#include "X86GenRegisterInfo.inc"
41
42static cl::opt<bool>
43EnableBasePointer("x86-use-base-pointer", cl::Hidden, cl::init(Val: true),
44 cl::desc("Enable use of a base pointer for complex stack frames"));
45
46static cl::opt<bool>
47 DisableRegAllocNDDHints("x86-disable-regalloc-hints-for-ndd", cl::Hidden,
48 cl::init(Val: false),
49 cl::desc("Disable two address hints for register "
50 "allocation"));
51
52static cl::opt<unsigned> SetjmpCSRWarningThreshold(
53 "x86-setjmp-csr-warning-threshold", cl::Hidden, cl::init(Val: 50),
54 cl::desc("Basic block count threshold for emitting a warning about "
55 "callee-saved registers reserved due to setjmp"));
56
57extern cl::opt<bool> X86EnableAPXForRelocation;
58
59X86RegisterInfo::X86RegisterInfo(const Triple &TT)
60 : X86GenRegisterInfo((TT.isX86_64() ? X86::RIP : X86::EIP),
61 X86_MC::getDwarfRegFlavour(TT, isEH: false),
62 X86_MC::getDwarfRegFlavour(TT, isEH: true),
63 (TT.isX86_64() ? X86::RIP : X86::EIP)) {
64 X86_MC::initLLVMToSEHAndCVRegMapping(MRI: this);
65
66 // Cache some information.
67 Is64Bit = TT.isX86_64();
68 IsTarget64BitLP64 = Is64Bit && !TT.isX32();
69 IsWin64 = Is64Bit && TT.isOSWindows();
70 IsUEFI64 = Is64Bit && TT.isUEFI();
71
72 // Use a callee-saved register as the base pointer. These registers must
73 // not conflict with any ABI requirements. For example, in 32-bit mode PIC
74 // requires GOT in the EBX register before function calls via PLT GOT pointer.
75 if (Is64Bit) {
76 SlotSize = 8;
77 // This matches the simplified 32-bit pointer code in the data layout
78 // computation.
79 // FIXME: Should use the data layout?
80 bool Use64BitReg = !TT.isX32();
81 StackPtr = Use64BitReg ? X86::RSP : X86::ESP;
82 FramePtr = Use64BitReg ? X86::RBP : X86::EBP;
83 BasePtr = Use64BitReg ? X86::RBX : X86::EBX;
84 } else {
85 SlotSize = 4;
86 StackPtr = X86::ESP;
87 FramePtr = X86::EBP;
88 BasePtr = X86::ESI;
89 }
90}
91
92const TargetRegisterClass *
93X86RegisterInfo::getSubClassWithSubReg(const TargetRegisterClass *RC,
94 unsigned Idx) const {
95 // The sub_8bit sub-register index is more constrained in 32-bit mode.
96 // It behaves just like the sub_8bit_hi index.
97 if (!Is64Bit && Idx == X86::sub_8bit)
98 Idx = X86::sub_8bit_hi;
99
100 // Forward to TableGen's default version.
101 return X86GenRegisterInfo::getSubClassWithSubReg(RC, Idx);
102}
103
104const TargetRegisterClass *
105X86RegisterInfo::getMatchingSuperRegClass(const TargetRegisterClass *A,
106 const TargetRegisterClass *B,
107 unsigned SubIdx) const {
108 // The sub_8bit sub-register index is more constrained in 32-bit mode.
109 if (!Is64Bit && SubIdx == X86::sub_8bit) {
110 A = X86GenRegisterInfo::getSubClassWithSubReg(RC: A, Idx: X86::sub_8bit_hi);
111 if (!A)
112 return nullptr;
113 }
114 return X86GenRegisterInfo::getMatchingSuperRegClass(A, B, Idx: SubIdx);
115}
116
117const TargetRegisterClass *
118X86RegisterInfo::getLargestLegalSuperClass(const TargetRegisterClass *RC,
119 const MachineFunction &MF) const {
120 // Don't allow super-classes of GR8_NOREX. This class is only used after
121 // extracting sub_8bit_hi sub-registers. The H sub-registers cannot be copied
122 // to the full GR8 register class in 64-bit mode, so we cannot allow the
123 // reigster class inflation.
124 //
125 // The GR8_NOREX class is always used in a way that won't be constrained to a
126 // sub-class, so sub-classes like GR8_ABCD_L are allowed to expand to the
127 // full GR8 class.
128 if (RC == &X86::GR8_NOREXRegClass)
129 return RC;
130
131 // Keep using non-rex2 register class when APX feature (EGPR/NDD/NF) is not
132 // enabled for relocation.
133 if (!X86EnableAPXForRelocation && isNonRex2RegClass(RC))
134 return RC;
135
136 const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
137
138 const TargetRegisterClass *Super = RC;
139 auto I = RC->superclasses().begin();
140 auto E = RC->superclasses().end();
141 do {
142 switch (Super->getID()) {
143 case X86::FR32RegClassID:
144 case X86::FR64RegClassID:
145 // If AVX-512 isn't supported we should only inflate to these classes.
146 if (!Subtarget.hasAVX512() &&
147 getRegSizeInBits(RC: *Super) == getRegSizeInBits(RC: *RC))
148 return Super;
149 break;
150 case X86::VR128RegClassID:
151 case X86::VR256RegClassID:
152 // If VLX isn't supported we should only inflate to these classes.
153 if (!Subtarget.hasVLX() &&
154 getRegSizeInBits(RC: *Super) == getRegSizeInBits(RC: *RC))
155 return Super;
156 break;
157 case X86::VR128XRegClassID:
158 case X86::VR256XRegClassID:
159 // If VLX isn't support we shouldn't inflate to these classes.
160 if (Subtarget.hasVLX() &&
161 getRegSizeInBits(RC: *Super) == getRegSizeInBits(RC: *RC))
162 return Super;
163 break;
164 case X86::FR32XRegClassID:
165 case X86::FR64XRegClassID:
166 // If AVX-512 isn't support we shouldn't inflate to these classes.
167 if (Subtarget.hasAVX512() &&
168 getRegSizeInBits(RC: *Super) == getRegSizeInBits(RC: *RC))
169 return Super;
170 break;
171 case X86::GR8RegClassID:
172 case X86::GR16RegClassID:
173 case X86::GR32RegClassID:
174 case X86::GR64RegClassID:
175 case X86::GR8_NOREX2RegClassID:
176 case X86::GR16_NOREX2RegClassID:
177 case X86::GR32_NOREX2RegClassID:
178 case X86::GR64_NOREX2RegClassID:
179 case X86::RFP32RegClassID:
180 case X86::RFP64RegClassID:
181 case X86::RFP80RegClassID:
182 case X86::VR512_0_15RegClassID:
183 case X86::VR512RegClassID:
184 // Don't return a super-class that would shrink the spill size.
185 // That can happen with the vector and float classes.
186 if (getRegSizeInBits(RC: *Super) == getRegSizeInBits(RC: *RC))
187 return Super;
188 }
189 if (I != E) {
190 Super = getRegClass(i: *I);
191 ++I;
192 } else {
193 Super = nullptr;
194 }
195 } while (Super);
196 return RC;
197}
198
199const TargetRegisterClass *
200X86RegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const {
201 if (RC == &X86::CCRRegClass) {
202 if (Is64Bit)
203 return &X86::GR64RegClass;
204 else
205 return &X86::GR32RegClass;
206 }
207 return RC;
208}
209
210unsigned
211X86RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
212 MachineFunction &MF) const {
213 const X86FrameLowering *TFI = getFrameLowering(MF);
214
215 unsigned FPDiff = TFI->hasFP(MF) ? 1 : 0;
216 switch (RC->getID()) {
217 default:
218 return 0;
219 case X86::GR32RegClassID:
220 return 4 - FPDiff;
221 case X86::GR64RegClassID:
222 return 12 - FPDiff;
223 case X86::VR128RegClassID:
224 return Is64Bit ? 10 : 4;
225 case X86::VR64RegClassID:
226 return 4;
227 }
228}
229
230const MCPhysReg *
231X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
232 assert(MF && "MachineFunction required");
233
234 const X86Subtarget &Subtarget = MF->getSubtarget<X86Subtarget>();
235 const Function &F = MF->getFunction();
236 bool HasSSE = Subtarget.hasSSE1();
237 bool HasAVX = Subtarget.hasAVX();
238 bool HasAVX512 = Subtarget.hasAVX512();
239 bool HasEGPR = Subtarget.hasEGPR();
240 bool CallsEHReturn = MF->callsEHReturn();
241
242 CallingConv::ID CC = F.getCallingConv();
243
244 // If attribute NoCallerSavedRegisters exists then we set X86_INTR calling
245 // convention because it has the CSR list.
246 if (MF->getFunction().hasFnAttribute(Kind: "no_caller_saved_registers"))
247 CC = CallingConv::X86_INTR;
248
249 // If atribute specified, override the CSRs normally specified by the
250 // calling convention and use the empty set instead.
251 if (MF->getFunction().hasFnAttribute(Kind: "no_callee_saved_registers"))
252 return CSR_NoRegs_SaveList;
253
254 switch (CC) {
255 case CallingConv::GHC:
256 case CallingConv::HiPE:
257 return CSR_NoRegs_SaveList;
258 case CallingConv::AnyReg:
259 if (HasAVX)
260 return CSR_64_AllRegs_AVX_SaveList;
261 return CSR_64_AllRegs_SaveList;
262 case CallingConv::PreserveMost:
263 if (IsWin64)
264 return HasEGPR ? CSR_Win64_APX_RT_MostRegs_SaveList
265 : CSR_Win64_RT_MostRegs_SaveList;
266 return CSR_64_RT_MostRegs_SaveList;
267 case CallingConv::PreserveAll:
268 if (HasAVX)
269 return CSR_64_RT_AllRegs_AVX_SaveList;
270 return CSR_64_RT_AllRegs_SaveList;
271 case CallingConv::PreserveNone:
272 return CSR_64_NoneRegs_SaveList;
273 case CallingConv::CXX_FAST_TLS:
274 if (Is64Bit)
275 return MF->getInfo<X86MachineFunctionInfo>()->isSplitCSR() ?
276 CSR_64_CXX_TLS_Darwin_PE_SaveList : CSR_64_TLS_Darwin_SaveList;
277 break;
278 case CallingConv::Intel_OCL_BI: {
279 if (HasAVX512 && IsWin64)
280 return HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX512_SaveList
281 : CSR_Win64_Intel_OCL_BI_AVX512_SaveList;
282 if (HasAVX512 && Is64Bit)
283 return CSR_64_Intel_OCL_BI_AVX512_SaveList;
284 if (HasAVX && IsWin64)
285 return HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX_SaveList
286 : CSR_Win64_Intel_OCL_BI_AVX_SaveList;
287 if (HasAVX && Is64Bit)
288 return CSR_64_Intel_OCL_BI_AVX_SaveList;
289 if (!HasAVX && !IsWin64 && Is64Bit)
290 return CSR_64_Intel_OCL_BI_SaveList;
291 break;
292 }
293 case CallingConv::X86_RegCall:
294 if (Is64Bit) {
295 if (IsWin64) {
296 if (HasSSE)
297 return HasEGPR ? CSR_Win64_APX_RegCall_SaveList
298 : CSR_Win64_RegCall_SaveList;
299 return CSR_Win64_RegCall_NoSSE_SaveList;
300 }
301 return HasSSE ? CSR_SysV64_RegCall_SaveList
302 : CSR_SysV64_RegCall_NoSSE_SaveList;
303 }
304 return HasSSE ? CSR_32_RegCall_SaveList : CSR_32_RegCall_NoSSE_SaveList;
305 case CallingConv::CFGuard_Check:
306 assert(!Is64Bit && "CFGuard check mechanism only used on 32-bit X86");
307 return HasSSE ? CSR_Win32_CFGuard_Check_SaveList
308 : CSR_Win32_CFGuard_Check_NoSSE_SaveList;
309 case CallingConv::Cold:
310 if (Is64Bit)
311 return CSR_64_MostRegs_SaveList;
312 break;
313 case CallingConv::Win64:
314 if (HasSSE)
315 return HasEGPR ? CSR_Win64_APX_SaveList : CSR_Win64_SaveList;
316 return CSR_Win64_NoSSE_SaveList;
317 case CallingConv::SwiftTail:
318 if (!Is64Bit)
319 return CSR_32_SaveList;
320 if (IsWin64)
321 return HasEGPR ? CSR_Win64_APX_SwiftTail_SaveList
322 : CSR_Win64_SwiftTail_SaveList;
323 return CSR_64_SwiftTail_SaveList;
324 case CallingConv::X86_64_SysV:
325 if (CallsEHReturn)
326 return CSR_64EHRet_SaveList;
327 return CSR_64_SaveList;
328 case CallingConv::X86_INTR:
329 if (Is64Bit) {
330 if (HasAVX512)
331 return CSR_64_AllRegs_AVX512_SaveList;
332 if (HasAVX)
333 return CSR_64_AllRegs_AVX_SaveList;
334 if (HasSSE)
335 return CSR_64_AllRegs_SaveList;
336 return CSR_64_AllRegs_NoSSE_SaveList;
337 }
338 if (HasAVX512)
339 return CSR_32_AllRegs_AVX512_SaveList;
340 if (HasAVX)
341 return CSR_32_AllRegs_AVX_SaveList;
342 if (HasSSE)
343 return CSR_32_AllRegs_SSE_SaveList;
344 return CSR_32_AllRegs_SaveList;
345 default:
346 break;
347 }
348
349 if (Is64Bit) {
350 bool IsSwiftCC = Subtarget.getTargetLowering()->supportSwiftError() &&
351 F.getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError);
352 if (IsSwiftCC) {
353 if (IsWin64)
354 return HasEGPR ? CSR_Win64_APX_SwiftError_SaveList
355 : CSR_Win64_SwiftError_SaveList;
356 return CSR_64_SwiftError_SaveList;
357 }
358
359 if (IsWin64 || IsUEFI64) {
360 if (HasSSE)
361 return HasEGPR ? CSR_Win64_APX_SaveList : CSR_Win64_SaveList;
362 return CSR_Win64_NoSSE_SaveList;
363 }
364 if (CallsEHReturn)
365 return CSR_64EHRet_SaveList;
366 return CSR_64_SaveList;
367 }
368
369 return CallsEHReturn ? CSR_32EHRet_SaveList : CSR_32_SaveList;
370}
371
372const MCPhysReg *
373X86RegisterInfo::getIPRACSRegs(const MachineFunction *MF) const {
374 return Is64Bit ? CSR_IPRA_64_SaveList : CSR_IPRA_32_SaveList;
375}
376
377const MCPhysReg *X86RegisterInfo::getCalleeSavedRegsViaCopy(
378 const MachineFunction *MF) const {
379 assert(MF && "Invalid MachineFunction pointer.");
380 if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
381 MF->getInfo<X86MachineFunctionInfo>()->isSplitCSR())
382 return CSR_64_CXX_TLS_Darwin_ViaCopy_SaveList;
383 return nullptr;
384}
385
386const uint32_t *
387X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
388 CallingConv::ID CC) const {
389 const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
390 bool HasSSE = Subtarget.hasSSE1();
391 bool HasAVX = Subtarget.hasAVX();
392 bool HasAVX512 = Subtarget.hasAVX512();
393 bool HasEGPR = Subtarget.hasEGPR();
394
395 switch (CC) {
396 case CallingConv::GHC:
397 case CallingConv::HiPE:
398 return CSR_NoRegs_RegMask;
399 case CallingConv::AnyReg:
400 if (HasAVX)
401 return CSR_64_AllRegs_AVX_RegMask;
402 return CSR_64_AllRegs_RegMask;
403 case CallingConv::PreserveMost:
404 if (IsWin64)
405 return HasEGPR ? CSR_Win64_APX_RT_MostRegs_RegMask
406 : CSR_Win64_RT_MostRegs_RegMask;
407 return CSR_64_RT_MostRegs_RegMask;
408 case CallingConv::PreserveAll:
409 if (HasAVX)
410 return CSR_64_RT_AllRegs_AVX_RegMask;
411 return CSR_64_RT_AllRegs_RegMask;
412 case CallingConv::PreserveNone:
413 return CSR_64_NoneRegs_RegMask;
414 case CallingConv::CXX_FAST_TLS:
415 if (Is64Bit)
416 return CSR_64_TLS_Darwin_RegMask;
417 break;
418 case CallingConv::Intel_OCL_BI: {
419 if (HasAVX512 && IsWin64)
420 return HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX512_RegMask
421 : CSR_Win64_Intel_OCL_BI_AVX512_RegMask;
422 if (HasAVX512 && Is64Bit)
423 return CSR_64_Intel_OCL_BI_AVX512_RegMask;
424 if (HasAVX && IsWin64)
425 return HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX_RegMask
426 : CSR_Win64_Intel_OCL_BI_AVX_RegMask;
427 if (HasAVX && Is64Bit)
428 return CSR_64_Intel_OCL_BI_AVX_RegMask;
429 if (!HasAVX && !IsWin64 && Is64Bit)
430 return CSR_64_Intel_OCL_BI_RegMask;
431 break;
432 }
433 case CallingConv::X86_RegCall:
434 if (Is64Bit) {
435 if (IsWin64) {
436 if (HasSSE)
437 return HasEGPR ? CSR_Win64_APX_RegCall_RegMask
438 : CSR_Win64_RegCall_RegMask;
439 return CSR_Win64_RegCall_NoSSE_RegMask;
440 }
441 return HasSSE ? CSR_SysV64_RegCall_RegMask
442 : CSR_SysV64_RegCall_NoSSE_RegMask;
443 }
444 return HasSSE ? CSR_32_RegCall_RegMask : CSR_32_RegCall_NoSSE_RegMask;
445 case CallingConv::CFGuard_Check:
446 if (Is64Bit) {
447 if (HasSSE)
448 return HasEGPR ? CSR_Win64_APX_CFGuard_Check_RegMask
449 : CSR_Win64_CFGuard_Check_RegMask;
450 return CSR_Win64_CFGuard_Check_NoSSE_RegMask;
451 }
452 return HasSSE ? CSR_Win32_CFGuard_Check_RegMask
453 : CSR_Win32_CFGuard_Check_NoSSE_RegMask;
454 case CallingConv::Cold:
455 if (Is64Bit)
456 return CSR_64_MostRegs_RegMask;
457 break;
458 case CallingConv::Win64:
459 return HasEGPR ? CSR_Win64_APX_RegMask : CSR_Win64_RegMask;
460 case CallingConv::SwiftTail:
461 if (!Is64Bit)
462 return CSR_32_RegMask;
463 if (IsWin64)
464 return HasEGPR ? CSR_Win64_APX_SwiftTail_RegMask
465 : CSR_Win64_SwiftTail_RegMask;
466 return CSR_64_SwiftTail_RegMask;
467 case CallingConv::X86_64_SysV:
468 return CSR_64_RegMask;
469 case CallingConv::X86_INTR:
470 if (Is64Bit) {
471 if (HasAVX512)
472 return CSR_64_AllRegs_AVX512_RegMask;
473 if (HasAVX)
474 return CSR_64_AllRegs_AVX_RegMask;
475 if (HasSSE)
476 return CSR_64_AllRegs_RegMask;
477 return CSR_64_AllRegs_NoSSE_RegMask;
478 }
479 if (HasAVX512)
480 return CSR_32_AllRegs_AVX512_RegMask;
481 if (HasAVX)
482 return CSR_32_AllRegs_AVX_RegMask;
483 if (HasSSE)
484 return CSR_32_AllRegs_SSE_RegMask;
485 return CSR_32_AllRegs_RegMask;
486 default:
487 break;
488 }
489
490 // Unlike getCalleeSavedRegs(), we don't have MMI so we can't check
491 // callsEHReturn().
492 if (Is64Bit) {
493 const Function &F = MF.getFunction();
494 bool IsSwiftCC = Subtarget.getTargetLowering()->supportSwiftError() &&
495 F.getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError);
496 if (IsSwiftCC) {
497 if (IsWin64)
498 return HasEGPR ? CSR_Win64_APX_SwiftError_RegMask
499 : CSR_Win64_SwiftError_RegMask;
500 return CSR_64_SwiftError_RegMask;
501 }
502
503 if (IsWin64 || IsUEFI64)
504 return HasEGPR ? CSR_Win64_APX_RegMask : CSR_Win64_RegMask;
505 return CSR_64_RegMask;
506 }
507
508 return CSR_32_RegMask;
509}
510
511const uint32_t*
512X86RegisterInfo::getNoPreservedMask() const {
513 return CSR_NoRegs_RegMask;
514}
515
516const uint32_t *X86RegisterInfo::getDarwinTLSCallPreservedMask() const {
517 return CSR_64_TLS_Darwin_RegMask;
518}
519
520BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
521 BitVector Reserved(getNumRegs());
522 const X86FrameLowering *TFI = getFrameLowering(MF);
523
524 // Set the floating point control register as reserved.
525 Reserved.set(X86::FPCW);
526
527 // Set the floating point status register as reserved.
528 Reserved.set(X86::FPSW);
529
530 // Set the SIMD floating point control register as reserved.
531 Reserved.set(X86::MXCSR);
532
533 // Set the stack-pointer register and its aliases as reserved.
534 for (const MCPhysReg &SubReg : subregs_inclusive(Reg: X86::RSP))
535 Reserved.set(SubReg);
536
537 // Set the Shadow Stack Pointer as reserved.
538 Reserved.set(X86::SSP);
539
540 auto &ST = MF.getSubtarget<X86Subtarget>();
541 if (ST.hasUserReservedRegisters()) {
542 if (ST.is64Bit()) {
543 // Set r# as reserved register if user required.
544 for (unsigned Reg = X86::R8; Reg <= X86::R15; ++Reg)
545 if (ST.isRegisterReservedByUser(i: Reg))
546 for (const MCPhysReg &SubReg : subregs_inclusive(Reg))
547 Reserved.set(SubReg);
548 if (ST.hasEGPR())
549 for (unsigned Reg = X86::R16; Reg <= X86::R31; ++Reg)
550 if (ST.isRegisterReservedByUser(i: Reg))
551 for (const MCPhysReg &SubReg : subregs_inclusive(Reg))
552 Reserved.set(SubReg);
553 } else {
554 if (ST.isRegisterReservedByUser(i: X86::EDI))
555 for (const MCPhysReg &SubReg : sub_and_superregs_inclusive(Reg: X86::EDI))
556 Reserved.set(SubReg);
557 }
558 }
559
560 // Set the instruction pointer register and its aliases as reserved.
561 for (const MCPhysReg &SubReg : subregs_inclusive(Reg: X86::RIP))
562 Reserved.set(SubReg);
563
564 // Set the frame-pointer register and its aliases as reserved if needed.
565 if (TFI->hasFP(MF) || MF.framePointerIsReserved()) {
566 if (MF.getInfo<X86MachineFunctionInfo>()->getFPClobberedByInvoke())
567 MF.getContext().reportError(
568 L: SMLoc(),
569 Msg: "Frame pointer clobbered by function invoke is not supported.");
570
571 for (const MCPhysReg &SubReg : subregs_inclusive(Reg: X86::RBP))
572 Reserved.set(SubReg);
573 }
574
575 // Set the base-pointer register and its aliases as reserved if needed.
576 if (hasBasePointer(MF)) {
577 if (MF.getInfo<X86MachineFunctionInfo>()->getBPClobberedByInvoke())
578 MF.getContext().reportError(L: SMLoc(),
579 Msg: "Stack realignment in presence of dynamic "
580 "allocas is not supported with "
581 "this calling convention.");
582
583 Register BasePtr = getX86SubSuperRegister(Reg: getBaseRegister(), Size: 64);
584 for (const MCPhysReg &SubReg : subregs_inclusive(Reg: BasePtr))
585 Reserved.set(SubReg);
586 }
587
588 // Mark the segment registers as reserved.
589 Reserved.set(X86::CS);
590 Reserved.set(X86::SS);
591 Reserved.set(X86::DS);
592 Reserved.set(X86::ES);
593 Reserved.set(X86::FS);
594 Reserved.set(X86::GS);
595
596 // Mark the floating point stack registers as reserved.
597 for (unsigned n = 0; n != 8; ++n)
598 Reserved.set(X86::ST0 + n);
599
600 // Without usable x87 (soft float or -mno-x87), reserve the allocatable FPn
601 // pseudos (FP0-FP6; FP7 is already non-allocatable) so they aren't scrubbed.
602 if (ST.useSoftFloat() || !ST.hasX87())
603 for (unsigned n = 0; n != 7; ++n)
604 Reserved.set(X86::FP0 + n);
605
606 // Reserve the registers that only exist in 64-bit mode.
607 if (!Is64Bit) {
608 // These 8-bit registers are part of the x86-64 extension even though their
609 // super-registers are old 32-bits.
610 Reserved.set(X86::SIL);
611 Reserved.set(X86::DIL);
612 Reserved.set(X86::BPL);
613 Reserved.set(X86::SPL);
614 Reserved.set(X86::SIH);
615 Reserved.set(X86::DIH);
616 Reserved.set(X86::BPH);
617 Reserved.set(X86::SPH);
618
619 for (unsigned n = 0; n != 8; ++n) {
620 // R8, R9, ...
621 for (MCRegAliasIterator AI(X86::R8 + n, this, true); AI.isValid(); ++AI)
622 Reserved.set(*AI);
623
624 // XMM8, XMM9, ...
625 for (MCRegAliasIterator AI(X86::XMM8 + n, this, true); AI.isValid(); ++AI)
626 Reserved.set(*AI);
627 }
628 }
629 if (!Is64Bit || !MF.getSubtarget<X86Subtarget>().hasAVX512()) {
630 for (unsigned n = 0; n != 16; ++n) {
631 for (MCRegAliasIterator AI(X86::XMM16 + n, this, true); AI.isValid();
632 ++AI)
633 Reserved.set(*AI);
634 }
635 }
636
637 // Reserve the extended general purpose registers.
638 if (!Is64Bit || !MF.getSubtarget<X86Subtarget>().hasEGPR())
639 Reserved.set(I: X86::R16, E: X86::R31WH + 1);
640
641 // Due to specifics of setjmp unwinding in Win64 APX ABI, the unwinder
642 // cannot restore R30/R31. Reserve them to prevent register allocation.
643 // https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention#setjmplongjmp
644 if (MF.exposesReturnsTwice() && ST.isTargetWin64()) {
645 unsigned NumReservedCSRs = 0;
646 for (unsigned Reg = X86::R16; Reg <= X86::R31; ++Reg)
647 if (isCalleeSavedPhysReg(PhysReg: Reg, MF)) {
648 ++NumReservedCSRs;
649 for (const MCPhysReg &SubReg : subregs_inclusive(Reg))
650 Reserved.set(SubReg);
651 }
652 if (NumReservedCSRs && MF.size() > SetjmpCSRWarningThreshold &&
653 !MF.getRegInfo().reservedRegsFrozen()) {
654 MF.getContext().reportWarning(
655 L: SMLoc(), Msg: Twine(NumReservedCSRs) +
656 " callee-saved register(s) reserved due to setjmp in '" +
657 MF.getName() +
658 "'; this may impact performance in large functions");
659 }
660 }
661
662 if (MF.getFunction().getCallingConv() == CallingConv::GRAAL) {
663 for (MCRegAliasIterator AI(X86::R14, this, true); AI.isValid(); ++AI)
664 Reserved.set(*AI);
665 for (MCRegAliasIterator AI(X86::R15, this, true); AI.isValid(); ++AI)
666 Reserved.set(*AI);
667 }
668
669 // Reserve registers for LFI sandboxing.
670 if (MF.getSubtarget<X86Subtarget>().isLFI()) {
671 for (MCRegAliasIterator AI(X86::R11, this, true); AI.isValid(); ++AI)
672 Reserved.set(*AI);
673 for (MCRegAliasIterator AI(X86::R14, this, true); AI.isValid(); ++AI)
674 Reserved.set(*AI);
675 for (MCRegAliasIterator AI(X86::R15, this, true); AI.isValid(); ++AI)
676 Reserved.set(*AI);
677 }
678
679 assert(checkAllSuperRegsMarked(Reserved,
680 {X86::SIL, X86::DIL, X86::BPL, X86::SPL,
681 X86::SIH, X86::DIH, X86::BPH, X86::SPH}));
682 return Reserved;
683}
684
685unsigned X86RegisterInfo::getNumSupportedRegs(const MachineFunction &MF) const {
686 // All existing Intel CPUs that support AMX support AVX512 and all existing
687 // Intel CPUs that support APX support AMX. AVX512 implies AVX.
688 //
689 // We enumerate the registers in X86GenRegisterInfo.inc in this order:
690 //
691 // Registers before AVX512,
692 // AVX512 registers (X/YMM16-31, ZMM0-31, K registers)
693 // AMX registers (TMM)
694 // APX registers (R16-R31)
695 //
696 // and try to return the minimum number of registers supported by the target.
697 static_assert((X86::R15WH + 1 == X86::YMM0) && (X86::YMM15 + 1 == X86::K0) &&
698 (X86::K6_K7 + 1 == X86::TMMCFG) &&
699 (X86::TMM7 + 1 == X86::R16) &&
700 (X86::R31WH + 1 == X86::NUM_TARGET_REGS),
701 "Register number may be incorrect");
702
703 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
704 if (ST.hasEGPR())
705 return X86::NUM_TARGET_REGS;
706 if (ST.hasAMXTILE())
707 return X86::TMM7 + 1;
708 if (ST.hasAVX512())
709 return X86::K6_K7 + 1;
710 if (ST.hasAVX())
711 return X86::YMM15 + 1;
712 return X86::R15WH + 1;
713}
714
715bool X86RegisterInfo::isArgumentRegister(const MachineFunction &MF,
716 MCRegister Reg) const {
717 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
718 const TargetRegisterInfo &TRI = *ST.getRegisterInfo();
719 auto IsSubReg = [&](MCRegister RegA, MCRegister RegB) {
720 return TRI.isSuperOrSubRegisterEq(RegA, RegB);
721 };
722
723 if (!ST.is64Bit())
724 return llvm::any_of(
725 Range: SmallVector<MCRegister>{X86::EAX, X86::ECX, X86::EDX},
726 P: [&](MCRegister &RegA) { return IsSubReg(RegA, Reg); }) ||
727 (ST.hasMMX() && X86::VR64RegClass.contains(Reg));
728
729 CallingConv::ID CC = MF.getFunction().getCallingConv();
730
731 if (CC == CallingConv::X86_64_SysV && IsSubReg(X86::RAX, Reg))
732 return true;
733
734 if (llvm::any_of(
735 Range: SmallVector<MCRegister>{X86::RDX, X86::RCX, X86::R8, X86::R9},
736 P: [&](MCRegister &RegA) { return IsSubReg(RegA, Reg); }))
737 return true;
738
739 if (CC != CallingConv::Win64 &&
740 llvm::any_of(Range: SmallVector<MCRegister>{X86::RDI, X86::RSI},
741 P: [&](MCRegister &RegA) { return IsSubReg(RegA, Reg); }))
742 return true;
743
744 if (ST.hasSSE1() &&
745 llvm::any_of(Range: SmallVector<MCRegister>{X86::XMM0, X86::XMM1, X86::XMM2,
746 X86::XMM3, X86::XMM4, X86::XMM5,
747 X86::XMM6, X86::XMM7},
748 P: [&](MCRegister &RegA) { return IsSubReg(RegA, Reg); }))
749 return true;
750
751 return X86GenRegisterInfo::isArgumentRegister(MF, PhysReg: Reg);
752}
753
754bool X86RegisterInfo::isFixedRegister(const MachineFunction &MF,
755 MCRegister PhysReg) const {
756 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
757 const TargetRegisterInfo &TRI = *ST.getRegisterInfo();
758
759 // Stack pointer.
760 if (TRI.isSuperOrSubRegisterEq(RegA: X86::RSP, RegB: PhysReg))
761 return true;
762
763 // Don't use the frame pointer if it's being used.
764 const X86FrameLowering &TFI = *getFrameLowering(MF);
765 if (TFI.hasFP(MF) && TRI.isSuperOrSubRegisterEq(RegA: X86::RBP, RegB: PhysReg))
766 return true;
767
768 return X86GenRegisterInfo::isFixedRegister(MF, PhysReg);
769}
770
771bool X86RegisterInfo::isTileRegisterClass(const TargetRegisterClass *RC) const {
772 return RC->getID() == X86::TILERegClassID;
773}
774
775void X86RegisterInfo::adjustStackMapLiveOutMask(uint32_t *Mask) const {
776 // Check if the EFLAGS register is marked as live-out. This shouldn't happen,
777 // because the calling convention defines the EFLAGS register as NOT
778 // preserved.
779 //
780 // Unfortunatelly the EFLAGS show up as live-out after branch folding. Adding
781 // an assert to track this and clear the register afterwards to avoid
782 // unnecessary crashes during release builds.
783 assert(!(Mask[X86::EFLAGS / 32] & (1U << (X86::EFLAGS % 32))) &&
784 "EFLAGS are not live-out from a patchpoint.");
785
786 // Also clean other registers that don't need preserving (IP).
787 for (auto Reg : {X86::EFLAGS, X86::RIP, X86::EIP, X86::IP})
788 Mask[Reg / 32] &= ~(1U << (Reg % 32));
789}
790
791//===----------------------------------------------------------------------===//
792// Stack Frame Processing methods
793//===----------------------------------------------------------------------===//
794
795static bool CantUseSP(const MachineFrameInfo &MFI) {
796 return MFI.hasVarSizedObjects() || MFI.hasOpaqueSPAdjustment();
797}
798
799bool X86RegisterInfo::hasBasePointer(const MachineFunction &MF) const {
800 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
801 // We have a virtual register to reference argument, and don't need base
802 // pointer.
803 if (X86FI->getStackPtrSaveMI() != nullptr)
804 return false;
805
806 if (X86FI->hasPreallocatedCall())
807 return true;
808
809 const MachineFrameInfo &MFI = MF.getFrameInfo();
810
811 if (!EnableBasePointer)
812 return false;
813
814 // When we need stack realignment, we can't address the stack from the frame
815 // pointer. When we have dynamic allocas or stack-adjusting inline asm, we
816 // can't address variables from the stack pointer. MS inline asm can
817 // reference locals while also adjusting the stack pointer. When we can't
818 // use both the SP and the FP, we need a separate base pointer register.
819 bool CantUseFP = hasStackRealignment(MF);
820 return CantUseFP && CantUseSP(MFI);
821}
822
823bool X86RegisterInfo::canRealignStack(const MachineFunction &MF) const {
824 if (!TargetRegisterInfo::canRealignStack(MF))
825 return false;
826
827 const MachineFrameInfo &MFI = MF.getFrameInfo();
828 const MachineRegisterInfo *MRI = &MF.getRegInfo();
829
830 // Stack realignment requires a frame pointer. If we already started
831 // register allocation with frame pointer elimination, it is too late now.
832 if (!MRI->canReserveReg(PhysReg: FramePtr))
833 return false;
834
835 // If a base pointer is necessary. Check that it isn't too late to reserve
836 // it.
837 if (CantUseSP(MFI))
838 return MRI->canReserveReg(PhysReg: BasePtr);
839 return true;
840}
841
842bool X86RegisterInfo::shouldRealignStack(const MachineFunction &MF) const {
843 if (TargetRegisterInfo::shouldRealignStack(MF))
844 return true;
845
846 return !Is64Bit && MF.getFunction().getCallingConv() == CallingConv::X86_INTR;
847}
848
849// tryOptimizeLEAtoMOV - helper function that tries to replace a LEA instruction
850// of the form 'lea (%esp), %ebx' --> 'mov %esp, %ebx'.
851// TODO: In this case we should be really trying first to entirely eliminate
852// this instruction which is a plain copy.
853static bool tryOptimizeLEAtoMOV(MachineBasicBlock::iterator II) {
854 MachineInstr &MI = *II;
855 unsigned Opc = II->getOpcode();
856 // Check if this is a LEA of the form 'lea (%esp), %ebx'
857 if ((Opc != X86::LEA32r && Opc != X86::LEA64r && Opc != X86::LEA64_32r) ||
858 MI.getOperand(i: 2).getImm() != 1 ||
859 MI.getOperand(i: 3).getReg() != X86::NoRegister ||
860 MI.getOperand(i: 4).getImm() != 0 ||
861 MI.getOperand(i: 5).getReg() != X86::NoRegister)
862 return false;
863 Register BasePtr = MI.getOperand(i: 1).getReg();
864 // In X32 mode, ensure the base-pointer is a 32-bit operand, so the LEA will
865 // be replaced with a 32-bit operand MOV which will zero extend the upper
866 // 32-bits of the super register.
867 if (Opc == X86::LEA64_32r)
868 BasePtr = getX86SubSuperRegister(Reg: BasePtr, Size: 32);
869 Register NewDestReg = MI.getOperand(i: 0).getReg();
870 const X86InstrInfo *TII =
871 MI.getParent()->getParent()->getSubtarget<X86Subtarget>().getInstrInfo();
872 TII->copyPhysReg(MBB&: *MI.getParent(), MI: II, DL: MI.getDebugLoc(), DestReg: NewDestReg, SrcReg: BasePtr,
873 KillSrc: MI.getOperand(i: 1).isKill());
874 MI.eraseFromParent();
875 return true;
876}
877
878static bool isFuncletReturnInstr(MachineInstr &MI) {
879 switch (MI.getOpcode()) {
880 case X86::CATCHRET:
881 case X86::CLEANUPRET:
882 return true;
883 default:
884 return false;
885 }
886 llvm_unreachable("impossible");
887}
888
889void X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
890 unsigned FIOperandNum,
891 Register BaseReg,
892 int FIOffset) const {
893 MachineInstr &MI = *II;
894 unsigned Opc = MI.getOpcode();
895 if (Opc == TargetOpcode::LOCAL_ESCAPE) {
896 MachineOperand &FI = MI.getOperand(i: FIOperandNum);
897 FI.ChangeToImmediate(ImmVal: FIOffset);
898 return;
899 }
900
901 MI.getOperand(i: FIOperandNum).ChangeToRegister(Reg: BaseReg, isDef: false);
902
903 // The frame index format for stackmaps and patchpoints is different from the
904 // X86 format. It only has a FI and an offset.
905 if (Opc == TargetOpcode::STACKMAP || Opc == TargetOpcode::PATCHPOINT) {
906 assert(BasePtr == FramePtr && "Expected the FP as base register");
907 int64_t Offset = MI.getOperand(i: FIOperandNum + 1).getImm() + FIOffset;
908 MI.getOperand(i: FIOperandNum + 1).ChangeToImmediate(ImmVal: Offset);
909 return;
910 }
911
912 if (MI.getOperand(i: FIOperandNum + 3).isImm()) {
913 // Offset is a 32-bit integer.
914 int Imm = (int)(MI.getOperand(i: FIOperandNum + 3).getImm());
915 int Offset = FIOffset + Imm;
916 assert((!Is64Bit || isInt<32>((long long)FIOffset + Imm)) &&
917 "Requesting 64-bit offset in 32-bit immediate!");
918 if (Offset != 0)
919 MI.getOperand(i: FIOperandNum + 3).ChangeToImmediate(ImmVal: Offset);
920 } else {
921 // Offset is symbolic. This is extremely rare.
922 uint64_t Offset =
923 FIOffset + (uint64_t)MI.getOperand(i: FIOperandNum + 3).getOffset();
924 MI.getOperand(i: FIOperandNum + 3).setOffset(Offset);
925 }
926}
927
928bool
929X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
930 int SPAdj, unsigned FIOperandNum,
931 RegScavenger *RS) const {
932 MachineInstr &MI = *II;
933 MachineBasicBlock &MBB = *MI.getParent();
934 MachineFunction &MF = *MBB.getParent();
935 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
936 bool IsEHFuncletEpilogue = MBBI == MBB.end() ? false
937 : isFuncletReturnInstr(MI&: *MBBI);
938 const X86FrameLowering *TFI = getFrameLowering(MF);
939 int FrameIndex = MI.getOperand(i: FIOperandNum).getIndex();
940
941 // Determine base register and offset.
942 int64_t FIOffset;
943 Register BasePtr;
944 if (MI.isReturn()) {
945 assert((!hasStackRealignment(MF) ||
946 MF.getFrameInfo().isFixedObjectIndex(FrameIndex)) &&
947 "Return instruction can only reference SP relative frame objects");
948 FIOffset =
949 TFI->getFrameIndexReferenceSP(MF, FI: FrameIndex, SPReg&: BasePtr, Adjustment: 0).getFixed();
950 } else if (TFI->Is64Bit && (MBB.isEHFuncletEntry() || IsEHFuncletEpilogue)) {
951 FIOffset = TFI->getWin64EHFrameIndexRef(MF, FI: FrameIndex, SPReg&: BasePtr);
952 } else {
953 FIOffset = TFI->getFrameIndexReference(MF, FI: FrameIndex, FrameReg&: BasePtr).getFixed();
954 }
955
956 // LOCAL_ESCAPE uses a single offset, with no register. It only works in the
957 // simple FP case, and doesn't work with stack realignment. On 32-bit, the
958 // offset is from the traditional base pointer location. On 64-bit, the
959 // offset is from the SP at the end of the prologue, not the FP location. This
960 // matches the behavior of llvm.frameaddress.
961 unsigned Opc = MI.getOpcode();
962 if (Opc == TargetOpcode::LOCAL_ESCAPE) {
963 MachineOperand &FI = MI.getOperand(i: FIOperandNum);
964 FI.ChangeToImmediate(ImmVal: FIOffset);
965 return false;
966 }
967
968 // For LEA64_32r when BasePtr is 32-bits (X32) we can use full-size 64-bit
969 // register as source operand, semantic is the same and destination is
970 // 32-bits. It saves one byte per lea in code since 0x67 prefix is avoided.
971 // Don't change BasePtr since it is used later for stack adjustment.
972 Register MachineBasePtr = BasePtr;
973 if (Opc == X86::LEA64_32r && X86::GR32RegClass.contains(Reg: BasePtr))
974 MachineBasePtr = getX86SubSuperRegister(Reg: BasePtr, Size: 64);
975
976 // This must be part of a four operand memory reference. Replace the
977 // FrameIndex with base register. Add an offset to the offset.
978 MI.getOperand(i: FIOperandNum).ChangeToRegister(Reg: MachineBasePtr, isDef: false);
979
980 if (BasePtr == StackPtr)
981 FIOffset += SPAdj;
982
983 // The frame index format for stackmaps and patchpoints is different from the
984 // X86 format. It only has a FI and an offset.
985 if (Opc == TargetOpcode::STACKMAP || Opc == TargetOpcode::PATCHPOINT) {
986 assert(BasePtr == FramePtr && "Expected the FP as base register");
987 int64_t Offset = MI.getOperand(i: FIOperandNum + 1).getImm() + FIOffset;
988 MI.getOperand(i: FIOperandNum + 1).ChangeToImmediate(ImmVal: Offset);
989 return false;
990 }
991
992 if (MI.getOperand(i: FIOperandNum+3).isImm()) {
993 const X86InstrInfo *TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
994 const DebugLoc &DL = MI.getDebugLoc();
995 int64_t Imm = MI.getOperand(i: FIOperandNum + 3).getImm();
996 int64_t Offset = FIOffset + Imm;
997 bool FitsIn32Bits = isInt<32>(x: Offset);
998 // If the offset will not fit in a 32-bit displacement, then for 64-bit
999 // targets, scavenge a register to hold it. Otherwise...
1000 if (Is64Bit && !FitsIn32Bits) {
1001 assert(RS && "RegisterScavenger was NULL");
1002
1003 RS->enterBasicBlockEnd(MBB);
1004 RS->backward(I: std::next(x: II));
1005
1006 Register ScratchReg = RS->scavengeRegisterBackwards(
1007 RC: X86::GR64RegClass, To: II, /*RestoreAfter=*/false, /*SPAdj=*/0,
1008 /*AllowSpill=*/true);
1009 assert(ScratchReg != 0 && "scratch reg was 0");
1010 RS->setRegUsed(Reg: ScratchReg);
1011
1012 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: TII->get(Opcode: X86::MOV64ri), DestReg: ScratchReg).addImm(Val: Offset);
1013
1014 MI.getOperand(i: FIOperandNum + 3).setImm(0);
1015 if (MI.getOperand(i: FIOperandNum + 2).getReg() == X86::NoRegister) {
1016 MI.getOperand(i: FIOperandNum + 2).setReg(ScratchReg);
1017 } else {
1018 // The index register slot is already in use, fold the offset into
1019 // the base register instead. LEA does not clobber EFLAGS.
1020 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: TII->get(Opcode: X86::LEA64r), DestReg: ScratchReg)
1021 .addReg(RegNo: MachineBasePtr)
1022 .addImm(Val: 1)
1023 .addReg(RegNo: ScratchReg)
1024 .addImm(Val: 0)
1025 .addReg(RegNo: X86::NoRegister);
1026 MI.getOperand(i: FIOperandNum).setReg(ScratchReg);
1027 }
1028
1029 return false;
1030 }
1031
1032 // ... for 32-bit targets, this is a bug!
1033 if (!Is64Bit && !FitsIn32Bits) {
1034 MI.emitGenericError(ErrMsg: "64-bit offset calculated but target is 32-bit");
1035 // Trap so that the instruction verification pass does not fail if run.
1036 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::TRAP));
1037 return false;
1038 }
1039
1040 if (Offset != 0 || !tryOptimizeLEAtoMOV(II))
1041 MI.getOperand(i: FIOperandNum + 3).ChangeToImmediate(ImmVal: Offset);
1042 } else {
1043 // Offset is symbolic. This is extremely rare.
1044 uint64_t Offset = FIOffset +
1045 (uint64_t)MI.getOperand(i: FIOperandNum+3).getOffset();
1046 MI.getOperand(i: FIOperandNum + 3).setOffset(Offset);
1047 }
1048 return false;
1049}
1050
1051unsigned X86RegisterInfo::findDeadCallerSavedReg(
1052 MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI) const {
1053 const MachineFunction *MF = MBB.getParent();
1054 const MachineRegisterInfo &MRI = MF->getRegInfo();
1055 if (MF->callsEHReturn())
1056 return 0;
1057
1058 if (MBBI == MBB.end())
1059 return 0;
1060
1061 switch (MBBI->getOpcode()) {
1062 default:
1063 return 0;
1064 case TargetOpcode::PATCHABLE_RET:
1065 case X86::RET:
1066 case X86::RET32:
1067 case X86::RET64:
1068 case X86::RETI32:
1069 case X86::RETI64:
1070 case X86::TCRETURNdi:
1071 case X86::TCRETURNri:
1072 case X86::TCRETURN_WIN64ri:
1073 case X86::TCRETURN_HIPE32ri:
1074 case X86::TCRETURNmi:
1075 case X86::TCRETURNdi64:
1076 case X86::TCRETURNri64:
1077 case X86::TCRETURNri64_ImpCall:
1078 case X86::TCRETURNmi64:
1079 case X86::TCRETURN_WINmi64:
1080 case X86::EH_RETURN:
1081 case X86::EH_RETURN64: {
1082 LiveRegUnits LRU(*this);
1083 LRU.addLiveOuts(MBB);
1084 LRU.stepBackward(MI: *MBBI);
1085
1086 const TargetRegisterClass &RC =
1087 Is64Bit ? X86::GR64_NOSPRegClass : X86::GR32_NOSPRegClass;
1088 for (MCRegister Reg : RC) {
1089 if (LRU.available(Reg) && !MRI.isReserved(PhysReg: Reg))
1090 return Reg;
1091 }
1092 }
1093 }
1094
1095 return 0;
1096}
1097
1098Register X86RegisterInfo::getFrameRegister(const MachineFunction &MF) const {
1099 const X86FrameLowering *TFI = getFrameLowering(MF);
1100 return TFI->hasFP(MF) ? FramePtr : StackPtr;
1101}
1102
1103Register
1104X86RegisterInfo::getPtrSizedFrameRegister(const MachineFunction &MF) const {
1105 const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
1106 Register FrameReg = getFrameRegister(MF);
1107 if (Subtarget.isTarget64BitILP32())
1108 FrameReg = getX86SubSuperRegister(Reg: FrameReg, Size: 32);
1109 return FrameReg;
1110}
1111
1112Register
1113X86RegisterInfo::getPtrSizedStackRegister(const MachineFunction &MF) const {
1114 const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
1115 Register StackReg = getStackRegister();
1116 if (Subtarget.isTarget64BitILP32())
1117 StackReg = getX86SubSuperRegister(Reg: StackReg, Size: 32);
1118 return StackReg;
1119}
1120
1121static ShapeT getTileShape(Register VirtReg, VirtRegMap *VRM,
1122 const MachineRegisterInfo *MRI) {
1123 if (VRM->hasShape(virtReg: VirtReg))
1124 return VRM->getShape(virtReg: VirtReg);
1125
1126 const MachineOperand &Def = *MRI->def_begin(RegNo: VirtReg);
1127 MachineInstr *MI = const_cast<MachineInstr *>(Def.getParent());
1128 unsigned OpCode = MI->getOpcode();
1129 switch (OpCode) {
1130 default:
1131 llvm_unreachable("Unexpected machine instruction on tile register!");
1132 break;
1133 case X86::COPY: {
1134 Register SrcReg = MI->getOperand(i: 1).getReg();
1135 ShapeT Shape = getTileShape(VirtReg: SrcReg, VRM, MRI);
1136 VRM->assignVirt2Shape(virtReg: VirtReg, shape: Shape);
1137 return Shape;
1138 }
1139 // We only collect the tile shape that is defined.
1140 case X86::PTILELOADDV:
1141 case X86::PTILELOADDT1V:
1142 case X86::PTDPBSSDV:
1143 case X86::PTDPBSUDV:
1144 case X86::PTDPBUSDV:
1145 case X86::PTDPBUUDV:
1146 case X86::PTILEZEROV:
1147 case X86::PTDPBF16PSV:
1148 case X86::PTDPFP16PSV:
1149 case X86::PTCMMIMFP16PSV:
1150 case X86::PTCMMRLFP16PSV:
1151 case X86::PTILELOADDRSV:
1152 case X86::PTILELOADDRST1V:
1153 case X86::PTDPBF8PSV:
1154 case X86::PTDPBHF8PSV:
1155 case X86::PTDPHBF8PSV:
1156 case X86::PTDPHF8PSV: {
1157 MachineOperand &MO1 = MI->getOperand(i: 1);
1158 MachineOperand &MO2 = MI->getOperand(i: 2);
1159 ShapeT Shape(&MO1, &MO2, MRI);
1160 VRM->assignVirt2Shape(virtReg: VirtReg, shape: Shape);
1161 return Shape;
1162 }
1163 }
1164}
1165
1166bool X86RegisterInfo::getRegAllocationHints(Register VirtReg,
1167 ArrayRef<MCPhysReg> Order,
1168 SmallVectorImpl<MCPhysReg> &Hints,
1169 const MachineFunction &MF,
1170 const VirtRegMap *VRM,
1171 const LiveRegMatrix *Matrix) const {
1172 const MachineRegisterInfo *MRI = &MF.getRegInfo();
1173 const TargetRegisterClass &RC = *MRI->getRegClass(Reg: VirtReg);
1174 bool BaseImplRetVal = TargetRegisterInfo::getRegAllocationHints(
1175 VirtReg, Order, Hints, MF, VRM, Matrix);
1176 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
1177 const TargetRegisterInfo &TRI = *ST.getRegisterInfo();
1178
1179 unsigned ID = RC.getID();
1180
1181 if (!VRM)
1182 return BaseImplRetVal;
1183
1184 if (ID != X86::TILERegClassID) {
1185 if (DisableRegAllocNDDHints || !ST.hasNDD() ||
1186 !TRI.isGeneralPurposeRegisterClass(RC: &RC))
1187 return BaseImplRetVal;
1188
1189 // Add any two address hints after any copy hints.
1190 SmallSet<unsigned, 4> TwoAddrHints;
1191
1192 auto TryAddNDDHint = [&](const MachineOperand &MO) {
1193 Register Reg = MO.getReg();
1194 Register PhysReg = Reg.isPhysical() ? Reg : Register(VRM->getPhys(virtReg: Reg));
1195 if (PhysReg && !MRI->isReserved(PhysReg) && !is_contained(Range&: Hints, Element: PhysReg))
1196 TwoAddrHints.insert(V: PhysReg);
1197 };
1198
1199 // NDD instructions is compressible when Op0 is allocated to the same
1200 // physic register as Op1 (or Op2 if it's commutable).
1201 for (auto &MO : MRI->reg_nodbg_operands(Reg: VirtReg)) {
1202 const MachineInstr &MI = *MO.getParent();
1203 if (!X86::getNonNDVariant(Opc: MI.getOpcode()))
1204 continue;
1205 unsigned OpIdx = MI.getOperandNo(I: &MO);
1206 if (OpIdx == 0) {
1207 assert(MI.getOperand(1).isReg());
1208 TryAddNDDHint(MI.getOperand(i: 1));
1209 if (MI.isCommutable()) {
1210 assert(MI.getOperand(2).isReg());
1211 TryAddNDDHint(MI.getOperand(i: 2));
1212 }
1213 } else if (OpIdx == 1) {
1214 TryAddNDDHint(MI.getOperand(i: 0));
1215 } else if (MI.isCommutable() && OpIdx == 2) {
1216 TryAddNDDHint(MI.getOperand(i: 0));
1217 }
1218 }
1219
1220 for (MCPhysReg OrderReg : Order)
1221 if (TwoAddrHints.count(V: OrderReg))
1222 Hints.push_back(Elt: OrderReg);
1223
1224 return BaseImplRetVal;
1225 }
1226
1227 ShapeT VirtShape = getTileShape(VirtReg, VRM: const_cast<VirtRegMap *>(VRM), MRI);
1228 auto AddHint = [&](MCPhysReg PhysReg) {
1229 Register VReg = Matrix->getOneVReg(PhysReg);
1230 if (VReg == MCRegister::NoRegister) { // Not allocated yet
1231 Hints.push_back(Elt: PhysReg);
1232 return;
1233 }
1234 ShapeT PhysShape = getTileShape(VirtReg: VReg, VRM: const_cast<VirtRegMap *>(VRM), MRI);
1235 if (PhysShape == VirtShape)
1236 Hints.push_back(Elt: PhysReg);
1237 };
1238
1239 SmallSet<MCPhysReg, 4> CopyHints(llvm::from_range, Hints);
1240 Hints.clear();
1241 for (auto Hint : CopyHints) {
1242 if (RC.contains(Reg: Hint) && !MRI->isReserved(PhysReg: Hint))
1243 AddHint(Hint);
1244 }
1245 for (MCPhysReg PhysReg : Order) {
1246 if (!CopyHints.count(V: PhysReg) && RC.contains(Reg: PhysReg) &&
1247 !MRI->isReserved(PhysReg))
1248 AddHint(PhysReg);
1249 }
1250
1251#define DEBUG_TYPE "tile-hint"
1252 LLVM_DEBUG({
1253 dbgs() << "Hints for virtual register " << format_hex(VirtReg, 8) << "\n";
1254 for (auto Hint : Hints) {
1255 dbgs() << "tmm" << Hint << ",";
1256 }
1257 dbgs() << "\n";
1258 });
1259#undef DEBUG_TYPE
1260
1261 return true;
1262}
1263
1264const TargetRegisterClass *X86RegisterInfo::constrainRegClassToNonRex2(
1265 const TargetRegisterClass *RC) const {
1266 switch (RC->getID()) {
1267 default:
1268 return RC;
1269 case X86::GR8RegClassID:
1270 return &X86::GR8_NOREX2RegClass;
1271 case X86::GR16RegClassID:
1272 return &X86::GR16_NOREX2RegClass;
1273 case X86::GR32RegClassID:
1274 return &X86::GR32_NOREX2RegClass;
1275 case X86::GR64RegClassID:
1276 return &X86::GR64_NOREX2RegClass;
1277 case X86::GR32_NOSPRegClassID:
1278 return &X86::GR32_NOREX2_NOSPRegClass;
1279 case X86::GR64_NOSPRegClassID:
1280 return &X86::GR64_NOREX2_NOSPRegClass;
1281 }
1282}
1283
1284bool X86RegisterInfo::isNonRex2RegClass(const TargetRegisterClass *RC) const {
1285 switch (RC->getID()) {
1286 default:
1287 return false;
1288 case X86::GR8_NOREX2RegClassID:
1289 case X86::GR16_NOREX2RegClassID:
1290 case X86::GR32_NOREX2RegClassID:
1291 case X86::GR64_NOREX2RegClassID:
1292 case X86::GR32_NOREX2_NOSPRegClassID:
1293 case X86::GR64_NOREX2_NOSPRegClassID:
1294 case X86::GR64_with_sub_16bit_in_GR16_NOREX2RegClassID:
1295 return true;
1296 }
1297}
1298