1//===-- X86TargetMachine.cpp - Define TargetMachine for the X86 -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the X86 specific subclass of TargetMachine.
10//
11//===----------------------------------------------------------------------===//
12
13#include "X86TargetMachine.h"
14#include "MCTargetDesc/X86MCTargetDesc.h"
15#include "TargetInfo/X86TargetInfo.h"
16#include "X86.h"
17#include "X86MachineFunctionInfo.h"
18#include "X86MacroFusion.h"
19#include "X86Subtarget.h"
20#include "X86TargetObjectFile.h"
21#include "X86TargetTransformInfo.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/Analysis/TargetTransformInfo.h"
25#include "llvm/CodeGen/ExecutionDomainFix.h"
26#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
27#include "llvm/CodeGen/GlobalISel/CallLowering.h"
28#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
29#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
30#include "llvm/CodeGen/GlobalISel/Legalizer.h"
31#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
32#include "llvm/CodeGen/MIRParser/MIParser.h"
33#include "llvm/CodeGen/MIRYamlMapping.h"
34#include "llvm/CodeGen/MachineScheduler.h"
35#include "llvm/CodeGen/Passes.h"
36#include "llvm/CodeGen/TargetPassConfig.h"
37#include "llvm/IR/Attributes.h"
38#include "llvm/IR/DataLayout.h"
39#include "llvm/IR/Function.h"
40#include "llvm/MC/MCAsmInfo.h"
41#include "llvm/MC/TargetRegistry.h"
42#include "llvm/Pass.h"
43#include "llvm/Support/CodeGen.h"
44#include "llvm/Support/CommandLine.h"
45#include "llvm/Support/ErrorHandling.h"
46#include "llvm/Target/TargetLoweringObjectFile.h"
47#include "llvm/Target/TargetOptions.h"
48#include "llvm/TargetParser/Triple.h"
49#include "llvm/Transforms/CFGuard.h"
50#include <memory>
51#include <optional>
52#include <string>
53
54using namespace llvm;
55
56static cl::opt<bool> EnableMachineCombinerPass("x86-machine-combiner",
57 cl::desc("Enable the machine combiner pass"),
58 cl::init(Val: true), cl::Hidden);
59
60static cl::opt<bool>
61 EnableTileRAPass("x86-tile-ra",
62 cl::desc("Enable the tile register allocation pass"),
63 cl::init(Val: true), cl::Hidden);
64
65extern "C" LLVM_C_ABI void LLVMInitializeX86Target() {
66 // Register the target.
67 RegisterTargetMachine<X86TargetMachine> X(getTheX86_32Target());
68 RegisterTargetMachine<X86TargetMachine> Y(getTheX86_64Target());
69
70 PassRegistry &PR = *PassRegistry::getPassRegistry();
71 initializeX86LowerAMXIntrinsicsLegacyPassPass(PR);
72 initializeX86LowerAMXTypeLegacyPassPass(PR);
73 initializeX86PreTileConfigPass(PR);
74 initializeGlobalISel(PR);
75 initializeWinEHStatePassPass(PR);
76 initializeFixupBWInstPassPass(PR);
77 initializeCompressEVEXPassPass(PR);
78 initializeFixupLEAPassPass(PR);
79 initializeFPSPass(PR);
80 initializeX86FixupSetCCPassPass(PR);
81 initializeX86CallFrameOptimizationPass(PR);
82 initializeX86CmovConverterPassPass(PR);
83 initializeX86TileConfigPass(PR);
84 initializeX86FastPreTileConfigPass(PR);
85 initializeX86FastTileConfigPass(PR);
86 initializeKCFIPass(PR);
87 initializeX86LowerTileCopyPass(PR);
88 initializeX86ExpandPseudoPass(PR);
89 initializeX86ExecutionDomainFixPass(PR);
90 initializeX86DomainReassignmentPass(PR);
91 initializeX86AvoidSFBPassPass(PR);
92 initializeX86AvoidTrailingCallPassPass(PR);
93 initializeX86SpeculativeLoadHardeningPassPass(PR);
94 initializeX86SpeculativeExecutionSideEffectSuppressionPass(PR);
95 initializeX86FlagsCopyLoweringPassPass(PR);
96 initializeX86LoadValueInjectionLoadHardeningPassPass(PR);
97 initializeX86LoadValueInjectionRetHardeningPassPass(PR);
98 initializeX86OptimizeLEAPassPass(PR);
99 initializeX86PartialReductionPass(PR);
100 initializePseudoProbeInserterPass(PR);
101 initializeX86ReturnThunksPass(PR);
102 initializeX86DAGToDAGISelLegacyPass(PR);
103 initializeX86ArgumentStackSlotPassPass(PR);
104 initializeX86AsmPrinterPass(PR);
105 initializeX86FixupInstTuningPassPass(PR);
106 initializeX86FixupVectorConstantsPassPass(PR);
107 initializeX86DynAllocaExpanderPass(PR);
108 initializeX86SuppressAPXForRelocationPassPass(PR);
109 initializeX86WinEHUnwindV2Pass(PR);
110}
111
112static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
113 if (TT.isOSBinFormatMachO()) {
114 if (TT.getArch() == Triple::x86_64)
115 return std::make_unique<X86_64MachoTargetObjectFile>();
116 return std::make_unique<TargetLoweringObjectFileMachO>();
117 }
118
119 if (TT.isOSBinFormatCOFF())
120 return std::make_unique<TargetLoweringObjectFileCOFF>();
121
122 if (TT.getArch() == Triple::x86_64)
123 return std::make_unique<X86_64ELFTargetObjectFile>();
124 return std::make_unique<X86ELFTargetObjectFile>();
125}
126
127static std::string computeDataLayout(const Triple &TT) {
128 // X86 is little endian
129 std::string Ret = "e";
130
131 Ret += DataLayout::getManglingComponent(T: TT);
132 // X86 and x32 have 32 bit pointers.
133 if (!TT.isArch64Bit() || TT.isX32() || TT.isOSNaCl())
134 Ret += "-p:32:32";
135
136 // Address spaces for 32 bit signed, 32 bit unsigned, and 64 bit pointers.
137 Ret += "-p270:32:32-p271:32:32-p272:64:64";
138
139 // Some ABIs align 64 bit integers and doubles to 64 bits, others to 32.
140 // 128 bit integers are not specified in the 32-bit ABIs but are used
141 // internally for lowering f128, so we match the alignment to that.
142 if (TT.isArch64Bit() || TT.isOSWindows() || TT.isOSNaCl())
143 Ret += "-i64:64-i128:128";
144 else if (TT.isOSIAMCU())
145 Ret += "-i64:32-f64:32";
146 else
147 Ret += "-i128:128-f64:32:64";
148
149 // Some ABIs align long double to 128 bits, others to 32.
150 if (TT.isOSNaCl() || TT.isOSIAMCU())
151 ; // No f80
152 else if (TT.isArch64Bit() || TT.isOSDarwin() || TT.isWindowsMSVCEnvironment())
153 Ret += "-f80:128";
154 else
155 Ret += "-f80:32";
156
157 if (TT.isOSIAMCU())
158 Ret += "-f128:32";
159
160 // The registers can hold 8, 16, 32 or, in x86-64, 64 bits.
161 if (TT.isArch64Bit())
162 Ret += "-n8:16:32:64";
163 else
164 Ret += "-n8:16:32";
165
166 // The stack is aligned to 32 bits on some ABIs and 128 bits on others.
167 if ((!TT.isArch64Bit() && TT.isOSWindows()) || TT.isOSIAMCU())
168 Ret += "-a:0:32-S32";
169 else
170 Ret += "-S128";
171
172 return Ret;
173}
174
175static Reloc::Model getEffectiveRelocModel(const Triple &TT, bool JIT,
176 std::optional<Reloc::Model> RM) {
177 bool is64Bit = TT.getArch() == Triple::x86_64;
178 if (!RM) {
179 // JIT codegen should use static relocations by default, since it's
180 // typically executed in process and not relocatable.
181 if (JIT)
182 return Reloc::Static;
183
184 // Darwin defaults to PIC in 64 bit mode and dynamic-no-pic in 32 bit mode.
185 // Win64 requires rip-rel addressing, thus we force it to PIC. Otherwise we
186 // use static relocation model by default.
187 if (TT.isOSDarwin()) {
188 if (is64Bit)
189 return Reloc::PIC_;
190 return Reloc::DynamicNoPIC;
191 }
192 if (TT.isOSWindows() && is64Bit)
193 return Reloc::PIC_;
194 return Reloc::Static;
195 }
196
197 // ELF and X86-64 don't have a distinct DynamicNoPIC model. DynamicNoPIC
198 // is defined as a model for code which may be used in static or dynamic
199 // executables but not necessarily a shared library. On X86-32 we just
200 // compile in -static mode, in x86-64 we use PIC.
201 if (*RM == Reloc::DynamicNoPIC) {
202 if (is64Bit)
203 return Reloc::PIC_;
204 if (!TT.isOSDarwin())
205 return Reloc::Static;
206 }
207
208 // If we are on Darwin, disallow static relocation model in X86-64 mode, since
209 // the Mach-O file format doesn't support it.
210 if (*RM == Reloc::Static && TT.isOSDarwin() && is64Bit)
211 return Reloc::PIC_;
212
213 return *RM;
214}
215
216static CodeModel::Model
217getEffectiveX86CodeModel(const Triple &TT, std::optional<CodeModel::Model> CM,
218 bool JIT) {
219 bool Is64Bit = TT.getArch() == Triple::x86_64;
220 if (CM) {
221 if (*CM == CodeModel::Tiny)
222 reportFatalUsageError(reason: "target does not support the tiny CodeModel");
223 return *CM;
224 }
225 if (JIT)
226 return Is64Bit ? CodeModel::Large : CodeModel::Small;
227 return CodeModel::Small;
228}
229
230/// Create an X86 target.
231///
232X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT,
233 StringRef CPU, StringRef FS,
234 const TargetOptions &Options,
235 std::optional<Reloc::Model> RM,
236 std::optional<CodeModel::Model> CM,
237 CodeGenOptLevel OL, bool JIT)
238 : CodeGenTargetMachineImpl(T, computeDataLayout(TT), TT, CPU, FS, Options,
239 getEffectiveRelocModel(TT, JIT, RM),
240 getEffectiveX86CodeModel(TT, CM, JIT), OL),
241 TLOF(createTLOF(TT: getTargetTriple())), IsJIT(JIT) {
242 // On PS4/PS5, the "return address" of a 'noreturn' call must still be within
243 // the calling function. Note that this also includes __stack_chk_fail,
244 // so there was some target-specific logic in the instruction selectors
245 // to handle that. That code has since been generalized, so the only thing
246 // needed is to set TrapUnreachable here.
247 if (TT.isPS() || TT.isOSBinFormatMachO()) {
248 this->Options.TrapUnreachable = true;
249 this->Options.NoTrapAfterNoreturn = TT.isOSBinFormatMachO();
250 }
251
252 setMachineOutliner(true);
253
254 // x86 supports the debug entry values.
255 setSupportsDebugEntryValues(true);
256
257 initAsmInfo();
258}
259
260X86TargetMachine::~X86TargetMachine() = default;
261
262const X86Subtarget *
263X86TargetMachine::getSubtargetImpl(const Function &F) const {
264 Attribute CPUAttr = F.getFnAttribute(Kind: "target-cpu");
265 Attribute TuneAttr = F.getFnAttribute(Kind: "tune-cpu");
266 Attribute FSAttr = F.getFnAttribute(Kind: "target-features");
267
268 StringRef CPU =
269 CPUAttr.isValid() ? CPUAttr.getValueAsString() : (StringRef)TargetCPU;
270 // "x86-64" is a default target setting for many front ends. In these cases,
271 // they actually request for "generic" tuning unless the "tune-cpu" was
272 // specified.
273 StringRef TuneCPU = TuneAttr.isValid() ? TuneAttr.getValueAsString()
274 : CPU == "x86-64" ? "generic"
275 : (StringRef)CPU;
276 StringRef FS =
277 FSAttr.isValid() ? FSAttr.getValueAsString() : (StringRef)TargetFS;
278
279 SmallString<512> Key;
280 // The additions here are ordered so that the definitely short strings are
281 // added first so we won't exceed the small size. We append the
282 // much longer FS string at the end so that we only heap allocate at most
283 // one time.
284
285 // Extract prefer-vector-width attribute.
286 unsigned PreferVectorWidthOverride = 0;
287 Attribute PreferVecWidthAttr = F.getFnAttribute(Kind: "prefer-vector-width");
288 if (PreferVecWidthAttr.isValid()) {
289 StringRef Val = PreferVecWidthAttr.getValueAsString();
290 unsigned Width;
291 if (!Val.getAsInteger(Radix: 0, Result&: Width)) {
292 Key += 'p';
293 Key += Val;
294 PreferVectorWidthOverride = Width;
295 }
296 }
297
298 // Extract min-legal-vector-width attribute.
299 unsigned RequiredVectorWidth = UINT32_MAX;
300 Attribute MinLegalVecWidthAttr = F.getFnAttribute(Kind: "min-legal-vector-width");
301 if (MinLegalVecWidthAttr.isValid()) {
302 StringRef Val = MinLegalVecWidthAttr.getValueAsString();
303 unsigned Width;
304 if (!Val.getAsInteger(Radix: 0, Result&: Width)) {
305 Key += 'm';
306 Key += Val;
307 RequiredVectorWidth = Width;
308 }
309 }
310
311 // Add CPU to the Key.
312 Key += CPU;
313
314 // Add tune CPU to the Key.
315 Key += TuneCPU;
316
317 // Keep track of the start of the feature portion of the string.
318 unsigned FSStart = Key.size();
319
320 // FIXME: This is related to the code below to reset the target options,
321 // we need to know whether or not the soft float flag is set on the
322 // function before we can generate a subtarget. We also need to use
323 // it as a key for the subtarget since that can be the only difference
324 // between two functions.
325 bool SoftFloat = F.getFnAttribute(Kind: "use-soft-float").getValueAsBool();
326 // If the soft float attribute is set on the function turn on the soft float
327 // subtarget feature.
328 if (SoftFloat)
329 Key += FS.empty() ? "+soft-float" : "+soft-float,";
330
331 Key += FS;
332
333 // We may have added +soft-float to the features so move the StringRef to
334 // point to the full string in the Key.
335 FS = Key.substr(Start: FSStart);
336
337 auto &I = SubtargetMap[Key];
338 if (!I) {
339 // This needs to be done before we create a new subtarget since any
340 // creation will depend on the TM and the code generation flags on the
341 // function that reside in TargetOptions.
342 resetTargetOptions(F);
343 I = std::make_unique<X86Subtarget>(
344 args: TargetTriple, args&: CPU, args&: TuneCPU, args&: FS, args: *this,
345 args: MaybeAlign(F.getParent()->getOverrideStackAlignment()),
346 args&: PreferVectorWidthOverride, args&: RequiredVectorWidth);
347 }
348 return I.get();
349}
350
351yaml::MachineFunctionInfo *X86TargetMachine::createDefaultFuncInfoYAML() const {
352 return new yaml::X86MachineFunctionInfo();
353}
354
355yaml::MachineFunctionInfo *
356X86TargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
357 const auto *MFI = MF.getInfo<X86MachineFunctionInfo>();
358 return new yaml::X86MachineFunctionInfo(*MFI);
359}
360
361bool X86TargetMachine::parseMachineFunctionInfo(
362 const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
363 SMDiagnostic &Error, SMRange &SourceRange) const {
364 const auto &YamlMFI = static_cast<const yaml::X86MachineFunctionInfo &>(MFI);
365 PFS.MF.getInfo<X86MachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
366 return false;
367}
368
369bool X86TargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
370 unsigned DestAS) const {
371 assert(SrcAS != DestAS && "Expected different address spaces!");
372 if (getPointerSize(AS: SrcAS) != getPointerSize(AS: DestAS))
373 return false;
374 return SrcAS < 256 && DestAS < 256;
375}
376
377void X86TargetMachine::reset() { SubtargetMap.clear(); }
378
379ScheduleDAGInstrs *
380X86TargetMachine::createMachineScheduler(MachineSchedContext *C) const {
381 ScheduleDAGMILive *DAG = createSchedLive(C);
382 DAG->addMutation(Mutation: createX86MacroFusionDAGMutation());
383 return DAG;
384}
385
386ScheduleDAGInstrs *
387X86TargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
388 ScheduleDAGMI *DAG = createSchedPostRA(C);
389 DAG->addMutation(Mutation: createX86MacroFusionDAGMutation());
390 return DAG;
391}
392
393//===----------------------------------------------------------------------===//
394// X86 TTI query.
395//===----------------------------------------------------------------------===//
396
397TargetTransformInfo
398X86TargetMachine::getTargetTransformInfo(const Function &F) const {
399 return TargetTransformInfo(std::make_unique<X86TTIImpl>(args: this, args: F));
400}
401
402//===----------------------------------------------------------------------===//
403// Pass Pipeline Configuration
404//===----------------------------------------------------------------------===//
405
406namespace {
407
408/// X86 Code Generator Pass Configuration Options.
409class X86PassConfig : public TargetPassConfig {
410public:
411 X86PassConfig(X86TargetMachine &TM, PassManagerBase &PM)
412 : TargetPassConfig(TM, PM) {}
413
414 X86TargetMachine &getX86TargetMachine() const {
415 return getTM<X86TargetMachine>();
416 }
417
418 void addIRPasses() override;
419 bool addInstSelector() override;
420 bool addIRTranslator() override;
421 bool addLegalizeMachineIR() override;
422 bool addRegBankSelect() override;
423 bool addGlobalInstructionSelect() override;
424 bool addILPOpts() override;
425 bool addPreISel() override;
426 void addMachineSSAOptimization() override;
427 void addPreRegAlloc() override;
428 bool addPostFastRegAllocRewrite() override;
429 void addPostRegAlloc() override;
430 void addPreEmitPass() override;
431 void addPreEmitPass2() override;
432 void addPreSched2() override;
433 bool addRegAssignAndRewriteOptimized() override;
434
435 std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
436};
437
438class X86ExecutionDomainFix : public ExecutionDomainFix {
439public:
440 static char ID;
441 X86ExecutionDomainFix() : ExecutionDomainFix(ID, X86::VR128XRegClass) {}
442 StringRef getPassName() const override {
443 return "X86 Execution Dependency Fix";
444 }
445};
446char X86ExecutionDomainFix::ID;
447
448} // end anonymous namespace
449
450INITIALIZE_PASS_BEGIN(X86ExecutionDomainFix, "x86-execution-domain-fix",
451 "X86 Execution Domain Fix", false, false)
452INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis)
453INITIALIZE_PASS_END(X86ExecutionDomainFix, "x86-execution-domain-fix",
454 "X86 Execution Domain Fix", false, false)
455
456TargetPassConfig *X86TargetMachine::createPassConfig(PassManagerBase &PM) {
457 return new X86PassConfig(*this, PM);
458}
459
460MachineFunctionInfo *X86TargetMachine::createMachineFunctionInfo(
461 BumpPtrAllocator &Allocator, const Function &F,
462 const TargetSubtargetInfo *STI) const {
463 return X86MachineFunctionInfo::create<X86MachineFunctionInfo>(Allocator, F,
464 STI);
465}
466
467void X86PassConfig::addIRPasses() {
468 addPass(P: createAtomicExpandLegacyPass());
469
470 // We add both pass anyway and when these two passes run, we skip the pass
471 // based on the option level and option attribute.
472 addPass(P: createX86LowerAMXIntrinsicsPass());
473 addPass(P: createX86LowerAMXTypePass());
474
475 TargetPassConfig::addIRPasses();
476
477 if (TM->getOptLevel() != CodeGenOptLevel::None) {
478 addPass(P: createInterleavedAccessPass());
479 addPass(P: createX86PartialReductionPass());
480 }
481
482 // Add passes that handle indirect branch removal and insertion of a retpoline
483 // thunk. These will be a no-op unless a function subtarget has the retpoline
484 // feature enabled.
485 addPass(P: createIndirectBrExpandPass());
486
487 // Add Control Flow Guard checks.
488 const Triple &TT = TM->getTargetTriple();
489 if (TT.isOSWindows()) {
490 if (TT.getArch() == Triple::x86_64) {
491 addPass(P: createCFGuardDispatchPass());
492 } else {
493 addPass(P: createCFGuardCheckPass());
494 }
495 }
496
497 if (TM->Options.JMCInstrument)
498 addPass(P: createJMCInstrumenterPass());
499}
500
501bool X86PassConfig::addInstSelector() {
502 // Install an instruction selector.
503 addPass(P: createX86ISelDag(TM&: getX86TargetMachine(), OptLevel: getOptLevel()));
504
505 // For ELF, cleanup any local-dynamic TLS accesses.
506 if (TM->getTargetTriple().isOSBinFormatELF() &&
507 getOptLevel() != CodeGenOptLevel::None)
508 addPass(P: createCleanupLocalDynamicTLSPass());
509
510 addPass(P: createX86GlobalBaseRegPass());
511 addPass(P: createX86ArgumentStackSlotPass());
512 return false;
513}
514
515bool X86PassConfig::addIRTranslator() {
516 addPass(P: new IRTranslator(getOptLevel()));
517 return false;
518}
519
520bool X86PassConfig::addLegalizeMachineIR() {
521 addPass(P: new Legalizer());
522 return false;
523}
524
525bool X86PassConfig::addRegBankSelect() {
526 addPass(P: new RegBankSelect());
527 return false;
528}
529
530bool X86PassConfig::addGlobalInstructionSelect() {
531 addPass(P: new InstructionSelect(getOptLevel()));
532 // Add GlobalBaseReg in case there is no SelectionDAG passes afterwards
533 if (isGlobalISelAbortEnabled())
534 addPass(P: createX86GlobalBaseRegPass());
535 return false;
536}
537
538bool X86PassConfig::addILPOpts() {
539 addPass(PassID: &EarlyIfConverterLegacyID);
540 if (EnableMachineCombinerPass)
541 addPass(PassID: &MachineCombinerID);
542 addPass(P: createX86CmovConverterPass());
543 return true;
544}
545
546bool X86PassConfig::addPreISel() {
547 // Only add this pass for 32-bit x86 Windows.
548 const Triple &TT = TM->getTargetTriple();
549 if (TT.isOSWindows() && TT.getArch() == Triple::x86)
550 addPass(P: createX86WinEHStatePass());
551 return true;
552}
553
554void X86PassConfig::addPreRegAlloc() {
555 if (getOptLevel() != CodeGenOptLevel::None) {
556 addPass(PassID: &LiveRangeShrinkID);
557 addPass(P: createX86FixupSetCC());
558 addPass(P: createX86OptimizeLEAs());
559 addPass(P: createX86CallFrameOptimization());
560 addPass(P: createX86AvoidStoreForwardingBlocks());
561 }
562
563 addPass(P: createX86SuppressAPXForRelocationPass());
564
565 addPass(P: createX86SpeculativeLoadHardeningPass());
566 addPass(P: createX86FlagsCopyLoweringPass());
567 addPass(P: createX86DynAllocaExpander());
568
569 if (getOptLevel() != CodeGenOptLevel::None)
570 addPass(P: createX86PreTileConfigPass());
571 else
572 addPass(P: createX86FastPreTileConfigPass());
573}
574
575void X86PassConfig::addMachineSSAOptimization() {
576 addPass(P: createX86DomainReassignmentPass());
577 TargetPassConfig::addMachineSSAOptimization();
578}
579
580void X86PassConfig::addPostRegAlloc() {
581 addPass(P: createX86LowerTileCopyPass());
582 addPass(P: createX86FloatingPointStackifierPass());
583 // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
584 // to using the Speculative Execution Side Effect Suppression pass for
585 // mitigation. This is to prevent slow downs due to
586 // analyses needed by the LVIHardening pass when compiling at -O0.
587 if (getOptLevel() != CodeGenOptLevel::None)
588 addPass(P: createX86LoadValueInjectionLoadHardeningPass());
589}
590
591void X86PassConfig::addPreSched2() {
592 addPass(P: createX86ExpandPseudoPass());
593 addPass(P: createKCFIPass());
594}
595
596void X86PassConfig::addPreEmitPass() {
597 if (getOptLevel() != CodeGenOptLevel::None) {
598 addPass(P: new X86ExecutionDomainFix());
599 addPass(P: createBreakFalseDeps());
600 }
601
602 addPass(P: createX86IndirectBranchTrackingPass());
603
604 addPass(P: createX86IssueVZeroUpperPass());
605
606 if (getOptLevel() != CodeGenOptLevel::None) {
607 addPass(P: createX86FixupBWInsts());
608 addPass(P: createX86PadShortFunctions());
609 addPass(P: createX86FixupLEAs());
610 addPass(P: createX86FixupInstTuning());
611 addPass(P: createX86FixupVectorConstants());
612 }
613 addPass(P: createX86CompressEVEXPass());
614 addPass(P: createX86DiscriminateMemOpsPass());
615 addPass(P: createX86InsertPrefetchPass());
616 addPass(P: createX86InsertX87waitPass());
617}
618
619void X86PassConfig::addPreEmitPass2() {
620 const Triple &TT = TM->getTargetTriple();
621 const MCAsmInfo *MAI = TM->getMCAsmInfo();
622
623 // The X86 Speculative Execution Pass must run after all control
624 // flow graph modifying passes. As a result it was listed to run right before
625 // the X86 Retpoline Thunks pass. The reason it must run after control flow
626 // graph modifications is that the model of LFENCE in LLVM has to be updated
627 // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the
628 // placement of this pass was hand checked to ensure that the subsequent
629 // passes don't move the code around the LFENCEs in a way that will hurt the
630 // correctness of this pass. This placement has been shown to work based on
631 // hand inspection of the codegen output.
632 addPass(P: createX86SpeculativeExecutionSideEffectSuppression());
633 addPass(P: createX86IndirectThunksPass());
634 addPass(P: createX86ReturnThunksPass());
635
636 // Insert extra int3 instructions after trailing call instructions to avoid
637 // issues in the unwinder.
638 if (TT.isOSWindows() && TT.getArch() == Triple::x86_64)
639 addPass(P: createX86AvoidTrailingCallPass());
640
641 // Verify basic block incoming and outgoing cfa offset and register values and
642 // correct CFA calculation rule where needed by inserting appropriate CFI
643 // instructions.
644 if (!TT.isOSDarwin() &&
645 (!TT.isOSWindows() ||
646 MAI->getExceptionHandlingType() == ExceptionHandling::DwarfCFI))
647 addPass(P: createCFIInstrInserter());
648
649 if (TT.isOSWindows()) {
650 // Identify valid longjmp targets for Windows Control Flow Guard.
651 addPass(P: createCFGuardLongjmpPass());
652 // Identify valid eh continuation targets for Windows EHCont Guard.
653 addPass(P: createEHContGuardTargetsPass());
654 }
655 addPass(P: createX86LoadValueInjectionRetHardeningPass());
656
657 // Insert pseudo probe annotation for callsite profiling
658 addPass(P: createPseudoProbeInserter());
659
660 // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms,
661 // also CALL_RVMARKER.
662 addPass(P: createUnpackMachineBundles(Ftor: [&TT](const MachineFunction &MF) {
663 // Only run bundle expansion if the module uses kcfi, or there are relevant
664 // ObjC runtime functions present in the module.
665 const Function &F = MF.getFunction();
666 const Module *M = F.getParent();
667 return M->getModuleFlag(Key: "kcfi") ||
668 (TT.isOSDarwin() &&
669 (M->getFunction(Name: "objc_retainAutoreleasedReturnValue") ||
670 M->getFunction(Name: "objc_unsafeClaimAutoreleasedReturnValue")));
671 }));
672
673 // Analyzes and emits pseudos to support Win x64 Unwind V2. This pass must run
674 // after all real instructions have been added to the epilog.
675 if (TT.isOSWindows() && (TT.getArch() == Triple::x86_64))
676 addPass(P: createX86WinEHUnwindV2Pass());
677}
678
679bool X86PassConfig::addPostFastRegAllocRewrite() {
680 addPass(P: createX86FastTileConfigPass());
681 return true;
682}
683
684std::unique_ptr<CSEConfigBase> X86PassConfig::getCSEConfig() const {
685 return getStandardCSEConfigForOpt(Level: TM->getOptLevel());
686}
687
688static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI,
689 const MachineRegisterInfo &MRI,
690 const Register Reg) {
691 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
692 return static_cast<const X86RegisterInfo &>(TRI).isTileRegisterClass(RC);
693}
694
695bool X86PassConfig::addRegAssignAndRewriteOptimized() {
696 // Don't support tile RA when RA is specified by command line "-regalloc".
697 if (!isCustomizedRegAlloc() && EnableTileRAPass) {
698 // Allocate tile register first.
699 addPass(P: createGreedyRegisterAllocator(F: onlyAllocateTileRegisters));
700 addPass(P: createX86TileConfigPass());
701 }
702 return TargetPassConfig::addRegAssignAndRewriteOptimized();
703}
704