1//===-- X86TargetMachine.cpp - Define TargetMachine for the X86 -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the X86 specific subclass of TargetMachine.
10//
11//===----------------------------------------------------------------------===//
12
13#include "X86TargetMachine.h"
14#include "MCTargetDesc/X86MCTargetDesc.h"
15#include "TargetInfo/X86TargetInfo.h"
16#include "X86.h"
17#include "X86MachineFunctionInfo.h"
18#include "X86MacroFusion.h"
19#include "X86Subtarget.h"
20#include "X86TargetObjectFile.h"
21#include "X86TargetTransformInfo.h"
22#include "llvm-c/Visibility.h"
23#include "llvm/ADT/SmallString.h"
24#include "llvm/ADT/StringRef.h"
25#include "llvm/Analysis/TargetTransformInfo.h"
26#include "llvm/CodeGen/ExecutionDomainFix.h"
27#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
28#include "llvm/CodeGen/GlobalISel/CallLowering.h"
29#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
30#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
31#include "llvm/CodeGen/GlobalISel/Legalizer.h"
32#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
33#include "llvm/CodeGen/MIRParser/MIParser.h"
34#include "llvm/CodeGen/MIRYamlMapping.h"
35#include "llvm/CodeGen/MachineScheduler.h"
36#include "llvm/CodeGen/Passes.h"
37#include "llvm/CodeGen/TargetPassConfig.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/Function.h"
41#include "llvm/MC/MCAsmInfo.h"
42#include "llvm/MC/TargetRegistry.h"
43#include "llvm/Pass.h"
44#include "llvm/Support/CodeGen.h"
45#include "llvm/Support/CommandLine.h"
46#include "llvm/Support/ErrorHandling.h"
47#include "llvm/Target/TargetLoweringObjectFile.h"
48#include "llvm/Target/TargetOptions.h"
49#include "llvm/TargetParser/Triple.h"
50#include "llvm/Transforms/CFGuard.h"
51#include <memory>
52#include <optional>
53
54using namespace llvm;
55
56cl::opt<bool>
57 X86EnableMachineCombinerPass("x86-machine-combiner",
58 cl::desc("Enable the machine combiner pass"),
59 cl::init(Val: true), cl::Hidden);
60
61static cl::opt<bool>
62 EnableTileRAPass("x86-tile-ra",
63 cl::desc("Enable the tile register allocation pass"),
64 cl::init(Val: true), cl::Hidden);
65
66extern "C" LLVM_C_ABI void LLVMInitializeX86Target() {
67 // Register the target.
68 RegisterTargetMachine<X86TargetMachine> X(getTheX86_32Target());
69 RegisterTargetMachine<X86TargetMachine> Y(getTheX86_64Target());
70
71 PassRegistry &PR = *PassRegistry::getPassRegistry();
72 initializeX86LowerAMXIntrinsicsLegacyPassPass(PR);
73 initializeX86LowerAMXTypeLegacyPassPass(PR);
74 initializeX86PreTileConfigLegacyPass(PR);
75 initializeGlobalISel(PR);
76 initializeWinEHStateLegacyPass(PR);
77 initializeX86FixupBWInstLegacyPass(PR);
78 initializeCompressEVEXLegacyPass(PR);
79 initializeFixupLEAsLegacyPass(PR);
80 initializeX86FPStackifierLegacyPass(PR);
81 initializeX86FixupSetCCLegacyPass(PR);
82 initializeX86CallFrameOptimizationLegacyPass(PR);
83 initializeX86CmovConversionLegacyPass(PR);
84 initializeX86TileConfigLegacyPass(PR);
85 initializeX86FastPreTileConfigLegacyPass(PR);
86 initializeX86FastTileConfigLegacyPass(PR);
87 initializeKCFIPass(PR);
88 initializeX86LowerTileCopyLegacyPass(PR);
89 initializeX86ExpandPseudoLegacyPass(PR);
90 initializeX86ExecutionDomainFixPass(PR);
91 initializeX86DomainReassignmentLegacyPass(PR);
92 initializeX86AvoidSFBLegacyPass(PR);
93 initializeX86AvoidTrailingCallLegacyPassPass(PR);
94 initializeX86SpeculativeLoadHardeningLegacyPass(PR);
95 initializeX86SpeculativeExecutionSideEffectSuppressionLegacyPass(PR);
96 initializeX86FlagsCopyLoweringLegacyPass(PR);
97 initializeX86LoadValueInjectionLoadHardeningLegacyPass(PR);
98 initializeX86LoadValueInjectionRetHardeningLegacyPass(PR);
99 initializeX86OptimizeLEAsLegacyPass(PR);
100 initializeX86PartialReductionLegacyPass(PR);
101 initializeX86ReturnThunksLegacyPass(PR);
102 initializeX86DAGToDAGISelLegacyPass(PR);
103 initializeX86ArgumentStackSlotLegacyPass(PR);
104 initializeX86AsmPrinterPass(PR);
105 initializeX86FixupInstTuningLegacyPass(PR);
106 initializeX86FixupVectorConstantsLegacyPass(PR);
107 initializeX86DynAllocaExpanderLegacyPass(PR);
108 initializeX86SuppressAPXForRelocationLegacyPass(PR);
109 initializeX86WinEHUnwindV2LegacyPass(PR);
110 initializeX86PreLegalizerCombinerPass(PR);
111 initializeX86PostLegalizerCombinerPass(PR);
112}
113
114static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
115 if (TT.isOSBinFormatMachO()) {
116 if (TT.isX86_64())
117 return std::make_unique<X86_64MachoTargetObjectFile>();
118 return std::make_unique<TargetLoweringObjectFileMachO>();
119 }
120
121 if (TT.isOSBinFormatCOFF())
122 return std::make_unique<TargetLoweringObjectFileCOFF>();
123
124 if (TT.isX86_64())
125 return std::make_unique<X86_64ELFTargetObjectFile>();
126 return std::make_unique<X86ELFTargetObjectFile>();
127}
128
129static Reloc::Model getEffectiveRelocModel(const Triple &TT, bool JIT,
130 std::optional<Reloc::Model> RM) {
131 bool is64Bit = TT.isX86_64();
132 if (!RM) {
133 // JIT codegen should use static relocations by default, since it's
134 // typically executed in process and not relocatable.
135 if (JIT)
136 return Reloc::Static;
137
138 // Darwin defaults to PIC in 64 bit mode and dynamic-no-pic in 32 bit mode.
139 // Win64 requires rip-rel addressing, thus we force it to PIC. Otherwise we
140 // use static relocation model by default.
141 if (TT.isOSDarwin()) {
142 if (is64Bit)
143 return Reloc::PIC_;
144 return Reloc::DynamicNoPIC;
145 }
146 if (TT.isOSWindows() && is64Bit)
147 return Reloc::PIC_;
148 return Reloc::Static;
149 }
150
151 // ELF and X86-64 don't have a distinct DynamicNoPIC model. DynamicNoPIC
152 // is defined as a model for code which may be used in static or dynamic
153 // executables but not necessarily a shared library. On X86-32 we just
154 // compile in -static mode, in x86-64 we use PIC.
155 if (*RM == Reloc::DynamicNoPIC) {
156 if (is64Bit)
157 return Reloc::PIC_;
158 if (!TT.isOSDarwin())
159 return Reloc::Static;
160 }
161
162 // If we are on Darwin, disallow static relocation model in X86-64 mode, since
163 // the Mach-O file format doesn't support it.
164 if (*RM == Reloc::Static && TT.isOSDarwin() && is64Bit)
165 return Reloc::PIC_;
166
167 return *RM;
168}
169
170static CodeModel::Model
171getEffectiveX86CodeModel(const Triple &TT, std::optional<CodeModel::Model> CM,
172 bool JIT) {
173 bool Is64Bit = TT.isX86_64();
174 if (CM) {
175 if (*CM == CodeModel::Tiny)
176 reportFatalUsageError(reason: "target does not support the tiny CodeModel");
177 return *CM;
178 }
179 if (JIT)
180 return Is64Bit ? CodeModel::Large : CodeModel::Small;
181 return CodeModel::Small;
182}
183
184/// Create an X86 target.
185///
186X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT,
187 StringRef CPU, StringRef FS,
188 const TargetOptions &Options,
189 std::optional<Reloc::Model> RM,
190 std::optional<CodeModel::Model> CM,
191 CodeGenOptLevel OL, bool JIT)
192 : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, FS, Options,
193 getEffectiveRelocModel(TT, JIT, RM),
194 getEffectiveX86CodeModel(TT, CM, JIT), OL),
195 TLOF(createTLOF(TT: getTargetTriple())), IsJIT(JIT) {
196 // On PS4/PS5, the "return address" of a 'noreturn' call must still be within
197 // the calling function. Note that this also includes __stack_chk_fail,
198 // so there was some target-specific logic in the instruction selectors
199 // to handle that. That code has since been generalized, so the only thing
200 // needed is to set TrapUnreachable here.
201 if (TT.isPS() || TT.isOSBinFormatMachO()) {
202 this->Options.TrapUnreachable = true;
203 this->Options.NoTrapAfterNoreturn = TT.isOSBinFormatMachO();
204 }
205
206 setMachineOutliner(true);
207
208 // x86 supports the debug entry values.
209 setSupportsDebugEntryValues(true);
210
211 initAsmInfo();
212}
213
214X86TargetMachine::~X86TargetMachine() = default;
215
216const X86Subtarget *
217X86TargetMachine::getSubtargetImpl(const Function &F) const {
218 Attribute CPUAttr = F.getFnAttribute(Kind: "target-cpu");
219 Attribute TuneAttr = F.getFnAttribute(Kind: "tune-cpu");
220 Attribute FSAttr = F.getFnAttribute(Kind: "target-features");
221
222 StringRef CPU =
223 CPUAttr.isValid() ? CPUAttr.getValueAsString() : (StringRef)TargetCPU;
224 // "x86-64" is a default target setting for many front ends. In these cases,
225 // they actually request for "generic" tuning unless the "tune-cpu" was
226 // specified.
227 StringRef TuneCPU = TuneAttr.isValid() ? TuneAttr.getValueAsString()
228 : CPU == "x86-64" ? "generic"
229 : (StringRef)CPU;
230 StringRef FS =
231 FSAttr.isValid() ? FSAttr.getValueAsString() : (StringRef)TargetFS;
232
233 SmallString<512> Key;
234 // The additions here are ordered so that the definitely short strings are
235 // added first so we won't exceed the small size. We append the
236 // much longer FS string at the end so that we only heap allocate at most
237 // one time.
238
239 // Extract prefer-vector-width attribute.
240 unsigned PreferVectorWidthOverride = 0;
241 Attribute PreferVecWidthAttr = F.getFnAttribute(Kind: "prefer-vector-width");
242 if (PreferVecWidthAttr.isValid()) {
243 StringRef Val = PreferVecWidthAttr.getValueAsString();
244 unsigned Width;
245 if (!Val.getAsInteger(Radix: 0, Result&: Width)) {
246 Key += 'p';
247 Key += Val;
248 PreferVectorWidthOverride = Width;
249 }
250 }
251
252 // Extract min-legal-vector-width attribute.
253 unsigned RequiredVectorWidth = UINT32_MAX;
254 Attribute MinLegalVecWidthAttr = F.getFnAttribute(Kind: "min-legal-vector-width");
255 if (MinLegalVecWidthAttr.isValid()) {
256 StringRef Val = MinLegalVecWidthAttr.getValueAsString();
257 unsigned Width;
258 if (!Val.getAsInteger(Radix: 0, Result&: Width)) {
259 Key += 'm';
260 Key += Val;
261 RequiredVectorWidth = Width;
262 }
263 }
264
265 // Add CPU to the Key.
266 Key += CPU;
267
268 // Add tune CPU to the Key.
269 Key += TuneCPU;
270
271 // Keep track of the start of the feature portion of the string.
272 unsigned FSStart = Key.size();
273
274 // FIXME: This is related to the code below to reset the target options,
275 // we need to know whether or not the soft float flag is set on the
276 // function before we can generate a subtarget. We also need to use
277 // it as a key for the subtarget since that can be the only difference
278 // between two functions.
279 bool SoftFloat = F.getFnAttribute(Kind: "use-soft-float").getValueAsBool();
280 // If the soft float attribute is set on the function turn on the soft float
281 // subtarget feature.
282 if (SoftFloat)
283 Key += FS.empty() ? "+soft-float" : "+soft-float,";
284
285 Key += FS;
286
287 // We may have added +soft-float to the features so move the StringRef to
288 // point to the full string in the Key.
289 FS = Key.substr(Start: FSStart);
290
291 auto &I = SubtargetMap[Key];
292 if (!I) {
293 // This needs to be done before we create a new subtarget since any
294 // creation will depend on the TM and the code generation flags on the
295 // function that reside in TargetOptions.
296 resetTargetOptions(F);
297 I = std::make_unique<X86Subtarget>(
298 args: TargetTriple, args&: CPU, args&: TuneCPU, args&: FS, args: *this,
299 args: MaybeAlign(F.getParent()->getOverrideStackAlignment()),
300 args&: PreferVectorWidthOverride, args&: RequiredVectorWidth);
301 }
302 return I.get();
303}
304
305yaml::MachineFunctionInfo *X86TargetMachine::createDefaultFuncInfoYAML() const {
306 return new yaml::X86MachineFunctionInfo();
307}
308
309yaml::MachineFunctionInfo *
310X86TargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
311 const auto *MFI = MF.getInfo<X86MachineFunctionInfo>();
312 return new yaml::X86MachineFunctionInfo(*MFI);
313}
314
315bool X86TargetMachine::parseMachineFunctionInfo(
316 const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
317 SMDiagnostic &Error, SMRange &SourceRange) const {
318 const auto &YamlMFI = static_cast<const yaml::X86MachineFunctionInfo &>(MFI);
319 PFS.MF.getInfo<X86MachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
320 return false;
321}
322
323bool X86TargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
324 unsigned DestAS) const {
325 assert(SrcAS != DestAS && "Expected different address spaces!");
326 if (getPointerSize(AS: SrcAS) != getPointerSize(AS: DestAS))
327 return false;
328 return SrcAS < 256 && DestAS < 256;
329}
330
331void X86TargetMachine::reset() { SubtargetMap.clear(); }
332
333ScheduleDAGInstrs *
334X86TargetMachine::createMachineScheduler(MachineSchedContext *C) const {
335 ScheduleDAGMILive *DAG = createSchedLive(C);
336 DAG->addMutation(Mutation: createX86MacroFusionDAGMutation());
337 return DAG;
338}
339
340ScheduleDAGInstrs *
341X86TargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
342 ScheduleDAGMI *DAG = createSchedPostRA(C);
343 DAG->addMutation(Mutation: createX86MacroFusionDAGMutation());
344 return DAG;
345}
346
347//===----------------------------------------------------------------------===//
348// X86 TTI query.
349//===----------------------------------------------------------------------===//
350
351TargetTransformInfo
352X86TargetMachine::getTargetTransformInfo(const Function &F) const {
353 return TargetTransformInfo(std::make_unique<X86TTIImpl>(args: this, args: F));
354}
355
356//===----------------------------------------------------------------------===//
357// Pass Pipeline Configuration
358//===----------------------------------------------------------------------===//
359
360namespace {
361
362/// X86 Code Generator Pass Configuration Options.
363class X86PassConfig : public TargetPassConfig {
364public:
365 X86PassConfig(X86TargetMachine &TM, PassManagerBase &PM)
366 : TargetPassConfig(TM, PM) {}
367
368 X86TargetMachine &getX86TargetMachine() const {
369 return getTM<X86TargetMachine>();
370 }
371
372 void addIRPasses() override;
373 bool addInstSelector() override;
374 bool addIRTranslator() override;
375 bool addLegalizeMachineIR() override;
376 void addPreRegBankSelect() override;
377 bool addRegBankSelect() override;
378 bool addGlobalInstructionSelect() override;
379 void addPreLegalizeMachineIR() override;
380 bool addILPOpts() override;
381 bool addPreISel() override;
382 void addMachineSSAOptimization() override;
383 void addPreRegAlloc() override;
384 bool addPostFastRegAllocRewrite() override;
385 void addPostRegAlloc() override;
386 void addPreEmitPass() override;
387 void addPreEmitPass2() override;
388 void addPreSched2() override;
389 bool addRegAssignAndRewriteOptimized() override;
390
391 std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
392};
393
394class X86ExecutionDomainFix : public ExecutionDomainFix {
395public:
396 static char ID;
397 X86ExecutionDomainFix() : ExecutionDomainFix(ID, X86::VR128XRegClass) {}
398 StringRef getPassName() const override {
399 return "X86 Execution Dependency Fix";
400 }
401};
402char X86ExecutionDomainFix::ID;
403
404} // end anonymous namespace
405
406INITIALIZE_PASS_BEGIN(X86ExecutionDomainFix, "x86-execution-domain-fix",
407 "X86 Execution Domain Fix", false, false)
408INITIALIZE_PASS_DEPENDENCY(ReachingDefInfoWrapperPass)
409INITIALIZE_PASS_END(X86ExecutionDomainFix, "x86-execution-domain-fix",
410 "X86 Execution Domain Fix", false, false)
411
412TargetPassConfig *X86TargetMachine::createPassConfig(PassManagerBase &PM) {
413 return new X86PassConfig(*this, PM);
414}
415
416MachineFunctionInfo *X86TargetMachine::createMachineFunctionInfo(
417 BumpPtrAllocator &Allocator, const Function &F,
418 const TargetSubtargetInfo *STI) const {
419 return X86MachineFunctionInfo::create<X86MachineFunctionInfo>(Allocator, F,
420 STI);
421}
422
423void X86PassConfig::addIRPasses() {
424 addPass(P: createAtomicExpandLegacyPass());
425
426 // We add both pass anyway and when these two passes run, we skip the pass
427 // based on the option level and option attribute.
428 addPass(P: createX86LowerAMXIntrinsicsLegacyPass());
429 addPass(P: createX86LowerAMXTypeLegacyPass());
430
431 TargetPassConfig::addIRPasses();
432
433 if (TM->getOptLevel() != CodeGenOptLevel::None) {
434 addPass(P: createInterleavedAccessPass());
435 addPass(P: createX86PartialReductionLegacyPass());
436 }
437
438 // Add passes that handle indirect branch removal and insertion of a retpoline
439 // thunk. These will be a no-op unless a function subtarget has the retpoline
440 // feature enabled.
441 addPass(P: createIndirectBrExpandPass());
442
443 // Add Control Flow Guard checks.
444 const Triple &TT = TM->getTargetTriple();
445 if (TT.isOSWindows()) {
446 if (TT.isX86_64()) {
447 addPass(P: createCFGuardDispatchPass());
448 } else {
449 addPass(P: createCFGuardCheckPass());
450 }
451 }
452
453 if (TM->Options.JMCInstrument)
454 addPass(P: createJMCInstrumenterPass());
455}
456
457bool X86PassConfig::addInstSelector() {
458 // Install an instruction selector.
459 addPass(P: createX86ISelDag(TM&: getX86TargetMachine(), OptLevel: getOptLevel()));
460
461 // For ELF, cleanup any local-dynamic TLS accesses.
462 if (TM->getTargetTriple().isOSBinFormatELF() &&
463 getOptLevel() != CodeGenOptLevel::None)
464 addPass(P: createCleanupLocalDynamicTLSLegacyPass());
465
466 addPass(P: createX86GlobalBaseRegLegacyPass());
467 addPass(P: createX86ArgumentStackSlotLegacyPass());
468 return false;
469}
470
471bool X86PassConfig::addIRTranslator() {
472 addPass(P: new IRTranslator(getOptLevel()));
473 return false;
474}
475
476void X86PassConfig::addPreRegBankSelect() {
477 bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;
478 if (!IsOptNone) {
479 addPass(P: createX86PostLegalizerCombiner());
480 }
481}
482bool X86PassConfig::addLegalizeMachineIR() {
483 addPass(P: new Legalizer());
484 return false;
485}
486
487bool X86PassConfig::addRegBankSelect() {
488 addPass(P: new RegBankSelect());
489 return false;
490}
491
492bool X86PassConfig::addGlobalInstructionSelect() {
493 addPass(P: new InstructionSelect(getOptLevel()));
494 // Add GlobalBaseReg in case there is no SelectionDAG passes afterwards
495 if (isGlobalISelAbortEnabled())
496 addPass(P: createX86GlobalBaseRegLegacyPass());
497 return false;
498}
499
500void X86PassConfig::addPreLegalizeMachineIR() {
501 if (getOptLevel() != CodeGenOptLevel::None) {
502 addPass(P: createX86PreLegalizerCombiner());
503 }
504}
505
506bool X86PassConfig::addILPOpts() {
507 addPass(PassID: &EarlyIfConverterLegacyID);
508 if (X86EnableMachineCombinerPass)
509 addPass(PassID: &MachineCombinerID);
510 addPass(P: createX86CmovConversionLegacyPass());
511 return true;
512}
513
514bool X86PassConfig::addPreISel() {
515 // Only add this pass for 32-bit x86 Windows.
516 const Triple &TT = TM->getTargetTriple();
517 if (TT.isOSWindows() && TT.isX86_32())
518 addPass(P: createX86WinEHStateLegacyPass());
519 return true;
520}
521
522void X86PassConfig::addPreRegAlloc() {
523 if (getOptLevel() != CodeGenOptLevel::None) {
524 addPass(PassID: &LiveRangeShrinkID);
525 addPass(P: createX86FixupSetCCLegacyPass());
526 addPass(P: createX86OptimizeLEAsLegacyPass());
527 addPass(P: createX86CallFrameOptimizationLegacyPass());
528 addPass(P: createX86AvoidStoreForwardingBlocksLegacyPass());
529 }
530
531 addPass(P: createX86SuppressAPXForRelocationLegacyPass());
532
533 addPass(P: createX86SpeculativeLoadHardeningLegacyPass());
534 addPass(P: createX86FlagsCopyLoweringLegacyPass());
535 addPass(P: createX86DynAllocaExpanderLegacyPass());
536
537 if (getOptLevel() != CodeGenOptLevel::None)
538 addPass(P: createX86PreTileConfigLegacyPass());
539 else
540 addPass(P: createX86FastPreTileConfigLegacyPass());
541}
542
543void X86PassConfig::addMachineSSAOptimization() {
544 addPass(P: createX86DomainReassignmentLegacyPass());
545 TargetPassConfig::addMachineSSAOptimization();
546}
547
548void X86PassConfig::addPostRegAlloc() {
549 addPass(P: createX86LowerTileCopyLegacyPass());
550 addPass(P: createX86FPStackifierLegacyPass());
551 // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
552 // to using the Speculative Execution Side Effect Suppression pass for
553 // mitigation. This is to prevent slow downs due to
554 // analyses needed by the LVIHardening pass when compiling at -O0.
555 if (getOptLevel() != CodeGenOptLevel::None)
556 addPass(P: createX86LoadValueInjectionLoadHardeningLegacyPass());
557}
558
559void X86PassConfig::addPreSched2() {
560 addPass(P: createX86ExpandPseudoLegacyPass());
561 addPass(P: createKCFIPass());
562}
563
564void X86PassConfig::addPreEmitPass() {
565 if (getOptLevel() != CodeGenOptLevel::None) {
566 addPass(P: new X86ExecutionDomainFix());
567 addPass(P: createBreakFalseDeps());
568 }
569
570 addPass(P: createX86IndirectBranchTrackingLegacyPass());
571
572 addPass(P: createX86IssueVZeroUpperPass());
573
574 if (getOptLevel() != CodeGenOptLevel::None) {
575 addPass(P: createX86FixupBWInstsLegacyPass());
576 addPass(P: createX86PadShortFunctions());
577 addPass(P: createX86FixupLEAsLegacyPass());
578 addPass(P: createX86FixupInstTuningLegacyPass());
579 addPass(P: createX86FixupVectorConstantsLegacyPass());
580 }
581 addPass(P: createX86CompressEVEXLegacyPass());
582 addPass(P: createX86InsertX87WaitLegacyPass());
583}
584
585void X86PassConfig::addPreEmitPass2() {
586 const Triple &TT = TM->getTargetTriple();
587 const MCAsmInfo *MAI = TM->getMCAsmInfo();
588
589 // The X86 Speculative Execution Pass must run after all control
590 // flow graph modifying passes. As a result it was listed to run right before
591 // the X86 Retpoline Thunks pass. The reason it must run after control flow
592 // graph modifications is that the model of LFENCE in LLVM has to be updated
593 // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the
594 // placement of this pass was hand checked to ensure that the subsequent
595 // passes don't move the code around the LFENCEs in a way that will hurt the
596 // correctness of this pass. This placement has been shown to work based on
597 // hand inspection of the codegen output.
598 addPass(P: createX86SpeculativeExecutionSideEffectSuppressionLegacyPass());
599 addPass(P: createX86IndirectThunksPass());
600 addPass(P: createX86ReturnThunksLegacyPass());
601
602 // Insert extra int3 instructions after trailing call instructions to avoid
603 // issues in the unwinder.
604 if (TT.isOSWindows() && TT.isX86_64())
605 addPass(P: createX86AvoidTrailingCallLegacyPass());
606
607 // Verify basic block incoming and outgoing cfa offset and register values and
608 // correct CFA calculation rule where needed by inserting appropriate CFI
609 // instructions.
610 if (!TT.isOSDarwin() &&
611 (!TT.isOSWindows() ||
612 MAI->getExceptionHandlingType() == ExceptionHandling::DwarfCFI))
613 addPass(P: createCFIInstrInserter());
614
615 if (TT.isOSWindows()) {
616 // Identify valid longjmp targets for Windows Control Flow Guard.
617 addPass(P: createCFGuardLongjmpPass());
618 // Identify valid eh continuation targets for Windows EHCont Guard.
619 addPass(P: createEHContGuardTargetsPass());
620 }
621 addPass(P: createX86LoadValueInjectionRetHardeningLegacyPass());
622
623 // Insert pseudo probe annotation for callsite profiling
624 addPass(P: createPseudoProbeInserter());
625
626 // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms,
627 // also CALL_RVMARKER.
628 addPass(P: createUnpackMachineBundlesLegacy(Ftor: [&TT](const MachineFunction &MF) {
629 // Only run bundle expansion if the module uses kcfi, or there are relevant
630 // ObjC runtime functions present in the module.
631 const Function &F = MF.getFunction();
632 const Module *M = F.getParent();
633 return M->getModuleFlag(Key: "kcfi") ||
634 (TT.isOSDarwin() &&
635 (M->getFunction(Name: "objc_retainAutoreleasedReturnValue") ||
636 M->getFunction(Name: "objc_unsafeClaimAutoreleasedReturnValue")));
637 }));
638
639 // Analyzes and emits pseudos to support Win x64 Unwind V2. This pass must run
640 // after all real instructions have been added to the epilog.
641 if (TT.isOSWindows() && TT.isX86_64())
642 addPass(P: createX86WinEHUnwindV2LegacyPass());
643}
644
645bool X86PassConfig::addPostFastRegAllocRewrite() {
646 addPass(P: createX86FastTileConfigLegacyPass());
647 return true;
648}
649
650std::unique_ptr<CSEConfigBase> X86PassConfig::getCSEConfig() const {
651 return getStandardCSEConfigForOpt(Level: TM->getOptLevel());
652}
653
654static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI,
655 const MachineRegisterInfo &MRI,
656 const Register Reg) {
657 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
658 return static_cast<const X86RegisterInfo &>(TRI).isTileRegisterClass(RC);
659}
660
661bool X86PassConfig::addRegAssignAndRewriteOptimized() {
662 // Don't support tile RA when RA is specified by command line "-regalloc".
663 if (!isCustomizedRegAlloc() && EnableTileRAPass) {
664 // Allocate tile register first.
665 addPass(P: createGreedyRegisterAllocator(F: onlyAllocateTileRegisters));
666 addPass(P: createX86TileConfigLegacyPass());
667 }
668 return TargetPassConfig::addRegAssignAndRewriteOptimized();
669}
670