1//===-- X86TargetMachine.cpp - Define TargetMachine for the X86 -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the X86 specific subclass of TargetMachine.
10//
11//===----------------------------------------------------------------------===//
12
13#include "X86TargetMachine.h"
14#include "MCTargetDesc/X86MCTargetDesc.h"
15#include "TargetInfo/X86TargetInfo.h"
16#include "X86.h"
17#include "X86MachineFunctionInfo.h"
18#include "X86MacroFusion.h"
19#include "X86Subtarget.h"
20#include "X86TargetObjectFile.h"
21#include "X86TargetTransformInfo.h"
22#include "llvm-c/Visibility.h"
23#include "llvm/ADT/SmallString.h"
24#include "llvm/ADT/StringRef.h"
25#include "llvm/Analysis/TargetTransformInfo.h"
26#include "llvm/CodeGen/ExecutionDomainFix.h"
27#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
28#include "llvm/CodeGen/GlobalISel/CallLowering.h"
29#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
30#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
31#include "llvm/CodeGen/GlobalISel/Legalizer.h"
32#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
33#include "llvm/CodeGen/MIRParser/MIParser.h"
34#include "llvm/CodeGen/MIRYamlMapping.h"
35#include "llvm/CodeGen/MachineScheduler.h"
36#include "llvm/CodeGen/Passes.h"
37#include "llvm/CodeGen/TargetPassConfig.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/Function.h"
41#include "llvm/MC/MCAsmInfo.h"
42#include "llvm/MC/TargetRegistry.h"
43#include "llvm/Option/LibraryOptions.h"
44#include "llvm/Pass.h"
45#include "llvm/Support/CodeGen.h"
46#include "llvm/Support/ErrorHandling.h"
47#include "llvm/Target/TargetLoweringObjectFile.h"
48#include "llvm/Target/TargetOptions.h"
49#include "llvm/TargetParser/Triple.h"
50#include "llvm/Transforms/CFGuard.h"
51#include <memory>
52#include <optional>
53
54using namespace llvm;
55
56extern "C" LLVM_C_ABI void LLVMInitializeX86Target() {
57 // Register the target.
58 RegisterTargetMachine<X86TargetMachine> X(getTheX86_32Target());
59 RegisterTargetMachine<X86TargetMachine> Y(getTheX86_64Target());
60 static opt::RegisterLibraryOptions<X86Options> O;
61
62 PassRegistry &PR = *PassRegistry::getPassRegistry();
63 initializeX86LowerAMXIntrinsicsLegacyPassPass(PR);
64 initializeX86LowerAMXTypeLegacyPassPass(PR);
65 initializeX86PreTileConfigLegacyPass(PR);
66 initializeGlobalISel(PR);
67 initializeWinEHStateLegacyPass(PR);
68 initializeX86FixupBWInstLegacyPass(PR);
69 initializeCompressEVEXLegacyPass(PR);
70 initializeFixupLEAsLegacyPass(PR);
71 initializeX86FPStackifierLegacyPass(PR);
72 initializeX86FixupSetCCLegacyPass(PR);
73 initializeX86CallFrameOptimizationLegacyPass(PR);
74 initializeX86CmovConversionLegacyPass(PR);
75 initializeX86TileConfigLegacyPass(PR);
76 initializeX86FastPreTileConfigLegacyPass(PR);
77 initializeX86FastTileConfigLegacyPass(PR);
78 initializeMachineKCFILegacyPass(PR);
79 initializeX86LowerTileCopyLegacyPass(PR);
80 initializeX86ExpandPseudoLegacyPass(PR);
81 initializeX86ExecutionDomainFixPass(PR);
82 initializeX86DomainReassignmentLegacyPass(PR);
83 initializeX86AvoidSFBLegacyPass(PR);
84 initializeX86AvoidTrailingCallLegacyPassPass(PR);
85 initializeX86SpeculativeLoadHardeningLegacyPass(PR);
86 initializeX86SpeculativeExecutionSideEffectSuppressionLegacyPass(PR);
87 initializeX86FlagsCopyLoweringLegacyPass(PR);
88 initializeX86LoadValueInjectionLoadHardeningLegacyPass(PR);
89 initializeX86LoadValueInjectionRetHardeningLegacyPass(PR);
90 initializeX86OptimizeLEAsLegacyPass(PR);
91 initializeX86PartialReductionLegacyPass(PR);
92 initializeX86ReturnThunksLegacyPass(PR);
93 initializeX86DAGToDAGISelLegacyPass(PR);
94 initializeX86ArgumentStackSlotLegacyPass(PR);
95 initializeX86AsmPrinterPass(PR);
96 initializeX86FixupInstTuningLegacyPass(PR);
97 initializeX86FixupVectorConstantsLegacyPass(PR);
98 initializeX86DynAllocaExpanderLegacyPass(PR);
99 initializeX86SuppressAPXForRelocationLegacyPass(PR);
100 initializeX86WinEHUnwindV2LegacyPass(PR);
101 initializeX86PreLegalizerCombinerLegacyPass(PR);
102 initializeX86PostLegalizerCombinerLegacyPass(PR);
103 initializeX86WinEHUnwindV3Pass(PR);
104}
105
106static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
107 if (TT.isOSBinFormatMachO()) {
108 if (TT.isX86_64())
109 return std::make_unique<X86_64MachoTargetObjectFile>();
110 return std::make_unique<TargetLoweringObjectFileMachO>();
111 }
112
113 if (TT.isOSBinFormatCOFF())
114 return std::make_unique<TargetLoweringObjectFileCOFF>();
115
116 if (TT.isX86_64())
117 return std::make_unique<X86_64ELFTargetObjectFile>();
118 return std::make_unique<X86ELFTargetObjectFile>();
119}
120
121static Reloc::Model getEffectiveRelocModel(const Triple &TT, bool JIT,
122 std::optional<Reloc::Model> RM) {
123 bool is64Bit = TT.isX86_64();
124 if (!RM) {
125 // JIT codegen should use static relocations by default, since it's
126 // typically executed in process and not relocatable.
127 if (JIT)
128 return Reloc::Static;
129
130 // Darwin defaults to PIC in 64 bit mode and dynamic-no-pic in 32 bit mode.
131 // Win64 requires rip-rel addressing, thus we force it to PIC. Otherwise we
132 // use static relocation model by default.
133 if (TT.isOSDarwin()) {
134 if (is64Bit)
135 return Reloc::PIC_;
136 return Reloc::DynamicNoPIC;
137 }
138 if (TT.isOSWindows() && is64Bit)
139 return Reloc::PIC_;
140 return Reloc::Static;
141 }
142
143 // ELF and X86-64 don't have a distinct DynamicNoPIC model. DynamicNoPIC
144 // is defined as a model for code which may be used in static or dynamic
145 // executables but not necessarily a shared library. On X86-32 we just
146 // compile in -static mode, in x86-64 we use PIC.
147 if (*RM == Reloc::DynamicNoPIC) {
148 if (is64Bit)
149 return Reloc::PIC_;
150 if (!TT.isOSDarwin())
151 return Reloc::Static;
152 }
153
154 // If we are on Darwin, disallow static relocation model in X86-64 mode, since
155 // the Mach-O file format doesn't support it.
156 if (*RM == Reloc::Static && TT.isOSDarwin() && is64Bit)
157 return Reloc::PIC_;
158
159 return *RM;
160}
161
162static CodeModel::Model
163getEffectiveX86CodeModel(const Triple &TT, std::optional<CodeModel::Model> CM,
164 bool JIT) {
165 bool Is64Bit = TT.isX86_64();
166 if (CM) {
167 if (*CM == CodeModel::Tiny)
168 reportFatalUsageError(reason: "target does not support the tiny CodeModel");
169 return *CM;
170 }
171 if (JIT)
172 return Is64Bit ? CodeModel::Large : CodeModel::Small;
173 return CodeModel::Small;
174}
175
176/// Create an X86 target.
177///
178X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT,
179 StringRef CPU, StringRef FS,
180 const TargetOptions &Options,
181 std::optional<Reloc::Model> RM,
182 std::optional<CodeModel::Model> CM,
183 CodeGenOptLevel OL, bool JIT)
184 : CodeGenTargetMachineImpl(T, TT, CPU, FS, Options,
185 getEffectiveRelocModel(TT, JIT, RM),
186 getEffectiveX86CodeModel(TT, CM, JIT), OL),
187 CLOpts(cl::LibraryOptionsContext::get<X86Options>(
188 C: Options.MCOptions.LibraryOptions)),
189 TLOF(createTLOF(TT: getTargetTriple())), IsJIT(JIT) {
190 // On PS4/PS5, the "return address" of a 'noreturn' call must still be within
191 // the calling function. Note that this also includes __stack_chk_fail,
192 // so there was some target-specific logic in the instruction selectors
193 // to handle that. That code has since been generalized, so the only thing
194 // needed is to set TrapUnreachable here.
195 if (TT.isPS() || TT.isOSBinFormatMachO()) {
196 this->Options.TrapUnreachable = true;
197 this->Options.NoTrapAfterNoreturn = TT.isOSBinFormatMachO();
198 }
199
200 setMachineOutliner(true);
201
202 // x86 supports the debug entry values.
203 setSupportsDebugEntryValues(true);
204
205 initAsmInfo();
206}
207
208X86TargetMachine::~X86TargetMachine() = default;
209
210const X86Subtarget *
211X86TargetMachine::getSubtargetImpl(const Function &F) const {
212 Attribute CPUAttr = F.getFnAttribute(Kind: "target-cpu");
213 Attribute TuneAttr = F.getFnAttribute(Kind: "tune-cpu");
214 Attribute FSAttr = F.getFnAttribute(Kind: "target-features");
215
216 StringRef CPU =
217 CPUAttr.isValid() ? CPUAttr.getValueAsString() : (StringRef)TargetCPU;
218 // "x86-64" is a default target setting for many front ends. In these cases,
219 // they actually request for "generic" tuning unless the "tune-cpu" was
220 // specified.
221 StringRef TuneCPU = TuneAttr.isValid() ? TuneAttr.getValueAsString()
222 : CPU == "x86-64" ? "generic"
223 : (StringRef)CPU;
224 StringRef FS =
225 FSAttr.isValid() ? FSAttr.getValueAsString() : (StringRef)TargetFS;
226
227 SmallString<512> Key;
228 // The additions here are ordered so that the definitely short strings are
229 // added first so we won't exceed the small size. We append the
230 // much longer FS string at the end so that we only heap allocate at most
231 // one time.
232
233 // Extract prefer-vector-width attribute.
234 unsigned PreferVectorWidthOverride = 0;
235 Attribute PreferVecWidthAttr = F.getFnAttribute(Kind: "prefer-vector-width");
236 if (PreferVecWidthAttr.isValid()) {
237 StringRef Val = PreferVecWidthAttr.getValueAsString();
238 unsigned Width;
239 if (!Val.getAsInteger(Radix: 0, Result&: Width)) {
240 Key += 'p';
241 Key += Val;
242 PreferVectorWidthOverride = Width;
243 }
244 }
245
246 // Extract min-legal-vector-width attribute.
247 unsigned RequiredVectorWidth = UINT32_MAX;
248 Attribute MinLegalVecWidthAttr = F.getFnAttribute(Kind: "min-legal-vector-width");
249 if (MinLegalVecWidthAttr.isValid()) {
250 StringRef Val = MinLegalVecWidthAttr.getValueAsString();
251 unsigned Width;
252 if (!Val.getAsInteger(Radix: 0, Result&: Width)) {
253 Key += 'm';
254 Key += Val;
255 RequiredVectorWidth = Width;
256 }
257 }
258
259 // Add CPU to the Key.
260 Key += CPU;
261
262 // Add tune CPU to the Key.
263 Key += TuneCPU;
264
265 // Keep track of the start of the feature portion of the string.
266 unsigned FSStart = Key.size();
267
268 // FIXME: This is related to the code below to reset the target options,
269 // we need to know whether or not the soft float flag is set on the
270 // function before we can generate a subtarget. We also need to use
271 // it as a key for the subtarget since that can be the only difference
272 // between two functions.
273 bool SoftFloat = F.getFnAttribute(Kind: "use-soft-float").getValueAsBool();
274 // If the soft float attribute is set on the function turn on the soft float
275 // subtarget feature.
276 if (SoftFloat)
277 Key += FS.empty() ? "+soft-float" : "+soft-float,";
278
279 Key += FS;
280
281 // We may have added +soft-float to the features so move the StringRef to
282 // point to the full string in the Key.
283 FS = Key.substr(Start: FSStart);
284
285 auto &I = SubtargetMap[Key];
286 if (!I) {
287 I = std::make_unique<X86Subtarget>(
288 args: TargetTriple, args&: CPU, args&: TuneCPU, args&: FS, args: *this,
289 args: MaybeAlign(F.getParent()->getOverrideStackAlignment()),
290 args&: PreferVectorWidthOverride, args&: RequiredVectorWidth);
291 }
292 return I.get();
293}
294
295yaml::MachineFunctionInfo *X86TargetMachine::createDefaultFuncInfoYAML() const {
296 return new yaml::X86MachineFunctionInfo();
297}
298
299yaml::MachineFunctionInfo *
300X86TargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
301 const auto *MFI = MF.getInfo<X86MachineFunctionInfo>();
302 return new yaml::X86MachineFunctionInfo(*MFI);
303}
304
305bool X86TargetMachine::parseMachineFunctionInfo(
306 const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
307 SMDiagnostic &Error, SMRange &SourceRange) const {
308 const auto &YamlMFI = static_cast<const yaml::X86MachineFunctionInfo &>(MFI);
309 PFS.MF.getInfo<X86MachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
310 return false;
311}
312
313bool X86TargetMachine::isNoopAddrSpaceCast(const DataLayout &DL, unsigned SrcAS,
314 unsigned DestAS) const {
315 assert(SrcAS != DestAS && "Expected different address spaces!");
316 if (DL.getPointerSize(AS: SrcAS) != DL.getPointerSize(AS: DestAS))
317 return false;
318 return SrcAS < 256 && DestAS < 256;
319}
320
321void X86TargetMachine::reset() { SubtargetMap.clear(); }
322
323ScheduleDAGInstrs *
324X86TargetMachine::createMachineScheduler(MachineSchedContext *C) const {
325 ScheduleDAGMILive *DAG = createSchedLive(C);
326 DAG->addMutation(Mutation: createX86MacroFusionDAGMutation());
327 return DAG;
328}
329
330ScheduleDAGInstrs *
331X86TargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
332 ScheduleDAGMI *DAG = createSchedPostRA(C);
333 DAG->addMutation(Mutation: createX86MacroFusionDAGMutation());
334 return DAG;
335}
336
337//===----------------------------------------------------------------------===//
338// X86 TTI query.
339//===----------------------------------------------------------------------===//
340
341TargetTransformInfo
342X86TargetMachine::getTargetTransformInfo(const Function &F) const {
343 return TargetTransformInfo(std::make_unique<X86TTIImpl>(args: this, args: F));
344}
345
346//===----------------------------------------------------------------------===//
347// Pass Pipeline Configuration
348//===----------------------------------------------------------------------===//
349
350namespace {
351
352/// X86 Code Generator Pass Configuration Options.
353class X86PassConfig : public TargetPassConfig {
354public:
355 X86PassConfig(X86TargetMachine &TM, PassManagerBase &PM)
356 : TargetPassConfig(TM, PM) {}
357
358 X86TargetMachine &getX86TargetMachine() const {
359 return getTM<X86TargetMachine>();
360 }
361
362 void addIRPasses() override;
363 bool addInstSelector() override;
364 bool addIRTranslator() override;
365 bool addLegalizeMachineIR() override;
366 void addPreRegBankSelect() override;
367 bool addRegBankSelect() override;
368 bool addGlobalInstructionSelect() override;
369 void addPreLegalizeMachineIR() override;
370 bool addILPOpts() override;
371 bool addPreISel() override;
372 void addMachineSSAOptimization() override;
373 void addPreRegAlloc() override;
374 bool addPostFastRegAllocRewrite() override;
375 void addPostRegAlloc() override;
376 void addPreEmitPass() override;
377 void addPreEmitPass2() override;
378 void addPreSched2() override;
379 bool addRegAssignAndRewriteOptimized() override;
380
381 std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
382};
383
384class X86ExecutionDomainFix : public ExecutionDomainFix {
385public:
386 static char ID;
387 X86ExecutionDomainFix() : ExecutionDomainFix(ID, X86::VR128XRegClass) {}
388 StringRef getPassName() const override {
389 return "X86 Execution Dependency Fix";
390 }
391};
392char X86ExecutionDomainFix::ID;
393
394} // end anonymous namespace
395
396INITIALIZE_PASS_BEGIN(X86ExecutionDomainFix, "x86-execution-domain-fix",
397 "X86 Execution Domain Fix", false, false)
398INITIALIZE_PASS_DEPENDENCY(ReachingDefInfoWrapperPass)
399INITIALIZE_PASS_END(X86ExecutionDomainFix, "x86-execution-domain-fix",
400 "X86 Execution Domain Fix", false, false)
401
402TargetPassConfig *X86TargetMachine::createPassConfig(PassManagerBase &PM) {
403 return new X86PassConfig(*this, PM);
404}
405
406MachineFunctionInfo *X86TargetMachine::createMachineFunctionInfo(
407 BumpPtrAllocator &Allocator, const Function &F,
408 const TargetSubtargetInfo *STI) const {
409 return X86MachineFunctionInfo::create<X86MachineFunctionInfo>(Allocator, F,
410 STI);
411}
412
413void X86PassConfig::addIRPasses() {
414 addPass(P: createAtomicExpandLegacyPass());
415
416 // We add both pass anyway and when these two passes run, we skip the pass
417 // based on the option level and option attribute.
418 addPass(P: createX86LowerAMXIntrinsicsLegacyPass());
419 addPass(P: createX86LowerAMXTypeLegacyPass());
420
421 TargetPassConfig::addIRPasses();
422
423 if (TM->getOptLevel() != CodeGenOptLevel::None) {
424 addPass(P: createInterleavedAccessPass());
425 addPass(P: createX86PartialReductionLegacyPass());
426 }
427
428 // Add passes that handle indirect branch removal and insertion of a retpoline
429 // thunk. These will be a no-op unless a function subtarget has the retpoline
430 // feature enabled.
431 addPass(P: createIndirectBrExpandPass(OptLevel: getOptLevel()));
432
433 // Add Control Flow Guard checks.
434 const Triple &TT = TM->getTargetTriple();
435 if (TT.isOSWindows()) {
436 addPass(P: createCFGuardPass());
437 }
438
439 if (TM->Options.JMCInstrument)
440 addPass(P: createJMCInstrumenterPass());
441}
442
443bool X86PassConfig::addInstSelector() {
444 // Install an instruction selector.
445 addPass(P: createX86ISelDag(TM&: getX86TargetMachine(), OptLevel: getOptLevel()));
446
447 // For ELF, cleanup any local-dynamic TLS accesses.
448 if (TM->getTargetTriple().isOSBinFormatELF() &&
449 getOptLevel() != CodeGenOptLevel::None)
450 addPass(P: createCleanupLocalDynamicTLSLegacyPass());
451
452 addPass(P: createX86GlobalBaseRegLegacyPass());
453 addPass(P: createX86ArgumentStackSlotLegacyPass());
454 return false;
455}
456
457bool X86PassConfig::addIRTranslator() {
458 addPass(P: new IRTranslatorLegacy(getOptLevel()));
459 return false;
460}
461
462void X86PassConfig::addPreRegBankSelect() {
463 bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;
464 if (!IsOptNone) {
465 addPass(P: createX86PostLegalizerCombinerLegacy());
466 }
467}
468bool X86PassConfig::addLegalizeMachineIR() {
469 addPass(P: new LegalizerLegacy());
470 return false;
471}
472
473bool X86PassConfig::addRegBankSelect() {
474 addPass(P: new RegBankSelectLegacy());
475 return false;
476}
477
478bool X86PassConfig::addGlobalInstructionSelect() {
479 addPass(P: new InstructionSelectLegacy(getOptLevel()));
480 // Add GlobalBaseReg in case there is no SelectionDAG passes afterwards
481 if (isGlobalISelAbortEnabled())
482 addPass(P: createX86GlobalBaseRegLegacyPass());
483 return false;
484}
485
486void X86PassConfig::addPreLegalizeMachineIR() {
487 if (getOptLevel() != CodeGenOptLevel::None) {
488 addPass(P: createX86PreLegalizerCombinerLegacy());
489 }
490}
491
492bool X86PassConfig::addILPOpts() {
493 addPass(PassID: &EarlyIfConverterLegacyID);
494 if (getX86TargetMachine().getCLOpts().machine_combiner)
495 addPass(PassID: &MachineCombinerID);
496 addPass(P: createX86CmovConversionLegacyPass());
497 return true;
498}
499
500bool X86PassConfig::addPreISel() {
501 // Only add this pass for 32-bit x86 Windows.
502 const Triple &TT = TM->getTargetTriple();
503 if (TT.isOSWindows() && TT.isX86_32())
504 addPass(P: createX86WinEHStateLegacyPass());
505 return true;
506}
507
508void X86PassConfig::addPreRegAlloc() {
509 if (getOptLevel() != CodeGenOptLevel::None) {
510 addPass(PassID: &LiveRangeShrinkID);
511 addPass(P: createX86FixupSetCCLegacyPass());
512 addPass(P: createX86OptimizeLEAsLegacyPass());
513 addPass(P: createX86CallFrameOptimizationLegacyPass());
514 addPass(P: createX86AvoidStoreForwardingBlocksLegacyPass());
515 }
516
517 addPass(P: createX86SuppressAPXForRelocationLegacyPass());
518
519 addPass(P: createX86SpeculativeLoadHardeningLegacyPass());
520 addPass(P: createX86FlagsCopyLoweringLegacyPass());
521 addPass(P: createX86DynAllocaExpanderLegacyPass());
522
523 if (getOptLevel() != CodeGenOptLevel::None)
524 addPass(P: createX86PreTileConfigLegacyPass());
525 else
526 addPass(P: createX86FastPreTileConfigLegacyPass());
527}
528
529void X86PassConfig::addMachineSSAOptimization() {
530 addPass(P: createX86DomainReassignmentLegacyPass());
531 TargetPassConfig::addMachineSSAOptimization();
532}
533
534void X86PassConfig::addPostRegAlloc() {
535 addPass(P: createX86LowerTileCopyLegacyPass());
536 addPass(P: createX86FPStackifierLegacyPass());
537 // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
538 // to using the Speculative Execution Side Effect Suppression pass for
539 // mitigation. This is to prevent slow downs due to
540 // analyses needed by the LVIHardening pass when compiling at -O0.
541 if (getOptLevel() != CodeGenOptLevel::None)
542 addPass(P: createX86LoadValueInjectionLoadHardeningLegacyPass());
543}
544
545void X86PassConfig::addPreSched2() {
546 addPass(P: createX86ExpandPseudoLegacyPass());
547 addPass(P: createKCFIPass());
548}
549
550void X86PassConfig::addPreEmitPass() {
551 if (getOptLevel() != CodeGenOptLevel::None) {
552 addPass(P: new X86ExecutionDomainFix());
553 addPass(P: createBreakFalseDepsLegacyPass());
554 }
555
556 addPass(P: createX86IndirectBranchTrackingLegacyPass());
557
558 addPass(P: createX86InsertVZeroUpperLegacyPass());
559
560 if (getOptLevel() != CodeGenOptLevel::None) {
561 addPass(P: createX86FixupBWInstsLegacyPass());
562 addPass(P: createX86PadShortFunctions());
563 addPass(P: createX86FixupLEAsLegacyPass());
564 addPass(P: createX86FixupInstTuningLegacyPass());
565 addPass(P: createX86FixupVectorConstantsLegacyPass());
566 }
567 addPass(P: createX86CompressEVEXLegacyPass());
568 addPass(P: createX86InsertX87WaitLegacyPass());
569
570 if (TM->getTargetTriple().isLFI())
571 addPass(P: createX86LFIRewritePass());
572}
573
574void X86PassConfig::addPreEmitPass2() {
575 const Triple &TT = TM->getTargetTriple();
576 const MCAsmInfo &MAI = TM->getMCAsmInfo();
577
578 // The X86 Speculative Execution Pass must run after all control
579 // flow graph modifying passes. As a result it was listed to run right before
580 // the X86 Retpoline Thunks pass. The reason it must run after control flow
581 // graph modifications is that the model of LFENCE in LLVM has to be updated
582 // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the
583 // placement of this pass was hand checked to ensure that the subsequent
584 // passes don't move the code around the LFENCEs in a way that will hurt the
585 // correctness of this pass. This placement has been shown to work based on
586 // hand inspection of the codegen output.
587 addPass(P: createX86SpeculativeExecutionSideEffectSuppressionLegacyPass());
588 addPass(P: createX86IndirectThunksPass());
589 addPass(P: createX86ReturnThunksLegacyPass());
590
591 // Insert extra int3 instructions after trailing call instructions to avoid
592 // issues in the unwinder.
593 if (TT.isOSWindows() && TT.isX86_64())
594 addPass(P: createX86AvoidTrailingCallLegacyPass());
595
596 // Verify basic block incoming and outgoing cfa offset and register values and
597 // correct CFA calculation rule where needed by inserting appropriate CFI
598 // instructions.
599 if (!TT.isOSDarwin() &&
600 (!TT.isOSWindows() ||
601 MAI.getExceptionHandlingType() == ExceptionHandling::DwarfCFI))
602 addPass(P: createCFIInstrInserterLegacy());
603
604 if (TT.isOSWindows()) {
605 // Identify valid longjmp targets for Windows Control Flow Guard.
606 addPass(P: createCFGuardLongjmpPass());
607 // Identify valid eh continuation targets for Windows EHCont Guard.
608 addPass(P: createEHContGuardTargetsLegacy());
609 }
610 addPass(P: createX86LoadValueInjectionRetHardeningLegacyPass());
611
612 // Insert pseudo probe annotation for callsite profiling
613 addPass(P: createPseudoProbeInserter());
614
615 // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms,
616 // also CALL_RVMARKER.
617 addPass(P: createUnpackMachineBundlesLegacy(Ftor: [&TT](const MachineFunction &MF) {
618 // Only run bundle expansion if the module uses kcfi, or there are relevant
619 // ObjC runtime functions present in the module.
620 const Function &F = MF.getFunction();
621 const Module *M = F.getParent();
622 return M->getModuleFlag(Key: "kcfi") ||
623 (TT.isOSDarwin() &&
624 (M->getFunction(Name: "objc_retainAutoreleasedReturnValue") ||
625 M->getFunction(Name: "objc_unsafeClaimAutoreleasedReturnValue")));
626 }));
627
628 // Analyzes and emits pseudos to support Win x64 Unwind V2. This pass must run
629 // after all real instructions have been added to the epilog.
630 if (TT.isOSWindows() && TT.isX86_64()) {
631 addPass(P: createX86WinEHUnwindV2LegacyPass());
632 addPass(P: createX86WinEHUnwindV3Pass());
633 }
634}
635
636bool X86PassConfig::addPostFastRegAllocRewrite() {
637 addPass(P: createX86FastTileConfigLegacyPass());
638 return true;
639}
640
641std::unique_ptr<CSEConfigBase> X86PassConfig::getCSEConfig() const {
642 return getStandardCSEConfigForOpt(Level: TM->getOptLevel());
643}
644
645static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI,
646 const MachineRegisterInfo &MRI,
647 const Register Reg) {
648 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
649 return static_cast<const X86RegisterInfo &>(TRI).isTileRegisterClass(RC);
650}
651
652bool X86PassConfig::addRegAssignAndRewriteOptimized() {
653 // Don't support tile RA when RA is specified by command line "-regalloc".
654 if (!isCustomizedRegAlloc() && getX86TargetMachine().getCLOpts().tile_ra) {
655 // Allocate tile register first.
656 addPass(P: createGreedyRegisterAllocator(F: onlyAllocateTileRegisters));
657 addPass(P: createX86TileConfigLegacyPass());
658 }
659 return TargetPassConfig::addRegAssignAndRewriteOptimized();
660}
661