1//===-- X86TargetMachine.cpp - Define TargetMachine for the X86 -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the X86 specific subclass of TargetMachine.
10//
11//===----------------------------------------------------------------------===//
12
13#include "X86TargetMachine.h"
14#include "MCTargetDesc/X86MCTargetDesc.h"
15#include "TargetInfo/X86TargetInfo.h"
16#include "X86.h"
17#include "X86MachineFunctionInfo.h"
18#include "X86MacroFusion.h"
19#include "X86Subtarget.h"
20#include "X86TargetObjectFile.h"
21#include "X86TargetTransformInfo.h"
22#include "llvm-c/Visibility.h"
23#include "llvm/ADT/SmallString.h"
24#include "llvm/ADT/StringRef.h"
25#include "llvm/Analysis/TargetTransformInfo.h"
26#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
27#include "llvm/CodeGen/GlobalISel/CallLowering.h"
28#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
29#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
30#include "llvm/CodeGen/GlobalISel/Legalizer.h"
31#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
32#include "llvm/CodeGen/MIRParser/MIParser.h"
33#include "llvm/CodeGen/MIRYamlMapping.h"
34#include "llvm/CodeGen/MachineScheduler.h"
35#include "llvm/CodeGen/Passes.h"
36#include "llvm/CodeGen/TargetPassConfig.h"
37#include "llvm/IR/Attributes.h"
38#include "llvm/IR/DataLayout.h"
39#include "llvm/IR/Function.h"
40#include "llvm/MC/MCAsmInfo.h"
41#include "llvm/MC/TargetRegistry.h"
42#include "llvm/Option/LibraryOptions.h"
43#include "llvm/Pass.h"
44#include "llvm/Support/CodeGen.h"
45#include "llvm/Support/ErrorHandling.h"
46#include "llvm/Target/TargetLoweringObjectFile.h"
47#include "llvm/Target/TargetOptions.h"
48#include "llvm/TargetParser/Triple.h"
49#include "llvm/Transforms/CFGuard.h"
50#include <memory>
51#include <optional>
52
53using namespace llvm;
54
55extern "C" LLVM_C_ABI void LLVMInitializeX86Target() {
56 // Register the target.
57 RegisterTargetMachine<X86TargetMachine> X(getTheX86_32Target());
58 RegisterTargetMachine<X86TargetMachine> Y(getTheX86_64Target());
59 static opt::RegisterLibraryOptions<X86Options> O;
60
61 PassRegistry &PR = *PassRegistry::getPassRegistry();
62 initializeX86LowerAMXIntrinsicsLegacyPassPass(PR);
63 initializeX86LowerAMXTypeLegacyPassPass(PR);
64 initializeX86PreTileConfigLegacyPass(PR);
65 initializeGlobalISel(PR);
66 initializeWinEHStateLegacyPass(PR);
67 initializeX86FixupBWInstLegacyPass(PR);
68 initializeCompressEVEXLegacyPass(PR);
69 initializeFixupLEAsLegacyPass(PR);
70 initializeX86FPStackifierLegacyPass(PR);
71 initializeX86FenceNonTemporalStoresLegacyPass(PR);
72 initializeX86FixupSetCCLegacyPass(PR);
73 initializeX86CallFrameOptimizationLegacyPass(PR);
74 initializeX86CmovConversionLegacyPass(PR);
75 initializeX86TileConfigLegacyPass(PR);
76 initializeX86FastPreTileConfigLegacyPass(PR);
77 initializeX86FastTileConfigLegacyPass(PR);
78 initializeMachineKCFILegacyPass(PR);
79 initializeX86LowerTileCopyLegacyPass(PR);
80 initializeX86ExpandPseudoLegacyPass(PR);
81 initializeX86DomainReassignmentLegacyPass(PR);
82 initializeX86AvoidSFBLegacyPass(PR);
83 initializeX86AvoidTrailingCallLegacyPassPass(PR);
84 initializeX86SpeculativeLoadHardeningLegacyPass(PR);
85 initializeX86SpeculativeExecutionSideEffectSuppressionLegacyPass(PR);
86 initializeX86FlagsCopyLoweringLegacyPass(PR);
87 initializeX86LoadValueInjectionLoadHardeningLegacyPass(PR);
88 initializeX86LoadValueInjectionRetHardeningLegacyPass(PR);
89 initializeX86OptimizeLEAsLegacyPass(PR);
90 initializeX86PartialReductionLegacyPass(PR);
91 initializeX86ReturnThunksLegacyPass(PR);
92 initializeX86DAGToDAGISelLegacyPass(PR);
93 initializeX86ArgumentStackSlotLegacyPass(PR);
94 initializeX86AsmPrinterPass(PR);
95 initializeX86FixupInstTuningLegacyPass(PR);
96 initializeX86FixupVectorConstantsLegacyPass(PR);
97 initializeX86DynAllocaExpanderLegacyPass(PR);
98 initializeX86SuppressAPXForRelocationLegacyPass(PR);
99 initializeX86WinEHUnwindV2LegacyPass(PR);
100 initializeX86PreLegalizerCombinerLegacyPass(PR);
101 initializeX86PostLegalizerCombinerLegacyPass(PR);
102 initializeX86WinEHUnwindV3Pass(PR);
103}
104
105static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
106 if (TT.isOSBinFormatMachO()) {
107 if (TT.isX86_64())
108 return std::make_unique<X86_64MachoTargetObjectFile>();
109 return std::make_unique<TargetLoweringObjectFileMachO>();
110 }
111
112 if (TT.isOSBinFormatCOFF())
113 return std::make_unique<TargetLoweringObjectFileCOFF>();
114
115 if (TT.isX86_64())
116 return std::make_unique<X86_64ELFTargetObjectFile>();
117 return std::make_unique<X86ELFTargetObjectFile>();
118}
119
120static Reloc::Model getEffectiveRelocModel(const Triple &TT, bool JIT,
121 std::optional<Reloc::Model> RM) {
122 bool is64Bit = TT.isX86_64();
123 if (!RM) {
124 // JIT codegen should use static relocations by default, since it's
125 // typically executed in process and not relocatable.
126 if (JIT)
127 return Reloc::Static;
128
129 // Darwin defaults to PIC in 64 bit mode and dynamic-no-pic in 32 bit mode.
130 // Win64 requires rip-rel addressing, thus we force it to PIC. Otherwise we
131 // use static relocation model by default.
132 if (TT.isOSDarwin()) {
133 if (is64Bit)
134 return Reloc::PIC_;
135 return Reloc::DynamicNoPIC;
136 }
137 if (TT.isOSWindows() && is64Bit)
138 return Reloc::PIC_;
139 return Reloc::Static;
140 }
141
142 // ELF and X86-64 don't have a distinct DynamicNoPIC model. DynamicNoPIC
143 // is defined as a model for code which may be used in static or dynamic
144 // executables but not necessarily a shared library. On X86-32 we just
145 // compile in -static mode, in x86-64 we use PIC.
146 if (*RM == Reloc::DynamicNoPIC) {
147 if (is64Bit)
148 return Reloc::PIC_;
149 if (!TT.isOSDarwin())
150 return Reloc::Static;
151 }
152
153 // If we are on Darwin, disallow static relocation model in X86-64 mode, since
154 // the Mach-O file format doesn't support it.
155 if (*RM == Reloc::Static && TT.isOSDarwin() && is64Bit)
156 return Reloc::PIC_;
157
158 return *RM;
159}
160
161static CodeModel::Model
162getEffectiveX86CodeModel(const Triple &TT, std::optional<CodeModel::Model> CM,
163 bool JIT) {
164 bool Is64Bit = TT.isX86_64();
165 if (CM) {
166 if (*CM == CodeModel::Tiny)
167 reportFatalUsageError(reason: "target does not support the tiny CodeModel");
168 return *CM;
169 }
170 if (JIT)
171 return Is64Bit ? CodeModel::Large : CodeModel::Small;
172 return CodeModel::Small;
173}
174
175/// Create an X86 target.
176///
177X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT,
178 StringRef CPU, StringRef FS,
179 const TargetOptions &Options,
180 std::optional<Reloc::Model> RM,
181 std::optional<CodeModel::Model> CM,
182 CodeGenOptLevel OL, bool JIT)
183 : CodeGenTargetMachineImpl(T, TT, CPU, FS, Options,
184 getEffectiveRelocModel(TT, JIT, RM),
185 getEffectiveX86CodeModel(TT, CM, JIT), OL),
186 CLOpts(cl::LibraryOptionsContext::get<X86Options>(
187 C: Options.MCOptions.LibraryOptions)),
188 TLOF(createTLOF(TT: getTargetTriple())), IsJIT(JIT) {
189 // On PS4/PS5, the "return address" of a 'noreturn' call must still be within
190 // the calling function. Note that this also includes __stack_chk_fail,
191 // so there was some target-specific logic in the instruction selectors
192 // to handle that. That code has since been generalized, so the only thing
193 // needed is to set TrapUnreachable here.
194 if (TT.isPS() || TT.isOSBinFormatMachO()) {
195 this->Options.TrapUnreachable = true;
196 this->Options.NoTrapAfterNoreturn = TT.isOSBinFormatMachO();
197 }
198
199 setMachineOutliner(true);
200
201 // x86 supports the debug entry values.
202 setSupportsDebugEntryValues(true);
203
204 initAsmInfo();
205}
206
207X86TargetMachine::~X86TargetMachine() = default;
208
209const X86Subtarget *
210X86TargetMachine::getSubtargetImpl(const Function &F) const {
211 AttributeSet FnAttrs = F.getAttributes().getFnAttrs();
212 const Module *M = F.getParent();
213 if (const X86Subtarget *ST = LastSubtarget.lookup(NewFnAttrs: FnAttrs, NewM: M))
214 return ST;
215
216 SubtargetAttrs Attrs;
217 Attrs.CPU = FnAttrs.getAttribute(Kind: "target-cpu");
218 Attrs.TuneCPU = FnAttrs.getAttribute(Kind: "tune-cpu");
219 Attrs.FS = FnAttrs.getAttribute(Kind: "target-features");
220 Attrs.PreferVecWidth = FnAttrs.getAttribute(Kind: "prefer-vector-width");
221 Attrs.MinLegalVecWidth = FnAttrs.getAttribute(Kind: "min-legal-vector-width");
222 Attrs.SoftFloat = FnAttrs.getAttribute(Kind: "use-soft-float");
223
224 if (const X86Subtarget *ST = LastSubtarget.lookup(NewFnAttrs: FnAttrs, NewM: M, NewAttrs: Attrs))
225 return ST;
226
227 StringRef CPU =
228 Attrs.CPU.isValid() ? Attrs.CPU.getValueAsString() : (StringRef)TargetCPU;
229 // "x86-64" is a default target setting for many front ends. In these cases,
230 // they actually request for "generic" tuning unless the "tune-cpu" was
231 // specified.
232 StringRef TuneCPU = Attrs.TuneCPU.isValid() ? Attrs.TuneCPU.getValueAsString()
233 : CPU == "x86-64" ? "generic"
234 : (StringRef)CPU;
235 StringRef FS =
236 Attrs.FS.isValid() ? Attrs.FS.getValueAsString() : (StringRef)TargetFS;
237
238 SmallString<512> Key;
239 // The additions here are ordered so that the definitely short strings are
240 // added first so we won't exceed the small size. We append the
241 // much longer FS string at the end so that we only heap allocate at most
242 // one time.
243
244 // Extract prefer-vector-width attribute.
245 unsigned PreferVectorWidthOverride = 0;
246 if (Attrs.PreferVecWidth.isValid()) {
247 StringRef Val = Attrs.PreferVecWidth.getValueAsString();
248 unsigned Width;
249 if (!Val.getAsInteger(Radix: 0, Result&: Width)) {
250 Key += 'p';
251 Key += Val;
252 PreferVectorWidthOverride = Width;
253 }
254 }
255
256 // Extract min-legal-vector-width attribute.
257 unsigned RequiredVectorWidth = UINT32_MAX;
258 if (Attrs.MinLegalVecWidth.isValid()) {
259 StringRef Val = Attrs.MinLegalVecWidth.getValueAsString();
260 unsigned Width;
261 if (!Val.getAsInteger(Radix: 0, Result&: Width)) {
262 Key += 'm';
263 Key += Val;
264 RequiredVectorWidth = Width;
265 }
266 }
267
268 // Add CPU to the Key.
269 Key += CPU;
270
271 // Add tune CPU to the Key.
272 Key += TuneCPU;
273
274 // Keep track of the start of the feature portion of the string.
275 unsigned FSStart = Key.size();
276
277 // FIXME: This is related to the code below to reset the target options,
278 // we need to know whether or not the soft float flag is set on the
279 // function before we can generate a subtarget. We also need to use
280 // it as a key for the subtarget since that can be the only difference
281 // between two functions.
282 bool SoftFloat = Attrs.SoftFloat.getValueAsBool();
283 // If the soft float attribute is set on the function turn on the soft float
284 // subtarget feature.
285 if (SoftFloat)
286 Key += FS.empty() ? "+soft-float" : "+soft-float,";
287
288 Key += FS;
289
290 // We may have added +soft-float to the features so move the StringRef to
291 // point to the full string in the Key.
292 FS = Key.substr(Start: FSStart);
293
294 auto &I = SubtargetMap[Key];
295 if (!I) {
296 I = std::make_unique<X86Subtarget>(
297 args: TargetTriple, args&: CPU, args&: TuneCPU, args&: FS, args: *this,
298 args: MaybeAlign(F.getParent()->getOverrideStackAlignment()),
299 args&: PreferVectorWidthOverride, args&: RequiredVectorWidth);
300 }
301 LastSubtarget.set(NewFnAttrs: FnAttrs, NewM: M, NewAttrs: Attrs, NewSubtarget: I.get());
302 return I.get();
303}
304
305yaml::MachineFunctionInfo *X86TargetMachine::createDefaultFuncInfoYAML() const {
306 return new yaml::X86MachineFunctionInfo();
307}
308
309yaml::MachineFunctionInfo *
310X86TargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
311 const auto *MFI = MF.getInfo<X86MachineFunctionInfo>();
312 return new yaml::X86MachineFunctionInfo(*MFI);
313}
314
315bool X86TargetMachine::parseMachineFunctionInfo(
316 const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
317 SMDiagnostic &Error, SMRange &SourceRange) const {
318 const auto &YamlMFI = static_cast<const yaml::X86MachineFunctionInfo &>(MFI);
319 PFS.MF.getInfo<X86MachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
320 return false;
321}
322
323bool X86TargetMachine::isNoopAddrSpaceCast(const DataLayout &DL, unsigned SrcAS,
324 unsigned DestAS) const {
325 assert(SrcAS != DestAS && "Expected different address spaces!");
326 if (DL.getPointerSize(AS: SrcAS) != DL.getPointerSize(AS: DestAS))
327 return false;
328 return SrcAS < 256 && DestAS < 256;
329}
330
331void X86TargetMachine::reset() {
332 SubtargetMap.clear();
333 LastSubtarget.clear();
334}
335
336ScheduleDAGInstrs *
337X86TargetMachine::createMachineScheduler(MachineSchedContext *C) const {
338 ScheduleDAGMILive *DAG = createSchedLive(C);
339 DAG->addMutation(Mutation: createX86MacroFusionDAGMutation());
340 return DAG;
341}
342
343ScheduleDAGInstrs *
344X86TargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
345 ScheduleDAGMI *DAG = createSchedPostRA(C);
346 DAG->addMutation(Mutation: createX86MacroFusionDAGMutation());
347 return DAG;
348}
349
350//===----------------------------------------------------------------------===//
351// X86 TTI query.
352//===----------------------------------------------------------------------===//
353
354TargetTransformInfo
355X86TargetMachine::getTargetTransformInfo(const Function &F) const {
356 return TargetTransformInfo(std::make_unique<X86TTIImpl>(args: this, args: F));
357}
358
359//===----------------------------------------------------------------------===//
360// Pass Pipeline Configuration
361//===----------------------------------------------------------------------===//
362
363namespace {
364
365/// X86 Code Generator Pass Configuration Options.
366class X86PassConfig : public TargetPassConfig {
367public:
368 X86PassConfig(X86TargetMachine &TM, PassManagerBase &PM)
369 : TargetPassConfig(TM, PM) {}
370
371 X86TargetMachine &getX86TargetMachine() const {
372 return getTM<X86TargetMachine>();
373 }
374
375 void addIRPasses() override;
376 bool addInstSelector() override;
377 bool addIRTranslator() override;
378 bool addLegalizeMachineIR() override;
379 void addPreRegBankSelect() override;
380 bool addRegBankSelect() override;
381 bool addGlobalInstructionSelect() override;
382 void addPreLegalizeMachineIR() override;
383 bool addILPOpts() override;
384 bool addPreISel() override;
385 void addMachineSSAOptimization() override;
386 void addPreRegAlloc() override;
387 bool addRegAssignAndRewriteFast() override;
388 void addPostRegAlloc() override;
389 void addPreEmitPass() override;
390 void addPreEmitPass2() override;
391 void addPreSched2() override;
392 bool addRegAssignAndRewriteOptimized() override;
393
394 std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
395};
396
397} // end anonymous namespace
398
399TargetPassConfig *X86TargetMachine::createPassConfig(PassManagerBase &PM) {
400 return new X86PassConfig(*this, PM);
401}
402
403MachineFunctionInfo *X86TargetMachine::createMachineFunctionInfo(
404 BumpPtrAllocator &Allocator, const Function &F,
405 const TargetSubtargetInfo *STI) const {
406 return X86MachineFunctionInfo::create<X86MachineFunctionInfo>(Allocator, F,
407 STI);
408}
409
410void X86PassConfig::addIRPasses() {
411 addPass(P: createAtomicExpandLegacyPass());
412
413 // We add both pass anyway and when these two passes run, we skip the pass
414 // based on the option level and option attribute.
415 addPass(P: createX86LowerAMXIntrinsicsLegacyPass());
416 addPass(P: createX86LowerAMXTypeLegacyPass());
417
418 TargetPassConfig::addIRPasses();
419
420 if (TM->getOptLevel() != CodeGenOptLevel::None) {
421 addPass(P: createInterleavedAccessPass());
422 addPass(P: createX86PartialReductionLegacyPass());
423 }
424
425 // Add passes that handle indirect branch removal and insertion of a retpoline
426 // thunk. These will be a no-op unless a function subtarget has the retpoline
427 // feature enabled.
428 addPass(P: createIndirectBrExpandPass(OptLevel: getOptLevel()));
429
430 // Add Control Flow Guard checks.
431 const Triple &TT = TM->getTargetTriple();
432 if (TT.isOSWindows()) {
433 addPass(P: createCFGuardPass());
434 }
435
436 if (TM->Options.JMCInstrument)
437 addPass(P: createJMCInstrumenterPass());
438
439 addPass(P: createX86FenceNonTemporalStoresLegacyPass());
440}
441
442bool X86PassConfig::addInstSelector() {
443 // Install an instruction selector.
444 addPass(P: createX86ISelDag(TM&: getX86TargetMachine(), OptLevel: getOptLevel()));
445
446 // For ELF, cleanup any local-dynamic TLS accesses.
447 if (TM->getTargetTriple().isOSBinFormatELF() &&
448 getOptLevel() != CodeGenOptLevel::None)
449 addPass(P: createCleanupLocalDynamicTLSLegacyPass());
450
451 addPass(P: createX86GlobalBaseRegLegacyPass());
452 addPass(P: createX86ArgumentStackSlotLegacyPass());
453 return false;
454}
455
456bool X86PassConfig::addIRTranslator() {
457 addPass(P: new IRTranslatorLegacy(getOptLevel()));
458 return false;
459}
460
461void X86PassConfig::addPreRegBankSelect() {
462 bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;
463 if (!IsOptNone) {
464 addPass(P: createX86PostLegalizerCombinerLegacy());
465 }
466}
467bool X86PassConfig::addLegalizeMachineIR() {
468 addPass(P: new LegalizerLegacy());
469 return false;
470}
471
472bool X86PassConfig::addRegBankSelect() {
473 addPass(P: new RegBankSelectLegacy());
474 return false;
475}
476
477bool X86PassConfig::addGlobalInstructionSelect() {
478 addPass(P: new InstructionSelectLegacy(getOptLevel()));
479 // Add GlobalBaseReg in case there is no SelectionDAG passes afterwards
480 if (isGlobalISelAbortEnabled())
481 addPass(P: createX86GlobalBaseRegLegacyPass());
482 return false;
483}
484
485void X86PassConfig::addPreLegalizeMachineIR() {
486 if (getOptLevel() != CodeGenOptLevel::None) {
487 addPass(P: createX86PreLegalizerCombinerLegacy());
488 }
489}
490
491bool X86PassConfig::addILPOpts() {
492 addPass(PassID: &EarlyIfConverterLegacyID);
493 if (getX86TargetMachine().getCLOpts().machine_combiner)
494 addPass(PassID: &MachineCombinerID);
495 addPass(P: createX86CmovConversionLegacyPass());
496 return true;
497}
498
499bool X86PassConfig::addPreISel() {
500 // Only add this pass for 32-bit x86 Windows.
501 const Triple &TT = TM->getTargetTriple();
502 if (TT.isOSWindows() && TT.isX86_32())
503 addPass(P: createX86WinEHStateLegacyPass());
504 return true;
505}
506
507void X86PassConfig::addPreRegAlloc() {
508 if (getOptLevel() != CodeGenOptLevel::None) {
509 addPass(PassID: &LiveRangeShrinkID);
510 addPass(P: createX86FixupSetCCLegacyPass());
511 addPass(P: createX86OptimizeLEAsLegacyPass());
512 addPass(P: createX86CallFrameOptimizationLegacyPass());
513 addPass(P: createX86AvoidStoreForwardingBlocksLegacyPass());
514 }
515
516 addPass(P: createX86SuppressAPXForRelocationLegacyPass());
517
518 addPass(P: createX86SpeculativeLoadHardeningLegacyPass());
519 addPass(P: createX86FlagsCopyLoweringLegacyPass());
520 addPass(P: createX86DynAllocaExpanderLegacyPass());
521
522 if (getOptLevel() != CodeGenOptLevel::None)
523 addPass(P: createX86PreTileConfigLegacyPass());
524 else
525 addPass(P: createX86FastPreTileConfigLegacyPass());
526}
527
528bool X86PassConfig::addRegAssignAndRewriteFast() {
529 bool ToReturn = TargetPassConfig::addRegAssignAndRewriteFast();
530 addPass(P: createX86FastTileConfigLegacyPass());
531 return ToReturn;
532}
533
534void X86PassConfig::addMachineSSAOptimization() {
535 addPass(P: createX86DomainReassignmentLegacyPass());
536 TargetPassConfig::addMachineSSAOptimization();
537}
538
539void X86PassConfig::addPostRegAlloc() {
540 addPass(P: createX86LowerTileCopyLegacyPass());
541 addPass(P: createX86FPStackifierLegacyPass());
542 // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
543 // to using the Speculative Execution Side Effect Suppression pass for
544 // mitigation. This is to prevent slow downs due to
545 // analyses needed by the LVIHardening pass when compiling at -O0.
546 if (getOptLevel() != CodeGenOptLevel::None)
547 addPass(P: createX86LoadValueInjectionLoadHardeningLegacyPass());
548}
549
550void X86PassConfig::addPreSched2() {
551 addPass(P: createX86ExpandPseudoLegacyPass());
552 addPass(P: createKCFIPass());
553}
554
555void X86PassConfig::addPreEmitPass() {
556 if (getOptLevel() != CodeGenOptLevel::None) {
557 addPass(P: createExecutionDomainFixLegacyPass());
558 addPass(P: createBreakFalseDepsLegacyPass());
559 }
560
561 addPass(P: createX86IndirectBranchTrackingLegacyPass());
562
563 addPass(P: createX86InsertVZeroUpperLegacyPass());
564
565 if (getOptLevel() != CodeGenOptLevel::None) {
566 addPass(P: createX86FixupBWInstsLegacyPass());
567 addPass(P: createX86PadShortFunctionsLegacyPass());
568 addPass(P: createX86FixupLEAsLegacyPass());
569 addPass(P: createX86FixupInstTuningLegacyPass());
570 addPass(P: createX86FixupVectorConstantsLegacyPass());
571 }
572 addPass(P: createX86CompressEVEXLegacyPass());
573 addPass(P: createX86InsertX87WaitLegacyPass());
574
575 if (TM->getTargetTriple().isLFI())
576 addPass(P: createX86LFIRewritePass());
577}
578
579void X86PassConfig::addPreEmitPass2() {
580 const Triple &TT = TM->getTargetTriple();
581
582 // The X86 Speculative Execution Pass must run after all control
583 // flow graph modifying passes. As a result it was listed to run right before
584 // the X86 Retpoline Thunks pass. The reason it must run after control flow
585 // graph modifications is that the model of LFENCE in LLVM has to be updated
586 // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the
587 // placement of this pass was hand checked to ensure that the subsequent
588 // passes don't move the code around the LFENCEs in a way that will hurt the
589 // correctness of this pass. This placement has been shown to work based on
590 // hand inspection of the codegen output.
591 addPass(P: createX86SpeculativeExecutionSideEffectSuppressionLegacyPass());
592 addPass(P: createX86IndirectThunksLegacyPass());
593 addPass(P: createX86ReturnThunksLegacyPass());
594
595 // Insert extra int3 instructions after trailing call instructions to avoid
596 // issues in the unwinder.
597 if (TT.isOSWindows() && TT.isX86_64())
598 addPass(P: createX86AvoidTrailingCallLegacyPass());
599
600 // Verify basic block incoming and outgoing cfa offset and register values and
601 // correct CFA calculation rule where needed by inserting appropriate CFI
602 // instructions. CFIInstrInserter self-skips targets that do not use DWARF
603 // CFI.
604 if (!TT.isOSDarwin())
605 addPass(P: createCFIInstrInserterLegacy());
606
607 if (TT.isOSWindows()) {
608 // Identify valid longjmp targets for Windows Control Flow Guard.
609 addPass(P: createCFGuardLongjmpLegacyPass());
610 // Identify valid eh continuation targets for Windows EHCont Guard.
611 addPass(P: createEHContGuardTargetsLegacy());
612 }
613 addPass(P: createX86LoadValueInjectionRetHardeningLegacyPass());
614
615 // Insert pseudo probe annotation for callsite profiling
616 addPass(P: createPseudoProbeInserterLegacyPass());
617
618 // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms,
619 // also CALL_RVMARKER.
620 addPass(P: createUnpackMachineBundlesLegacy(Ftor: [&TT](const MachineFunction &MF) {
621 // Only run bundle expansion if the module uses kcfi, or there are relevant
622 // ObjC runtime functions present in the module.
623 const Function &F = MF.getFunction();
624 const Module *M = F.getParent();
625 return M->getModuleFlag(Key: "kcfi") ||
626 (TT.isOSDarwin() &&
627 (M->getFunction(Name: "objc_retainAutoreleasedReturnValue") ||
628 M->getFunction(Name: "objc_unsafeClaimAutoreleasedReturnValue")));
629 }));
630
631 // Analyzes and emits pseudos to support Win x64 Unwind V2. This pass must run
632 // after all real instructions have been added to the epilog.
633 if (TT.isOSWindows() && TT.isX86_64()) {
634 addPass(P: createX86WinEHUnwindV2LegacyPass());
635 addPass(P: createX86WinEHUnwindV3Pass());
636 }
637}
638
639std::unique_ptr<CSEConfigBase> X86PassConfig::getCSEConfig() const {
640 return getStandardCSEConfigForOpt(Level: TM->getOptLevel());
641}
642
643static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI,
644 const MachineRegisterInfo &MRI,
645 const Register Reg) {
646 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
647 return static_cast<const X86RegisterInfo &>(TRI).isTileRegisterClass(RC);
648}
649
650bool X86PassConfig::addRegAssignAndRewriteOptimized() {
651 // Don't support tile RA when RA is specified by command line "-regalloc".
652 if (!isCustomizedRegAlloc() && getX86TargetMachine().getCLOpts().tile_ra) {
653 // Allocate tile register first.
654 addPass(P: createGreedyRegisterAllocator(F: onlyAllocateTileRegisters));
655 addPass(P: createX86TileConfigLegacyPass());
656 }
657 return TargetPassConfig::addRegAssignAndRewriteOptimized();
658}
659