| 1 | //===-- X86CodeGenPassBuilder.cpp ---------------------------------*- C++ -*-=// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// \file |
| 9 | /// This file contains X86 CodeGen pipeline builder. |
| 10 | /// |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "X86.h" |
| 14 | #include "X86AsmPrinter.h" |
| 15 | #include "X86TargetMachine.h" |
| 16 | |
| 17 | #include "llvm/CodeGen/AtomicExpand.h" |
| 18 | #include "llvm/CodeGen/BreakFalseDeps.h" |
| 19 | #include "llvm/CodeGen/CFGuardLongjmp.h" |
| 20 | #include "llvm/CodeGen/CFIInstrInserter.h" |
| 21 | #include "llvm/CodeGen/EHContGuardTargets.h" |
| 22 | #include "llvm/CodeGen/EarlyIfConversion.h" |
| 23 | #include "llvm/CodeGen/ExecutionDomainFix.h" |
| 24 | #include "llvm/CodeGen/GlobalISel/IRTranslator.h" |
| 25 | #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" |
| 26 | #include "llvm/CodeGen/GlobalISel/Legalizer.h" |
| 27 | #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" |
| 28 | #include "llvm/CodeGen/IndirectBrExpand.h" |
| 29 | #include "llvm/CodeGen/InterleavedAccess.h" |
| 30 | #include "llvm/CodeGen/JMCInstrumenter.h" |
| 31 | #include "llvm/CodeGen/KCFI.h" |
| 32 | #include "llvm/CodeGen/MachineCombiner.h" |
| 33 | #include "llvm/CodeGen/PseudoProbeInserter.h" |
| 34 | #include "llvm/CodeGen/RegAllocGreedyPass.h" |
| 35 | #include "llvm/MC/MCAsmInfo.h" |
| 36 | #include "llvm/MC/MCStreamer.h" |
| 37 | #include "llvm/Pass.h" |
| 38 | #include "llvm/Passes/CodeGenPassBuilder.h" |
| 39 | #include "llvm/Passes/PassBuilder.h" |
| 40 | #include "llvm/Support/CodeGen.h" |
| 41 | #include "llvm/Target/CGPassBuilderOption.h" |
| 42 | #include "llvm/Transforms/CFGuard.h" |
| 43 | |
| 44 | using namespace llvm; |
| 45 | |
| 46 | namespace { |
| 47 | |
| 48 | class X86CodeGenPassBuilder : public CodeGenPassBuilder { |
| 49 | using Base = CodeGenPassBuilder; |
| 50 | |
| 51 | X86TargetMachine &getTM() const { |
| 52 | return static_cast<X86TargetMachine &>(TM); |
| 53 | } |
| 54 | |
| 55 | public: |
| 56 | explicit X86CodeGenPassBuilder(X86TargetMachine &TM, |
| 57 | const CGPassBuilderOption &Opts, |
| 58 | PassInstrumentationCallbacks *PIC) |
| 59 | : CodeGenPassBuilder(TM, Opts, PIC) {} |
| 60 | |
| 61 | void addIRPasses(PassManagerWrapper &PMW) override; |
| 62 | void addPreISel(PassManagerWrapper &PMW) override; |
| 63 | Error addInstSelector(PassManagerWrapper &PMW) override; |
| 64 | |
| 65 | Error addIRTranslator(PassManagerWrapper &PMW) override; |
| 66 | void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override; |
| 67 | Error addLegalizeMachineIR(PassManagerWrapper &PMW) override; |
| 68 | void addPreRegBankSelect(PassManagerWrapper &PMW) override; |
| 69 | Error addRegBankSelect(PassManagerWrapper &PMW) override; |
| 70 | Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override; |
| 71 | |
| 72 | void addMachineSSAOptimization(PassManagerWrapper &PMW) override; |
| 73 | void addILPOpts(PassManagerWrapper &PMW) override; |
| 74 | void addPreRegAlloc(PassManagerWrapper &PMW) override; |
| 75 | Error addRegAssignAndRewriteFast(PassManagerWrapper &PMW) override; |
| 76 | Expected<bool> |
| 77 | addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) override; |
| 78 | void addPostRegAlloc(PassManagerWrapper &PMW) override; |
| 79 | void addPreSched2(PassManagerWrapper &PMW) override; |
| 80 | void addPreEmitPass(PassManagerWrapper &PMW) override; |
| 81 | void addPreEmitPass2(PassManagerWrapper &PMW) override; |
| 82 | void addAsmPrinterBegin(PassManagerWrapper &PMW) override; |
| 83 | void addAsmPrinter(PassManagerWrapper &PMW) override; |
| 84 | void addAsmPrinterEnd(PassManagerWrapper &PMW) override; |
| 85 | }; |
| 86 | |
| 87 | void X86CodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) { |
| 88 | addFunctionPass(Pass: AtomicExpandPass(TM), PMW); |
| 89 | |
| 90 | // We add both pass anyway and when these two passes run, one will be a |
| 91 | // no-op based on the optimization level/attributes. |
| 92 | addFunctionPass(Pass: X86LowerAMXIntrinsicsPass(&TM), PMW); |
| 93 | addFunctionPass(Pass: X86LowerAMXTypePass(&TM), PMW); |
| 94 | |
| 95 | Base::addIRPasses(PMW); |
| 96 | |
| 97 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 98 | addFunctionPass(Pass: InterleavedAccessPass(TM), PMW); |
| 99 | addFunctionPass(Pass: X86PartialReductionPass(&getTM()), PMW); |
| 100 | } |
| 101 | |
| 102 | // Add passes that handle indirect branch removal and insertion of a retpoline |
| 103 | // thunk. These will be a no-op unless a function subtarget has the retpoline |
| 104 | // feature enabled. |
| 105 | addFunctionPass(Pass: IndirectBrExpandPass(TM), PMW); |
| 106 | |
| 107 | // Add Control Flow Guard checks. |
| 108 | const Triple &TT = TM.getTargetTriple(); |
| 109 | if (TT.isOSWindows()) |
| 110 | addFunctionPass(Pass: CFGuardPass(), PMW); |
| 111 | |
| 112 | if (TM.Options.JMCInstrument) { |
| 113 | flushFPMsToMPM(PMW); |
| 114 | addModulePass(Pass: JMCInstrumenterPass(), PMW); |
| 115 | } |
| 116 | |
| 117 | addFunctionPass(Pass: X86FenceNonTemporalStoresPass(&getTM()), PMW); |
| 118 | } |
| 119 | |
| 120 | void X86CodeGenPassBuilder::addPreISel(PassManagerWrapper &PMW) { |
| 121 | // Only add this pass for 32-bit x86 Windows. |
| 122 | const Triple &TT = TM.getTargetTriple(); |
| 123 | if (TT.isOSWindows() && TT.isX86_32()) { |
| 124 | flushFPMsToMPM(PMW); |
| 125 | addModulePass(Pass: X86WinEHStatePass(), PMW); |
| 126 | } |
| 127 | } |
| 128 | |
| 129 | Error X86CodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) { |
| 130 | addMachineFunctionPass(Pass: X86ISelDAGToDAGPass(getTM()), PMW); |
| 131 | |
| 132 | // For ELF, cleanup any local-dynamic TLS accesses |
| 133 | if (TM.getTargetTriple().isOSBinFormatELF() && |
| 134 | getOptLevel() != CodeGenOptLevel::None) { |
| 135 | addMachineFunctionPass(Pass: X86CleanupLocalDynamicTLSPass(), PMW); |
| 136 | } |
| 137 | |
| 138 | addMachineFunctionPass(Pass: X86GlobalBaseRegPass(), PMW); |
| 139 | addMachineFunctionPass(Pass: X86ArgumentStackSlotPass(), PMW); |
| 140 | return Error::success(); |
| 141 | } |
| 142 | |
| 143 | Error X86CodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) { |
| 144 | addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW); |
| 145 | return Error::success(); |
| 146 | } |
| 147 | |
| 148 | void X86CodeGenPassBuilder::addPreLegalizeMachineIR(PassManagerWrapper &PMW) { |
| 149 | if (getOptLevel() != CodeGenOptLevel::None) |
| 150 | addMachineFunctionPass(Pass: X86PreLegalizerCombinerPass(), PMW); |
| 151 | } |
| 152 | |
| 153 | Error X86CodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) { |
| 154 | addMachineFunctionPass(Pass: LegalizerPass(), PMW); |
| 155 | return Error::success(); |
| 156 | } |
| 157 | |
| 158 | void X86CodeGenPassBuilder::addPreRegBankSelect(PassManagerWrapper &PMW) { |
| 159 | if (getOptLevel() != CodeGenOptLevel::None) |
| 160 | addMachineFunctionPass(Pass: X86PostLegalizerCombinerPass(), PMW); |
| 161 | } |
| 162 | |
| 163 | Error X86CodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) { |
| 164 | addMachineFunctionPass(Pass: RegBankSelectPass(), PMW); |
| 165 | return Error::success(); |
| 166 | } |
| 167 | |
| 168 | Error X86CodeGenPassBuilder::addGlobalInstructionSelect( |
| 169 | PassManagerWrapper &PMW) { |
| 170 | addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW); |
| 171 | // Add GlobalBaseReg in case there is no SelectionDAG passes afterwards |
| 172 | if (isGlobalISelAbortEnabled()) |
| 173 | addMachineFunctionPass(Pass: X86GlobalBaseRegPass(), PMW); |
| 174 | return Error::success(); |
| 175 | } |
| 176 | |
| 177 | void X86CodeGenPassBuilder::addMachineSSAOptimization(PassManagerWrapper &PMW) { |
| 178 | addMachineFunctionPass(Pass: X86DomainReassignmentPass(), PMW); |
| 179 | Base::addMachineSSAOptimization(PMW); |
| 180 | } |
| 181 | |
| 182 | void X86CodeGenPassBuilder::addILPOpts(PassManagerWrapper &PMW) { |
| 183 | addMachineFunctionPass(Pass: EarlyIfConverterPass(), PMW); |
| 184 | if (getTM().getCLOpts().machine_combiner) |
| 185 | addMachineFunctionPass(Pass: MachineCombinerPass(), PMW); |
| 186 | addMachineFunctionPass(Pass: X86CmovConversionPass(), PMW); |
| 187 | } |
| 188 | |
| 189 | void X86CodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) { |
| 190 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 191 | addMachineFunctionPass(Pass: LiveRangeShrinkPass(), PMW); |
| 192 | addMachineFunctionPass(Pass: X86FixupSetCCPass(), PMW); |
| 193 | addMachineFunctionPass(Pass: X86OptimizeLEAsPass(), PMW); |
| 194 | addMachineFunctionPass(Pass: X86CallFrameOptimizationPass(), PMW); |
| 195 | addMachineFunctionPass(Pass: X86AvoidStoreForwardingBlocksPass(), PMW); |
| 196 | } |
| 197 | |
| 198 | addMachineFunctionPass(Pass: X86SuppressAPXForRelocationPass(), PMW); |
| 199 | addMachineFunctionPass(Pass: X86SpeculativeLoadHardeningPass(), PMW); |
| 200 | addMachineFunctionPass(Pass: X86FlagsCopyLoweringPass(), PMW); |
| 201 | addMachineFunctionPass(Pass: X86DynAllocaExpanderPass(), PMW); |
| 202 | |
| 203 | if (getOptLevel() != CodeGenOptLevel::None) |
| 204 | addMachineFunctionPass(Pass: X86PreTileConfigPass(), PMW); |
| 205 | else |
| 206 | addMachineFunctionPass(Pass: X86FastPreTileConfigPass(), PMW); |
| 207 | } |
| 208 | |
| 209 | static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI, |
| 210 | const MachineRegisterInfo &MRI, |
| 211 | const Register Reg) { |
| 212 | const TargetRegisterClass *RC = MRI.getRegClass(Reg); |
| 213 | return static_cast<const X86RegisterInfo &>(TRI).isTileRegisterClass(RC); |
| 214 | } |
| 215 | |
| 216 | Error X86CodeGenPassBuilder::addRegAssignAndRewriteFast( |
| 217 | PassManagerWrapper &PMW) { |
| 218 | Error PossibleError = Base::addRegAssignAndRewriteFast(PMW); |
| 219 | addMachineFunctionPass(Pass: X86FastTileConfigPass(), PMW); |
| 220 | return PossibleError; |
| 221 | } |
| 222 | |
| 223 | Expected<bool> X86CodeGenPassBuilder::addRegAssignAndRewriteOptimized( |
| 224 | PassManagerWrapper &PMW) { |
| 225 | // Don't support tile RA when RA is specified by command line "-regalloc". |
| 226 | if (Opt.RegAlloc == RegAllocType::Default && getTM().getCLOpts().tile_ra) { |
| 227 | // Allocate tile register first. |
| 228 | addMachineFunctionPass( |
| 229 | Pass: RAGreedyPass({RAGreedyPass::Options(onlyAllocateTileRegisters)}), PMW); |
| 230 | addMachineFunctionPass(Pass: X86TileConfigPass(), PMW); |
| 231 | } |
| 232 | return Base::addRegAssignAndRewriteOptimized(PMW); |
| 233 | } |
| 234 | |
| 235 | void X86CodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) { |
| 236 | addMachineFunctionPass(Pass: X86LowerTileCopyPass(), PMW); |
| 237 | addMachineFunctionPass(Pass: X86FPStackifierPass(), PMW); |
| 238 | // When -O0 is enabled, the Load Value Injection Hardening pass will fall back |
| 239 | // to using the Speculative Execution Side Effect Suppression pass for |
| 240 | // mitigation. This is to prevent slow downs due to |
| 241 | // analyses needed by the LVIHardening pass when compiling at -O0. |
| 242 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 243 | addMachineFunctionPass(Pass: X86LoadValueInjectionLoadHardeningPass(), PMW); |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | void X86CodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) { |
| 248 | addMachineFunctionPass(Pass: X86ExpandPseudoPass(), PMW); |
| 249 | addMachineFunctionPass(Pass: MachineKCFIPass(), PMW); |
| 250 | } |
| 251 | |
| 252 | void X86CodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) { |
| 253 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 254 | addMachineFunctionPass(Pass: ExecutionDomainFixPass(), PMW); |
| 255 | addMachineFunctionPass(Pass: BreakFalseDepsPass(), PMW); |
| 256 | } |
| 257 | |
| 258 | addMachineFunctionPass(Pass: X86IndirectBranchTrackingPass(), PMW); |
| 259 | addMachineFunctionPass(Pass: X86InsertVZeroUpperPass(), PMW); |
| 260 | |
| 261 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 262 | addMachineFunctionPass(Pass: X86FixupBWInstsPass(), PMW); |
| 263 | addMachineFunctionPass(Pass: X86PadShortFunctionsPass(), PMW); |
| 264 | addMachineFunctionPass(Pass: X86FixupLEAsPass(), PMW); |
| 265 | addMachineFunctionPass(Pass: X86FixupInstTuningPass(), PMW); |
| 266 | addMachineFunctionPass(Pass: X86FixupVectorConstantsPass(), PMW); |
| 267 | } |
| 268 | addMachineFunctionPass(Pass: X86CompressEVEXPass(), PMW); |
| 269 | addMachineFunctionPass(Pass: X86InsertX87WaitPass(), PMW); |
| 270 | |
| 271 | if (TM.getTargetTriple().isLFI()) |
| 272 | addMachineFunctionPass(Pass: X86LFIRewritePass(), PMW); |
| 273 | } |
| 274 | |
| 275 | void X86CodeGenPassBuilder::addPreEmitPass2(PassManagerWrapper &PMW) { |
| 276 | const Triple &TT = TM.getTargetTriple(); |
| 277 | |
| 278 | // The X86 Speculative Execution Pass must run after all control |
| 279 | // flow graph modifying passes. As a result it was listed to run right before |
| 280 | // the X86 Retpoline Thunks pass. The reason it must run after control flow |
| 281 | // graph modifications is that the model of LFENCE in LLVM has to be updated |
| 282 | // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the |
| 283 | // placement of this pass was hand checked to ensure that the subsequent |
| 284 | // passes don't move the code around the LFENCEs in a way that will hurt the |
| 285 | // correctness of this pass. This placement has been shown to work based on |
| 286 | // hand inspection of the codegen output. |
| 287 | addMachineFunctionPass(Pass: X86SpeculativeExecutionSideEffectSuppressionPass(), |
| 288 | PMW); |
| 289 | addMachineFunctionPass(Pass: X86IndirectThunksPass(), PMW); |
| 290 | addMachineFunctionPass(Pass: X86ReturnThunksPass(), PMW); |
| 291 | |
| 292 | // Insert extra int3 instructions after trailing call instructions to avoid |
| 293 | // issues in the unwinder. |
| 294 | if (TT.isOSWindows() && TT.isX86_64()) |
| 295 | addMachineFunctionPass(Pass: X86AvoidTrailingCallPass(), PMW); |
| 296 | |
| 297 | // Verify basic block incoming and outgoing cfa offset and register values and |
| 298 | // correct CFA calculation rule where needed by inserting appropriate CFI |
| 299 | // instructions. CFIInstrInserter self-skips targets that do not use DWARF |
| 300 | // CFI. |
| 301 | if (!TT.isOSDarwin()) |
| 302 | addMachineFunctionPass(Pass: CFIInstrInserterPass(), PMW); |
| 303 | |
| 304 | if (TT.isOSWindows()) { |
| 305 | // Identify valid longjmp targets for Windows Control Flow Guard. |
| 306 | addMachineFunctionPass(Pass: CFGuardLongjmpPass(), PMW); |
| 307 | // Identify valid eh continuation targets for Windows EHCont Guard. |
| 308 | addMachineFunctionPass(Pass: EHContGuardTargetsPass(), PMW); |
| 309 | } |
| 310 | |
| 311 | addMachineFunctionPass(Pass: X86LoadValueInjectionRetHardeningPass(), PMW); |
| 312 | |
| 313 | // Insert pseudo probe annotation for callsite profiling |
| 314 | addMachineFunctionPass(Pass: PseudoProbeInserterPass(), PMW); |
| 315 | |
| 316 | // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms, |
| 317 | // also CALL_RVMARKER. |
| 318 | addMachineFunctionPass( |
| 319 | Pass: UnpackMachineBundlesPass([&TT](const MachineFunction &MF) { |
| 320 | // Only run bundle expansion if the module uses kcfi, or there are |
| 321 | // relevant ObjC runtime functions present in the module. |
| 322 | const Function &F = MF.getFunction(); |
| 323 | const Module *M = F.getParent(); |
| 324 | return M->getModuleFlag(Key: "kcfi" ) || |
| 325 | (TT.isOSDarwin() && |
| 326 | (M->getFunction(Name: "objc_retainAutoreleasedReturnValue" ) || |
| 327 | M->getFunction(Name: "objc_unsafeClaimAutoreleasedReturnValue" ))); |
| 328 | }), |
| 329 | PMW); |
| 330 | |
| 331 | // Analyzes and emits pseudos to support Win x64 Unwind V2. This pass must run |
| 332 | // after all real instructions have been added to the epilog. |
| 333 | if (TT.isOSWindows() && TT.isX86_64()) { |
| 334 | addMachineFunctionPass(Pass: X86WinEHUnwindV2Pass(), PMW); |
| 335 | } |
| 336 | } |
| 337 | |
| 338 | void X86CodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) { |
| 339 | addModulePass(Pass: X86AsmPrinterBeginPass(), PMW, /*Force=*/true); |
| 340 | } |
| 341 | |
| 342 | void X86CodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) { |
| 343 | addMachineFunctionPass(Pass: X86AsmPrinterPass(), PMW); |
| 344 | } |
| 345 | |
| 346 | void X86CodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) { |
| 347 | addModulePass(Pass: X86AsmPrinterEndPass(), PMW, /*Force=*/true); |
| 348 | } |
| 349 | |
| 350 | } // namespace |
| 351 | |
| 352 | void X86TargetMachine::registerPassBuilderCallbacks(PassBuilder &PB){ |
| 353 | #define GET_PASS_REGISTRY "X86PassRegistry.def" |
| 354 | #include "llvm/Passes/TargetPassRegistry.inc" |
| 355 | } |
| 356 | |
| 357 | Error X86TargetMachine::buildCodeGenPipeline( |
| 358 | ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out, |
| 359 | raw_pwrite_stream *DwoOut, CodeGenFileType FileType, |
| 360 | const CGPassBuilderOption &Opt, MCContext &Ctx, |
| 361 | PassInstrumentationCallbacks *PIC) { |
| 362 | auto CGPB = X86CodeGenPassBuilder(*this, Opt, PIC); |
| 363 | return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx); |
| 364 | } |
| 365 | |