1//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the IRTranslator class.
10//===----------------------------------------------------------------------===//
11
12#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
13#include "llvm/ADT/PostOrderIterator.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/Analysis/AliasAnalysis.h"
18#include "llvm/Analysis/AssumptionCache.h"
19#include "llvm/Analysis/BranchProbabilityInfo.h"
20#include "llvm/Analysis/Loads.h"
21#include "llvm/Analysis/OptimizationRemarkEmitter.h"
22#include "llvm/Analysis/ValueTracking.h"
23#include "llvm/Analysis/VectorUtils.h"
24#include "llvm/CodeGen/Analysis.h"
25#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
26#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
27#include "llvm/CodeGen/GlobalISel/CallLowering.h"
28#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
29#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
30#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
31#include "llvm/CodeGen/LowLevelTypeUtils.h"
32#include "llvm/CodeGen/MachineBasicBlock.h"
33#include "llvm/CodeGen/MachineFrameInfo.h"
34#include "llvm/CodeGen/MachineFunction.h"
35#include "llvm/CodeGen/MachineInstrBuilder.h"
36#include "llvm/CodeGen/MachineMemOperand.h"
37#include "llvm/CodeGen/MachineModuleInfo.h"
38#include "llvm/CodeGen/MachineOperand.h"
39#include "llvm/CodeGen/MachineRegisterInfo.h"
40#include "llvm/CodeGen/StackProtector.h"
41#include "llvm/CodeGen/SwitchLoweringUtils.h"
42#include "llvm/CodeGen/TargetFrameLowering.h"
43#include "llvm/CodeGen/TargetInstrInfo.h"
44#include "llvm/CodeGen/TargetLowering.h"
45#include "llvm/CodeGen/TargetOpcodes.h"
46#include "llvm/CodeGen/TargetPassConfig.h"
47#include "llvm/CodeGen/TargetRegisterInfo.h"
48#include "llvm/CodeGen/TargetSubtargetInfo.h"
49#include "llvm/CodeGenTypes/LowLevelType.h"
50#include "llvm/IR/BasicBlock.h"
51#include "llvm/IR/CFG.h"
52#include "llvm/IR/Constant.h"
53#include "llvm/IR/Constants.h"
54#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DerivedTypes.h"
56#include "llvm/IR/DiagnosticInfo.h"
57#include "llvm/IR/Function.h"
58#include "llvm/IR/GetElementPtrTypeIterator.h"
59#include "llvm/IR/InlineAsm.h"
60#include "llvm/IR/InstrTypes.h"
61#include "llvm/IR/Instructions.h"
62#include "llvm/IR/IntrinsicInst.h"
63#include "llvm/IR/Intrinsics.h"
64#include "llvm/IR/IntrinsicsAMDGPU.h"
65#include "llvm/IR/LLVMContext.h"
66#include "llvm/IR/Metadata.h"
67#include "llvm/IR/PatternMatch.h"
68#include "llvm/IR/Statepoint.h"
69#include "llvm/IR/Type.h"
70#include "llvm/IR/User.h"
71#include "llvm/IR/Value.h"
72#include "llvm/InitializePasses.h"
73#include "llvm/MC/MCContext.h"
74#include "llvm/Pass.h"
75#include "llvm/Support/Casting.h"
76#include "llvm/Support/CodeGen.h"
77#include "llvm/Support/Debug.h"
78#include "llvm/Support/ErrorHandling.h"
79#include "llvm/Support/MathExtras.h"
80#include "llvm/Support/raw_ostream.h"
81#include "llvm/Target/TargetMachine.h"
82#include "llvm/Transforms/Utils/Local.h"
83#include "llvm/Transforms/Utils/MemoryOpRemark.h"
84#include <algorithm>
85#include <cassert>
86#include <cstdint>
87#include <iterator>
88#include <optional>
89#include <string>
90#include <utility>
91#include <vector>
92
93#define DEBUG_TYPE "irtranslator"
94
95using namespace llvm;
96
97static cl::opt<bool>
98 EnableCSEInIRTranslator("enable-cse-in-irtranslator",
99 cl::desc("Should enable CSE in irtranslator"),
100 cl::Optional, cl::init(Val: false));
101char IRTranslator::ID = 0;
102
103INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
104 false, false)
105INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
106INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
107INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
108INITIALIZE_PASS_DEPENDENCY(StackProtector)
109INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
110INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
111 false, false)
112
113static void reportTranslationError(MachineFunction &MF,
114 OptimizationRemarkEmitter &ORE,
115 OptimizationRemarkMissed &R) {
116 MF.getProperties().setFailedISel();
117 bool IsGlobalISelAbortEnabled =
118 MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
119
120 // Print the function name explicitly if we don't have a debug location (which
121 // makes the diagnostic less useful) or if we're going to emit a raw error.
122 if (!R.getLocation().isValid() || IsGlobalISelAbortEnabled)
123 R << (" (in function: " + MF.getName() + ")").str();
124
125 if (IsGlobalISelAbortEnabled)
126 report_fatal_error(reason: Twine(R.getMsg()));
127 else
128 ORE.emit(OptDiag&: R);
129}
130
131IRTranslator::IRTranslator(CodeGenOptLevel optlevel)
132 : MachineFunctionPass(ID), OptLevel(optlevel) {}
133
134#ifndef NDEBUG
135namespace {
136/// Verify that every instruction created has the same DILocation as the
137/// instruction being translated.
138class DILocationVerifier : public GISelChangeObserver {
139 const Instruction *CurrInst = nullptr;
140
141public:
142 DILocationVerifier() = default;
143 ~DILocationVerifier() override = default;
144
145 const Instruction *getCurrentInst() const { return CurrInst; }
146 void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
147
148 void erasingInstr(MachineInstr &MI) override {}
149 void changingInstr(MachineInstr &MI) override {}
150 void changedInstr(MachineInstr &MI) override {}
151
152 void createdInstr(MachineInstr &MI) override {
153 assert(getCurrentInst() && "Inserted instruction without a current MI");
154
155 // Only print the check message if we're actually checking it.
156#ifndef NDEBUG
157 LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
158 << " was copied to " << MI);
159#endif
160 // We allow insts in the entry block to have no debug loc because
161 // they could have originated from constants, and we don't want a jumpy
162 // debug experience.
163 assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
164 (MI.getParent()->isEntryBlock() && !MI.getDebugLoc()) ||
165 (MI.isDebugInstr())) &&
166 "Line info was not transferred to all instructions");
167 }
168};
169} // namespace
170#endif // ifndef NDEBUG
171
172
173void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const {
174 AU.addRequired<StackProtector>();
175 AU.addRequired<TargetPassConfig>();
176 AU.addRequired<GISelCSEAnalysisWrapperPass>();
177 AU.addRequired<AssumptionCacheTracker>();
178 if (OptLevel != CodeGenOptLevel::None) {
179 AU.addRequired<BranchProbabilityInfoWrapperPass>();
180 AU.addRequired<AAResultsWrapperPass>();
181 }
182 AU.addRequired<TargetLibraryInfoWrapperPass>();
183 AU.addPreserved<TargetLibraryInfoWrapperPass>();
184 AU.addRequired<LibcallLoweringInfoWrapper>();
185
186 getSelectionDAGFallbackAnalysisUsage(AU);
187 MachineFunctionPass::getAnalysisUsage(AU);
188}
189
190IRTranslator::ValueToVRegInfo::VRegListT &
191IRTranslator::allocateVRegs(const Value &Val) {
192 auto VRegsIt = VMap.findVRegs(V: Val);
193 if (VRegsIt != VMap.vregs_end())
194 return *VRegsIt->second;
195 auto *Regs = VMap.getVRegs(V: Val);
196 auto *Offsets = VMap.getOffsets(V: Val);
197 SmallVector<LLT, 4> SplitTys;
198 computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueLLTs&: SplitTys,
199 FixedOffsets: Offsets->empty() ? Offsets : nullptr);
200 for (unsigned i = 0; i < SplitTys.size(); ++i)
201 Regs->push_back(Elt: 0);
202 return *Regs;
203}
204
205ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
206 auto VRegsIt = VMap.findVRegs(V: Val);
207 if (VRegsIt != VMap.vregs_end())
208 return *VRegsIt->second;
209
210 if (Val.getType()->isVoidTy())
211 return *VMap.getVRegs(V: Val);
212
213 // Create entry for this type.
214 auto *VRegs = VMap.getVRegs(V: Val);
215 auto *Offsets = VMap.getOffsets(V: Val);
216
217 if (!Val.getType()->isTokenTy())
218 assert(Val.getType()->isSized() &&
219 "Don't know how to create an empty vreg");
220
221 SmallVector<LLT, 4> SplitTys;
222 computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueLLTs&: SplitTys,
223 FixedOffsets: Offsets->empty() ? Offsets : nullptr);
224
225 if (!isa<Constant>(Val)) {
226 for (auto Ty : SplitTys)
227 VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty));
228 return *VRegs;
229 }
230
231 if (Val.getType()->isAggregateType()) {
232 // UndefValue, ConstantAggregateZero
233 auto &C = cast<Constant>(Val);
234 unsigned Idx = 0;
235 while (auto Elt = C.getAggregateElement(Elt: Idx++)) {
236 auto EltRegs = getOrCreateVRegs(Val: *Elt);
237 llvm::append_range(C&: *VRegs, R&: EltRegs);
238 }
239 } else {
240 assert(SplitTys.size() == 1 && "unexpectedly split LLT");
241 VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty: SplitTys[0]));
242 bool Success = translate(C: cast<Constant>(Val), Reg: VRegs->front());
243 if (!Success) {
244 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
245 MF->getFunction().getSubprogram(),
246 &MF->getFunction().getEntryBlock());
247 R << "unable to translate constant: " << ore::NV("Type", Val.getType());
248 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
249 return *VRegs;
250 }
251 }
252
253 return *VRegs;
254}
255
256int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
257 auto [MapEntry, Inserted] = FrameIndices.try_emplace(Key: &AI);
258 if (!Inserted)
259 return MapEntry->second;
260
261 uint64_t Size =
262 AI.getAllocationSize(DL: *DL).value_or(u: TypeSize::getZero()).getFixedValue();
263
264 // Always allocate at least one byte.
265 Size = std::max<uint64_t>(a: Size, b: 1u);
266
267 int &FI = MapEntry->second;
268 FI = MF->getFrameInfo().CreateStackObject(Size, Alignment: AI.getAlign(), isSpillSlot: false, Alloca: &AI);
269 return FI;
270}
271
272Align IRTranslator::getMemOpAlign(const Instruction &I) {
273 if (const StoreInst *SI = dyn_cast<StoreInst>(Val: &I))
274 return SI->getAlign();
275 if (const LoadInst *LI = dyn_cast<LoadInst>(Val: &I))
276 return LI->getAlign();
277 if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(Val: &I))
278 return AI->getAlign();
279 if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(Val: &I))
280 return AI->getAlign();
281
282 OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
283 R << "unable to translate memop: " << ore::NV("Opcode", &I);
284 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
285 return Align(1);
286}
287
288MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
289 MachineBasicBlock *MBB = FuncInfo.getMBB(BB: &BB);
290 assert(MBB && "BasicBlock was not encountered before");
291 return *MBB;
292}
293
294void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
295 assert(NewPred && "new predecessor must be a real MachineBasicBlock");
296 MachinePreds[Edge].push_back(Elt: NewPred);
297}
298
299static bool targetSupportsBF16Type(const MachineFunction *MF) {
300 return MF->getTarget().getTargetTriple().isSPIRV();
301}
302
303static bool containsBF16Type(const User &U) {
304 // BF16 cannot currently be represented by LLT, to avoid miscompiles we
305 // prevent any instructions using them. FIXME: This can be removed once LLT
306 // supports bfloat.
307 return U.getType()->getScalarType()->isBFloatTy() ||
308 any_of(Range: U.operands(), P: [](Value *V) {
309 return V->getType()->getScalarType()->isBFloatTy();
310 });
311}
312
313bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
314 MachineIRBuilder &MIRBuilder) {
315 if (containsBF16Type(U) && !targetSupportsBF16Type(MF))
316 return false;
317
318 // Get or create a virtual register for each value.
319 // Unless the value is a Constant => loadimm cst?
320 // or inline constant each time?
321 // Creation of a virtual register needs to have a size.
322 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
323 Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
324 Register Res = getOrCreateVReg(Val: U);
325 uint32_t Flags = 0;
326 if (isa<Instruction>(Val: U)) {
327 const Instruction &I = cast<Instruction>(Val: U);
328 Flags = MachineInstr::copyFlagsFromInstruction(I);
329 }
330
331 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0, Op1}, Flags);
332 return true;
333}
334
335bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
336 MachineIRBuilder &MIRBuilder) {
337 if (containsBF16Type(U) && !targetSupportsBF16Type(MF))
338 return false;
339
340 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
341 Register Res = getOrCreateVReg(Val: U);
342 uint32_t Flags = 0;
343 if (isa<Instruction>(Val: U)) {
344 const Instruction &I = cast<Instruction>(Val: U);
345 Flags = MachineInstr::copyFlagsFromInstruction(I);
346 }
347 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0}, Flags);
348 return true;
349}
350
351bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
352 return translateUnaryOp(Opcode: TargetOpcode::G_FNEG, U, MIRBuilder);
353}
354
355bool IRTranslator::translateCompare(const User &U,
356 MachineIRBuilder &MIRBuilder) {
357 if (containsBF16Type(U) && !targetSupportsBF16Type(MF))
358 return false;
359
360 auto *CI = cast<CmpInst>(Val: &U);
361 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
362 Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
363 Register Res = getOrCreateVReg(Val: U);
364 CmpInst::Predicate Pred = CI->getPredicate();
365 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: *CI);
366 if (CmpInst::isIntPredicate(P: Pred))
367 MIRBuilder.buildICmp(Pred, Res, Op0, Op1, Flags);
368 else if (Pred == CmpInst::FCMP_FALSE)
369 MIRBuilder.buildCopy(
370 Res, Op: getOrCreateVReg(Val: *Constant::getNullValue(Ty: U.getType())));
371 else if (Pred == CmpInst::FCMP_TRUE)
372 MIRBuilder.buildCopy(
373 Res, Op: getOrCreateVReg(Val: *Constant::getAllOnesValue(Ty: U.getType())));
374 else
375 MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
376
377 return true;
378}
379
380bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
381 const ReturnInst &RI = cast<ReturnInst>(Val: U);
382 const Value *Ret = RI.getReturnValue();
383 if (Ret && DL->getTypeStoreSize(Ty: Ret->getType()).isZero())
384 Ret = nullptr;
385
386 ArrayRef<Register> VRegs;
387 if (Ret)
388 VRegs = getOrCreateVRegs(Val: *Ret);
389
390 Register SwiftErrorVReg = 0;
391 if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
392 SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
393 &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
394 }
395
396 // The target may mess up with the insertion point, but
397 // this is not important as a return is the last instruction
398 // of the block anyway.
399 return CLI->lowerReturn(MIRBuilder, Val: Ret, VRegs, FLI&: FuncInfo, SwiftErrorVReg);
400}
401
402void IRTranslator::emitBranchForMergedCondition(
403 const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
404 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
405 BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
406 // If the leaf of the tree is a comparison, merge the condition into
407 // the caseblock.
408 if (const CmpInst *BOp = dyn_cast<CmpInst>(Val: Cond)) {
409 CmpInst::Predicate Condition;
410 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Val: Cond)) {
411 Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
412 } else {
413 const FCmpInst *FC = cast<FCmpInst>(Val: Cond);
414 Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
415 }
416
417 SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(i_nocapture: 0),
418 BOp->getOperand(i_nocapture: 1), nullptr, TBB, FBB, CurBB,
419 CurBuilder->getDebugLoc(), TProb, FProb);
420 SL->SwitchCases.push_back(x: CB);
421 return;
422 }
423
424 // Create a CaseBlock record representing this branch.
425 CmpInst::Predicate Pred = InvertCond ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ;
426 SwitchCG::CaseBlock CB(
427 Pred, false, Cond, ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
428 nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
429 SL->SwitchCases.push_back(x: CB);
430}
431
432static bool isValInBlock(const Value *V, const BasicBlock *BB) {
433 if (const Instruction *I = dyn_cast<Instruction>(Val: V))
434 return I->getParent() == BB;
435 return true;
436}
437
438void IRTranslator::findMergedConditions(
439 const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
440 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
441 Instruction::BinaryOps Opc, BranchProbability TProb,
442 BranchProbability FProb, bool InvertCond) {
443 using namespace PatternMatch;
444 assert((Opc == Instruction::And || Opc == Instruction::Or) &&
445 "Expected Opc to be AND/OR");
446 // Skip over not part of the tree and remember to invert op and operands at
447 // next level.
448 Value *NotCond;
449 if (match(V: Cond, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: NotCond)))) &&
450 isValInBlock(V: NotCond, BB: CurBB->getBasicBlock())) {
451 findMergedConditions(Cond: NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
452 InvertCond: !InvertCond);
453 return;
454 }
455
456 const Instruction *BOp = dyn_cast<Instruction>(Val: Cond);
457 const Value *BOpOp0, *BOpOp1;
458 // Compute the effective opcode for Cond, taking into account whether it needs
459 // to be inverted, e.g.
460 // and (not (or A, B)), C
461 // gets lowered as
462 // and (and (not A, not B), C)
463 Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
464 if (BOp) {
465 BOpc = match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
466 ? Instruction::And
467 : (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
468 ? Instruction::Or
469 : (Instruction::BinaryOps)0);
470 if (InvertCond) {
471 if (BOpc == Instruction::And)
472 BOpc = Instruction::Or;
473 else if (BOpc == Instruction::Or)
474 BOpc = Instruction::And;
475 }
476 }
477
478 // If this node is not part of the or/and tree, emit it as a branch.
479 // Note that all nodes in the tree should have same opcode.
480 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
481 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
482 !isValInBlock(V: BOpOp0, BB: CurBB->getBasicBlock()) ||
483 !isValInBlock(V: BOpOp1, BB: CurBB->getBasicBlock())) {
484 emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
485 InvertCond);
486 return;
487 }
488
489 // Create TmpBB after CurBB.
490 MachineFunction::iterator BBI(CurBB);
491 MachineBasicBlock *TmpBB =
492 MF->CreateMachineBasicBlock(BB: CurBB->getBasicBlock());
493 CurBB->getParent()->insert(MBBI: ++BBI, MBB: TmpBB);
494
495 if (Opc == Instruction::Or) {
496 // Codegen X | Y as:
497 // BB1:
498 // jmp_if_X TBB
499 // jmp TmpBB
500 // TmpBB:
501 // jmp_if_Y TBB
502 // jmp FBB
503 //
504
505 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
506 // The requirement is that
507 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
508 // = TrueProb for original BB.
509 // Assuming the original probabilities are A and B, one choice is to set
510 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
511 // A/(1+B) and 2B/(1+B). This choice assumes that
512 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
513 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
514 // TmpBB, but the math is more complicated.
515
516 auto NewTrueProb = TProb / 2;
517 auto NewFalseProb = TProb / 2 + FProb;
518 // Emit the LHS condition.
519 findMergedConditions(Cond: BOpOp0, TBB, FBB: TmpBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
520 FProb: NewFalseProb, InvertCond);
521
522 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
523 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
524 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
525 // Emit the RHS condition into TmpBB.
526 findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
527 FProb: Probs[1], InvertCond);
528 } else {
529 assert(Opc == Instruction::And && "Unknown merge op!");
530 // Codegen X & Y as:
531 // BB1:
532 // jmp_if_X TmpBB
533 // jmp FBB
534 // TmpBB:
535 // jmp_if_Y TBB
536 // jmp FBB
537 //
538 // This requires creation of TmpBB after CurBB.
539
540 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
541 // The requirement is that
542 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
543 // = FalseProb for original BB.
544 // Assuming the original probabilities are A and B, one choice is to set
545 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
546 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
547 // TrueProb for BB1 * FalseProb for TmpBB.
548
549 auto NewTrueProb = TProb + FProb / 2;
550 auto NewFalseProb = FProb / 2;
551 // Emit the LHS condition.
552 findMergedConditions(Cond: BOpOp0, TBB: TmpBB, FBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
553 FProb: NewFalseProb, InvertCond);
554
555 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
556 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
557 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
558 // Emit the RHS condition into TmpBB.
559 findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
560 FProb: Probs[1], InvertCond);
561 }
562}
563
564bool IRTranslator::shouldEmitAsBranches(
565 const std::vector<SwitchCG::CaseBlock> &Cases) {
566 // For multiple cases, it's better to emit as branches.
567 if (Cases.size() != 2)
568 return true;
569
570 // If this is two comparisons of the same values or'd or and'd together, they
571 // will get folded into a single comparison, so don't emit two blocks.
572 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
573 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
574 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
575 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
576 return false;
577 }
578
579 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
580 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
581 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
582 Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
583 isa<Constant>(Val: Cases[0].CmpRHS) &&
584 cast<Constant>(Val: Cases[0].CmpRHS)->isNullValue()) {
585 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
586 Cases[0].TrueBB == Cases[1].ThisBB)
587 return false;
588 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
589 Cases[0].FalseBB == Cases[1].ThisBB)
590 return false;
591 }
592
593 return true;
594}
595
596bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
597 const BranchInst &BrInst = cast<BranchInst>(Val: U);
598 auto &CurMBB = MIRBuilder.getMBB();
599 auto *Succ0MBB = &getMBB(BB: *BrInst.getSuccessor(i: 0));
600
601 if (BrInst.isUnconditional()) {
602 // If the unconditional target is the layout successor, fallthrough.
603 if (OptLevel == CodeGenOptLevel::None ||
604 !CurMBB.isLayoutSuccessor(MBB: Succ0MBB))
605 MIRBuilder.buildBr(Dest&: *Succ0MBB);
606
607 // Link successors.
608 for (const BasicBlock *Succ : successors(I: &BrInst))
609 CurMBB.addSuccessor(Succ: &getMBB(BB: *Succ));
610 return true;
611 }
612
613 // If this condition is one of the special cases we handle, do special stuff
614 // now.
615 const Value *CondVal = BrInst.getCondition();
616 MachineBasicBlock *Succ1MBB = &getMBB(BB: *BrInst.getSuccessor(i: 1));
617
618 // If this is a series of conditions that are or'd or and'd together, emit
619 // this as a sequence of branches instead of setcc's with and/or operations.
620 // As long as jumps are not expensive (exceptions for multi-use logic ops,
621 // unpredictable branches, and vector extracts because those jumps are likely
622 // expensive for any target), this should improve performance.
623 // For example, instead of something like:
624 // cmp A, B
625 // C = seteq
626 // cmp D, E
627 // F = setle
628 // or C, F
629 // jnz foo
630 // Emit:
631 // cmp A, B
632 // je foo
633 // cmp D, E
634 // jle foo
635 using namespace PatternMatch;
636 const Instruction *CondI = dyn_cast<Instruction>(Val: CondVal);
637 if (!TLI->isJumpExpensive() && CondI && CondI->hasOneUse() &&
638 !BrInst.hasMetadata(KindID: LLVMContext::MD_unpredictable)) {
639 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
640 Value *Vec;
641 const Value *BOp0, *BOp1;
642 if (match(V: CondI, P: m_LogicalAnd(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
643 Opcode = Instruction::And;
644 else if (match(V: CondI, P: m_LogicalOr(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
645 Opcode = Instruction::Or;
646
647 if (Opcode && !(match(V: BOp0, P: m_ExtractElt(Val: m_Value(V&: Vec), Idx: m_Value())) &&
648 match(V: BOp1, P: m_ExtractElt(Val: m_Specific(V: Vec), Idx: m_Value())))) {
649 findMergedConditions(Cond: CondI, TBB: Succ0MBB, FBB: Succ1MBB, CurBB: &CurMBB, SwitchBB: &CurMBB, Opc: Opcode,
650 TProb: getEdgeProbability(Src: &CurMBB, Dst: Succ0MBB),
651 FProb: getEdgeProbability(Src: &CurMBB, Dst: Succ1MBB),
652 /*InvertCond=*/false);
653 assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
654
655 // Allow some cases to be rejected.
656 if (shouldEmitAsBranches(Cases: SL->SwitchCases)) {
657 // Emit the branch for this block.
658 emitSwitchCase(CB&: SL->SwitchCases[0], SwitchBB: &CurMBB, MIB&: *CurBuilder);
659 SL->SwitchCases.erase(position: SL->SwitchCases.begin());
660 return true;
661 }
662
663 // Okay, we decided not to do this, remove any inserted MBB's and clear
664 // SwitchCases.
665 for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
666 MF->erase(MBBI: SL->SwitchCases[I].ThisBB);
667
668 SL->SwitchCases.clear();
669 }
670 }
671
672 // Create a CaseBlock record representing this branch.
673 SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
674 ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
675 nullptr, Succ0MBB, Succ1MBB, &CurMBB,
676 CurBuilder->getDebugLoc());
677
678 // Use emitSwitchCase to actually insert the fast branch sequence for this
679 // cond branch.
680 emitSwitchCase(CB, SwitchBB: &CurMBB, MIB&: *CurBuilder);
681 return true;
682}
683
684void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
685 MachineBasicBlock *Dst,
686 BranchProbability Prob) {
687 if (!FuncInfo.BPI) {
688 Src->addSuccessorWithoutProb(Succ: Dst);
689 return;
690 }
691 if (Prob.isUnknown())
692 Prob = getEdgeProbability(Src, Dst);
693 Src->addSuccessor(Succ: Dst, Prob);
694}
695
696BranchProbability
697IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
698 const MachineBasicBlock *Dst) const {
699 const BasicBlock *SrcBB = Src->getBasicBlock();
700 const BasicBlock *DstBB = Dst->getBasicBlock();
701 if (!FuncInfo.BPI) {
702 // If BPI is not available, set the default probability as 1 / N, where N is
703 // the number of successors.
704 auto SuccSize = std::max<uint32_t>(a: succ_size(BB: SrcBB), b: 1);
705 return BranchProbability(1, SuccSize);
706 }
707 return FuncInfo.BPI->getEdgeProbability(Src: SrcBB, Dst: DstBB);
708}
709
710bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
711 using namespace SwitchCG;
712 // Extract cases from the switch.
713 const SwitchInst &SI = cast<SwitchInst>(Val: U);
714 BranchProbabilityInfo *BPI = FuncInfo.BPI;
715 CaseClusterVector Clusters;
716 Clusters.reserve(n: SI.getNumCases());
717 for (const auto &I : SI.cases()) {
718 MachineBasicBlock *Succ = &getMBB(BB: *I.getCaseSuccessor());
719 assert(Succ && "Could not find successor mbb in mapping");
720 const ConstantInt *CaseVal = I.getCaseValue();
721 BranchProbability Prob =
722 BPI ? BPI->getEdgeProbability(Src: SI.getParent(), IndexInSuccessors: I.getSuccessorIndex())
723 : BranchProbability(1, SI.getNumCases() + 1);
724 Clusters.push_back(x: CaseCluster::range(Low: CaseVal, High: CaseVal, MBB: Succ, Prob));
725 }
726
727 MachineBasicBlock *DefaultMBB = &getMBB(BB: *SI.getDefaultDest());
728
729 // Cluster adjacent cases with the same destination. We do this at all
730 // optimization levels because it's cheap to do and will make codegen faster
731 // if there are many clusters.
732 sortAndRangeify(Clusters);
733
734 MachineBasicBlock *SwitchMBB = &getMBB(BB: *SI.getParent());
735
736 // If there is only the default destination, jump there directly.
737 if (Clusters.empty()) {
738 SwitchMBB->addSuccessor(Succ: DefaultMBB);
739 if (DefaultMBB != SwitchMBB->getNextNode())
740 MIB.buildBr(Dest&: *DefaultMBB);
741 return true;
742 }
743
744 SL->findJumpTables(Clusters, SI: &SI, SL: std::nullopt, DefaultMBB, PSI: nullptr, BFI: nullptr);
745 SL->findBitTestClusters(Clusters, SI: &SI);
746
747 LLVM_DEBUG({
748 dbgs() << "Case clusters: ";
749 for (const CaseCluster &C : Clusters) {
750 if (C.Kind == CC_JumpTable)
751 dbgs() << "JT:";
752 if (C.Kind == CC_BitTests)
753 dbgs() << "BT:";
754
755 C.Low->getValue().print(dbgs(), true);
756 if (C.Low != C.High) {
757 dbgs() << '-';
758 C.High->getValue().print(dbgs(), true);
759 }
760 dbgs() << ' ';
761 }
762 dbgs() << '\n';
763 });
764
765 assert(!Clusters.empty());
766 SwitchWorkList WorkList;
767 CaseClusterIt First = Clusters.begin();
768 CaseClusterIt Last = Clusters.end() - 1;
769 auto DefaultProb = getEdgeProbability(Src: SwitchMBB, Dst: DefaultMBB);
770 WorkList.push_back(Elt: {.MBB: SwitchMBB, .FirstCluster: First, .LastCluster: Last, .GE: nullptr, .LT: nullptr, .DefaultProb: DefaultProb});
771
772 while (!WorkList.empty()) {
773 SwitchWorkListItem W = WorkList.pop_back_val();
774
775 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
776 // For optimized builds, lower large range as a balanced binary tree.
777 if (NumClusters > 3 &&
778 MF->getTarget().getOptLevel() != CodeGenOptLevel::None &&
779 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
780 splitWorkItem(WorkList, W, Cond: SI.getCondition(), SwitchMBB, MIB);
781 continue;
782 }
783
784 if (!lowerSwitchWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
785 return false;
786 }
787 return true;
788}
789
790void IRTranslator::splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
791 const SwitchCG::SwitchWorkListItem &W,
792 Value *Cond, MachineBasicBlock *SwitchMBB,
793 MachineIRBuilder &MIB) {
794 using namespace SwitchCG;
795 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
796 "Clusters not sorted?");
797 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
798
799 auto [LastLeft, FirstRight, LeftProb, RightProb] =
800 SL->computeSplitWorkItemInfo(W);
801
802 // Use the first element on the right as pivot since we will make less-than
803 // comparisons against it.
804 CaseClusterIt PivotCluster = FirstRight;
805 assert(PivotCluster > W.FirstCluster);
806 assert(PivotCluster <= W.LastCluster);
807
808 CaseClusterIt FirstLeft = W.FirstCluster;
809 CaseClusterIt LastRight = W.LastCluster;
810
811 const ConstantInt *Pivot = PivotCluster->Low;
812
813 // New blocks will be inserted immediately after the current one.
814 MachineFunction::iterator BBI(W.MBB);
815 ++BBI;
816
817 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
818 // we can branch to its destination directly if it's squeezed exactly in
819 // between the known lower bound and Pivot - 1.
820 MachineBasicBlock *LeftMBB;
821 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
822 FirstLeft->Low == W.GE &&
823 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
824 LeftMBB = FirstLeft->MBB;
825 } else {
826 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
827 FuncInfo.MF->insert(MBBI: BBI, MBB: LeftMBB);
828 WorkList.push_back(
829 Elt: {.MBB: LeftMBB, .FirstCluster: FirstLeft, .LastCluster: LastLeft, .GE: W.GE, .LT: Pivot, .DefaultProb: W.DefaultProb / 2});
830 }
831
832 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
833 // single cluster, RHS.Low == Pivot, and we can branch to its destination
834 // directly if RHS.High equals the current upper bound.
835 MachineBasicBlock *RightMBB;
836 if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT &&
837 (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
838 RightMBB = FirstRight->MBB;
839 } else {
840 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
841 FuncInfo.MF->insert(MBBI: BBI, MBB: RightMBB);
842 WorkList.push_back(
843 Elt: {.MBB: RightMBB, .FirstCluster: FirstRight, .LastCluster: LastRight, .GE: Pivot, .LT: W.LT, .DefaultProb: W.DefaultProb / 2});
844 }
845
846 // Create the CaseBlock record that will be used to lower the branch.
847 CaseBlock CB(ICmpInst::Predicate::ICMP_SLT, false, Cond, Pivot, nullptr,
848 LeftMBB, RightMBB, W.MBB, MIB.getDebugLoc(), LeftProb,
849 RightProb);
850
851 if (W.MBB == SwitchMBB)
852 emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
853 else
854 SL->SwitchCases.push_back(x: CB);
855}
856
857void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
858 MachineBasicBlock *MBB) {
859 // Emit the code for the jump table
860 assert(JT.Reg && "Should lower JT Header first!");
861 MachineIRBuilder MIB(*MBB->getParent());
862 MIB.setMBB(*MBB);
863 MIB.setDebugLoc(CurBuilder->getDebugLoc());
864
865 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
866 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
867
868 auto Table = MIB.buildJumpTable(PtrTy, JTI: JT.JTI);
869 MIB.buildBrJT(TablePtr: Table.getReg(Idx: 0), JTI: JT.JTI, IndexReg: JT.Reg);
870}
871
872bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
873 SwitchCG::JumpTableHeader &JTH,
874 MachineBasicBlock *HeaderBB) {
875 MachineIRBuilder MIB(*HeaderBB->getParent());
876 MIB.setMBB(*HeaderBB);
877 MIB.setDebugLoc(CurBuilder->getDebugLoc());
878
879 const Value &SValue = *JTH.SValue;
880 // Subtract the lowest switch case value from the value being switched on.
881 const LLT SwitchTy = getLLTForType(Ty&: *SValue.getType(), DL: *DL);
882 Register SwitchOpReg = getOrCreateVReg(Val: SValue);
883 auto FirstCst = MIB.buildConstant(Res: SwitchTy, Val: JTH.First);
884 auto Sub = MIB.buildSub(Dst: {SwitchTy}, Src0: SwitchOpReg, Src1: FirstCst);
885
886 // This value may be smaller or larger than the target's pointer type, and
887 // therefore require extension or truncating.
888 auto *PtrIRTy = PointerType::getUnqual(C&: SValue.getContext());
889 const LLT PtrScalarTy = LLT::scalar(SizeInBits: DL->getTypeSizeInBits(Ty: PtrIRTy));
890 Sub = MIB.buildZExtOrTrunc(Res: PtrScalarTy, Op: Sub);
891
892 JT.Reg = Sub.getReg(Idx: 0);
893
894 if (JTH.FallthroughUnreachable) {
895 if (JT.MBB != HeaderBB->getNextNode())
896 MIB.buildBr(Dest&: *JT.MBB);
897 return true;
898 }
899
900 // Emit the range check for the jump table, and branch to the default block
901 // for the switch statement if the value being switched on exceeds the
902 // largest case in the switch.
903 auto Cst = getOrCreateVReg(
904 Val: *ConstantInt::get(Ty: SValue.getType(), V: JTH.Last - JTH.First));
905 Cst = MIB.buildZExtOrTrunc(Res: PtrScalarTy, Op: Cst).getReg(Idx: 0);
906 auto Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_UGT, Res: LLT::scalar(SizeInBits: 1), Op0: Sub, Op1: Cst);
907
908 auto BrCond = MIB.buildBrCond(Tst: Cmp.getReg(Idx: 0), Dest&: *JT.Default);
909
910 // Avoid emitting unnecessary branches to the next block.
911 if (JT.MBB != HeaderBB->getNextNode())
912 BrCond = MIB.buildBr(Dest&: *JT.MBB);
913 return true;
914}
915
916void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
917 MachineBasicBlock *SwitchBB,
918 MachineIRBuilder &MIB) {
919 Register CondLHS = getOrCreateVReg(Val: *CB.CmpLHS);
920 Register Cond;
921 DebugLoc OldDbgLoc = MIB.getDebugLoc();
922 MIB.setDebugLoc(CB.DbgLoc);
923 MIB.setMBB(*CB.ThisBB);
924
925 if (CB.PredInfo.NoCmp) {
926 // Branch or fall through to TrueBB.
927 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
928 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
929 NewPred: CB.ThisBB);
930 CB.ThisBB->normalizeSuccProbs();
931 if (CB.TrueBB != CB.ThisBB->getNextNode())
932 MIB.buildBr(Dest&: *CB.TrueBB);
933 MIB.setDebugLoc(OldDbgLoc);
934 return;
935 }
936
937 const LLT i1Ty = LLT::scalar(SizeInBits: 1);
938 // Build the compare.
939 if (!CB.CmpMHS) {
940 const auto *CI = dyn_cast<ConstantInt>(Val: CB.CmpRHS);
941 // For conditional branch lowering, we might try to do something silly like
942 // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
943 // just re-use the existing condition vreg.
944 if (MRI->getType(Reg: CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
945 CB.PredInfo.Pred == CmpInst::ICMP_EQ) {
946 Cond = CondLHS;
947 } else {
948 Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
949 if (CmpInst::isFPPredicate(P: CB.PredInfo.Pred))
950 Cond =
951 MIB.buildFCmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
952 else
953 Cond =
954 MIB.buildICmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
955 }
956 } else {
957 assert(CB.PredInfo.Pred == CmpInst::ICMP_SLE &&
958 "Can only handle SLE ranges");
959
960 const APInt& Low = cast<ConstantInt>(Val: CB.CmpLHS)->getValue();
961 const APInt& High = cast<ConstantInt>(Val: CB.CmpRHS)->getValue();
962
963 Register CmpOpReg = getOrCreateVReg(Val: *CB.CmpMHS);
964 if (cast<ConstantInt>(Val: CB.CmpLHS)->isMinValue(IsSigned: true)) {
965 Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
966 Cond =
967 MIB.buildICmp(Pred: CmpInst::ICMP_SLE, Res: i1Ty, Op0: CmpOpReg, Op1: CondRHS).getReg(Idx: 0);
968 } else {
969 const LLT CmpTy = MRI->getType(Reg: CmpOpReg);
970 auto Sub = MIB.buildSub(Dst: {CmpTy}, Src0: CmpOpReg, Src1: CondLHS);
971 auto Diff = MIB.buildConstant(Res: CmpTy, Val: High - Low);
972 Cond = MIB.buildICmp(Pred: CmpInst::ICMP_ULE, Res: i1Ty, Op0: Sub, Op1: Diff).getReg(Idx: 0);
973 }
974 }
975
976 // Update successor info
977 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
978
979 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
980 NewPred: CB.ThisBB);
981
982 // TrueBB and FalseBB are always different unless the incoming IR is
983 // degenerate. This only happens when running llc on weird IR.
984 if (CB.TrueBB != CB.FalseBB)
985 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.FalseBB, Prob: CB.FalseProb);
986 CB.ThisBB->normalizeSuccProbs();
987
988 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
989 NewPred: CB.ThisBB);
990
991 MIB.buildBrCond(Tst: Cond, Dest&: *CB.TrueBB);
992 MIB.buildBr(Dest&: *CB.FalseBB);
993 MIB.setDebugLoc(OldDbgLoc);
994}
995
996bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
997 MachineBasicBlock *SwitchMBB,
998 MachineBasicBlock *CurMBB,
999 MachineBasicBlock *DefaultMBB,
1000 MachineIRBuilder &MIB,
1001 MachineFunction::iterator BBI,
1002 BranchProbability UnhandledProbs,
1003 SwitchCG::CaseClusterIt I,
1004 MachineBasicBlock *Fallthrough,
1005 bool FallthroughUnreachable) {
1006 using namespace SwitchCG;
1007 MachineFunction *CurMF = SwitchMBB->getParent();
1008 // FIXME: Optimize away range check based on pivot comparisons.
1009 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
1010 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
1011 BranchProbability DefaultProb = W.DefaultProb;
1012
1013 // The jump block hasn't been inserted yet; insert it here.
1014 MachineBasicBlock *JumpMBB = JT->MBB;
1015 CurMF->insert(MBBI: BBI, MBB: JumpMBB);
1016
1017 // Since the jump table block is separate from the switch block, we need
1018 // to keep track of it as a machine predecessor to the default block,
1019 // otherwise we lose the phi edges.
1020 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1021 NewPred: CurMBB);
1022 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1023 NewPred: JumpMBB);
1024
1025 auto JumpProb = I->Prob;
1026 auto FallthroughProb = UnhandledProbs;
1027
1028 // If the default statement is a target of the jump table, we evenly
1029 // distribute the default probability to successors of CurMBB. Also
1030 // update the probability on the edge from JumpMBB to Fallthrough.
1031 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
1032 SE = JumpMBB->succ_end();
1033 SI != SE; ++SI) {
1034 if (*SI == DefaultMBB) {
1035 JumpProb += DefaultProb / 2;
1036 FallthroughProb -= DefaultProb / 2;
1037 JumpMBB->setSuccProbability(I: SI, Prob: DefaultProb / 2);
1038 JumpMBB->normalizeSuccProbs();
1039 } else {
1040 // Also record edges from the jump table block to it's successors.
1041 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
1042 NewPred: JumpMBB);
1043 }
1044 }
1045
1046 if (FallthroughUnreachable)
1047 JTH->FallthroughUnreachable = true;
1048
1049 if (!JTH->FallthroughUnreachable)
1050 addSuccessorWithProb(Src: CurMBB, Dst: Fallthrough, Prob: FallthroughProb);
1051 addSuccessorWithProb(Src: CurMBB, Dst: JumpMBB, Prob: JumpProb);
1052 CurMBB->normalizeSuccProbs();
1053
1054 // The jump table header will be inserted in our current block, do the
1055 // range check, and fall through to our fallthrough block.
1056 JTH->HeaderBB = CurMBB;
1057 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
1058
1059 // If we're in the right place, emit the jump table header right now.
1060 if (CurMBB == SwitchMBB) {
1061 if (!emitJumpTableHeader(JT&: *JT, JTH&: *JTH, HeaderBB: CurMBB))
1062 return false;
1063 JTH->Emitted = true;
1064 }
1065 return true;
1066}
1067bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
1068 Value *Cond,
1069 MachineBasicBlock *Fallthrough,
1070 bool FallthroughUnreachable,
1071 BranchProbability UnhandledProbs,
1072 MachineBasicBlock *CurMBB,
1073 MachineIRBuilder &MIB,
1074 MachineBasicBlock *SwitchMBB) {
1075 using namespace SwitchCG;
1076 const Value *RHS, *LHS, *MHS;
1077 CmpInst::Predicate Pred;
1078 if (I->Low == I->High) {
1079 // Check Cond == I->Low.
1080 Pred = CmpInst::ICMP_EQ;
1081 LHS = Cond;
1082 RHS = I->Low;
1083 MHS = nullptr;
1084 } else {
1085 // Check I->Low <= Cond <= I->High.
1086 Pred = CmpInst::ICMP_SLE;
1087 LHS = I->Low;
1088 MHS = Cond;
1089 RHS = I->High;
1090 }
1091
1092 // If Fallthrough is unreachable, fold away the comparison.
1093 // The false probability is the sum of all unhandled cases.
1094 CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
1095 CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
1096
1097 emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
1098 return true;
1099}
1100
1101void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B,
1102 MachineBasicBlock *SwitchBB) {
1103 MachineIRBuilder &MIB = *CurBuilder;
1104 MIB.setMBB(*SwitchBB);
1105
1106 // Subtract the minimum value.
1107 Register SwitchOpReg = getOrCreateVReg(Val: *B.SValue);
1108
1109 LLT SwitchOpTy = MRI->getType(Reg: SwitchOpReg);
1110 Register MinValReg = MIB.buildConstant(Res: SwitchOpTy, Val: B.First).getReg(Idx: 0);
1111 auto RangeSub = MIB.buildSub(Dst: SwitchOpTy, Src0: SwitchOpReg, Src1: MinValReg);
1112
1113 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
1114 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1115
1116 LLT MaskTy = SwitchOpTy;
1117 if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
1118 !llvm::has_single_bit<uint32_t>(Value: MaskTy.getSizeInBits()))
1119 MaskTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
1120 else {
1121 // Ensure that the type will fit the mask value.
1122 for (const SwitchCG::BitTestCase &Case : B.Cases) {
1123 if (!isUIntN(N: SwitchOpTy.getSizeInBits(), x: Case.Mask)) {
1124 // Switch table case range are encoded into series of masks.
1125 // Just use pointer type, it's guaranteed to fit.
1126 MaskTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
1127 break;
1128 }
1129 }
1130 }
1131 Register SubReg = RangeSub.getReg(Idx: 0);
1132 if (SwitchOpTy != MaskTy)
1133 SubReg = MIB.buildZExtOrTrunc(Res: MaskTy, Op: SubReg).getReg(Idx: 0);
1134
1135 B.RegVT = getMVTForLLT(Ty: MaskTy);
1136 B.Reg = SubReg;
1137
1138 MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1139
1140 if (!B.FallthroughUnreachable)
1141 addSuccessorWithProb(Src: SwitchBB, Dst: B.Default, Prob: B.DefaultProb);
1142 addSuccessorWithProb(Src: SwitchBB, Dst: MBB, Prob: B.Prob);
1143
1144 SwitchBB->normalizeSuccProbs();
1145
1146 if (!B.FallthroughUnreachable) {
1147 // Conditional branch to the default block.
1148 auto RangeCst = MIB.buildConstant(Res: SwitchOpTy, Val: B.Range);
1149 auto RangeCmp = MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: LLT::scalar(SizeInBits: 1),
1150 Op0: RangeSub, Op1: RangeCst);
1151 MIB.buildBrCond(Tst: RangeCmp, Dest&: *B.Default);
1152 }
1153
1154 // Avoid emitting unnecessary branches to the next block.
1155 if (MBB != SwitchBB->getNextNode())
1156 MIB.buildBr(Dest&: *MBB);
1157}
1158
1159void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1160 MachineBasicBlock *NextMBB,
1161 BranchProbability BranchProbToNext,
1162 Register Reg, SwitchCG::BitTestCase &B,
1163 MachineBasicBlock *SwitchBB) {
1164 MachineIRBuilder &MIB = *CurBuilder;
1165 MIB.setMBB(*SwitchBB);
1166
1167 LLT SwitchTy = getLLTForMVT(Ty: BB.RegVT);
1168 Register Cmp;
1169 unsigned PopCount = llvm::popcount(Value: B.Mask);
1170 if (PopCount == 1) {
1171 // Testing for a single bit; just compare the shift count with what it
1172 // would need to be to shift a 1 bit in that position.
1173 auto MaskTrailingZeros =
1174 MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_zero(Val: B.Mask));
1175 Cmp =
1176 MIB.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: LLT::scalar(SizeInBits: 1), Op0: Reg, Op1: MaskTrailingZeros)
1177 .getReg(Idx: 0);
1178 } else if (PopCount == BB.Range) {
1179 // There is only one zero bit in the range, test for it directly.
1180 auto MaskTrailingOnes =
1181 MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_one(Value: B.Mask));
1182 Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::scalar(SizeInBits: 1), Op0: Reg, Op1: MaskTrailingOnes)
1183 .getReg(Idx: 0);
1184 } else {
1185 // Make desired shift.
1186 auto CstOne = MIB.buildConstant(Res: SwitchTy, Val: 1);
1187 auto SwitchVal = MIB.buildShl(Dst: SwitchTy, Src0: CstOne, Src1: Reg);
1188
1189 // Emit bit tests and jumps.
1190 auto CstMask = MIB.buildConstant(Res: SwitchTy, Val: B.Mask);
1191 auto AndOp = MIB.buildAnd(Dst: SwitchTy, Src0: SwitchVal, Src1: CstMask);
1192 auto CstZero = MIB.buildConstant(Res: SwitchTy, Val: 0);
1193 Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::scalar(SizeInBits: 1), Op0: AndOp, Op1: CstZero)
1194 .getReg(Idx: 0);
1195 }
1196
1197 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1198 addSuccessorWithProb(Src: SwitchBB, Dst: B.TargetBB, Prob: B.ExtraProb);
1199 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1200 addSuccessorWithProb(Src: SwitchBB, Dst: NextMBB, Prob: BranchProbToNext);
1201 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1202 // one as they are relative probabilities (and thus work more like weights),
1203 // and hence we need to normalize them to let the sum of them become one.
1204 SwitchBB->normalizeSuccProbs();
1205
1206 // Record the fact that the IR edge from the header to the bit test target
1207 // will go through our new block. Neeeded for PHIs to have nodes added.
1208 addMachineCFGPred(Edge: {BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1209 NewPred: SwitchBB);
1210
1211 MIB.buildBrCond(Tst: Cmp, Dest&: *B.TargetBB);
1212
1213 // Avoid emitting unnecessary branches to the next block.
1214 if (NextMBB != SwitchBB->getNextNode())
1215 MIB.buildBr(Dest&: *NextMBB);
1216}
1217
1218bool IRTranslator::lowerBitTestWorkItem(
1219 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
1220 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1221 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
1222 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1223 SwitchCG::CaseClusterIt I, MachineBasicBlock *Fallthrough,
1224 bool FallthroughUnreachable) {
1225 using namespace SwitchCG;
1226 MachineFunction *CurMF = SwitchMBB->getParent();
1227 // FIXME: Optimize away range check based on pivot comparisons.
1228 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
1229 // The bit test blocks haven't been inserted yet; insert them here.
1230 for (BitTestCase &BTC : BTB->Cases)
1231 CurMF->insert(MBBI: BBI, MBB: BTC.ThisBB);
1232
1233 // Fill in fields of the BitTestBlock.
1234 BTB->Parent = CurMBB;
1235 BTB->Default = Fallthrough;
1236
1237 BTB->DefaultProb = UnhandledProbs;
1238 // If the cases in bit test don't form a contiguous range, we evenly
1239 // distribute the probability on the edge to Fallthrough to two
1240 // successors of CurMBB.
1241 if (!BTB->ContiguousRange) {
1242 BTB->Prob += DefaultProb / 2;
1243 BTB->DefaultProb -= DefaultProb / 2;
1244 }
1245
1246 if (FallthroughUnreachable)
1247 BTB->FallthroughUnreachable = true;
1248
1249 // If we're in the right place, emit the bit test header right now.
1250 if (CurMBB == SwitchMBB) {
1251 emitBitTestHeader(B&: *BTB, SwitchBB: SwitchMBB);
1252 BTB->Emitted = true;
1253 }
1254 return true;
1255}
1256
1257bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
1258 Value *Cond,
1259 MachineBasicBlock *SwitchMBB,
1260 MachineBasicBlock *DefaultMBB,
1261 MachineIRBuilder &MIB) {
1262 using namespace SwitchCG;
1263 MachineFunction *CurMF = FuncInfo.MF;
1264 MachineBasicBlock *NextMBB = nullptr;
1265 MachineFunction::iterator BBI(W.MBB);
1266 if (++BBI != FuncInfo.MF->end())
1267 NextMBB = &*BBI;
1268
1269 if (EnableOpts) {
1270 // Here, we order cases by probability so the most likely case will be
1271 // checked first. However, two clusters can have the same probability in
1272 // which case their relative ordering is non-deterministic. So we use Low
1273 // as a tie-breaker as clusters are guaranteed to never overlap.
1274 llvm::sort(Start: W.FirstCluster, End: W.LastCluster + 1,
1275 Comp: [](const CaseCluster &a, const CaseCluster &b) {
1276 return a.Prob != b.Prob
1277 ? a.Prob > b.Prob
1278 : a.Low->getValue().slt(RHS: b.Low->getValue());
1279 });
1280
1281 // Rearrange the case blocks so that the last one falls through if possible
1282 // without changing the order of probabilities.
1283 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
1284 --I;
1285 if (I->Prob > W.LastCluster->Prob)
1286 break;
1287 if (I->Kind == CC_Range && I->MBB == NextMBB) {
1288 std::swap(a&: *I, b&: *W.LastCluster);
1289 break;
1290 }
1291 }
1292 }
1293
1294 // Compute total probability.
1295 BranchProbability DefaultProb = W.DefaultProb;
1296 BranchProbability UnhandledProbs = DefaultProb;
1297 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
1298 UnhandledProbs += I->Prob;
1299
1300 MachineBasicBlock *CurMBB = W.MBB;
1301 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
1302 bool FallthroughUnreachable = false;
1303 MachineBasicBlock *Fallthrough;
1304 if (I == W.LastCluster) {
1305 // For the last cluster, fall through to the default destination.
1306 Fallthrough = DefaultMBB;
1307 FallthroughUnreachable = isa<UnreachableInst>(
1308 Val: DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
1309 } else {
1310 Fallthrough = CurMF->CreateMachineBasicBlock(BB: CurMBB->getBasicBlock());
1311 CurMF->insert(MBBI: BBI, MBB: Fallthrough);
1312 }
1313 UnhandledProbs -= I->Prob;
1314
1315 switch (I->Kind) {
1316 case CC_BitTests: {
1317 if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1318 DefaultProb, UnhandledProbs, I, Fallthrough,
1319 FallthroughUnreachable)) {
1320 LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
1321 return false;
1322 }
1323 break;
1324 }
1325
1326 case CC_JumpTable: {
1327 if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1328 UnhandledProbs, I, Fallthrough,
1329 FallthroughUnreachable)) {
1330 LLVM_DEBUG(dbgs() << "Failed to lower jump table");
1331 return false;
1332 }
1333 break;
1334 }
1335 case CC_Range: {
1336 if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
1337 FallthroughUnreachable, UnhandledProbs,
1338 CurMBB, MIB, SwitchMBB)) {
1339 LLVM_DEBUG(dbgs() << "Failed to lower switch range");
1340 return false;
1341 }
1342 break;
1343 }
1344 }
1345 CurMBB = Fallthrough;
1346 }
1347
1348 return true;
1349}
1350
1351bool IRTranslator::translateIndirectBr(const User &U,
1352 MachineIRBuilder &MIRBuilder) {
1353 const IndirectBrInst &BrInst = cast<IndirectBrInst>(Val: U);
1354
1355 const Register Tgt = getOrCreateVReg(Val: *BrInst.getAddress());
1356 MIRBuilder.buildBrIndirect(Tgt);
1357
1358 // Link successors.
1359 SmallPtrSet<const BasicBlock *, 32> AddedSuccessors;
1360 MachineBasicBlock &CurBB = MIRBuilder.getMBB();
1361 for (const BasicBlock *Succ : successors(I: &BrInst)) {
1362 // It's legal for indirectbr instructions to have duplicate blocks in the
1363 // destination list. We don't allow this in MIR. Skip anything that's
1364 // already a successor.
1365 if (!AddedSuccessors.insert(Ptr: Succ).second)
1366 continue;
1367 CurBB.addSuccessor(Succ: &getMBB(BB: *Succ));
1368 }
1369
1370 return true;
1371}
1372
1373static bool isSwiftError(const Value *V) {
1374 if (auto Arg = dyn_cast<Argument>(Val: V))
1375 return Arg->hasSwiftErrorAttr();
1376 if (auto AI = dyn_cast<AllocaInst>(Val: V))
1377 return AI->isSwiftError();
1378 return false;
1379}
1380
1381bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
1382 const LoadInst &LI = cast<LoadInst>(Val: U);
1383 TypeSize StoreSize = DL->getTypeStoreSize(Ty: LI.getType());
1384 if (StoreSize.isZero())
1385 return true;
1386
1387 ArrayRef<Register> Regs = getOrCreateVRegs(Val: LI);
1388 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: LI);
1389 Register Base = getOrCreateVReg(Val: *LI.getPointerOperand());
1390 AAMDNodes AAInfo = LI.getAAMetadata();
1391
1392 const Value *Ptr = LI.getPointerOperand();
1393 Type *OffsetIRTy = DL->getIndexType(PtrTy: Ptr->getType());
1394 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1395
1396 if (CLI->supportSwiftError() && isSwiftError(V: Ptr)) {
1397 assert(Regs.size() == 1 && "swifterror should be single pointer");
1398 Register VReg =
1399 SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);
1400 MIRBuilder.buildCopy(Res: Regs[0], Op: VReg);
1401 return true;
1402 }
1403
1404 MachineMemOperand::Flags Flags =
1405 TLI->getLoadMemOperandFlags(LI, DL: *DL, AC, LibInfo);
1406 if (AA && !(Flags & MachineMemOperand::MOInvariant)) {
1407 if (AA->pointsToConstantMemory(
1408 Loc: MemoryLocation(Ptr, LocationSize::precise(Value: StoreSize), AAInfo))) {
1409 Flags |= MachineMemOperand::MOInvariant;
1410 }
1411 }
1412
1413 const MDNode *Ranges =
1414 Regs.size() == 1 ? LI.getMetadata(KindID: LLVMContext::MD_range) : nullptr;
1415 for (unsigned i = 0; i < Regs.size(); ++i) {
1416 Register Addr;
1417 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i]);
1418
1419 MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i]);
1420 Align BaseAlign = getMemOpAlign(I: LI);
1421 auto MMO =
1422 MF->getMachineMemOperand(PtrInfo: Ptr, f: Flags, MemTy: MRI->getType(Reg: Regs[i]),
1423 base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]), AAInfo,
1424 Ranges, SSID: LI.getSyncScopeID(), Ordering: LI.getOrdering());
1425 MIRBuilder.buildLoad(Res: Regs[i], Addr, MMO&: *MMO);
1426 }
1427
1428 return true;
1429}
1430
1431bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
1432 const StoreInst &SI = cast<StoreInst>(Val: U);
1433 if (DL->getTypeStoreSize(Ty: SI.getValueOperand()->getType()).isZero())
1434 return true;
1435
1436 ArrayRef<Register> Vals = getOrCreateVRegs(Val: *SI.getValueOperand());
1437 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *SI.getValueOperand());
1438 Register Base = getOrCreateVReg(Val: *SI.getPointerOperand());
1439
1440 Type *OffsetIRTy = DL->getIndexType(PtrTy: SI.getPointerOperandType());
1441 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1442
1443 if (CLI->supportSwiftError() && isSwiftError(V: SI.getPointerOperand())) {
1444 assert(Vals.size() == 1 && "swifterror should be single pointer");
1445
1446 Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
1447 SI.getPointerOperand());
1448 MIRBuilder.buildCopy(Res: VReg, Op: Vals[0]);
1449 return true;
1450 }
1451
1452 MachineMemOperand::Flags Flags = TLI->getStoreMemOperandFlags(SI, DL: *DL);
1453
1454 for (unsigned i = 0; i < Vals.size(); ++i) {
1455 Register Addr;
1456 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i]);
1457
1458 MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i]);
1459 Align BaseAlign = getMemOpAlign(I: SI);
1460 auto MMO = MF->getMachineMemOperand(PtrInfo: Ptr, f: Flags, MemTy: MRI->getType(Reg: Vals[i]),
1461 base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]),
1462 AAInfo: SI.getAAMetadata(), Ranges: nullptr,
1463 SSID: SI.getSyncScopeID(), Ordering: SI.getOrdering());
1464 MIRBuilder.buildStore(Val: Vals[i], Addr, MMO&: *MMO);
1465 }
1466 return true;
1467}
1468
1469static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
1470 const Value *Src = U.getOperand(i: 0);
1471 Type *Int32Ty = Type::getInt32Ty(C&: U.getContext());
1472
1473 // getIndexedOffsetInType is designed for GEPs, so the first index is the
1474 // usual array element rather than looking into the actual aggregate.
1475 SmallVector<Value *, 1> Indices;
1476 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: 0));
1477
1478 if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: &U)) {
1479 for (auto Idx : EVI->indices())
1480 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
1481 } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(Val: &U)) {
1482 for (auto Idx : IVI->indices())
1483 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
1484 } else {
1485 llvm::append_range(C&: Indices, R: drop_begin(RangeOrContainer: U.operands()));
1486 }
1487
1488 return static_cast<uint64_t>(
1489 DL.getIndexedOffsetInType(ElemTy: Src->getType(), Indices));
1490}
1491
1492bool IRTranslator::translateExtractValue(const User &U,
1493 MachineIRBuilder &MIRBuilder) {
1494 const Value *Src = U.getOperand(i: 0);
1495 uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
1496 ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
1497 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *Src);
1498 unsigned Idx = llvm::lower_bound(Range&: Offsets, Value&: Offset) - Offsets.begin();
1499 auto &DstRegs = allocateVRegs(Val: U);
1500
1501 for (unsigned i = 0; i < DstRegs.size(); ++i)
1502 DstRegs[i] = SrcRegs[Idx++];
1503
1504 return true;
1505}
1506
1507bool IRTranslator::translateInsertValue(const User &U,
1508 MachineIRBuilder &MIRBuilder) {
1509 const Value *Src = U.getOperand(i: 0);
1510 uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
1511 auto &DstRegs = allocateVRegs(Val: U);
1512 ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(V: U);
1513 ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
1514 ArrayRef<Register> InsertedRegs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
1515 auto *InsertedIt = InsertedRegs.begin();
1516
1517 for (unsigned i = 0; i < DstRegs.size(); ++i) {
1518 if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
1519 DstRegs[i] = *InsertedIt++;
1520 else
1521 DstRegs[i] = SrcRegs[i];
1522 }
1523
1524 return true;
1525}
1526
1527bool IRTranslator::translateSelect(const User &U,
1528 MachineIRBuilder &MIRBuilder) {
1529 Register Tst = getOrCreateVReg(Val: *U.getOperand(i: 0));
1530 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: U);
1531 ArrayRef<Register> Op0Regs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
1532 ArrayRef<Register> Op1Regs = getOrCreateVRegs(Val: *U.getOperand(i: 2));
1533
1534 uint32_t Flags = 0;
1535 if (const SelectInst *SI = dyn_cast<SelectInst>(Val: &U))
1536 Flags = MachineInstr::copyFlagsFromInstruction(I: *SI);
1537
1538 for (unsigned i = 0; i < ResRegs.size(); ++i) {
1539 MIRBuilder.buildSelect(Res: ResRegs[i], Tst, Op0: Op0Regs[i], Op1: Op1Regs[i], Flags);
1540 }
1541
1542 return true;
1543}
1544
1545bool IRTranslator::translateCopy(const User &U, const Value &V,
1546 MachineIRBuilder &MIRBuilder) {
1547 Register Src = getOrCreateVReg(Val: V);
1548 auto &Regs = *VMap.getVRegs(V: U);
1549 if (Regs.empty()) {
1550 Regs.push_back(Elt: Src);
1551 VMap.getOffsets(V: U)->push_back(Elt: 0);
1552 } else {
1553 // If we already assigned a vreg for this instruction, we can't change that.
1554 // Emit a copy to satisfy the users we already emitted.
1555 MIRBuilder.buildCopy(Res: Regs[0], Op: Src);
1556 }
1557 return true;
1558}
1559
1560bool IRTranslator::translateBitCast(const User &U,
1561 MachineIRBuilder &MIRBuilder) {
1562 // If we're bitcasting to the source type, we can reuse the source vreg.
1563 if (getLLTForType(Ty&: *U.getOperand(i: 0)->getType(), DL: *DL) ==
1564 getLLTForType(Ty&: *U.getType(), DL: *DL)) {
1565 // If the source is a ConstantInt then it was probably created by
1566 // ConstantHoisting and we should leave it alone.
1567 if (isa<ConstantInt>(Val: U.getOperand(i: 0)))
1568 return translateCast(Opcode: TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
1569 MIRBuilder);
1570 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
1571 }
1572
1573 return translateCast(Opcode: TargetOpcode::G_BITCAST, U, MIRBuilder);
1574}
1575
1576bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1577 MachineIRBuilder &MIRBuilder) {
1578 if (containsBF16Type(U) && !targetSupportsBF16Type(MF))
1579 return false;
1580
1581 uint32_t Flags = 0;
1582 if (const Instruction *I = dyn_cast<Instruction>(Val: &U))
1583 Flags = MachineInstr::copyFlagsFromInstruction(I: *I);
1584
1585 Register Op = getOrCreateVReg(Val: *U.getOperand(i: 0));
1586 Register Res = getOrCreateVReg(Val: U);
1587 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op}, Flags);
1588 return true;
1589}
1590
1591bool IRTranslator::translateGetElementPtr(const User &U,
1592 MachineIRBuilder &MIRBuilder) {
1593 Value &Op0 = *U.getOperand(i: 0);
1594 Register BaseReg = getOrCreateVReg(Val: Op0);
1595 Type *PtrIRTy = Op0.getType();
1596 LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1597 Type *OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
1598 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1599
1600 uint32_t PtrAddFlags = 0;
1601 // Each PtrAdd generated to implement the GEP inherits its nuw, nusw, inbounds
1602 // flags.
1603 if (const Instruction *I = dyn_cast<Instruction>(Val: &U))
1604 PtrAddFlags = MachineInstr::copyFlagsFromInstruction(I: *I);
1605
1606 auto PtrAddFlagsWithConst = [&](int64_t Offset) {
1607 // For nusw/inbounds GEP with an offset that is nonnegative when interpreted
1608 // as signed, assume there is no unsigned overflow.
1609 if (Offset >= 0 && (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap))
1610 return PtrAddFlags | MachineInstr::MIFlag::NoUWrap;
1611 return PtrAddFlags;
1612 };
1613
1614 // Normalize Vector GEP - all scalar operands should be converted to the
1615 // splat vector.
1616 unsigned VectorWidth = 0;
1617
1618 // True if we should use a splat vector; using VectorWidth alone is not
1619 // sufficient.
1620 bool WantSplatVector = false;
1621 if (auto *VT = dyn_cast<VectorType>(Val: U.getType())) {
1622 VectorWidth = cast<FixedVectorType>(Val: VT)->getNumElements();
1623 // We don't produce 1 x N vectors; those are treated as scalars.
1624 WantSplatVector = VectorWidth > 1;
1625 }
1626
1627 // We might need to splat the base pointer into a vector if the offsets
1628 // are vectors.
1629 if (WantSplatVector && !PtrTy.isVector()) {
1630 BaseReg = MIRBuilder
1631 .buildSplatBuildVector(Res: LLT::fixed_vector(NumElements: VectorWidth, ScalarTy: PtrTy),
1632 Src: BaseReg)
1633 .getReg(Idx: 0);
1634 PtrIRTy = FixedVectorType::get(ElementType: PtrIRTy, NumElts: VectorWidth);
1635 PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1636 OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
1637 OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1638 }
1639
1640 int64_t Offset = 0;
1641 for (gep_type_iterator GTI = gep_type_begin(GEP: &U), E = gep_type_end(GEP: &U);
1642 GTI != E; ++GTI) {
1643 const Value *Idx = GTI.getOperand();
1644 if (StructType *StTy = GTI.getStructTypeOrNull()) {
1645 unsigned Field = cast<Constant>(Val: Idx)->getUniqueInteger().getZExtValue();
1646 Offset += DL->getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
1647 continue;
1648 } else {
1649 uint64_t ElementSize = GTI.getSequentialElementStride(DL: *DL);
1650
1651 // If this is a scalar constant or a splat vector of constants,
1652 // handle it quickly.
1653 if (const auto *CI = dyn_cast<ConstantInt>(Val: Idx)) {
1654 if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {
1655 Offset += ElementSize * *Val;
1656 continue;
1657 }
1658 }
1659
1660 if (Offset != 0) {
1661 auto OffsetMIB = MIRBuilder.buildConstant(Res: {OffsetTy}, Val: Offset);
1662 BaseReg = MIRBuilder
1663 .buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0),
1664 Flags: PtrAddFlagsWithConst(Offset))
1665 .getReg(Idx: 0);
1666 Offset = 0;
1667 }
1668
1669 Register IdxReg = getOrCreateVReg(Val: *Idx);
1670 LLT IdxTy = MRI->getType(Reg: IdxReg);
1671 if (IdxTy != OffsetTy) {
1672 if (!IdxTy.isVector() && WantSplatVector) {
1673 IdxReg = MIRBuilder
1674 .buildSplatBuildVector(Res: OffsetTy.changeElementType(NewEltTy: IdxTy),
1675 Src: IdxReg)
1676 .getReg(Idx: 0);
1677 }
1678
1679 IdxReg = MIRBuilder.buildSExtOrTrunc(Res: OffsetTy, Op: IdxReg).getReg(Idx: 0);
1680 }
1681
1682 // N = N + Idx * ElementSize;
1683 // Avoid doing it for ElementSize of 1.
1684 Register GepOffsetReg;
1685 if (ElementSize != 1) {
1686 auto ElementSizeMIB = MIRBuilder.buildConstant(
1687 Res: getLLTForType(Ty&: *OffsetIRTy, DL: *DL), Val: ElementSize);
1688
1689 // The multiplication is NUW if the GEP is NUW and NSW if the GEP is
1690 // NUSW.
1691 uint32_t ScaleFlags = PtrAddFlags & MachineInstr::MIFlag::NoUWrap;
1692 if (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap)
1693 ScaleFlags |= MachineInstr::MIFlag::NoSWrap;
1694
1695 GepOffsetReg =
1696 MIRBuilder.buildMul(Dst: OffsetTy, Src0: IdxReg, Src1: ElementSizeMIB, Flags: ScaleFlags)
1697 .getReg(Idx: 0);
1698 } else {
1699 GepOffsetReg = IdxReg;
1700 }
1701
1702 BaseReg =
1703 MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: GepOffsetReg, Flags: PtrAddFlags)
1704 .getReg(Idx: 0);
1705 }
1706 }
1707
1708 if (Offset != 0) {
1709 auto OffsetMIB =
1710 MIRBuilder.buildConstant(Res: OffsetTy, Val: Offset);
1711
1712 MIRBuilder.buildPtrAdd(Res: getOrCreateVReg(Val: U), Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0),
1713 Flags: PtrAddFlagsWithConst(Offset));
1714 return true;
1715 }
1716
1717 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: U), Op: BaseReg);
1718 return true;
1719}
1720
1721bool IRTranslator::translateMemFunc(const CallInst &CI,
1722 MachineIRBuilder &MIRBuilder,
1723 unsigned Opcode) {
1724 const Value *SrcPtr = CI.getArgOperand(i: 1);
1725 // If the source is undef, then just emit a nop.
1726 if (isa<UndefValue>(Val: SrcPtr))
1727 return true;
1728
1729 SmallVector<Register, 3> SrcRegs;
1730
1731 unsigned MinPtrSize = UINT_MAX;
1732 for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(x: AI) != AE; ++AI) {
1733 Register SrcReg = getOrCreateVReg(Val: **AI);
1734 LLT SrcTy = MRI->getType(Reg: SrcReg);
1735 if (SrcTy.isPointer())
1736 MinPtrSize = std::min<unsigned>(a: SrcTy.getSizeInBits(), b: MinPtrSize);
1737 SrcRegs.push_back(Elt: SrcReg);
1738 }
1739
1740 LLT SizeTy = LLT::scalar(SizeInBits: MinPtrSize);
1741
1742 // The size operand should be the minimum of the pointer sizes.
1743 Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
1744 if (MRI->getType(Reg: SizeOpReg) != SizeTy)
1745 SizeOpReg = MIRBuilder.buildZExtOrTrunc(Res: SizeTy, Op: SizeOpReg).getReg(Idx: 0);
1746
1747 auto ICall = MIRBuilder.buildInstr(Opcode);
1748 for (Register SrcReg : SrcRegs)
1749 ICall.addUse(RegNo: SrcReg);
1750
1751 Align DstAlign;
1752 Align SrcAlign;
1753 unsigned IsVol =
1754 cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1))->getZExtValue();
1755
1756 ConstantInt *CopySize = nullptr;
1757
1758 if (auto *MCI = dyn_cast<MemCpyInst>(Val: &CI)) {
1759 DstAlign = MCI->getDestAlign().valueOrOne();
1760 SrcAlign = MCI->getSourceAlign().valueOrOne();
1761 CopySize = dyn_cast<ConstantInt>(Val: MCI->getArgOperand(i: 2));
1762 } else if (auto *MMI = dyn_cast<MemMoveInst>(Val: &CI)) {
1763 DstAlign = MMI->getDestAlign().valueOrOne();
1764 SrcAlign = MMI->getSourceAlign().valueOrOne();
1765 CopySize = dyn_cast<ConstantInt>(Val: MMI->getArgOperand(i: 2));
1766 } else {
1767 auto *MSI = cast<MemSetInst>(Val: &CI);
1768 DstAlign = MSI->getDestAlign().valueOrOne();
1769 }
1770
1771 if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {
1772 // We need to propagate the tail call flag from the IR inst as an argument.
1773 // Otherwise, we have to pessimize and assume later that we cannot tail call
1774 // any memory intrinsics.
1775 ICall.addImm(Val: CI.isTailCall() ? 1 : 0);
1776 }
1777
1778 // Create mem operands to store the alignment and volatile info.
1779 MachineMemOperand::Flags LoadFlags = MachineMemOperand::MOLoad;
1780 MachineMemOperand::Flags StoreFlags = MachineMemOperand::MOStore;
1781 if (IsVol) {
1782 LoadFlags |= MachineMemOperand::MOVolatile;
1783 StoreFlags |= MachineMemOperand::MOVolatile;
1784 }
1785
1786 AAMDNodes AAInfo = CI.getAAMetadata();
1787 if (AA && CopySize &&
1788 AA->pointsToConstantMemory(Loc: MemoryLocation(
1789 SrcPtr, LocationSize::precise(Value: CopySize->getZExtValue()), AAInfo))) {
1790 LoadFlags |= MachineMemOperand::MOInvariant;
1791
1792 // FIXME: pointsToConstantMemory probably does not imply dereferenceable,
1793 // but the previous usage implied it did. Probably should check
1794 // isDereferenceableAndAlignedPointer.
1795 LoadFlags |= MachineMemOperand::MODereferenceable;
1796 }
1797
1798 ICall.addMemOperand(
1799 MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(CI.getArgOperand(i: 0)),
1800 F: StoreFlags, Size: 1, BaseAlignment: DstAlign, AAInfo));
1801 if (Opcode != TargetOpcode::G_MEMSET)
1802 ICall.addMemOperand(MMO: MF->getMachineMemOperand(
1803 PtrInfo: MachinePointerInfo(SrcPtr), F: LoadFlags, Size: 1, BaseAlignment: SrcAlign, AAInfo));
1804
1805 return true;
1806}
1807
1808bool IRTranslator::translateTrap(const CallInst &CI,
1809 MachineIRBuilder &MIRBuilder,
1810 unsigned Opcode) {
1811 StringRef TrapFuncName =
1812 CI.getAttributes().getFnAttr(Kind: "trap-func-name").getValueAsString();
1813 if (TrapFuncName.empty()) {
1814 if (Opcode == TargetOpcode::G_UBSANTRAP) {
1815 uint64_t Code = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 0))->getZExtValue();
1816 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: ArrayRef<llvm::SrcOp>{Code});
1817 } else {
1818 MIRBuilder.buildInstr(Opcode);
1819 }
1820 return true;
1821 }
1822
1823 CallLowering::CallLoweringInfo Info;
1824 if (Opcode == TargetOpcode::G_UBSANTRAP)
1825 Info.OrigArgs.push_back(Elt: {getOrCreateVRegs(Val: *CI.getArgOperand(i: 0)),
1826 CI.getArgOperand(i: 0)->getType(), 0});
1827
1828 Info.Callee = MachineOperand::CreateES(SymName: TrapFuncName.data());
1829 Info.CB = &CI;
1830 Info.OrigRet = {Register(), Type::getVoidTy(C&: CI.getContext()), 0};
1831 return CLI->lowerCall(MIRBuilder, Info);
1832}
1833
1834bool IRTranslator::translateVectorInterleave2Intrinsic(
1835 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
1836 assert(CI.getIntrinsicID() == Intrinsic::vector_interleave2 &&
1837 "This function can only be called on the interleave2 intrinsic!");
1838 // Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG).
1839 Register Op0 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
1840 Register Op1 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1));
1841 Register Res = getOrCreateVReg(Val: CI);
1842
1843 LLT OpTy = MRI->getType(Reg: Op0);
1844 MIRBuilder.buildShuffleVector(Res, Src1: Op0, Src2: Op1,
1845 Mask: createInterleaveMask(VF: OpTy.getNumElements(), NumVecs: 2));
1846
1847 return true;
1848}
1849
1850bool IRTranslator::translateVectorDeinterleave2Intrinsic(
1851 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
1852 assert(CI.getIntrinsicID() == Intrinsic::vector_deinterleave2 &&
1853 "This function can only be called on the deinterleave2 intrinsic!");
1854 // Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to
1855 // SelectionDAG).
1856 Register Op = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
1857 auto Undef = MIRBuilder.buildUndef(Res: MRI->getType(Reg: Op));
1858 ArrayRef<Register> Res = getOrCreateVRegs(Val: CI);
1859
1860 LLT ResTy = MRI->getType(Reg: Res[0]);
1861 MIRBuilder.buildShuffleVector(Res: Res[0], Src1: Op, Src2: Undef,
1862 Mask: createStrideMask(Start: 0, Stride: 2, VF: ResTy.getNumElements()));
1863 MIRBuilder.buildShuffleVector(Res: Res[1], Src1: Op, Src2: Undef,
1864 Mask: createStrideMask(Start: 1, Stride: 2, VF: ResTy.getNumElements()));
1865
1866 return true;
1867}
1868
1869void IRTranslator::getStackGuard(Register DstReg,
1870 MachineIRBuilder &MIRBuilder) {
1871 Value *Global =
1872 TLI->getSDagStackGuard(M: *MF->getFunction().getParent(), Libcalls: *Libcalls);
1873 if (!Global) {
1874 LLVMContext &Ctx = MIRBuilder.getContext();
1875 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
1876 MIRBuilder.buildUndef(Res: DstReg);
1877 return;
1878 }
1879
1880 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1881 MRI->setRegClass(Reg: DstReg, RC: TRI->getPointerRegClass());
1882 auto MIB =
1883 MIRBuilder.buildInstr(Opc: TargetOpcode::LOAD_STACK_GUARD, DstOps: {DstReg}, SrcOps: {});
1884
1885 unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
1886 LLT PtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL->getPointerSizeInBits(AS: AddrSpace));
1887
1888 MachinePointerInfo MPInfo(Global);
1889 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
1890 MachineMemOperand::MODereferenceable;
1891 MachineMemOperand *MemRef = MF->getMachineMemOperand(
1892 PtrInfo: MPInfo, f: Flags, MemTy: PtrTy, base_alignment: DL->getPointerABIAlignment(AS: AddrSpace));
1893 MIB.setMemRefs({MemRef});
1894}
1895
1896bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1897 MachineIRBuilder &MIRBuilder) {
1898 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: CI);
1899 MIRBuilder.buildInstr(
1900 Opc: Op, DstOps: {ResRegs[0], ResRegs[1]},
1901 SrcOps: {getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)), getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1))});
1902
1903 return true;
1904}
1905
1906bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
1907 MachineIRBuilder &MIRBuilder) {
1908 Register Dst = getOrCreateVReg(Val: CI);
1909 Register Src0 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
1910 Register Src1 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1));
1911 uint64_t Scale = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
1912 MIRBuilder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: { Src0, Src1, Scale });
1913 return true;
1914}
1915
1916unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1917 switch (ID) {
1918 default:
1919 break;
1920 case Intrinsic::acos:
1921 return TargetOpcode::G_FACOS;
1922 case Intrinsic::asin:
1923 return TargetOpcode::G_FASIN;
1924 case Intrinsic::atan:
1925 return TargetOpcode::G_FATAN;
1926 case Intrinsic::atan2:
1927 return TargetOpcode::G_FATAN2;
1928 case Intrinsic::bswap:
1929 return TargetOpcode::G_BSWAP;
1930 case Intrinsic::bitreverse:
1931 return TargetOpcode::G_BITREVERSE;
1932 case Intrinsic::fshl:
1933 return TargetOpcode::G_FSHL;
1934 case Intrinsic::fshr:
1935 return TargetOpcode::G_FSHR;
1936 case Intrinsic::ceil:
1937 return TargetOpcode::G_FCEIL;
1938 case Intrinsic::cos:
1939 return TargetOpcode::G_FCOS;
1940 case Intrinsic::cosh:
1941 return TargetOpcode::G_FCOSH;
1942 case Intrinsic::ctpop:
1943 return TargetOpcode::G_CTPOP;
1944 case Intrinsic::exp:
1945 return TargetOpcode::G_FEXP;
1946 case Intrinsic::exp2:
1947 return TargetOpcode::G_FEXP2;
1948 case Intrinsic::exp10:
1949 return TargetOpcode::G_FEXP10;
1950 case Intrinsic::fabs:
1951 return TargetOpcode::G_FABS;
1952 case Intrinsic::copysign:
1953 return TargetOpcode::G_FCOPYSIGN;
1954 case Intrinsic::minnum:
1955 return TargetOpcode::G_FMINNUM;
1956 case Intrinsic::maxnum:
1957 return TargetOpcode::G_FMAXNUM;
1958 case Intrinsic::minimum:
1959 return TargetOpcode::G_FMINIMUM;
1960 case Intrinsic::maximum:
1961 return TargetOpcode::G_FMAXIMUM;
1962 case Intrinsic::minimumnum:
1963 return TargetOpcode::G_FMINIMUMNUM;
1964 case Intrinsic::maximumnum:
1965 return TargetOpcode::G_FMAXIMUMNUM;
1966 case Intrinsic::canonicalize:
1967 return TargetOpcode::G_FCANONICALIZE;
1968 case Intrinsic::floor:
1969 return TargetOpcode::G_FFLOOR;
1970 case Intrinsic::fma:
1971 return TargetOpcode::G_FMA;
1972 case Intrinsic::log:
1973 return TargetOpcode::G_FLOG;
1974 case Intrinsic::log2:
1975 return TargetOpcode::G_FLOG2;
1976 case Intrinsic::log10:
1977 return TargetOpcode::G_FLOG10;
1978 case Intrinsic::ldexp:
1979 return TargetOpcode::G_FLDEXP;
1980 case Intrinsic::nearbyint:
1981 return TargetOpcode::G_FNEARBYINT;
1982 case Intrinsic::pow:
1983 return TargetOpcode::G_FPOW;
1984 case Intrinsic::powi:
1985 return TargetOpcode::G_FPOWI;
1986 case Intrinsic::rint:
1987 return TargetOpcode::G_FRINT;
1988 case Intrinsic::round:
1989 return TargetOpcode::G_INTRINSIC_ROUND;
1990 case Intrinsic::roundeven:
1991 return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
1992 case Intrinsic::sin:
1993 return TargetOpcode::G_FSIN;
1994 case Intrinsic::sinh:
1995 return TargetOpcode::G_FSINH;
1996 case Intrinsic::sqrt:
1997 return TargetOpcode::G_FSQRT;
1998 case Intrinsic::tan:
1999 return TargetOpcode::G_FTAN;
2000 case Intrinsic::tanh:
2001 return TargetOpcode::G_FTANH;
2002 case Intrinsic::trunc:
2003 return TargetOpcode::G_INTRINSIC_TRUNC;
2004 case Intrinsic::readcyclecounter:
2005 return TargetOpcode::G_READCYCLECOUNTER;
2006 case Intrinsic::readsteadycounter:
2007 return TargetOpcode::G_READSTEADYCOUNTER;
2008 case Intrinsic::ptrmask:
2009 return TargetOpcode::G_PTRMASK;
2010 case Intrinsic::lrint:
2011 return TargetOpcode::G_INTRINSIC_LRINT;
2012 case Intrinsic::llrint:
2013 return TargetOpcode::G_INTRINSIC_LLRINT;
2014 // FADD/FMUL require checking the FMF, so are handled elsewhere.
2015 case Intrinsic::vector_reduce_fmin:
2016 return TargetOpcode::G_VECREDUCE_FMIN;
2017 case Intrinsic::vector_reduce_fmax:
2018 return TargetOpcode::G_VECREDUCE_FMAX;
2019 case Intrinsic::vector_reduce_fminimum:
2020 return TargetOpcode::G_VECREDUCE_FMINIMUM;
2021 case Intrinsic::vector_reduce_fmaximum:
2022 return TargetOpcode::G_VECREDUCE_FMAXIMUM;
2023 case Intrinsic::vector_reduce_add:
2024 return TargetOpcode::G_VECREDUCE_ADD;
2025 case Intrinsic::vector_reduce_mul:
2026 return TargetOpcode::G_VECREDUCE_MUL;
2027 case Intrinsic::vector_reduce_and:
2028 return TargetOpcode::G_VECREDUCE_AND;
2029 case Intrinsic::vector_reduce_or:
2030 return TargetOpcode::G_VECREDUCE_OR;
2031 case Intrinsic::vector_reduce_xor:
2032 return TargetOpcode::G_VECREDUCE_XOR;
2033 case Intrinsic::vector_reduce_smax:
2034 return TargetOpcode::G_VECREDUCE_SMAX;
2035 case Intrinsic::vector_reduce_smin:
2036 return TargetOpcode::G_VECREDUCE_SMIN;
2037 case Intrinsic::vector_reduce_umax:
2038 return TargetOpcode::G_VECREDUCE_UMAX;
2039 case Intrinsic::vector_reduce_umin:
2040 return TargetOpcode::G_VECREDUCE_UMIN;
2041 case Intrinsic::experimental_vector_compress:
2042 return TargetOpcode::G_VECTOR_COMPRESS;
2043 case Intrinsic::lround:
2044 return TargetOpcode::G_LROUND;
2045 case Intrinsic::llround:
2046 return TargetOpcode::G_LLROUND;
2047 case Intrinsic::get_fpenv:
2048 return TargetOpcode::G_GET_FPENV;
2049 case Intrinsic::get_fpmode:
2050 return TargetOpcode::G_GET_FPMODE;
2051 }
2052 return Intrinsic::not_intrinsic;
2053}
2054
2055bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
2056 Intrinsic::ID ID,
2057 MachineIRBuilder &MIRBuilder) {
2058
2059 unsigned Op = getSimpleIntrinsicOpcode(ID);
2060
2061 // Is this a simple intrinsic?
2062 if (Op == Intrinsic::not_intrinsic)
2063 return false;
2064
2065 // Yes. Let's translate it.
2066 SmallVector<llvm::SrcOp, 4> VRegs;
2067 for (const auto &Arg : CI.args())
2068 VRegs.push_back(Elt: getOrCreateVReg(Val: *Arg));
2069
2070 MIRBuilder.buildInstr(Opc: Op, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: VRegs,
2071 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2072 return true;
2073}
2074
2075// TODO: Include ConstainedOps.def when all strict instructions are defined.
2076static unsigned getConstrainedOpcode(Intrinsic::ID ID) {
2077 switch (ID) {
2078 case Intrinsic::experimental_constrained_fadd:
2079 return TargetOpcode::G_STRICT_FADD;
2080 case Intrinsic::experimental_constrained_fsub:
2081 return TargetOpcode::G_STRICT_FSUB;
2082 case Intrinsic::experimental_constrained_fmul:
2083 return TargetOpcode::G_STRICT_FMUL;
2084 case Intrinsic::experimental_constrained_fdiv:
2085 return TargetOpcode::G_STRICT_FDIV;
2086 case Intrinsic::experimental_constrained_frem:
2087 return TargetOpcode::G_STRICT_FREM;
2088 case Intrinsic::experimental_constrained_fma:
2089 return TargetOpcode::G_STRICT_FMA;
2090 case Intrinsic::experimental_constrained_sqrt:
2091 return TargetOpcode::G_STRICT_FSQRT;
2092 case Intrinsic::experimental_constrained_ldexp:
2093 return TargetOpcode::G_STRICT_FLDEXP;
2094 default:
2095 return 0;
2096 }
2097}
2098
2099bool IRTranslator::translateConstrainedFPIntrinsic(
2100 const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
2101 fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
2102
2103 unsigned Opcode = getConstrainedOpcode(ID: FPI.getIntrinsicID());
2104 if (!Opcode)
2105 return false;
2106
2107 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: FPI);
2108 if (EB == fp::ExceptionBehavior::ebIgnore)
2109 Flags |= MachineInstr::NoFPExcept;
2110
2111 SmallVector<llvm::SrcOp, 4> VRegs;
2112 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
2113 VRegs.push_back(Elt: getOrCreateVReg(Val: *FPI.getArgOperand(i: I)));
2114
2115 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: FPI)}, SrcOps: VRegs, Flags);
2116 return true;
2117}
2118
2119std::optional<MCRegister> IRTranslator::getArgPhysReg(Argument &Arg) {
2120 auto VRegs = getOrCreateVRegs(Val: Arg);
2121 if (VRegs.size() != 1)
2122 return std::nullopt;
2123
2124 // Arguments are lowered as a copy of a livein physical register.
2125 auto *VRegDef = MF->getRegInfo().getVRegDef(Reg: VRegs[0]);
2126 if (!VRegDef || !VRegDef->isCopy())
2127 return std::nullopt;
2128 return VRegDef->getOperand(i: 1).getReg().asMCReg();
2129}
2130
2131bool IRTranslator::translateIfEntryValueArgument(bool isDeclare, Value *Val,
2132 const DILocalVariable *Var,
2133 const DIExpression *Expr,
2134 const DebugLoc &DL,
2135 MachineIRBuilder &MIRBuilder) {
2136 auto *Arg = dyn_cast<Argument>(Val);
2137 if (!Arg)
2138 return false;
2139
2140 if (!Expr->isEntryValue())
2141 return false;
2142
2143 std::optional<MCRegister> PhysReg = getArgPhysReg(Arg&: *Arg);
2144 if (!PhysReg) {
2145 LLVM_DEBUG(dbgs() << "Dropping dbg." << (isDeclare ? "declare" : "value")
2146 << ": expression is entry_value but "
2147 << "couldn't find a physical register\n");
2148 LLVM_DEBUG(dbgs() << *Var << "\n");
2149 return true;
2150 }
2151
2152 if (isDeclare) {
2153 // Append an op deref to account for the fact that this is a dbg_declare.
2154 Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
2155 MF->setVariableDbgInfo(Var, Expr, Reg: *PhysReg, Loc: DL);
2156 } else {
2157 MIRBuilder.buildDirectDbgValue(Reg: *PhysReg, Variable: Var, Expr);
2158 }
2159
2160 return true;
2161}
2162
2163static unsigned getConvOpcode(Intrinsic::ID ID) {
2164 switch (ID) {
2165 default:
2166 llvm_unreachable("Unexpected intrinsic");
2167 case Intrinsic::experimental_convergence_anchor:
2168 return TargetOpcode::CONVERGENCECTRL_ANCHOR;
2169 case Intrinsic::experimental_convergence_entry:
2170 return TargetOpcode::CONVERGENCECTRL_ENTRY;
2171 case Intrinsic::experimental_convergence_loop:
2172 return TargetOpcode::CONVERGENCECTRL_LOOP;
2173 }
2174}
2175
2176bool IRTranslator::translateConvergenceControlIntrinsic(
2177 const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
2178 MachineInstrBuilder MIB = MIRBuilder.buildInstr(Opcode: getConvOpcode(ID));
2179 Register OutputReg = getOrCreateConvergenceTokenVReg(Token: CI);
2180 MIB.addDef(RegNo: OutputReg);
2181
2182 if (ID == Intrinsic::experimental_convergence_loop) {
2183 auto Bundle = CI.getOperandBundle(ID: LLVMContext::OB_convergencectrl);
2184 assert(Bundle && "Expected a convergence control token.");
2185 Register InputReg =
2186 getOrCreateConvergenceTokenVReg(Token: *Bundle->Inputs[0].get());
2187 MIB.addUse(RegNo: InputReg);
2188 }
2189
2190 return true;
2191}
2192
2193bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
2194 MachineIRBuilder &MIRBuilder) {
2195 if (auto *MI = dyn_cast<AnyMemIntrinsic>(Val: &CI)) {
2196 if (ORE->enabled()) {
2197 if (MemoryOpRemark::canHandle(I: MI, TLI: *LibInfo)) {
2198 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2199 R.visit(I: MI);
2200 }
2201 }
2202 }
2203
2204 // If this is a simple intrinsic (that is, we just need to add a def of
2205 // a vreg, and uses for each arg operand, then translate it.
2206 if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
2207 return true;
2208
2209 switch (ID) {
2210 default:
2211 break;
2212 case Intrinsic::lifetime_start:
2213 case Intrinsic::lifetime_end: {
2214 // No stack colouring in O0, discard region information.
2215 if (MF->getTarget().getOptLevel() == CodeGenOptLevel::None ||
2216 MF->getFunction().hasOptNone())
2217 return true;
2218
2219 unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
2220 : TargetOpcode::LIFETIME_END;
2221
2222 const AllocaInst *AI = dyn_cast<AllocaInst>(Val: CI.getArgOperand(i: 0));
2223 if (!AI || !AI->isStaticAlloca())
2224 return true;
2225
2226 MIRBuilder.buildInstr(Opcode: Op).addFrameIndex(Idx: getOrCreateFrameIndex(AI: *AI));
2227 return true;
2228 }
2229 case Intrinsic::fake_use: {
2230 SmallVector<llvm::SrcOp, 4> VRegs;
2231 for (const auto &Arg : CI.args())
2232 llvm::append_range(C&: VRegs, R: getOrCreateVRegs(Val: *Arg));
2233 MIRBuilder.buildInstr(Opc: TargetOpcode::FAKE_USE, DstOps: {}, SrcOps: VRegs);
2234 MF->setHasFakeUses(true);
2235 return true;
2236 }
2237 case Intrinsic::dbg_declare: {
2238 const DbgDeclareInst &DI = cast<DbgDeclareInst>(Val: CI);
2239 assert(DI.getVariable() && "Missing variable");
2240 translateDbgDeclareRecord(Address: DI.getAddress(), HasArgList: DI.hasArgList(), Variable: DI.getVariable(),
2241 Expression: DI.getExpression(), DL: DI.getDebugLoc(), MIRBuilder);
2242 return true;
2243 }
2244 case Intrinsic::dbg_label: {
2245 const DbgLabelInst &DI = cast<DbgLabelInst>(Val: CI);
2246 assert(DI.getLabel() && "Missing label");
2247
2248 assert(DI.getLabel()->isValidLocationForIntrinsic(
2249 MIRBuilder.getDebugLoc()) &&
2250 "Expected inlined-at fields to agree");
2251
2252 MIRBuilder.buildDbgLabel(Label: DI.getLabel());
2253 return true;
2254 }
2255 case Intrinsic::vaend:
2256 // No target I know of cares about va_end. Certainly no in-tree target
2257 // does. Simplest intrinsic ever!
2258 return true;
2259 case Intrinsic::vastart: {
2260 Value *Ptr = CI.getArgOperand(i: 0);
2261 unsigned ListSize = TLI->getVaListSizeInBits(DL: *DL) / 8;
2262 Align Alignment = getKnownAlignment(V: Ptr, DL: *DL);
2263
2264 MIRBuilder.buildInstr(Opc: TargetOpcode::G_VASTART, DstOps: {}, SrcOps: {getOrCreateVReg(Val: *Ptr)})
2265 .addMemOperand(MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Ptr),
2266 F: MachineMemOperand::MOStore,
2267 Size: ListSize, BaseAlignment: Alignment));
2268 return true;
2269 }
2270 case Intrinsic::dbg_assign:
2271 // A dbg.assign is a dbg.value with more information about stack locations,
2272 // typically produced during optimisation of variables with leaked
2273 // addresses. We can treat it like a normal dbg_value intrinsic here; to
2274 // benefit from the full analysis of stack/SSA locations, GlobalISel would
2275 // need to register for and use the AssignmentTrackingAnalysis pass.
2276 [[fallthrough]];
2277 case Intrinsic::dbg_value: {
2278 // This form of DBG_VALUE is target-independent.
2279 const DbgValueInst &DI = cast<DbgValueInst>(Val: CI);
2280 translateDbgValueRecord(V: DI.getValue(), HasArgList: DI.hasArgList(), Variable: DI.getVariable(),
2281 Expression: DI.getExpression(), DL: DI.getDebugLoc(), MIRBuilder);
2282 return true;
2283 }
2284 case Intrinsic::uadd_with_overflow:
2285 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UADDO, MIRBuilder);
2286 case Intrinsic::sadd_with_overflow:
2287 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SADDO, MIRBuilder);
2288 case Intrinsic::usub_with_overflow:
2289 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_USUBO, MIRBuilder);
2290 case Intrinsic::ssub_with_overflow:
2291 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SSUBO, MIRBuilder);
2292 case Intrinsic::umul_with_overflow:
2293 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UMULO, MIRBuilder);
2294 case Intrinsic::smul_with_overflow:
2295 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SMULO, MIRBuilder);
2296 case Intrinsic::uadd_sat:
2297 return translateBinaryOp(Opcode: TargetOpcode::G_UADDSAT, U: CI, MIRBuilder);
2298 case Intrinsic::sadd_sat:
2299 return translateBinaryOp(Opcode: TargetOpcode::G_SADDSAT, U: CI, MIRBuilder);
2300 case Intrinsic::usub_sat:
2301 return translateBinaryOp(Opcode: TargetOpcode::G_USUBSAT, U: CI, MIRBuilder);
2302 case Intrinsic::ssub_sat:
2303 return translateBinaryOp(Opcode: TargetOpcode::G_SSUBSAT, U: CI, MIRBuilder);
2304 case Intrinsic::ushl_sat:
2305 return translateBinaryOp(Opcode: TargetOpcode::G_USHLSAT, U: CI, MIRBuilder);
2306 case Intrinsic::sshl_sat:
2307 return translateBinaryOp(Opcode: TargetOpcode::G_SSHLSAT, U: CI, MIRBuilder);
2308 case Intrinsic::umin:
2309 return translateBinaryOp(Opcode: TargetOpcode::G_UMIN, U: CI, MIRBuilder);
2310 case Intrinsic::umax:
2311 return translateBinaryOp(Opcode: TargetOpcode::G_UMAX, U: CI, MIRBuilder);
2312 case Intrinsic::smin:
2313 return translateBinaryOp(Opcode: TargetOpcode::G_SMIN, U: CI, MIRBuilder);
2314 case Intrinsic::smax:
2315 return translateBinaryOp(Opcode: TargetOpcode::G_SMAX, U: CI, MIRBuilder);
2316 case Intrinsic::abs:
2317 // TODO: Preserve "int min is poison" arg in GMIR?
2318 return translateUnaryOp(Opcode: TargetOpcode::G_ABS, U: CI, MIRBuilder);
2319 case Intrinsic::smul_fix:
2320 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIX, CI, MIRBuilder);
2321 case Intrinsic::umul_fix:
2322 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIX, CI, MIRBuilder);
2323 case Intrinsic::smul_fix_sat:
2324 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
2325 case Intrinsic::umul_fix_sat:
2326 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
2327 case Intrinsic::sdiv_fix:
2328 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
2329 case Intrinsic::udiv_fix:
2330 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
2331 case Intrinsic::sdiv_fix_sat:
2332 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
2333 case Intrinsic::udiv_fix_sat:
2334 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
2335 case Intrinsic::fmuladd: {
2336 const TargetMachine &TM = MF->getTarget();
2337 Register Dst = getOrCreateVReg(Val: CI);
2338 Register Op0 = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
2339 Register Op1 = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
2340 Register Op2 = getOrCreateVReg(Val: *CI.getArgOperand(i: 2));
2341 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
2342 TLI->isFMAFasterThanFMulAndFAdd(MF: *MF,
2343 TLI->getValueType(DL: *DL, Ty: CI.getType()))) {
2344 // TODO: Revisit this to see if we should move this part of the
2345 // lowering to the combiner.
2346 MIRBuilder.buildFMA(Dst, Src0: Op0, Src1: Op1, Src2: Op2,
2347 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2348 } else {
2349 LLT Ty = getLLTForType(Ty&: *CI.getType(), DL: *DL);
2350 auto FMul = MIRBuilder.buildFMul(
2351 Dst: Ty, Src0: Op0, Src1: Op1, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2352 MIRBuilder.buildFAdd(Dst, Src0: FMul, Src1: Op2,
2353 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2354 }
2355 return true;
2356 }
2357 case Intrinsic::frexp: {
2358 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
2359 MIRBuilder.buildFFrexp(Fract: VRegs[0], Exp: VRegs[1],
2360 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
2361 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2362 return true;
2363 }
2364 case Intrinsic::modf: {
2365 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
2366 MIRBuilder.buildModf(Fract: VRegs[0], Int: VRegs[1],
2367 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
2368 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2369 return true;
2370 }
2371 case Intrinsic::sincos: {
2372 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
2373 MIRBuilder.buildFSincos(Sin: VRegs[0], Cos: VRegs[1],
2374 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
2375 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2376 return true;
2377 }
2378 case Intrinsic::fptosi_sat:
2379 MIRBuilder.buildFPTOSI_SAT(Dst: getOrCreateVReg(Val: CI),
2380 Src0: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
2381 return true;
2382 case Intrinsic::fptoui_sat:
2383 MIRBuilder.buildFPTOUI_SAT(Dst: getOrCreateVReg(Val: CI),
2384 Src0: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
2385 return true;
2386 case Intrinsic::memcpy_inline:
2387 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY_INLINE);
2388 case Intrinsic::memcpy:
2389 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY);
2390 case Intrinsic::memmove:
2391 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMMOVE);
2392 case Intrinsic::memset:
2393 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMSET);
2394 case Intrinsic::eh_typeid_for: {
2395 GlobalValue *GV = ExtractTypeInfo(V: CI.getArgOperand(i: 0));
2396 Register Reg = getOrCreateVReg(Val: CI);
2397 unsigned TypeID = MF->getTypeIDFor(TI: GV);
2398 MIRBuilder.buildConstant(Res: Reg, Val: TypeID);
2399 return true;
2400 }
2401 case Intrinsic::objectsize:
2402 llvm_unreachable("llvm.objectsize.* should have been lowered already");
2403
2404 case Intrinsic::is_constant:
2405 llvm_unreachable("llvm.is.constant.* should have been lowered already");
2406
2407 case Intrinsic::stackguard:
2408 getStackGuard(DstReg: getOrCreateVReg(Val: CI), MIRBuilder);
2409 return true;
2410 case Intrinsic::stackprotector: {
2411 LLT PtrTy = getLLTForType(Ty&: *CI.getArgOperand(i: 0)->getType(), DL: *DL);
2412 Register GuardVal;
2413 if (TLI->useLoadStackGuardNode(M: *CI.getModule())) {
2414 GuardVal = MRI->createGenericVirtualRegister(Ty: PtrTy);
2415 getStackGuard(DstReg: GuardVal, MIRBuilder);
2416 } else
2417 GuardVal = getOrCreateVReg(Val: *CI.getArgOperand(i: 0)); // The guard's value.
2418
2419 AllocaInst *Slot = cast<AllocaInst>(Val: CI.getArgOperand(i: 1));
2420 int FI = getOrCreateFrameIndex(AI: *Slot);
2421 MF->getFrameInfo().setStackProtectorIndex(FI);
2422
2423 MIRBuilder.buildStore(
2424 Val: GuardVal, Addr: getOrCreateVReg(Val: *Slot),
2425 MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI),
2426 f: MachineMemOperand::MOStore |
2427 MachineMemOperand::MOVolatile,
2428 MemTy: PtrTy, base_alignment: Align(8)));
2429 return true;
2430 }
2431 case Intrinsic::stacksave: {
2432 MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKSAVE, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {});
2433 return true;
2434 }
2435 case Intrinsic::stackrestore: {
2436 MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKRESTORE, DstOps: {},
2437 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
2438 return true;
2439 }
2440 case Intrinsic::cttz:
2441 case Intrinsic::ctlz: {
2442 ConstantInt *Cst = cast<ConstantInt>(Val: CI.getArgOperand(i: 1));
2443 bool isTrailing = ID == Intrinsic::cttz;
2444 unsigned Opcode = isTrailing
2445 ? Cst->isZero() ? TargetOpcode::G_CTTZ
2446 : TargetOpcode::G_CTTZ_ZERO_UNDEF
2447 : Cst->isZero() ? TargetOpcode::G_CTLZ
2448 : TargetOpcode::G_CTLZ_ZERO_UNDEF;
2449 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: CI)},
2450 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
2451 return true;
2452 }
2453 case Intrinsic::invariant_start: {
2454 MIRBuilder.buildUndef(Res: getOrCreateVReg(Val: CI));
2455 return true;
2456 }
2457 case Intrinsic::invariant_end:
2458 return true;
2459 case Intrinsic::expect:
2460 case Intrinsic::expect_with_probability:
2461 case Intrinsic::annotation:
2462 case Intrinsic::ptr_annotation:
2463 case Intrinsic::launder_invariant_group:
2464 case Intrinsic::strip_invariant_group: {
2465 // Drop the intrinsic, but forward the value.
2466 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: CI),
2467 Op: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
2468 return true;
2469 }
2470 case Intrinsic::assume:
2471 case Intrinsic::experimental_noalias_scope_decl:
2472 case Intrinsic::var_annotation:
2473 case Intrinsic::sideeffect:
2474 // Discard annotate attributes, assumptions, and artificial side-effects.
2475 return true;
2476 case Intrinsic::read_volatile_register:
2477 case Intrinsic::read_register: {
2478 Value *Arg = CI.getArgOperand(i: 0);
2479 MIRBuilder
2480 .buildInstr(Opc: TargetOpcode::G_READ_REGISTER, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {})
2481 .addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()));
2482 return true;
2483 }
2484 case Intrinsic::write_register: {
2485 Value *Arg = CI.getArgOperand(i: 0);
2486 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_WRITE_REGISTER)
2487 .addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()))
2488 .addUse(RegNo: getOrCreateVReg(Val: *CI.getArgOperand(i: 1)));
2489 return true;
2490 }
2491 case Intrinsic::localescape: {
2492 MachineBasicBlock &EntryMBB = MF->front();
2493 StringRef EscapedName = GlobalValue::dropLLVMManglingEscape(Name: MF->getName());
2494
2495 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
2496 // is the same on all targets.
2497 for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
2498 Value *Arg = CI.getArgOperand(i: Idx)->stripPointerCasts();
2499 if (isa<ConstantPointerNull>(Val: Arg))
2500 continue; // Skip null pointers. They represent a hole in index space.
2501
2502 int FI = getOrCreateFrameIndex(AI: *cast<AllocaInst>(Val: Arg));
2503 MCSymbol *FrameAllocSym =
2504 MF->getContext().getOrCreateFrameAllocSymbol(FuncName: EscapedName, Idx);
2505
2506 // This should be inserted at the start of the entry block.
2507 auto LocalEscape =
2508 MIRBuilder.buildInstrNoInsert(Opcode: TargetOpcode::LOCAL_ESCAPE)
2509 .addSym(Sym: FrameAllocSym)
2510 .addFrameIndex(Idx: FI);
2511
2512 EntryMBB.insert(I: EntryMBB.begin(), MI: LocalEscape);
2513 }
2514
2515 return true;
2516 }
2517 case Intrinsic::vector_reduce_fadd:
2518 case Intrinsic::vector_reduce_fmul: {
2519 // Need to check for the reassoc flag to decide whether we want a
2520 // sequential reduction opcode or not.
2521 Register Dst = getOrCreateVReg(Val: CI);
2522 Register ScalarSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
2523 Register VecSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
2524 unsigned Opc = 0;
2525 if (!CI.hasAllowReassoc()) {
2526 // The sequential ordering case.
2527 Opc = ID == Intrinsic::vector_reduce_fadd
2528 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
2529 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
2530 if (!MRI->getType(Reg: VecSrc).isVector())
2531 Opc = ID == Intrinsic::vector_reduce_fadd ? TargetOpcode::G_FADD
2532 : TargetOpcode::G_FMUL;
2533 MIRBuilder.buildInstr(Opc, DstOps: {Dst}, SrcOps: {ScalarSrc, VecSrc},
2534 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2535 return true;
2536 }
2537 // We split the operation into a separate G_FADD/G_FMUL + the reduce,
2538 // since the associativity doesn't matter.
2539 unsigned ScalarOpc;
2540 if (ID == Intrinsic::vector_reduce_fadd) {
2541 Opc = TargetOpcode::G_VECREDUCE_FADD;
2542 ScalarOpc = TargetOpcode::G_FADD;
2543 } else {
2544 Opc = TargetOpcode::G_VECREDUCE_FMUL;
2545 ScalarOpc = TargetOpcode::G_FMUL;
2546 }
2547 LLT DstTy = MRI->getType(Reg: Dst);
2548 auto Rdx = MIRBuilder.buildInstr(
2549 Opc, DstOps: {DstTy}, SrcOps: {VecSrc}, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2550 MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {Dst}, SrcOps: {ScalarSrc, Rdx},
2551 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2552
2553 return true;
2554 }
2555 case Intrinsic::trap:
2556 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_TRAP);
2557 case Intrinsic::debugtrap:
2558 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_DEBUGTRAP);
2559 case Intrinsic::ubsantrap:
2560 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_UBSANTRAP);
2561 case Intrinsic::allow_runtime_check:
2562 case Intrinsic::allow_ubsan_check:
2563 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: CI),
2564 Op: getOrCreateVReg(Val: *ConstantInt::getTrue(Ty: CI.getType())));
2565 return true;
2566 case Intrinsic::amdgcn_cs_chain:
2567 case Intrinsic::amdgcn_call_whole_wave:
2568 return translateCallBase(CB: CI, MIRBuilder);
2569 case Intrinsic::fptrunc_round: {
2570 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: CI);
2571
2572 // Convert the metadata argument to a constant integer
2573 Metadata *MD = cast<MetadataAsValue>(Val: CI.getArgOperand(i: 1))->getMetadata();
2574 std::optional<RoundingMode> RoundMode =
2575 convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
2576
2577 // Add the Rounding mode as an integer
2578 MIRBuilder
2579 .buildInstr(Opc: TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
2580 DstOps: {getOrCreateVReg(Val: CI)},
2581 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))}, Flags)
2582 .addImm(Val: (int)*RoundMode);
2583
2584 return true;
2585 }
2586 case Intrinsic::is_fpclass: {
2587 Value *FpValue = CI.getOperand(i_nocapture: 0);
2588 ConstantInt *TestMaskValue = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1));
2589
2590 MIRBuilder
2591 .buildInstr(Opc: TargetOpcode::G_IS_FPCLASS, DstOps: {getOrCreateVReg(Val: CI)},
2592 SrcOps: {getOrCreateVReg(Val: *FpValue)})
2593 .addImm(Val: TestMaskValue->getZExtValue());
2594
2595 return true;
2596 }
2597 case Intrinsic::set_fpenv: {
2598 Value *FPEnv = CI.getOperand(i_nocapture: 0);
2599 MIRBuilder.buildSetFPEnv(Src: getOrCreateVReg(Val: *FPEnv));
2600 return true;
2601 }
2602 case Intrinsic::reset_fpenv:
2603 MIRBuilder.buildResetFPEnv();
2604 return true;
2605 case Intrinsic::set_fpmode: {
2606 Value *FPState = CI.getOperand(i_nocapture: 0);
2607 MIRBuilder.buildSetFPMode(Src: getOrCreateVReg(Val: *FPState));
2608 return true;
2609 }
2610 case Intrinsic::reset_fpmode:
2611 MIRBuilder.buildResetFPMode();
2612 return true;
2613 case Intrinsic::get_rounding:
2614 MIRBuilder.buildGetRounding(Dst: getOrCreateVReg(Val: CI));
2615 return true;
2616 case Intrinsic::set_rounding:
2617 MIRBuilder.buildSetRounding(Src: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)));
2618 return true;
2619 case Intrinsic::vscale: {
2620 MIRBuilder.buildVScale(Res: getOrCreateVReg(Val: CI), MinElts: 1);
2621 return true;
2622 }
2623 case Intrinsic::scmp:
2624 MIRBuilder.buildSCmp(Res: getOrCreateVReg(Val: CI),
2625 Op0: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)),
2626 Op1: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1)));
2627 return true;
2628 case Intrinsic::ucmp:
2629 MIRBuilder.buildUCmp(Res: getOrCreateVReg(Val: CI),
2630 Op0: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)),
2631 Op1: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1)));
2632 return true;
2633 case Intrinsic::vector_extract:
2634 return translateExtractVector(U: CI, MIRBuilder);
2635 case Intrinsic::vector_insert:
2636 return translateInsertVector(U: CI, MIRBuilder);
2637 case Intrinsic::stepvector: {
2638 MIRBuilder.buildStepVector(Res: getOrCreateVReg(Val: CI), Step: 1);
2639 return true;
2640 }
2641 case Intrinsic::prefetch: {
2642 Value *Addr = CI.getOperand(i_nocapture: 0);
2643 unsigned RW = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1))->getZExtValue();
2644 unsigned Locality = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
2645 unsigned CacheType = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 3))->getZExtValue();
2646
2647 auto Flags = RW ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
2648 auto &MMO = *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Addr), f: Flags,
2649 MemTy: LLT(), base_alignment: Align());
2650
2651 MIRBuilder.buildPrefetch(Addr: getOrCreateVReg(Val: *Addr), RW, Locality, CacheType,
2652 MMO);
2653
2654 return true;
2655 }
2656
2657 case Intrinsic::vector_interleave2:
2658 case Intrinsic::vector_deinterleave2: {
2659 // Both intrinsics have at least one operand.
2660 Value *Op0 = CI.getOperand(i_nocapture: 0);
2661 LLT ResTy = getLLTForType(Ty&: *Op0->getType(), DL: MIRBuilder.getDataLayout());
2662 if (!ResTy.isFixedVector())
2663 return false;
2664
2665 if (CI.getIntrinsicID() == Intrinsic::vector_interleave2)
2666 return translateVectorInterleave2Intrinsic(CI, MIRBuilder);
2667
2668 return translateVectorDeinterleave2Intrinsic(CI, MIRBuilder);
2669 }
2670
2671#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
2672 case Intrinsic::INTRINSIC:
2673#include "llvm/IR/ConstrainedOps.def"
2674 return translateConstrainedFPIntrinsic(FPI: cast<ConstrainedFPIntrinsic>(Val: CI),
2675 MIRBuilder);
2676 case Intrinsic::experimental_convergence_anchor:
2677 case Intrinsic::experimental_convergence_entry:
2678 case Intrinsic::experimental_convergence_loop:
2679 return translateConvergenceControlIntrinsic(CI, ID, MIRBuilder);
2680 case Intrinsic::reloc_none: {
2681 Metadata *MD = cast<MetadataAsValue>(Val: CI.getArgOperand(i: 0))->getMetadata();
2682 StringRef SymbolName = cast<MDString>(Val: MD)->getString();
2683 MIRBuilder.buildInstr(Opcode: TargetOpcode::RELOC_NONE)
2684 .addExternalSymbol(FnName: SymbolName.data());
2685 return true;
2686 }
2687 }
2688 return false;
2689}
2690
2691bool IRTranslator::translateInlineAsm(const CallBase &CB,
2692 MachineIRBuilder &MIRBuilder) {
2693 if (containsBF16Type(U: CB) && !targetSupportsBF16Type(MF))
2694 return false;
2695
2696 const InlineAsmLowering *ALI = MF->getSubtarget().getInlineAsmLowering();
2697
2698 if (!ALI) {
2699 LLVM_DEBUG(
2700 dbgs() << "Inline asm lowering is not supported for this target yet\n");
2701 return false;
2702 }
2703
2704 return ALI->lowerInlineAsm(
2705 MIRBuilder, CB, GetOrCreateVRegs: [&](const Value &Val) { return getOrCreateVRegs(Val); });
2706}
2707
2708bool IRTranslator::translateCallBase(const CallBase &CB,
2709 MachineIRBuilder &MIRBuilder) {
2710 ArrayRef<Register> Res = getOrCreateVRegs(Val: CB);
2711
2712 SmallVector<ArrayRef<Register>, 8> Args;
2713 Register SwiftInVReg = 0;
2714 Register SwiftErrorVReg = 0;
2715 for (const auto &Arg : CB.args()) {
2716 if (CLI->supportSwiftError() && isSwiftError(V: Arg)) {
2717 assert(SwiftInVReg == 0 && "Expected only one swift error argument");
2718 LLT Ty = getLLTForType(Ty&: *Arg->getType(), DL: *DL);
2719 SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
2720 MIRBuilder.buildCopy(Res: SwiftInVReg, Op: SwiftError.getOrCreateVRegUseAt(
2721 &CB, &MIRBuilder.getMBB(), Arg));
2722 Args.emplace_back(Args: ArrayRef(SwiftInVReg));
2723 SwiftErrorVReg =
2724 SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
2725 continue;
2726 }
2727 Args.push_back(Elt: getOrCreateVRegs(Val: *Arg));
2728 }
2729
2730 if (auto *CI = dyn_cast<CallInst>(Val: &CB)) {
2731 if (ORE->enabled()) {
2732 if (MemoryOpRemark::canHandle(I: CI, TLI: *LibInfo)) {
2733 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2734 R.visit(I: CI);
2735 }
2736 }
2737 }
2738
2739 std::optional<CallLowering::PtrAuthInfo> PAI;
2740 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_ptrauth)) {
2741 // Functions should never be ptrauth-called directly.
2742 assert(!CB.getCalledFunction() && "invalid direct ptrauth call");
2743
2744 const Value *Key = Bundle->Inputs[0];
2745 const Value *Discriminator = Bundle->Inputs[1];
2746
2747 // Look through ptrauth constants to try to eliminate the matching bundle
2748 // and turn this into a direct call with no ptrauth.
2749 // CallLowering will use the raw pointer if it doesn't find the PAI.
2750 const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(Val: CB.getCalledOperand());
2751 if (!CalleeCPA || !isa<Function>(Val: CalleeCPA->getPointer()) ||
2752 !CalleeCPA->isKnownCompatibleWith(Key, Discriminator, DL: *DL)) {
2753 // If we can't make it direct, package the bundle into PAI.
2754 Register DiscReg = getOrCreateVReg(Val: *Discriminator);
2755 PAI = CallLowering::PtrAuthInfo{.Key: cast<ConstantInt>(Val: Key)->getZExtValue(),
2756 .Discriminator: DiscReg};
2757 }
2758 }
2759
2760 Register ConvergenceCtrlToken = 0;
2761 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
2762 const auto &Token = *Bundle->Inputs[0].get();
2763 ConvergenceCtrlToken = getOrCreateConvergenceTokenVReg(Token);
2764 }
2765
2766 // We don't set HasCalls on MFI here yet because call lowering may decide to
2767 // optimize into tail calls. Instead, we defer that to selection where a final
2768 // scan is done to check if any instructions are calls.
2769 bool Success = CLI->lowerCall(
2770 MIRBuilder, Call: CB, ResRegs: Res, ArgRegs: Args, SwiftErrorVReg, PAI, ConvergenceCtrlToken,
2771 GetCalleeReg: [&]() { return getOrCreateVReg(Val: *CB.getCalledOperand()); });
2772
2773 // Check if we just inserted a tail call.
2774 if (Success) {
2775 assert(!HasTailCall && "Can't tail call return twice from block?");
2776 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
2777 HasTailCall = TII->isTailCall(Inst: *std::prev(x: MIRBuilder.getInsertPt()));
2778 }
2779
2780 return Success;
2781}
2782
2783bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
2784 if (containsBF16Type(U) && !targetSupportsBF16Type(MF))
2785 return false;
2786
2787 const CallInst &CI = cast<CallInst>(Val: U);
2788 const Function *F = CI.getCalledFunction();
2789
2790 // FIXME: support Windows dllimport function calls and calls through
2791 // weak symbols.
2792 if (F && (F->hasDLLImportStorageClass() ||
2793 (MF->getTarget().getTargetTriple().isOSWindows() &&
2794 F->hasExternalWeakLinkage())))
2795 return false;
2796
2797 // FIXME: support control flow guard targets.
2798 if (CI.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
2799 return false;
2800
2801 // FIXME: support statepoints and related.
2802 if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(Val: U))
2803 return false;
2804
2805 if (CI.isInlineAsm())
2806 return translateInlineAsm(CB: CI, MIRBuilder);
2807
2808 Intrinsic::ID ID = F ? F->getIntrinsicID() : Intrinsic::not_intrinsic;
2809 if (!F || ID == Intrinsic::not_intrinsic) {
2810 if (translateCallBase(CB: CI, MIRBuilder)) {
2811 diagnoseDontCall(CI);
2812 return true;
2813 }
2814 return false;
2815 }
2816
2817 assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
2818
2819 if (translateKnownIntrinsic(CI, ID, MIRBuilder))
2820 return true;
2821
2822 SmallVector<TargetLowering::IntrinsicInfo> Infos;
2823 TLI->getTgtMemIntrinsic(Infos, I: CI, MF&: *MF, Intrinsic: ID);
2824
2825 return translateIntrinsic(CB: CI, ID, MIRBuilder, TgtMemIntrinsicInfos: Infos);
2826}
2827
2828/// Translate a call or callbr to an intrinsic.
2829bool IRTranslator::translateIntrinsic(
2830 const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
2831 ArrayRef<TargetLowering::IntrinsicInfo> TgtMemIntrinsicInfos) {
2832 ArrayRef<Register> ResultRegs;
2833 if (!CB.getType()->isVoidTy())
2834 ResultRegs = getOrCreateVRegs(Val: CB);
2835
2836 // Ignore the callsite attributes. Backend code is most likely not expecting
2837 // an intrinsic to sometimes have side effects and sometimes not.
2838 MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, Res: ResultRegs);
2839 if (isa<FPMathOperator>(Val: CB))
2840 MIB->copyIRFlags(I: CB);
2841
2842 for (const auto &Arg : enumerate(First: CB.args())) {
2843 // If this is required to be an immediate, don't materialize it in a
2844 // register.
2845 if (CB.paramHasAttr(ArgNo: Arg.index(), Kind: Attribute::ImmArg)) {
2846 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Arg.value())) {
2847 // imm arguments are more convenient than cimm (and realistically
2848 // probably sufficient), so use them.
2849 assert(CI->getBitWidth() <= 64 &&
2850 "large intrinsic immediates not handled");
2851 MIB.addImm(Val: CI->getSExtValue());
2852 } else {
2853 MIB.addFPImm(Val: cast<ConstantFP>(Val: Arg.value()));
2854 }
2855 } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Val: Arg.value())) {
2856 auto *MD = MDVal->getMetadata();
2857 auto *MDN = dyn_cast<MDNode>(Val: MD);
2858 if (!MDN) {
2859 if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(Val: MD))
2860 MDN = MDNode::get(Context&: MF->getFunction().getContext(), MDs: ConstMD);
2861 else // This was probably an MDString.
2862 return false;
2863 }
2864 MIB.addMetadata(MD: MDN);
2865 } else {
2866 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: *Arg.value());
2867 if (VRegs.size() > 1)
2868 return false;
2869 MIB.addUse(RegNo: VRegs[0]);
2870 }
2871 }
2872
2873 // Add MachineMemOperands for each memory access described by the target.
2874 for (const auto &Info : TgtMemIntrinsicInfos) {
2875 Align Alignment = Info.align.value_or(
2876 u: DL->getABITypeAlign(Ty: Info.memVT.getTypeForEVT(Context&: CB.getContext())));
2877 LLT MemTy = Info.memVT.isSimple()
2878 ? getLLTForMVT(Ty: Info.memVT.getSimpleVT())
2879 : LLT::scalar(SizeInBits: Info.memVT.getStoreSizeInBits());
2880
2881 // TODO: We currently just fallback to address space 0 if
2882 // getTgtMemIntrinsic didn't yield anything useful.
2883 MachinePointerInfo MPI;
2884 if (Info.ptrVal) {
2885 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
2886 } else if (Info.fallbackAddressSpace) {
2887 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
2888 }
2889 MIB.addMemOperand(MMO: MF->getMachineMemOperand(
2890 PtrInfo: MPI, f: Info.flags, MemTy, base_alignment: Alignment, AAInfo: CB.getAAMetadata(),
2891 /*Ranges=*/nullptr, SSID: Info.ssid, Ordering: Info.order, FailureOrdering: Info.failureOrder));
2892 }
2893
2894 if (CB.isConvergent()) {
2895 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
2896 auto *Token = Bundle->Inputs[0].get();
2897 Register TokenReg = getOrCreateVReg(Val: *Token);
2898 MIB.addUse(RegNo: TokenReg, Flags: RegState::Implicit);
2899 }
2900 }
2901
2902 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol))
2903 MIB->setDeactivationSymbol(MF&: *MF, DS: Bundle->Inputs[0].get());
2904
2905 return true;
2906}
2907
2908bool IRTranslator::findUnwindDestinations(
2909 const BasicBlock *EHPadBB,
2910 BranchProbability Prob,
2911 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2912 &UnwindDests) {
2913 EHPersonality Personality = classifyEHPersonality(
2914 Pers: EHPadBB->getParent()->getFunction().getPersonalityFn());
2915 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2916 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2917 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2918 bool IsSEH = isAsynchronousEHPersonality(Pers: Personality);
2919
2920 if (IsWasmCXX) {
2921 // Ignore this for now.
2922 return false;
2923 }
2924
2925 while (EHPadBB) {
2926 BasicBlock::const_iterator Pad = EHPadBB->getFirstNonPHIIt();
2927 BasicBlock *NewEHPadBB = nullptr;
2928 if (isa<LandingPadInst>(Val: Pad)) {
2929 // Stop on landingpads. They are not funclets.
2930 UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
2931 break;
2932 }
2933 if (isa<CleanupPadInst>(Val: Pad)) {
2934 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2935 // personalities.
2936 UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
2937 UnwindDests.back().first->setIsEHScopeEntry();
2938 UnwindDests.back().first->setIsEHFuncletEntry();
2939 break;
2940 }
2941 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: Pad)) {
2942 // Add the catchpad handlers to the possible destinations.
2943 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2944 UnwindDests.emplace_back(Args: &getMBB(BB: *CatchPadBB), Args&: Prob);
2945 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2946 if (IsMSVCCXX || IsCoreCLR)
2947 UnwindDests.back().first->setIsEHFuncletEntry();
2948 if (!IsSEH)
2949 UnwindDests.back().first->setIsEHScopeEntry();
2950 }
2951 NewEHPadBB = CatchSwitch->getUnwindDest();
2952 } else {
2953 continue;
2954 }
2955
2956 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2957 if (BPI && NewEHPadBB)
2958 Prob *= BPI->getEdgeProbability(Src: EHPadBB, Dst: NewEHPadBB);
2959 EHPadBB = NewEHPadBB;
2960 }
2961 return true;
2962}
2963
2964bool IRTranslator::translateInvoke(const User &U,
2965 MachineIRBuilder &MIRBuilder) {
2966 const InvokeInst &I = cast<InvokeInst>(Val: U);
2967 MCContext &Context = MF->getContext();
2968
2969 const BasicBlock *ReturnBB = I.getSuccessor(i: 0);
2970 const BasicBlock *EHPadBB = I.getSuccessor(i: 1);
2971
2972 const Function *Fn = I.getCalledFunction();
2973
2974 // FIXME: support invoking patchpoint and statepoint intrinsics.
2975 if (Fn && Fn->isIntrinsic())
2976 return false;
2977
2978 // FIXME: support whatever these are.
2979 if (I.hasDeoptState())
2980 return false;
2981
2982 // FIXME: support control flow guard targets.
2983 if (I.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
2984 return false;
2985
2986 // FIXME: support Windows exception handling.
2987 if (!isa<LandingPadInst>(Val: EHPadBB->getFirstNonPHIIt()))
2988 return false;
2989
2990 // FIXME: support Windows dllimport function calls and calls through
2991 // weak symbols.
2992 if (Fn && (Fn->hasDLLImportStorageClass() ||
2993 (MF->getTarget().getTargetTriple().isOSWindows() &&
2994 Fn->hasExternalWeakLinkage())))
2995 return false;
2996
2997 bool LowerInlineAsm = I.isInlineAsm();
2998 bool NeedEHLabel = true;
2999
3000 // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
3001 // the region covered by the try.
3002 MCSymbol *BeginSymbol = nullptr;
3003 if (NeedEHLabel) {
3004 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_INVOKE_REGION_START);
3005 BeginSymbol = Context.createTempSymbol();
3006 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: BeginSymbol);
3007 }
3008
3009 if (LowerInlineAsm) {
3010 if (!translateInlineAsm(CB: I, MIRBuilder))
3011 return false;
3012 } else if (!translateCallBase(CB: I, MIRBuilder))
3013 return false;
3014
3015 MCSymbol *EndSymbol = nullptr;
3016 if (NeedEHLabel) {
3017 EndSymbol = Context.createTempSymbol();
3018 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: EndSymbol);
3019 }
3020
3021 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
3022 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3023 MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
3024 BranchProbability EHPadBBProb =
3025 BPI ? BPI->getEdgeProbability(Src: InvokeMBB->getBasicBlock(), Dst: EHPadBB)
3026 : BranchProbability::getZero();
3027
3028 if (!findUnwindDestinations(EHPadBB, Prob: EHPadBBProb, UnwindDests))
3029 return false;
3030
3031 MachineBasicBlock &EHPadMBB = getMBB(BB: *EHPadBB),
3032 &ReturnMBB = getMBB(BB: *ReturnBB);
3033 // Update successor info.
3034 addSuccessorWithProb(Src: InvokeMBB, Dst: &ReturnMBB);
3035 for (auto &UnwindDest : UnwindDests) {
3036 UnwindDest.first->setIsEHPad();
3037 addSuccessorWithProb(Src: InvokeMBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
3038 }
3039 InvokeMBB->normalizeSuccProbs();
3040
3041 if (NeedEHLabel) {
3042 assert(BeginSymbol && "Expected a begin symbol!");
3043 assert(EndSymbol && "Expected an end symbol!");
3044 MF->addInvoke(LandingPad: &EHPadMBB, BeginLabel: BeginSymbol, EndLabel: EndSymbol);
3045 }
3046
3047 MIRBuilder.buildBr(Dest&: ReturnMBB);
3048 return true;
3049}
3050
3051/// The intrinsics currently supported by callbr are implicit control flow
3052/// intrinsics such as amdgcn.kill.
3053bool IRTranslator::translateCallBr(const User &U,
3054 MachineIRBuilder &MIRBuilder) {
3055 if (containsBF16Type(U))
3056 return false; // see translateCall
3057
3058 const CallBrInst &I = cast<CallBrInst>(Val: U);
3059 MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB();
3060
3061 Intrinsic::ID IID = I.getIntrinsicID();
3062 if (I.isInlineAsm()) {
3063 // FIXME: inline asm is not yet supported for callbr in GlobalISel. As soon
3064 // as we add support, we need to handle the indirect asm targets, see
3065 // SelectionDAGBuilder::visitCallBr().
3066 return false;
3067 }
3068 if (!translateIntrinsic(CB: I, ID: IID, MIRBuilder))
3069 return false;
3070
3071 // Retrieve successors.
3072 SmallPtrSet<BasicBlock *, 8> Dests = {I.getDefaultDest()};
3073 MachineBasicBlock *Return = &getMBB(BB: *I.getDefaultDest());
3074
3075 // Update successor info.
3076 addSuccessorWithProb(Src: CallBrMBB, Dst: Return, Prob: BranchProbability::getOne());
3077
3078 // Add indirect targets as successors. For intrinsic callbr, these represent
3079 // implicit control flow (e.g., the "kill" path for amdgcn.kill). We mark them
3080 // with setIsInlineAsmBrIndirectTarget so the machine verifier accepts them as
3081 // valid successors, even though they're not from inline asm.
3082 for (BasicBlock *Dest : I.getIndirectDests()) {
3083 MachineBasicBlock &Target = getMBB(BB: *Dest);
3084 Target.setIsInlineAsmBrIndirectTarget();
3085 Target.setLabelMustBeEmitted();
3086 // Don't add duplicate machine successors.
3087 if (Dests.insert(Ptr: Dest).second)
3088 addSuccessorWithProb(Src: CallBrMBB, Dst: &Target, Prob: BranchProbability::getZero());
3089 }
3090
3091 CallBrMBB->normalizeSuccProbs();
3092
3093 // Drop into default successor.
3094 MIRBuilder.buildBr(Dest&: *Return);
3095
3096 return true;
3097}
3098
3099bool IRTranslator::translateLandingPad(const User &U,
3100 MachineIRBuilder &MIRBuilder) {
3101 const LandingPadInst &LP = cast<LandingPadInst>(Val: U);
3102
3103 MachineBasicBlock &MBB = MIRBuilder.getMBB();
3104
3105 MBB.setIsEHPad();
3106
3107 // If there aren't registers to copy the values into (e.g., during SjLj
3108 // exceptions), then don't bother.
3109 const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
3110 if (TLI->getExceptionPointerRegister(PersonalityFn) == 0 &&
3111 TLI->getExceptionSelectorRegister(PersonalityFn) == 0)
3112 return true;
3113
3114 // If landingpad's return type is token type, we don't create DAG nodes
3115 // for its exception pointer and selector value. The extraction of exception
3116 // pointer or selector value from token type landingpads is not currently
3117 // supported.
3118 if (LP.getType()->isTokenTy())
3119 return true;
3120
3121 // Add a label to mark the beginning of the landing pad. Deletion of the
3122 // landing pad can thus be detected via the MachineModuleInfo.
3123 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL)
3124 .addSym(Sym: MF->addLandingPad(LandingPad: &MBB));
3125
3126 // If the unwinder does not preserve all registers, ensure that the
3127 // function marks the clobbered registers as used.
3128 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
3129 if (auto *RegMask = TRI.getCustomEHPadPreservedMask(MF: *MF))
3130 MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask);
3131
3132 LLT Ty = getLLTForType(Ty&: *LP.getType(), DL: *DL);
3133 Register Undef = MRI->createGenericVirtualRegister(Ty);
3134 MIRBuilder.buildUndef(Res: Undef);
3135
3136 SmallVector<LLT, 2> Tys;
3137 for (Type *Ty : cast<StructType>(Val: LP.getType())->elements())
3138 Tys.push_back(Elt: getLLTForType(Ty&: *Ty, DL: *DL));
3139 assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
3140
3141 // Mark exception register as live in.
3142 Register ExceptionReg = TLI->getExceptionPointerRegister(PersonalityFn);
3143 if (!ExceptionReg)
3144 return false;
3145
3146 MBB.addLiveIn(PhysReg: ExceptionReg);
3147 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: LP);
3148 MIRBuilder.buildCopy(Res: ResRegs[0], Op: ExceptionReg);
3149
3150 Register SelectorReg = TLI->getExceptionSelectorRegister(PersonalityFn);
3151 if (!SelectorReg)
3152 return false;
3153
3154 MBB.addLiveIn(PhysReg: SelectorReg);
3155 Register PtrVReg = MRI->createGenericVirtualRegister(Ty: Tys[0]);
3156 MIRBuilder.buildCopy(Res: PtrVReg, Op: SelectorReg);
3157 MIRBuilder.buildCast(Dst: ResRegs[1], Src: PtrVReg);
3158
3159 return true;
3160}
3161
3162bool IRTranslator::translateAlloca(const User &U,
3163 MachineIRBuilder &MIRBuilder) {
3164 auto &AI = cast<AllocaInst>(Val: U);
3165
3166 if (AI.isSwiftError())
3167 return true;
3168
3169 if (AI.isStaticAlloca()) {
3170 Register Res = getOrCreateVReg(Val: AI);
3171 int FI = getOrCreateFrameIndex(AI);
3172 MIRBuilder.buildFrameIndex(Res, Idx: FI);
3173 return true;
3174 }
3175
3176 // FIXME: support stack probing for Windows.
3177 if (MF->getTarget().getTargetTriple().isOSWindows())
3178 return false;
3179
3180 // Now we're in the harder dynamic case.
3181 Register NumElts = getOrCreateVReg(Val: *AI.getArraySize());
3182 Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
3183 LLT IntPtrTy = getLLTForType(Ty&: *IntPtrIRTy, DL: *DL);
3184 if (MRI->getType(Reg: NumElts) != IntPtrTy) {
3185 Register ExtElts = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
3186 MIRBuilder.buildZExtOrTrunc(Res: ExtElts, Op: NumElts);
3187 NumElts = ExtElts;
3188 }
3189
3190 Type *Ty = AI.getAllocatedType();
3191
3192 Register AllocSize = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
3193 Register TySize =
3194 getOrCreateVReg(Val: *ConstantInt::get(Ty: IntPtrIRTy, V: DL->getTypeAllocSize(Ty)));
3195 MIRBuilder.buildMul(Dst: AllocSize, Src0: NumElts, Src1: TySize);
3196
3197 // Round the size of the allocation up to the stack alignment size
3198 // by add SA-1 to the size. This doesn't overflow because we're computing
3199 // an address inside an alloca.
3200 Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();
3201 auto SAMinusOne = MIRBuilder.buildConstant(Res: IntPtrTy, Val: StackAlign.value() - 1);
3202 auto AllocAdd = MIRBuilder.buildAdd(Dst: IntPtrTy, Src0: AllocSize, Src1: SAMinusOne,
3203 Flags: MachineInstr::NoUWrap);
3204 auto AlignCst =
3205 MIRBuilder.buildConstant(Res: IntPtrTy, Val: ~(uint64_t)(StackAlign.value() - 1));
3206 auto AlignedAlloc = MIRBuilder.buildAnd(Dst: IntPtrTy, Src0: AllocAdd, Src1: AlignCst);
3207
3208 Align Alignment = AI.getAlign();
3209 if (Alignment <= StackAlign)
3210 Alignment = Align(1);
3211 MIRBuilder.buildDynStackAlloc(Res: getOrCreateVReg(Val: AI), Size: AlignedAlloc, Alignment);
3212
3213 MF->getFrameInfo().CreateVariableSizedObject(Alignment, Alloca: &AI);
3214 assert(MF->getFrameInfo().hasVarSizedObjects());
3215 return true;
3216}
3217
3218bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
3219 // FIXME: We may need more info about the type. Because of how LLT works,
3220 // we're completely discarding the i64/double distinction here (amongst
3221 // others). Fortunately the ABIs I know of where that matters don't use va_arg
3222 // anyway but that's not guaranteed.
3223 MIRBuilder.buildInstr(Opc: TargetOpcode::G_VAARG, DstOps: {getOrCreateVReg(Val: U)},
3224 SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
3225 DL->getABITypeAlign(Ty: U.getType()).value()});
3226 return true;
3227}
3228
3229bool IRTranslator::translateUnreachable(const User &U,
3230 MachineIRBuilder &MIRBuilder) {
3231 auto &UI = cast<UnreachableInst>(Val: U);
3232 if (!UI.shouldLowerToTrap(TrapUnreachable: MF->getTarget().Options.TrapUnreachable,
3233 NoTrapAfterNoreturn: MF->getTarget().Options.NoTrapAfterNoreturn))
3234 return true;
3235
3236 MIRBuilder.buildTrap();
3237 return true;
3238}
3239
3240bool IRTranslator::translateInsertElement(const User &U,
3241 MachineIRBuilder &MIRBuilder) {
3242 // If it is a <1 x Ty> vector, use the scalar as it is
3243 // not a legal vector type in LLT.
3244 if (auto *FVT = dyn_cast<FixedVectorType>(Val: U.getType());
3245 FVT && FVT->getNumElements() == 1)
3246 return translateCopy(U, V: *U.getOperand(i: 1), MIRBuilder);
3247
3248 Register Res = getOrCreateVReg(Val: U);
3249 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
3250 Register Elt = getOrCreateVReg(Val: *U.getOperand(i: 1));
3251 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
3252 Register Idx;
3253 if (auto *CI = dyn_cast<ConstantInt>(Val: U.getOperand(i: 2))) {
3254 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3255 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
3256 auto *NewIdxCI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
3257 Idx = getOrCreateVReg(Val: *NewIdxCI);
3258 }
3259 }
3260 if (!Idx)
3261 Idx = getOrCreateVReg(Val: *U.getOperand(i: 2));
3262 if (MRI->getType(Reg: Idx).getSizeInBits() != PreferredVecIdxWidth) {
3263 const LLT VecIdxTy = LLT::scalar(SizeInBits: PreferredVecIdxWidth);
3264 Idx = MIRBuilder.buildZExtOrTrunc(Res: VecIdxTy, Op: Idx).getReg(Idx: 0);
3265 }
3266 MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
3267 return true;
3268}
3269
3270bool IRTranslator::translateInsertVector(const User &U,
3271 MachineIRBuilder &MIRBuilder) {
3272 Register Dst = getOrCreateVReg(Val: U);
3273 Register Vec = getOrCreateVReg(Val: *U.getOperand(i: 0));
3274 Register Elt = getOrCreateVReg(Val: *U.getOperand(i: 1));
3275
3276 ConstantInt *CI = cast<ConstantInt>(Val: U.getOperand(i: 2));
3277 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
3278
3279 // Resize Index to preferred index width.
3280 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3281 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
3282 CI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
3283 }
3284
3285 // If it is a <1 x Ty> vector, we have to use other means.
3286 if (auto *ResultType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 1)->getType());
3287 ResultType && ResultType->getNumElements() == 1) {
3288 if (auto *InputType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType());
3289 InputType && InputType->getNumElements() == 1) {
3290 // We are inserting an illegal fixed vector into an illegal
3291 // fixed vector, use the scalar as it is not a legal vector type
3292 // in LLT.
3293 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
3294 }
3295 if (isa<FixedVectorType>(Val: U.getOperand(i: 0)->getType())) {
3296 // We are inserting an illegal fixed vector into a legal fixed
3297 // vector, use the scalar as it is not a legal vector type in
3298 // LLT.
3299 Register Idx = getOrCreateVReg(Val: *CI);
3300 MIRBuilder.buildInsertVectorElement(Res: Dst, Val: Vec, Elt, Idx);
3301 return true;
3302 }
3303 if (isa<ScalableVectorType>(Val: U.getOperand(i: 0)->getType())) {
3304 // We are inserting an illegal fixed vector into a scalable
3305 // vector, use a scalar element insert.
3306 LLT VecIdxTy = LLT::scalar(SizeInBits: PreferredVecIdxWidth);
3307 Register Idx = getOrCreateVReg(Val: *CI);
3308 auto ScaledIndex = MIRBuilder.buildMul(
3309 Dst: VecIdxTy, Src0: MIRBuilder.buildVScale(Res: VecIdxTy, MinElts: 1), Src1: Idx);
3310 MIRBuilder.buildInsertVectorElement(Res: Dst, Val: Vec, Elt, Idx: ScaledIndex);
3311 return true;
3312 }
3313 }
3314
3315 MIRBuilder.buildInsertSubvector(
3316 Res: getOrCreateVReg(Val: U), Src0: getOrCreateVReg(Val: *U.getOperand(i: 0)),
3317 Src1: getOrCreateVReg(Val: *U.getOperand(i: 1)), Index: CI->getZExtValue());
3318 return true;
3319}
3320
3321bool IRTranslator::translateExtractElement(const User &U,
3322 MachineIRBuilder &MIRBuilder) {
3323 // If it is a <1 x Ty> vector, use the scalar as it is
3324 // not a legal vector type in LLT.
3325 if (const FixedVectorType *FVT =
3326 dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType()))
3327 if (FVT->getNumElements() == 1)
3328 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
3329
3330 Register Res = getOrCreateVReg(Val: U);
3331 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
3332 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
3333 Register Idx;
3334 if (auto *CI = dyn_cast<ConstantInt>(Val: U.getOperand(i: 1))) {
3335 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3336 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
3337 auto *NewIdxCI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
3338 Idx = getOrCreateVReg(Val: *NewIdxCI);
3339 }
3340 }
3341 if (!Idx)
3342 Idx = getOrCreateVReg(Val: *U.getOperand(i: 1));
3343 if (MRI->getType(Reg: Idx).getSizeInBits() != PreferredVecIdxWidth) {
3344 const LLT VecIdxTy = LLT::scalar(SizeInBits: PreferredVecIdxWidth);
3345 Idx = MIRBuilder.buildZExtOrTrunc(Res: VecIdxTy, Op: Idx).getReg(Idx: 0);
3346 }
3347 MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
3348 return true;
3349}
3350
3351bool IRTranslator::translateExtractVector(const User &U,
3352 MachineIRBuilder &MIRBuilder) {
3353 Register Res = getOrCreateVReg(Val: U);
3354 Register Vec = getOrCreateVReg(Val: *U.getOperand(i: 0));
3355 ConstantInt *CI = cast<ConstantInt>(Val: U.getOperand(i: 1));
3356 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
3357
3358 // Resize Index to preferred index width.
3359 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3360 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
3361 CI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
3362 }
3363
3364 // If it is a <1 x Ty> vector, we have to use other means.
3365 if (auto *ResultType = dyn_cast<FixedVectorType>(Val: U.getType());
3366 ResultType && ResultType->getNumElements() == 1) {
3367 if (auto *InputType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType());
3368 InputType && InputType->getNumElements() == 1) {
3369 // We are extracting an illegal fixed vector from an illegal fixed vector,
3370 // use the scalar as it is not a legal vector type in LLT.
3371 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
3372 }
3373 if (isa<FixedVectorType>(Val: U.getOperand(i: 0)->getType())) {
3374 // We are extracting an illegal fixed vector from a legal fixed
3375 // vector, use the scalar as it is not a legal vector type in
3376 // LLT.
3377 Register Idx = getOrCreateVReg(Val: *CI);
3378 MIRBuilder.buildExtractVectorElement(Res, Val: Vec, Idx);
3379 return true;
3380 }
3381 if (isa<ScalableVectorType>(Val: U.getOperand(i: 0)->getType())) {
3382 // We are extracting an illegal fixed vector from a scalable
3383 // vector, use a scalar element extract.
3384 LLT VecIdxTy = LLT::scalar(SizeInBits: PreferredVecIdxWidth);
3385 Register Idx = getOrCreateVReg(Val: *CI);
3386 auto ScaledIndex = MIRBuilder.buildMul(
3387 Dst: VecIdxTy, Src0: MIRBuilder.buildVScale(Res: VecIdxTy, MinElts: 1), Src1: Idx);
3388 MIRBuilder.buildExtractVectorElement(Res, Val: Vec, Idx: ScaledIndex);
3389 return true;
3390 }
3391 }
3392
3393 MIRBuilder.buildExtractSubvector(Res: getOrCreateVReg(Val: U),
3394 Src: getOrCreateVReg(Val: *U.getOperand(i: 0)),
3395 Index: CI->getZExtValue());
3396 return true;
3397}
3398
3399bool IRTranslator::translateShuffleVector(const User &U,
3400 MachineIRBuilder &MIRBuilder) {
3401 // A ShuffleVector that operates on scalable vectors is a splat vector where
3402 // the value of the splat vector is the 0th element of the first operand,
3403 // since the index mask operand is the zeroinitializer (undef and
3404 // poison are treated as zeroinitializer here).
3405 if (U.getOperand(i: 0)->getType()->isScalableTy()) {
3406 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
3407 auto SplatVal = MIRBuilder.buildExtractVectorElementConstant(
3408 Res: MRI->getType(Reg: Val).getElementType(), Val, Idx: 0);
3409 MIRBuilder.buildSplatVector(Res: getOrCreateVReg(Val: U), Val: SplatVal);
3410 return true;
3411 }
3412
3413 ArrayRef<int> Mask;
3414 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: &U))
3415 Mask = SVI->getShuffleMask();
3416 else
3417 Mask = cast<ConstantExpr>(Val: U).getShuffleMask();
3418
3419 // As GISel does not represent <1 x > vectors as a separate type from scalars,
3420 // we transform shuffle_vector with a scalar output to an
3421 // ExtractVectorElement. If the input type is also scalar it becomes a Copy.
3422 unsigned DstElts = cast<FixedVectorType>(Val: U.getType())->getNumElements();
3423 unsigned SrcElts =
3424 cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType())->getNumElements();
3425 if (DstElts == 1) {
3426 unsigned M = Mask[0];
3427 if (SrcElts == 1) {
3428 if (M == 0 || M == 1)
3429 return translateCopy(U, V: *U.getOperand(i: M), MIRBuilder);
3430 MIRBuilder.buildUndef(Res: getOrCreateVReg(Val: U));
3431 } else {
3432 Register Dst = getOrCreateVReg(Val: U);
3433 if (M < SrcElts) {
3434 MIRBuilder.buildExtractVectorElementConstant(
3435 Res: Dst, Val: getOrCreateVReg(Val: *U.getOperand(i: 0)), Idx: M);
3436 } else if (M < SrcElts * 2) {
3437 MIRBuilder.buildExtractVectorElementConstant(
3438 Res: Dst, Val: getOrCreateVReg(Val: *U.getOperand(i: 1)), Idx: M - SrcElts);
3439 } else {
3440 MIRBuilder.buildUndef(Res: Dst);
3441 }
3442 }
3443 return true;
3444 }
3445
3446 // A single element src is transformed to a build_vector.
3447 if (SrcElts == 1) {
3448 SmallVector<Register> Ops;
3449 Register Undef;
3450 for (int M : Mask) {
3451 LLT SrcTy = getLLTForType(Ty&: *U.getOperand(i: 0)->getType(), DL: *DL);
3452 if (M == 0 || M == 1) {
3453 Ops.push_back(Elt: getOrCreateVReg(Val: *U.getOperand(i: M)));
3454 } else {
3455 if (!Undef.isValid()) {
3456 Undef = MRI->createGenericVirtualRegister(Ty: SrcTy);
3457 MIRBuilder.buildUndef(Res: Undef);
3458 }
3459 Ops.push_back(Elt: Undef);
3460 }
3461 }
3462 MIRBuilder.buildBuildVector(Res: getOrCreateVReg(Val: U), Ops);
3463 return true;
3464 }
3465
3466 ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
3467 MIRBuilder
3468 .buildInstr(Opc: TargetOpcode::G_SHUFFLE_VECTOR, DstOps: {getOrCreateVReg(Val: U)},
3469 SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
3470 getOrCreateVReg(Val: *U.getOperand(i: 1))})
3471 .addShuffleMask(Val: MaskAlloc);
3472 return true;
3473}
3474
3475bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
3476 const PHINode &PI = cast<PHINode>(Val: U);
3477
3478 SmallVector<MachineInstr *, 4> Insts;
3479 for (auto Reg : getOrCreateVRegs(Val: PI)) {
3480 auto MIB = MIRBuilder.buildInstr(Opc: TargetOpcode::G_PHI, DstOps: {Reg}, SrcOps: {});
3481 Insts.push_back(Elt: MIB.getInstr());
3482 }
3483
3484 PendingPHIs.emplace_back(Args: &PI, Args: std::move(Insts));
3485 return true;
3486}
3487
3488bool IRTranslator::translateAtomicCmpXchg(const User &U,
3489 MachineIRBuilder &MIRBuilder) {
3490 const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(Val: U);
3491
3492 auto Flags = TLI->getAtomicMemOperandFlags(AI: I, DL: *DL);
3493
3494 auto Res = getOrCreateVRegs(Val: I);
3495 Register OldValRes = Res[0];
3496 Register SuccessRes = Res[1];
3497 Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
3498 Register Cmp = getOrCreateVReg(Val: *I.getCompareOperand());
3499 Register NewVal = getOrCreateVReg(Val: *I.getNewValOperand());
3500
3501 MIRBuilder.buildAtomicCmpXchgWithSuccess(
3502 OldValRes, SuccessRes, Addr, CmpVal: Cmp, NewVal,
3503 MMO&: *MF->getMachineMemOperand(
3504 PtrInfo: MachinePointerInfo(I.getPointerOperand()), f: Flags, MemTy: MRI->getType(Reg: Cmp),
3505 base_alignment: getMemOpAlign(I), AAInfo: I.getAAMetadata(), Ranges: nullptr, SSID: I.getSyncScopeID(),
3506 Ordering: I.getSuccessOrdering(), FailureOrdering: I.getFailureOrdering()));
3507 return true;
3508}
3509
3510bool IRTranslator::translateAtomicRMW(const User &U,
3511 MachineIRBuilder &MIRBuilder) {
3512 if (containsBF16Type(U) && !targetSupportsBF16Type(MF))
3513 return false;
3514
3515 const AtomicRMWInst &I = cast<AtomicRMWInst>(Val: U);
3516 auto Flags = TLI->getAtomicMemOperandFlags(AI: I, DL: *DL);
3517
3518 Register Res = getOrCreateVReg(Val: I);
3519 Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
3520 Register Val = getOrCreateVReg(Val: *I.getValOperand());
3521
3522 unsigned Opcode = 0;
3523 switch (I.getOperation()) {
3524 default:
3525 return false;
3526 case AtomicRMWInst::Xchg:
3527 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
3528 break;
3529 case AtomicRMWInst::Add:
3530 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
3531 break;
3532 case AtomicRMWInst::Sub:
3533 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
3534 break;
3535 case AtomicRMWInst::And:
3536 Opcode = TargetOpcode::G_ATOMICRMW_AND;
3537 break;
3538 case AtomicRMWInst::Nand:
3539 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
3540 break;
3541 case AtomicRMWInst::Or:
3542 Opcode = TargetOpcode::G_ATOMICRMW_OR;
3543 break;
3544 case AtomicRMWInst::Xor:
3545 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
3546 break;
3547 case AtomicRMWInst::Max:
3548 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
3549 break;
3550 case AtomicRMWInst::Min:
3551 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
3552 break;
3553 case AtomicRMWInst::UMax:
3554 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
3555 break;
3556 case AtomicRMWInst::UMin:
3557 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
3558 break;
3559 case AtomicRMWInst::FAdd:
3560 Opcode = TargetOpcode::G_ATOMICRMW_FADD;
3561 break;
3562 case AtomicRMWInst::FSub:
3563 Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
3564 break;
3565 case AtomicRMWInst::FMax:
3566 Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
3567 break;
3568 case AtomicRMWInst::FMin:
3569 Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
3570 break;
3571 case AtomicRMWInst::FMaximum:
3572 Opcode = TargetOpcode::G_ATOMICRMW_FMAXIMUM;
3573 break;
3574 case AtomicRMWInst::FMinimum:
3575 Opcode = TargetOpcode::G_ATOMICRMW_FMINIMUM;
3576 break;
3577 case AtomicRMWInst::UIncWrap:
3578 Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
3579 break;
3580 case AtomicRMWInst::UDecWrap:
3581 Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
3582 break;
3583 case AtomicRMWInst::USubCond:
3584 Opcode = TargetOpcode::G_ATOMICRMW_USUB_COND;
3585 break;
3586 case AtomicRMWInst::USubSat:
3587 Opcode = TargetOpcode::G_ATOMICRMW_USUB_SAT;
3588 break;
3589 }
3590
3591 MIRBuilder.buildAtomicRMW(
3592 Opcode, OldValRes: Res, Addr, Val,
3593 MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(I.getPointerOperand()),
3594 f: Flags, MemTy: MRI->getType(Reg: Val), base_alignment: getMemOpAlign(I),
3595 AAInfo: I.getAAMetadata(), Ranges: nullptr, SSID: I.getSyncScopeID(),
3596 Ordering: I.getOrdering()));
3597 return true;
3598}
3599
3600bool IRTranslator::translateFence(const User &U,
3601 MachineIRBuilder &MIRBuilder) {
3602 const FenceInst &Fence = cast<FenceInst>(Val: U);
3603 MIRBuilder.buildFence(Ordering: static_cast<unsigned>(Fence.getOrdering()),
3604 Scope: Fence.getSyncScopeID());
3605 return true;
3606}
3607
3608bool IRTranslator::translateFreeze(const User &U,
3609 MachineIRBuilder &MIRBuilder) {
3610 const ArrayRef<Register> DstRegs = getOrCreateVRegs(Val: U);
3611 const ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *U.getOperand(i: 0));
3612
3613 assert(DstRegs.size() == SrcRegs.size() &&
3614 "Freeze with different source and destination type?");
3615
3616 for (unsigned I = 0; I < DstRegs.size(); ++I) {
3617 MIRBuilder.buildFreeze(Dst: DstRegs[I], Src: SrcRegs[I]);
3618 }
3619
3620 return true;
3621}
3622
3623void IRTranslator::finishPendingPhis() {
3624#ifndef NDEBUG
3625 DILocationVerifier Verifier;
3626 GISelObserverWrapper WrapperObserver(&Verifier);
3627 RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);
3628#endif // ifndef NDEBUG
3629 for (auto &Phi : PendingPHIs) {
3630 const PHINode *PI = Phi.first;
3631 if (PI->getType()->isEmptyTy())
3632 continue;
3633 ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
3634 MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
3635 EntryBuilder->setDebugLoc(PI->getDebugLoc());
3636#ifndef NDEBUG
3637 Verifier.setCurrentInst(PI);
3638#endif // ifndef NDEBUG
3639
3640 SmallPtrSet<const MachineBasicBlock *, 16> SeenPreds;
3641 for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
3642 auto IRPred = PI->getIncomingBlock(i);
3643 ArrayRef<Register> ValRegs = getOrCreateVRegs(Val: *PI->getIncomingValue(i));
3644 for (auto *Pred : getMachinePredBBs(Edge: {IRPred, PI->getParent()})) {
3645 if (SeenPreds.count(Ptr: Pred) || !PhiMBB->isPredecessor(MBB: Pred))
3646 continue;
3647 SeenPreds.insert(Ptr: Pred);
3648 for (unsigned j = 0; j < ValRegs.size(); ++j) {
3649 MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
3650 MIB.addUse(RegNo: ValRegs[j]);
3651 MIB.addMBB(MBB: Pred);
3652 }
3653 }
3654 }
3655 }
3656}
3657
3658void IRTranslator::translateDbgValueRecord(Value *V, bool HasArgList,
3659 const DILocalVariable *Variable,
3660 const DIExpression *Expression,
3661 const DebugLoc &DL,
3662 MachineIRBuilder &MIRBuilder) {
3663 assert(Variable->isValidLocationForIntrinsic(DL) &&
3664 "Expected inlined-at fields to agree");
3665 // Act as if we're handling a debug intrinsic.
3666 MIRBuilder.setDebugLoc(DL);
3667
3668 if (!V || HasArgList) {
3669 // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
3670 // terminate any prior location.
3671 MIRBuilder.buildIndirectDbgValue(Reg: 0, Variable, Expr: Expression);
3672 return;
3673 }
3674
3675 if (const auto *CI = dyn_cast<Constant>(Val: V)) {
3676 MIRBuilder.buildConstDbgValue(C: *CI, Variable, Expr: Expression);
3677 return;
3678 }
3679
3680 if (auto *AI = dyn_cast<AllocaInst>(Val: V);
3681 AI && AI->isStaticAlloca() && Expression->startsWithDeref()) {
3682 // If the value is an alloca and the expression starts with a
3683 // dereference, track a stack slot instead of a register, as registers
3684 // may be clobbered.
3685 auto ExprOperands = Expression->getElements();
3686 auto *ExprDerefRemoved =
3687 DIExpression::get(Context&: AI->getContext(), Elements: ExprOperands.drop_front());
3688 MIRBuilder.buildFIDbgValue(FI: getOrCreateFrameIndex(AI: *AI), Variable,
3689 Expr: ExprDerefRemoved);
3690 return;
3691 }
3692 if (translateIfEntryValueArgument(isDeclare: false, Val: V, Var: Variable, Expr: Expression, DL,
3693 MIRBuilder))
3694 return;
3695 for (Register Reg : getOrCreateVRegs(Val: *V)) {
3696 // FIXME: This does not handle register-indirect values at offset 0. The
3697 // direct/indirect thing shouldn't really be handled by something as
3698 // implicit as reg+noreg vs reg+imm in the first place, but it seems
3699 // pretty baked in right now.
3700 MIRBuilder.buildDirectDbgValue(Reg, Variable, Expr: Expression);
3701 }
3702}
3703
3704void IRTranslator::translateDbgDeclareRecord(Value *Address, bool HasArgList,
3705 const DILocalVariable *Variable,
3706 const DIExpression *Expression,
3707 const DebugLoc &DL,
3708 MachineIRBuilder &MIRBuilder) {
3709 if (!Address || isa<UndefValue>(Val: Address)) {
3710 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *Variable << "\n");
3711 return;
3712 }
3713
3714 assert(Variable->isValidLocationForIntrinsic(DL) &&
3715 "Expected inlined-at fields to agree");
3716 auto AI = dyn_cast<AllocaInst>(Val: Address);
3717 if (AI && AI->isStaticAlloca()) {
3718 // Static allocas are tracked at the MF level, no need for DBG_VALUE
3719 // instructions (in fact, they get ignored if they *do* exist).
3720 MF->setVariableDbgInfo(Var: Variable, Expr: Expression,
3721 Slot: getOrCreateFrameIndex(AI: *AI), Loc: DL);
3722 return;
3723 }
3724
3725 if (translateIfEntryValueArgument(isDeclare: true, Val: Address, Var: Variable,
3726 Expr: Expression, DL,
3727 MIRBuilder))
3728 return;
3729
3730 // A dbg.declare describes the address of a source variable, so lower it
3731 // into an indirect DBG_VALUE.
3732 MIRBuilder.setDebugLoc(DL);
3733 MIRBuilder.buildIndirectDbgValue(Reg: getOrCreateVReg(Val: *Address), Variable,
3734 Expr: Expression);
3735}
3736
3737void IRTranslator::translateDbgInfo(const Instruction &Inst,
3738 MachineIRBuilder &MIRBuilder) {
3739 for (DbgRecord &DR : Inst.getDbgRecordRange()) {
3740 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
3741 MIRBuilder.setDebugLoc(DLR->getDebugLoc());
3742 assert(DLR->getLabel() && "Missing label");
3743 assert(DLR->getLabel()->isValidLocationForIntrinsic(
3744 MIRBuilder.getDebugLoc()) &&
3745 "Expected inlined-at fields to agree");
3746 MIRBuilder.buildDbgLabel(Label: DLR->getLabel());
3747 continue;
3748 }
3749 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
3750 const DILocalVariable *Variable = DVR.getVariable();
3751 const DIExpression *Expression = DVR.getExpression();
3752 Value *V = DVR.getVariableLocationOp(OpIdx: 0);
3753 if (DVR.isDbgDeclare())
3754 translateDbgDeclareRecord(Address: V, HasArgList: DVR.hasArgList(), Variable, Expression,
3755 DL: DVR.getDebugLoc(), MIRBuilder);
3756 else
3757 translateDbgValueRecord(V, HasArgList: DVR.hasArgList(), Variable, Expression,
3758 DL: DVR.getDebugLoc(), MIRBuilder);
3759 }
3760}
3761
3762bool IRTranslator::translate(const Instruction &Inst) {
3763 CurBuilder->setDebugLoc(Inst.getDebugLoc());
3764 CurBuilder->setPCSections(Inst.getMetadata(KindID: LLVMContext::MD_pcsections));
3765 CurBuilder->setMMRAMetadata(Inst.getMetadata(KindID: LLVMContext::MD_mmra));
3766
3767 if (TLI->fallBackToDAGISel(Inst))
3768 return false;
3769
3770 switch (Inst.getOpcode()) {
3771#define HANDLE_INST(NUM, OPCODE, CLASS) \
3772 case Instruction::OPCODE: \
3773 return translate##OPCODE(Inst, *CurBuilder.get());
3774#include "llvm/IR/Instruction.def"
3775 default:
3776 return false;
3777 }
3778}
3779
3780bool IRTranslator::translate(const Constant &C, Register Reg) {
3781 // We only emit constants into the entry block from here. To prevent jumpy
3782 // debug behaviour remove debug line.
3783 if (auto CurrInstDL = CurBuilder->getDL())
3784 EntryBuilder->setDebugLoc(DebugLoc());
3785
3786 if (auto CI = dyn_cast<ConstantInt>(Val: &C)) {
3787 // buildConstant expects a to-be-splatted scalar ConstantInt.
3788 if (isa<VectorType>(Val: CI->getType()))
3789 CI = ConstantInt::get(Context&: CI->getContext(), V: CI->getValue());
3790 EntryBuilder->buildConstant(Res: Reg, Val: *CI);
3791 } else if (auto CF = dyn_cast<ConstantFP>(Val: &C)) {
3792 // buildFConstant expects a to-be-splatted scalar ConstantFP.
3793 if (isa<VectorType>(Val: CF->getType()))
3794 CF = ConstantFP::get(Context&: CF->getContext(), V: CF->getValue());
3795 EntryBuilder->buildFConstant(Res: Reg, Val: *CF);
3796 } else if (isa<UndefValue>(Val: C))
3797 EntryBuilder->buildUndef(Res: Reg);
3798 else if (isa<ConstantPointerNull>(Val: C))
3799 EntryBuilder->buildConstant(Res: Reg, Val: 0);
3800 else if (auto GV = dyn_cast<GlobalValue>(Val: &C))
3801 EntryBuilder->buildGlobalValue(Res: Reg, GV);
3802 else if (auto CPA = dyn_cast<ConstantPtrAuth>(Val: &C)) {
3803 Register Addr = getOrCreateVReg(Val: *CPA->getPointer());
3804 Register AddrDisc = getOrCreateVReg(Val: *CPA->getAddrDiscriminator());
3805 EntryBuilder->buildConstantPtrAuth(Res: Reg, CPA, Addr, AddrDisc);
3806 } else if (auto CAZ = dyn_cast<ConstantAggregateZero>(Val: &C)) {
3807 Constant &Elt = *CAZ->getElementValue(Idx: 0u);
3808 if (isa<ScalableVectorType>(Val: CAZ->getType())) {
3809 EntryBuilder->buildSplatVector(Res: Reg, Val: getOrCreateVReg(Val: Elt));
3810 return true;
3811 }
3812 // Return the scalar if it is a <1 x Ty> vector.
3813 unsigned NumElts = CAZ->getElementCount().getFixedValue();
3814 if (NumElts == 1)
3815 return translateCopy(U: C, V: Elt, MIRBuilder&: *EntryBuilder);
3816 // All elements are zero so we can just use the first one.
3817 EntryBuilder->buildSplatBuildVector(Res: Reg, Src: getOrCreateVReg(Val: Elt));
3818 } else if (auto CV = dyn_cast<ConstantDataVector>(Val: &C)) {
3819 // Return the scalar if it is a <1 x Ty> vector.
3820 if (CV->getNumElements() == 1)
3821 return translateCopy(U: C, V: *CV->getElementAsConstant(i: 0), MIRBuilder&: *EntryBuilder);
3822 SmallVector<Register, 4> Ops;
3823 for (unsigned i = 0; i < CV->getNumElements(); ++i) {
3824 Constant &Elt = *CV->getElementAsConstant(i);
3825 Ops.push_back(Elt: getOrCreateVReg(Val: Elt));
3826 }
3827 EntryBuilder->buildBuildVector(Res: Reg, Ops);
3828 } else if (auto CE = dyn_cast<ConstantExpr>(Val: &C)) {
3829 switch(CE->getOpcode()) {
3830#define HANDLE_INST(NUM, OPCODE, CLASS) \
3831 case Instruction::OPCODE: \
3832 return translate##OPCODE(*CE, *EntryBuilder.get());
3833#include "llvm/IR/Instruction.def"
3834 default:
3835 return false;
3836 }
3837 } else if (auto CV = dyn_cast<ConstantVector>(Val: &C)) {
3838 if (CV->getNumOperands() == 1)
3839 return translateCopy(U: C, V: *CV->getOperand(i_nocapture: 0), MIRBuilder&: *EntryBuilder);
3840 SmallVector<Register, 4> Ops;
3841 for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
3842 Ops.push_back(Elt: getOrCreateVReg(Val: *CV->getOperand(i_nocapture: i)));
3843 }
3844 EntryBuilder->buildBuildVector(Res: Reg, Ops);
3845 } else if (auto *BA = dyn_cast<BlockAddress>(Val: &C)) {
3846 EntryBuilder->buildBlockAddress(Res: Reg, BA);
3847 } else
3848 return false;
3849
3850 return true;
3851}
3852
3853bool IRTranslator::finalizeBasicBlock(const BasicBlock &BB,
3854 MachineBasicBlock &MBB) {
3855 for (auto &BTB : SL->BitTestCases) {
3856 // Emit header first, if it wasn't already emitted.
3857 if (!BTB.Emitted)
3858 emitBitTestHeader(B&: BTB, SwitchBB: BTB.Parent);
3859
3860 BranchProbability UnhandledProb = BTB.Prob;
3861 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
3862 UnhandledProb -= BTB.Cases[j].ExtraProb;
3863 // Set the current basic block to the mbb we wish to insert the code into
3864 MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
3865 // If all cases cover a contiguous range, it is not necessary to jump to
3866 // the default block after the last bit test fails. This is because the
3867 // range check during bit test header creation has guaranteed that every
3868 // case here doesn't go outside the range. In this case, there is no need
3869 // to perform the last bit test, as it will always be true. Instead, make
3870 // the second-to-last bit-test fall through to the target of the last bit
3871 // test, and delete the last bit test.
3872
3873 MachineBasicBlock *NextMBB;
3874 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3875 // Second-to-last bit-test with contiguous range: fall through to the
3876 // target of the final bit test.
3877 NextMBB = BTB.Cases[j + 1].TargetBB;
3878 } else if (j + 1 == ej) {
3879 // For the last bit test, fall through to Default.
3880 NextMBB = BTB.Default;
3881 } else {
3882 // Otherwise, fall through to the next bit test.
3883 NextMBB = BTB.Cases[j + 1].ThisBB;
3884 }
3885
3886 emitBitTestCase(BB&: BTB, NextMBB, BranchProbToNext: UnhandledProb, Reg: BTB.Reg, B&: BTB.Cases[j], SwitchBB: MBB);
3887
3888 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3889 // We need to record the replacement phi edge here that normally
3890 // happens in emitBitTestCase before we delete the case, otherwise the
3891 // phi edge will be lost.
3892 addMachineCFGPred(Edge: {BTB.Parent->getBasicBlock(),
3893 BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
3894 NewPred: MBB);
3895 // Since we're not going to use the final bit test, remove it.
3896 BTB.Cases.pop_back();
3897 break;
3898 }
3899 }
3900 // This is "default" BB. We have two jumps to it. From "header" BB and from
3901 // last "case" BB, unless the latter was skipped.
3902 CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
3903 BTB.Default->getBasicBlock()};
3904 addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Parent);
3905 if (!BTB.ContiguousRange) {
3906 addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Cases.back().ThisBB);
3907 }
3908 }
3909 SL->BitTestCases.clear();
3910
3911 for (auto &JTCase : SL->JTCases) {
3912 // Emit header first, if it wasn't already emitted.
3913 if (!JTCase.first.Emitted)
3914 emitJumpTableHeader(JT&: JTCase.second, JTH&: JTCase.first, HeaderBB: JTCase.first.HeaderBB);
3915
3916 emitJumpTable(JT&: JTCase.second, MBB: JTCase.second.MBB);
3917 }
3918 SL->JTCases.clear();
3919
3920 for (auto &SwCase : SL->SwitchCases)
3921 emitSwitchCase(CB&: SwCase, SwitchBB: &CurBuilder->getMBB(), MIB&: *CurBuilder);
3922 SL->SwitchCases.clear();
3923
3924 // Check if we need to generate stack-protector guard checks.
3925 StackProtector &SP = getAnalysis<StackProtector>();
3926 if (SP.shouldEmitSDCheck(BB)) {
3927 bool FunctionBasedInstrumentation =
3928 TLI->getSSPStackGuardCheck(M: *MF->getFunction().getParent(), Libcalls: *Libcalls);
3929 SPDescriptor.initialize(BB: &BB, MBB: &MBB, FunctionBasedInstrumentation);
3930 }
3931 // Handle stack protector.
3932 if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
3933 LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
3934 return false;
3935 } else if (SPDescriptor.shouldEmitStackProtector()) {
3936 MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
3937 MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
3938
3939 // Find the split point to split the parent mbb. At the same time copy all
3940 // physical registers used in the tail of parent mbb into virtual registers
3941 // before the split point and back into physical registers after the split
3942 // point. This prevents us needing to deal with Live-ins and many other
3943 // register allocation issues caused by us splitting the parent mbb. The
3944 // register allocator will clean up said virtual copies later on.
3945 MachineBasicBlock::iterator SplitPoint = findSplitPointForStackProtector(
3946 BB: ParentMBB, TII: *MF->getSubtarget().getInstrInfo());
3947
3948 // Splice the terminator of ParentMBB into SuccessMBB.
3949 SuccessMBB->splice(Where: SuccessMBB->end(), Other: ParentMBB, From: SplitPoint,
3950 To: ParentMBB->end());
3951
3952 // Add compare/jump on neq/jump to the parent BB.
3953 if (!emitSPDescriptorParent(SPD&: SPDescriptor, ParentBB: ParentMBB))
3954 return false;
3955
3956 // CodeGen Failure MBB if we have not codegened it yet.
3957 MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
3958 if (FailureMBB->empty()) {
3959 if (!emitSPDescriptorFailure(SPD&: SPDescriptor, FailureBB: FailureMBB))
3960 return false;
3961 }
3962
3963 // Clear the Per-BB State.
3964 SPDescriptor.resetPerBBState();
3965 }
3966 return true;
3967}
3968
3969bool IRTranslator::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
3970 MachineBasicBlock *ParentBB) {
3971 CurBuilder->setInsertPt(MBB&: *ParentBB, II: ParentBB->end());
3972 // First create the loads to the guard/stack slot for the comparison.
3973 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
3974 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
3975 LLT PtrMemTy = getLLTForMVT(Ty: TLI->getPointerMemTy(DL: *DL));
3976
3977 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3978 int FI = MFI.getStackProtectorIndex();
3979
3980 Register Guard;
3981 Register StackSlotPtr = CurBuilder->buildFrameIndex(Res: PtrTy, Idx: FI).getReg(Idx: 0);
3982 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3983 Align Align = DL->getPrefTypeAlign(Ty: PointerType::getUnqual(C&: M.getContext()));
3984
3985 // Generate code to load the content of the guard slot.
3986 Register GuardVal =
3987 CurBuilder
3988 ->buildLoad(Res: PtrMemTy, Addr: StackSlotPtr,
3989 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
3990 MMOFlags: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile)
3991 .getReg(Idx: 0);
3992
3993 if (TLI->useStackGuardXorFP()) {
3994 LLVM_DEBUG(dbgs() << "Stack protector xor'ing with FP not yet implemented");
3995 return false;
3996 }
3997
3998 // Retrieve guard check function, nullptr if instrumentation is inlined.
3999 if (const Function *GuardCheckFn = TLI->getSSPStackGuardCheck(M, Libcalls: *Libcalls)) {
4000 // This path is currently untestable on GlobalISel, since the only platform
4001 // that needs this seems to be Windows, and we fall back on that currently.
4002 // The code still lives here in case that changes.
4003 // Silence warning about unused variable until the code below that uses
4004 // 'GuardCheckFn' is enabled.
4005 (void)GuardCheckFn;
4006 return false;
4007#if 0
4008 // The target provides a guard check function to validate the guard value.
4009 // Generate a call to that function with the content of the guard slot as
4010 // argument.
4011 FunctionType *FnTy = GuardCheckFn->getFunctionType();
4012 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
4013 ISD::ArgFlagsTy Flags;
4014 if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
4015 Flags.setInReg();
4016 CallLowering::ArgInfo GuardArgInfo(
4017 {GuardVal, FnTy->getParamType(0), {Flags}});
4018
4019 CallLowering::CallLoweringInfo Info;
4020 Info.OrigArgs.push_back(GuardArgInfo);
4021 Info.CallConv = GuardCheckFn->getCallingConv();
4022 Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
4023 Info.OrigRet = {Register(), FnTy->getReturnType()};
4024 if (!CLI->lowerCall(MIRBuilder, Info)) {
4025 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
4026 return false;
4027 }
4028 return true;
4029#endif
4030 }
4031
4032 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
4033 // Otherwise, emit a volatile load to retrieve the stack guard value.
4034 if (TLI->useLoadStackGuardNode(M: *ParentBB->getBasicBlock()->getModule())) {
4035 Guard =
4036 MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: PtrTy.getSizeInBits()));
4037 getStackGuard(DstReg: Guard, MIRBuilder&: *CurBuilder);
4038 } else {
4039 // TODO: test using android subtarget when we support @llvm.thread.pointer.
4040 const Value *IRGuard = TLI->getSDagStackGuard(M, Libcalls: *Libcalls);
4041 Register GuardPtr = getOrCreateVReg(Val: *IRGuard);
4042
4043 Guard = CurBuilder
4044 ->buildLoad(Res: PtrMemTy, Addr: GuardPtr,
4045 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
4046 MMOFlags: MachineMemOperand::MOLoad |
4047 MachineMemOperand::MOVolatile)
4048 .getReg(Idx: 0);
4049 }
4050
4051 // Perform the comparison.
4052 auto Cmp =
4053 CurBuilder->buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::scalar(SizeInBits: 1), Op0: Guard, Op1: GuardVal);
4054 // If the guard/stackslot do not equal, branch to failure MBB.
4055 CurBuilder->buildBrCond(Tst: Cmp, Dest&: *SPD.getFailureMBB());
4056 // Otherwise branch to success MBB.
4057 CurBuilder->buildBr(Dest&: *SPD.getSuccessMBB());
4058 return true;
4059}
4060
4061bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
4062 MachineBasicBlock *FailureBB) {
4063 const RTLIB::LibcallImpl LibcallImpl =
4064 Libcalls->getLibcallImpl(Call: RTLIB::STACKPROTECTOR_CHECK_FAIL);
4065 if (LibcallImpl == RTLIB::Unsupported)
4066 return false;
4067
4068 CurBuilder->setInsertPt(MBB&: *FailureBB, II: FailureBB->end());
4069
4070 CallLowering::CallLoweringInfo Info;
4071 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: LibcallImpl);
4072
4073 StringRef LibcallName =
4074 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LibcallImpl);
4075 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
4076 Info.OrigRet = {Register(), Type::getVoidTy(C&: MF->getFunction().getContext()),
4077 0};
4078 if (!CLI->lowerCall(MIRBuilder&: *CurBuilder, Info)) {
4079 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
4080 return false;
4081 }
4082
4083 // Emit a trap instruction if we are required to do so.
4084 const TargetOptions &TargetOpts = TLI->getTargetMachine().Options;
4085 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
4086 CurBuilder->buildInstr(Opcode: TargetOpcode::G_TRAP);
4087
4088 return true;
4089}
4090
4091void IRTranslator::finalizeFunction() {
4092 // Release the memory used by the different maps we
4093 // needed during the translation.
4094 PendingPHIs.clear();
4095 VMap.reset();
4096 FrameIndices.clear();
4097 MachinePreds.clear();
4098 // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
4099 // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
4100 // destroying it twice (in ~IRTranslator() and ~LLVMContext())
4101 EntryBuilder.reset();
4102 CurBuilder.reset();
4103 FuncInfo.clear();
4104 SPDescriptor.resetPerFunctionState();
4105}
4106
4107/// Returns true if a BasicBlock \p BB within a variadic function contains a
4108/// variadic musttail call.
4109static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
4110 if (!IsVarArg)
4111 return false;
4112
4113 // Walk the block backwards, because tail calls usually only appear at the end
4114 // of a block.
4115 return llvm::any_of(Range: llvm::reverse(C: BB), P: [](const Instruction &I) {
4116 const auto *CI = dyn_cast<CallInst>(Val: &I);
4117 return CI && CI->isMustTailCall();
4118 });
4119}
4120
4121bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
4122 MF = &CurMF;
4123 const Function &F = MF->getFunction();
4124 ORE = std::make_unique<OptimizationRemarkEmitter>(args: &F);
4125 CLI = MF->getSubtarget().getCallLowering();
4126
4127 if (CLI->fallBackToDAGISel(MF: *MF)) {
4128 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
4129 F.getSubprogram(), &F.getEntryBlock());
4130 R << "unable to lower function: "
4131 << ore::NV("Prototype", F.getFunctionType());
4132
4133 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
4134 return false;
4135 }
4136
4137 GISelCSEAnalysisWrapper &Wrapper =
4138 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
4139 // Set the CSEConfig and run the analysis.
4140 GISelCSEInfo *CSEInfo = nullptr;
4141 TPC = &getAnalysis<TargetPassConfig>();
4142
4143 bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
4144 ? EnableCSEInIRTranslator
4145 : TPC->isGISelCSEEnabled();
4146
4147 const TargetSubtargetInfo &Subtarget = MF->getSubtarget();
4148 TLI = Subtarget.getTargetLowering();
4149
4150 if (EnableCSE) {
4151 EntryBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
4152 CSEInfo = &Wrapper.get(CSEOpt: TPC->getCSEConfig());
4153 EntryBuilder->setCSEInfo(CSEInfo);
4154 CurBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
4155 CurBuilder->setCSEInfo(CSEInfo);
4156 } else {
4157 EntryBuilder = std::make_unique<MachineIRBuilder>();
4158 CurBuilder = std::make_unique<MachineIRBuilder>();
4159 }
4160 CLI = Subtarget.getCallLowering();
4161 CurBuilder->setMF(*MF);
4162 EntryBuilder->setMF(*MF);
4163 MRI = &MF->getRegInfo();
4164 DL = &F.getDataLayout();
4165 const TargetMachine &TM = MF->getTarget();
4166 TM.resetTargetOptions(F);
4167 EnableOpts = OptLevel != CodeGenOptLevel::None && !skipFunction(F);
4168 FuncInfo.MF = MF;
4169 if (EnableOpts) {
4170 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
4171 FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
4172 } else {
4173 AA = nullptr;
4174 FuncInfo.BPI = nullptr;
4175 }
4176
4177 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
4178 F&: MF->getFunction());
4179 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
4180 Libcalls = &getAnalysis<LibcallLoweringInfoWrapper>().getLibcallLowering(
4181 M: *F.getParent(), Subtarget);
4182
4183 FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(MF&: *MF);
4184
4185 SL = std::make_unique<GISelSwitchLowering>(args: this, args&: FuncInfo);
4186 SL->init(tli: *TLI, tm: TM, dl: *DL);
4187
4188 assert(PendingPHIs.empty() && "stale PHIs");
4189
4190 // Targets which want to use big endian can enable it using
4191 // enableBigEndian()
4192 if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
4193 // Currently we don't properly handle big endian code.
4194 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
4195 F.getSubprogram(), &F.getEntryBlock());
4196 R << "unable to translate in big endian mode";
4197 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
4198 return false;
4199 }
4200
4201 // Release the per-function state when we return, whether we succeeded or not.
4202 llvm::scope_exit FinalizeOnReturn([this]() { finalizeFunction(); });
4203
4204 // Setup a separate basic-block for the arguments and constants
4205 MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
4206 MF->push_back(MBB: EntryBB);
4207 EntryBuilder->setMBB(*EntryBB);
4208
4209 DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHIIt()->getDebugLoc();
4210 SwiftError.setFunction(CurMF);
4211 SwiftError.createEntriesInEntryBlock(DbgLoc);
4212
4213 bool IsVarArg = F.isVarArg();
4214 bool HasMustTailInVarArgFn = false;
4215
4216 // Create all blocks, in IR order, to preserve the layout.
4217 FuncInfo.MBBMap.resize(N: F.getMaxBlockNumber());
4218 for (const BasicBlock &BB: F) {
4219 auto *&MBB = FuncInfo.MBBMap[BB.getNumber()];
4220
4221 MBB = MF->CreateMachineBasicBlock(BB: &BB);
4222 MF->push_back(MBB);
4223
4224 // Only mark the block if the BlockAddress actually has users. The
4225 // hasAddressTaken flag may be stale if the BlockAddress was optimized away
4226 // but the constant still exists in the uniquing table.
4227 if (BB.hasAddressTaken()) {
4228 if (BlockAddress *BA = BlockAddress::lookup(BB: &BB))
4229 if (!BA->hasZeroLiveUses())
4230 MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
4231 }
4232
4233 if (!HasMustTailInVarArgFn)
4234 HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
4235 }
4236
4237 MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
4238
4239 // Make our arguments/constants entry block fallthrough to the IR entry block.
4240 EntryBB->addSuccessor(Succ: &getMBB(BB: F.front()));
4241
4242 // Lower the actual args into this basic block.
4243 SmallVector<ArrayRef<Register>, 8> VRegArgs;
4244 for (const Argument &Arg: F.args()) {
4245 if (DL->getTypeStoreSize(Ty: Arg.getType()).isZero())
4246 continue; // Don't handle zero sized types.
4247 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: Arg);
4248 VRegArgs.push_back(Elt: VRegs);
4249
4250 if (Arg.hasSwiftErrorAttr()) {
4251 assert(VRegs.size() == 1 && "Too many vregs for Swift error");
4252 SwiftError.setCurrentVReg(MBB: EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
4253 }
4254 }
4255
4256 if (!CLI->lowerFormalArguments(MIRBuilder&: *EntryBuilder, F, VRegs: VRegArgs, FLI&: FuncInfo)) {
4257 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
4258 F.getSubprogram(), &F.getEntryBlock());
4259 R << "unable to lower arguments: "
4260 << ore::NV("Prototype", F.getFunctionType());
4261 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
4262 return false;
4263 }
4264
4265 // Need to visit defs before uses when translating instructions.
4266 GISelObserverWrapper WrapperObserver;
4267 if (EnableCSE && CSEInfo)
4268 WrapperObserver.addObserver(O: CSEInfo);
4269 {
4270 ReversePostOrderTraversal<const Function *> RPOT(&F);
4271#ifndef NDEBUG
4272 DILocationVerifier Verifier;
4273 WrapperObserver.addObserver(&Verifier);
4274#endif // ifndef NDEBUG
4275 RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);
4276 for (const BasicBlock *BB : RPOT) {
4277 MachineBasicBlock &MBB = getMBB(BB: *BB);
4278 // Set the insertion point of all the following translations to
4279 // the end of this basic block.
4280 CurBuilder->setMBB(MBB);
4281 HasTailCall = false;
4282 for (const Instruction &Inst : *BB) {
4283 // If we translated a tail call in the last step, then we know
4284 // everything after the call is either a return, or something that is
4285 // handled by the call itself. (E.g. a lifetime marker or assume
4286 // intrinsic.) In this case, we should stop translating the block and
4287 // move on.
4288 if (HasTailCall)
4289 break;
4290#ifndef NDEBUG
4291 Verifier.setCurrentInst(&Inst);
4292#endif // ifndef NDEBUG
4293
4294 // Translate any debug-info attached to the instruction.
4295 translateDbgInfo(Inst, MIRBuilder&: *CurBuilder);
4296
4297 if (translate(Inst))
4298 continue;
4299
4300 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
4301 Inst.getDebugLoc(), BB);
4302 R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
4303
4304 if (ORE->allowExtraAnalysis(PassName: "gisel-irtranslator")) {
4305 std::string InstStrStorage;
4306 raw_string_ostream InstStr(InstStrStorage);
4307 InstStr << Inst;
4308
4309 R << ": '" << InstStrStorage << "'";
4310 }
4311
4312 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
4313 return false;
4314 }
4315
4316 if (!finalizeBasicBlock(BB: *BB, MBB)) {
4317 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
4318 BB->getTerminator()->getDebugLoc(), BB);
4319 R << "unable to translate basic block";
4320 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
4321 return false;
4322 }
4323 }
4324#ifndef NDEBUG
4325 WrapperObserver.removeObserver(&Verifier);
4326#endif
4327 }
4328
4329 finishPendingPhis();
4330
4331 SwiftError.propagateVRegs();
4332
4333 // Merge the argument lowering and constants block with its single
4334 // successor, the LLVM-IR entry block. We want the basic block to
4335 // be maximal.
4336 assert(EntryBB->succ_size() == 1 &&
4337 "Custom BB used for lowering should have only one successor");
4338 // Get the successor of the current entry block.
4339 MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
4340 assert(NewEntryBB.pred_size() == 1 &&
4341 "LLVM-IR entry block has a predecessor!?");
4342 // Move all the instruction from the current entry block to the
4343 // new entry block.
4344 NewEntryBB.splice(Where: NewEntryBB.begin(), Other: EntryBB, From: EntryBB->begin(),
4345 To: EntryBB->end());
4346
4347 // Update the live-in information for the new entry block.
4348 for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
4349 NewEntryBB.addLiveIn(RegMaskPair: LiveIn);
4350 NewEntryBB.sortUniqueLiveIns();
4351
4352 // Get rid of the now empty basic block.
4353 EntryBB->removeSuccessor(Succ: &NewEntryBB);
4354 MF->remove(MBBI: EntryBB);
4355 MF->deleteMachineBasicBlock(MBB: EntryBB);
4356
4357 assert(&MF->front() == &NewEntryBB &&
4358 "New entry wasn't next in the list of basic block!");
4359
4360 // Initialize stack protector information.
4361 StackProtector &SP = getAnalysis<StackProtector>();
4362 SP.copyToMachineFrameInfo(MFI&: MF->getFrameInfo());
4363
4364 return false;
4365}
4366