1//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the IRTranslator class.
10//===----------------------------------------------------------------------===//
11
12#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
13#include "llvm/ADT/PostOrderIterator.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/Analysis/AliasAnalysis.h"
18#include "llvm/Analysis/AssumptionCache.h"
19#include "llvm/Analysis/BranchProbabilityInfo.h"
20#include "llvm/Analysis/Loads.h"
21#include "llvm/Analysis/OptimizationRemarkEmitter.h"
22#include "llvm/Analysis/ValueTracking.h"
23#include "llvm/Analysis/VectorUtils.h"
24#include "llvm/CodeGen/Analysis.h"
25#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
26#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
27#include "llvm/CodeGen/GlobalISel/CallLowering.h"
28#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
29#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
30#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
31#include "llvm/CodeGen/LowLevelTypeUtils.h"
32#include "llvm/CodeGen/MachineBasicBlock.h"
33#include "llvm/CodeGen/MachineFrameInfo.h"
34#include "llvm/CodeGen/MachineFunction.h"
35#include "llvm/CodeGen/MachineInstrBuilder.h"
36#include "llvm/CodeGen/MachineMemOperand.h"
37#include "llvm/CodeGen/MachineModuleInfo.h"
38#include "llvm/CodeGen/MachineOperand.h"
39#include "llvm/CodeGen/MachineRegisterInfo.h"
40#include "llvm/CodeGen/StackProtector.h"
41#include "llvm/CodeGen/SwitchLoweringUtils.h"
42#include "llvm/CodeGen/TargetFrameLowering.h"
43#include "llvm/CodeGen/TargetInstrInfo.h"
44#include "llvm/CodeGen/TargetLowering.h"
45#include "llvm/CodeGen/TargetOpcodes.h"
46#include "llvm/CodeGen/TargetPassConfig.h"
47#include "llvm/CodeGen/TargetRegisterInfo.h"
48#include "llvm/CodeGen/TargetSubtargetInfo.h"
49#include "llvm/CodeGenTypes/LowLevelType.h"
50#include "llvm/IR/BasicBlock.h"
51#include "llvm/IR/CFG.h"
52#include "llvm/IR/Constant.h"
53#include "llvm/IR/Constants.h"
54#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DerivedTypes.h"
56#include "llvm/IR/DiagnosticInfo.h"
57#include "llvm/IR/Function.h"
58#include "llvm/IR/GetElementPtrTypeIterator.h"
59#include "llvm/IR/InlineAsm.h"
60#include "llvm/IR/InstrTypes.h"
61#include "llvm/IR/Instructions.h"
62#include "llvm/IR/IntrinsicInst.h"
63#include "llvm/IR/Intrinsics.h"
64#include "llvm/IR/IntrinsicsAMDGPU.h"
65#include "llvm/IR/LLVMContext.h"
66#include "llvm/IR/Metadata.h"
67#include "llvm/IR/PatternMatch.h"
68#include "llvm/IR/Statepoint.h"
69#include "llvm/IR/Type.h"
70#include "llvm/IR/User.h"
71#include "llvm/IR/Value.h"
72#include "llvm/InitializePasses.h"
73#include "llvm/MC/MCContext.h"
74#include "llvm/Pass.h"
75#include "llvm/Support/Casting.h"
76#include "llvm/Support/CodeGen.h"
77#include "llvm/Support/Debug.h"
78#include "llvm/Support/ErrorHandling.h"
79#include "llvm/Support/MathExtras.h"
80#include "llvm/Support/raw_ostream.h"
81#include "llvm/Target/TargetMachine.h"
82#include "llvm/Transforms/Utils/Local.h"
83#include "llvm/Transforms/Utils/MemoryOpRemark.h"
84#include <algorithm>
85#include <cassert>
86#include <cstdint>
87#include <iterator>
88#include <optional>
89#include <string>
90#include <utility>
91#include <vector>
92
93#define DEBUG_TYPE "irtranslator"
94
95using namespace llvm;
96
97static cl::opt<bool>
98 EnableCSEInIRTranslator("enable-cse-in-irtranslator",
99 cl::desc("Should enable CSE in irtranslator"),
100 cl::Optional, cl::init(Val: false));
101char IRTranslator::ID = 0;
102
103INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
104 false, false)
105INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
106INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
107INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
108INITIALIZE_PASS_DEPENDENCY(StackProtector)
109INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
110INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
111 false, false)
112
113static void reportTranslationError(MachineFunction &MF,
114 OptimizationRemarkEmitter &ORE,
115 OptimizationRemarkMissed &R) {
116 MF.getProperties().setFailedISel();
117 bool IsGlobalISelAbortEnabled =
118 MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
119
120 // Print the function name explicitly if we don't have a debug location (which
121 // makes the diagnostic less useful) or if we're going to emit a raw error.
122 if (!R.getLocation().isValid() || IsGlobalISelAbortEnabled)
123 R << (" (in function: " + MF.getName() + ")").str();
124
125 if (IsGlobalISelAbortEnabled)
126 report_fatal_error(reason: Twine(R.getMsg()));
127 else
128 ORE.emit(OptDiag&: R);
129}
130
131IRTranslator::IRTranslator(CodeGenOptLevel optlevel)
132 : MachineFunctionPass(ID), OptLevel(optlevel) {}
133
134#ifndef NDEBUG
135namespace {
136/// Verify that every instruction created has the same DILocation as the
137/// instruction being translated.
138class DILocationVerifier : public GISelChangeObserver {
139 const Instruction *CurrInst = nullptr;
140
141public:
142 DILocationVerifier() = default;
143 ~DILocationVerifier() override = default;
144
145 const Instruction *getCurrentInst() const { return CurrInst; }
146 void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
147
148 void erasingInstr(MachineInstr &MI) override {}
149 void changingInstr(MachineInstr &MI) override {}
150 void changedInstr(MachineInstr &MI) override {}
151
152 void createdInstr(MachineInstr &MI) override {
153 assert(getCurrentInst() && "Inserted instruction without a current MI");
154
155 // Only print the check message if we're actually checking it.
156#ifndef NDEBUG
157 LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
158 << " was copied to " << MI);
159#endif
160 // We allow insts in the entry block to have no debug loc because
161 // they could have originated from constants, and we don't want a jumpy
162 // debug experience.
163 assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
164 (MI.getParent()->isEntryBlock() && !MI.getDebugLoc()) ||
165 (MI.isDebugInstr())) &&
166 "Line info was not transferred to all instructions");
167 }
168};
169} // namespace
170#endif // ifndef NDEBUG
171
172
173void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const {
174 AU.addRequired<StackProtector>();
175 AU.addRequired<TargetPassConfig>();
176 AU.addRequired<GISelCSEAnalysisWrapperPass>();
177 if (OptLevel != CodeGenOptLevel::None) {
178 AU.addRequired<AssumptionCacheTracker>();
179 AU.addRequired<BranchProbabilityInfoWrapperPass>();
180 AU.addRequired<AAResultsWrapperPass>();
181 }
182 AU.addRequired<TargetLibraryInfoWrapperPass>();
183 AU.addPreserved<TargetLibraryInfoWrapperPass>();
184 AU.addRequired<LibcallLoweringInfoWrapper>();
185
186 getSelectionDAGFallbackAnalysisUsage(AU);
187 MachineFunctionPass::getAnalysisUsage(AU);
188}
189
190IRTranslator::ValueToVRegInfo::VRegListT &
191IRTranslator::allocateVRegs(const Value &Val) {
192 auto VRegsIt = VMap.findVRegs(V: Val);
193 if (VRegsIt != VMap.vregs_end())
194 return *VRegsIt->second;
195 auto *Regs = VMap.getVRegs(V: Val);
196 auto *Offsets = VMap.getOffsets(V: Val);
197 SmallVector<LLT, 4> SplitTys;
198 computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueLLTs&: SplitTys,
199 FixedOffsets: Offsets->empty() ? Offsets : nullptr);
200 for (unsigned i = 0; i < SplitTys.size(); ++i)
201 Regs->push_back(Elt: 0);
202 return *Regs;
203}
204
205ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
206 auto VRegsIt = VMap.findVRegs(V: Val);
207 if (VRegsIt != VMap.vregs_end())
208 return *VRegsIt->second;
209
210 if (Val.getType()->isVoidTy())
211 return *VMap.getVRegs(V: Val);
212
213 // Create entry for this type.
214 auto *VRegs = VMap.getVRegs(V: Val);
215 auto *Offsets = VMap.getOffsets(V: Val);
216
217 if (!Val.getType()->isTokenTy())
218 assert(Val.getType()->isSized() &&
219 "Don't know how to create an empty vreg");
220
221 // Fast-path values that lower to a single vreg.
222 if (!Val.getType()->isAggregateType()) {
223 LLT Ty = getLLTForType(Ty&: *Val.getType(), DL: *DL);
224 if (Offsets->empty())
225 Offsets->push_back(Elt: 0);
226 VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty));
227 if (isa<Constant>(Val)) {
228 bool Success = translate(C: cast<Constant>(Val), Reg: VRegs->front());
229 if (!Success) {
230 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
231 MF->getFunction().getSubprogram(),
232 &MF->getFunction().getEntryBlock());
233 R << "unable to translate constant: " << ore::NV("Type", Val.getType());
234 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
235 }
236 }
237 return *VRegs;
238 }
239
240 SmallVector<LLT, 4> SplitTys;
241 computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueLLTs&: SplitTys,
242 FixedOffsets: Offsets->empty() ? Offsets : nullptr);
243
244 if (!isa<Constant>(Val)) {
245 for (auto Ty : SplitTys)
246 VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty));
247 return *VRegs;
248 }
249
250 // UndefValue, ConstantAggregateZero
251 auto &C = cast<Constant>(Val);
252 unsigned Idx = 0;
253 while (auto Elt = C.getAggregateElement(Elt: Idx++)) {
254 auto EltRegs = getOrCreateVRegs(Val: *Elt);
255 llvm::append_range(C&: *VRegs, R&: EltRegs);
256 }
257
258 return *VRegs;
259}
260
261int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
262 auto [MapEntry, Inserted] = FrameIndices.try_emplace(Key: &AI);
263 if (!Inserted)
264 return MapEntry->second;
265
266 TypeSize TySize = AI.getAllocationSize(DL: *DL).value_or(u: TypeSize::getZero());
267 uint64_t Size = TySize.getKnownMinValue();
268
269 // Always allocate at least one byte.
270 Size = std::max<uint64_t>(a: Size, b: 1u);
271
272 int &FI = MapEntry->second;
273 FI = MF->getFrameInfo().CreateStackObject(Size, Alignment: AI.getAlign(), isSpillSlot: false, Alloca: &AI);
274
275 // Scalable vectors and structures that contain scalable vectors may
276 // need a special StackID to distinguish them from other (fixed size)
277 // stack objects.
278 if (TySize.isScalable()) {
279 auto StackID =
280 MF->getSubtarget().getFrameLowering()->getStackIDForScalableVectors();
281 MF->getFrameInfo().setStackID(ObjectIdx: FI, ID: StackID);
282 }
283
284 return FI;
285}
286
287Align IRTranslator::getMemOpAlign(const Instruction &I) {
288 if (const StoreInst *SI = dyn_cast<StoreInst>(Val: &I))
289 return SI->getAlign();
290 if (const LoadInst *LI = dyn_cast<LoadInst>(Val: &I))
291 return LI->getAlign();
292 if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(Val: &I))
293 return AI->getAlign();
294 if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(Val: &I))
295 return AI->getAlign();
296
297 OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
298 R << "unable to translate memop: " << ore::NV("Opcode", &I);
299 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
300 return Align(1);
301}
302
303MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
304 MachineBasicBlock *MBB = FuncInfo.getMBB(BB: &BB);
305 assert(MBB && "BasicBlock was not encountered before");
306 return *MBB;
307}
308
309void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
310 assert(NewPred && "new predecessor must be a real MachineBasicBlock");
311 MachinePreds[Edge].push_back(Elt: NewPred);
312}
313
314bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
315 MachineIRBuilder &MIRBuilder) {
316 if (!mayTranslateUserTypes(U))
317 return false;
318
319 // Get or create a virtual register for each value.
320 // Unless the value is a Constant => loadimm cst?
321 // or inline constant each time?
322 // Creation of a virtual register needs to have a size.
323 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
324 Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
325 Register Res = getOrCreateVReg(Val: U);
326 uint32_t Flags = 0;
327 if (isa<Instruction>(Val: U)) {
328 const Instruction &I = cast<Instruction>(Val: U);
329 Flags = MachineInstr::copyFlagsFromInstruction(I);
330 }
331
332 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0, Op1}, Flags);
333 return true;
334}
335
336bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
337 MachineIRBuilder &MIRBuilder) {
338 if (!mayTranslateUserTypes(U))
339 return false;
340
341 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
342 Register Res = getOrCreateVReg(Val: U);
343 uint32_t Flags = 0;
344 if (isa<Instruction>(Val: U)) {
345 const Instruction &I = cast<Instruction>(Val: U);
346 Flags = MachineInstr::copyFlagsFromInstruction(I);
347 }
348 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0}, Flags);
349 return true;
350}
351
352bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
353 return translateUnaryOp(Opcode: TargetOpcode::G_FNEG, U, MIRBuilder);
354}
355
356bool IRTranslator::translateCompare(const User &U,
357 MachineIRBuilder &MIRBuilder) {
358 if (!mayTranslateUserTypes(U))
359 return false;
360
361 auto *CI = cast<CmpInst>(Val: &U);
362 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
363 Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
364 Register Res = getOrCreateVReg(Val: U);
365 CmpInst::Predicate Pred = CI->getPredicate();
366 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: *CI);
367 if (CmpInst::isIntPredicate(P: Pred))
368 MIRBuilder.buildICmp(Pred, Res, Op0, Op1, Flags);
369 else if (Pred == CmpInst::FCMP_FALSE)
370 MIRBuilder.buildCopy(
371 Res, Op: getOrCreateVReg(Val: *Constant::getNullValue(Ty: U.getType())));
372 else if (Pred == CmpInst::FCMP_TRUE)
373 MIRBuilder.buildCopy(
374 Res, Op: getOrCreateVReg(Val: *Constant::getAllOnesValue(Ty: U.getType())));
375 else
376 MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
377
378 return true;
379}
380
381bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
382 const ReturnInst &RI = cast<ReturnInst>(Val: U);
383 const Value *Ret = RI.getReturnValue();
384 if (Ret && DL->getTypeStoreSize(Ty: Ret->getType()).isZero())
385 Ret = nullptr;
386
387 ArrayRef<Register> VRegs;
388 if (Ret)
389 VRegs = getOrCreateVRegs(Val: *Ret);
390
391 Register SwiftErrorVReg = 0;
392 if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
393 SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
394 &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
395 }
396
397 // The target may mess up with the insertion point, but
398 // this is not important as a return is the last instruction
399 // of the block anyway.
400 return CLI->lowerReturn(MIRBuilder, Val: Ret, VRegs, FLI&: FuncInfo, SwiftErrorVReg);
401}
402
403void IRTranslator::emitBranchForMergedCondition(
404 const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
405 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
406 BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
407 // If the leaf of the tree is a comparison, merge the condition into
408 // the caseblock.
409 if (const CmpInst *BOp = dyn_cast<CmpInst>(Val: Cond)) {
410 CmpInst::Predicate Condition;
411 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Val: Cond)) {
412 Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
413 } else {
414 const FCmpInst *FC = cast<FCmpInst>(Val: Cond);
415 Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
416 }
417
418 SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(i_nocapture: 0),
419 BOp->getOperand(i_nocapture: 1), nullptr, TBB, FBB, CurBB,
420 CurBuilder->getDebugLoc(), TProb, FProb);
421 SL->SwitchCases.push_back(x: CB);
422 return;
423 }
424
425 // Create a CaseBlock record representing this branch.
426 CmpInst::Predicate Pred = InvertCond ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ;
427 SwitchCG::CaseBlock CB(
428 Pred, false, Cond, ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
429 nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
430 SL->SwitchCases.push_back(x: CB);
431}
432
433static bool isValInBlock(const Value *V, const BasicBlock *BB) {
434 if (const Instruction *I = dyn_cast<Instruction>(Val: V))
435 return I->getParent() == BB;
436 return true;
437}
438
439void IRTranslator::findMergedConditions(
440 const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
441 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
442 Instruction::BinaryOps Opc, BranchProbability TProb,
443 BranchProbability FProb, bool InvertCond) {
444 using namespace PatternMatch;
445 assert((Opc == Instruction::And || Opc == Instruction::Or) &&
446 "Expected Opc to be AND/OR");
447 // Skip over not part of the tree and remember to invert op and operands at
448 // next level.
449 Value *NotCond;
450 if (match(V: Cond, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: NotCond)))) &&
451 isValInBlock(V: NotCond, BB: CurBB->getBasicBlock())) {
452 findMergedConditions(Cond: NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
453 InvertCond: !InvertCond);
454 return;
455 }
456
457 const Instruction *BOp = dyn_cast<Instruction>(Val: Cond);
458 const Value *BOpOp0, *BOpOp1;
459 // Compute the effective opcode for Cond, taking into account whether it needs
460 // to be inverted, e.g.
461 // and (not (or A, B)), C
462 // gets lowered as
463 // and (and (not A, not B), C)
464 Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
465 if (BOp) {
466 BOpc = match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
467 ? Instruction::And
468 : (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
469 ? Instruction::Or
470 : (Instruction::BinaryOps)0);
471 if (InvertCond) {
472 if (BOpc == Instruction::And)
473 BOpc = Instruction::Or;
474 else if (BOpc == Instruction::Or)
475 BOpc = Instruction::And;
476 }
477 }
478
479 // If this node is not part of the or/and tree, emit it as a branch.
480 // Note that all nodes in the tree should have same opcode.
481 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
482 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
483 !isValInBlock(V: BOpOp0, BB: CurBB->getBasicBlock()) ||
484 !isValInBlock(V: BOpOp1, BB: CurBB->getBasicBlock())) {
485 emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
486 InvertCond);
487 return;
488 }
489
490 // Create TmpBB after CurBB.
491 MachineFunction::iterator BBI(CurBB);
492 MachineBasicBlock *TmpBB =
493 MF->CreateMachineBasicBlock(BB: CurBB->getBasicBlock());
494 CurBB->getParent()->insert(MBBI: ++BBI, MBB: TmpBB);
495
496 if (Opc == Instruction::Or) {
497 // Codegen X | Y as:
498 // BB1:
499 // jmp_if_X TBB
500 // jmp TmpBB
501 // TmpBB:
502 // jmp_if_Y TBB
503 // jmp FBB
504 //
505
506 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
507 // The requirement is that
508 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
509 // = TrueProb for original BB.
510 // Assuming the original probabilities are A and B, one choice is to set
511 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
512 // A/(1+B) and 2B/(1+B). This choice assumes that
513 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
514 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
515 // TmpBB, but the math is more complicated.
516
517 auto NewTrueProb = TProb / 2;
518 auto NewFalseProb = TProb / 2 + FProb;
519 // Emit the LHS condition.
520 findMergedConditions(Cond: BOpOp0, TBB, FBB: TmpBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
521 FProb: NewFalseProb, InvertCond);
522
523 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
524 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
525 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
526 // Emit the RHS condition into TmpBB.
527 findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
528 FProb: Probs[1], InvertCond);
529 } else {
530 assert(Opc == Instruction::And && "Unknown merge op!");
531 // Codegen X & Y as:
532 // BB1:
533 // jmp_if_X TmpBB
534 // jmp FBB
535 // TmpBB:
536 // jmp_if_Y TBB
537 // jmp FBB
538 //
539 // This requires creation of TmpBB after CurBB.
540
541 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
542 // The requirement is that
543 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
544 // = FalseProb for original BB.
545 // Assuming the original probabilities are A and B, one choice is to set
546 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
547 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
548 // TrueProb for BB1 * FalseProb for TmpBB.
549
550 auto NewTrueProb = TProb + FProb / 2;
551 auto NewFalseProb = FProb / 2;
552 // Emit the LHS condition.
553 findMergedConditions(Cond: BOpOp0, TBB: TmpBB, FBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
554 FProb: NewFalseProb, InvertCond);
555
556 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
557 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
558 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
559 // Emit the RHS condition into TmpBB.
560 findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
561 FProb: Probs[1], InvertCond);
562 }
563}
564
565bool IRTranslator::shouldEmitAsBranches(
566 const std::vector<SwitchCG::CaseBlock> &Cases) {
567 // For multiple cases, it's better to emit as branches.
568 if (Cases.size() != 2)
569 return true;
570
571 // If this is two comparisons of the same values or'd or and'd together, they
572 // will get folded into a single comparison, so don't emit two blocks.
573 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
574 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
575 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
576 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
577 return false;
578 }
579
580 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
581 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
582 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
583 Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
584 isa<Constant>(Val: Cases[0].CmpRHS) &&
585 cast<Constant>(Val: Cases[0].CmpRHS)->isNullValue()) {
586 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
587 Cases[0].TrueBB == Cases[1].ThisBB)
588 return false;
589 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
590 Cases[0].FalseBB == Cases[1].ThisBB)
591 return false;
592 }
593
594 return true;
595}
596
597bool IRTranslator::translateUncondBr(const User &U,
598 MachineIRBuilder &MIRBuilder) {
599 const UncondBrInst &BrInst = cast<UncondBrInst>(Val: U);
600 auto &CurMBB = MIRBuilder.getMBB();
601 auto *Succ0MBB = &getMBB(BB: *BrInst.getSuccessor(i: 0));
602
603 // If the unconditional target is the layout successor, fallthrough.
604 if (OptLevel == CodeGenOptLevel::None || !CurMBB.isLayoutSuccessor(MBB: Succ0MBB))
605 MIRBuilder.buildBr(Dest&: *Succ0MBB);
606
607 // Link successors.
608 for (const BasicBlock *Succ : successors(I: &BrInst))
609 CurMBB.addSuccessor(Succ: &getMBB(BB: *Succ));
610 return true;
611}
612
613bool IRTranslator::translateCondBr(const User &U,
614 MachineIRBuilder &MIRBuilder) {
615 const CondBrInst &BrInst = cast<CondBrInst>(Val: U);
616 auto &CurMBB = MIRBuilder.getMBB();
617 auto *Succ0MBB = &getMBB(BB: *BrInst.getSuccessor(i: 0));
618
619 // If this condition is one of the special cases we handle, do special stuff
620 // now.
621 const Value *CondVal = BrInst.getCondition();
622 MachineBasicBlock *Succ1MBB = &getMBB(BB: *BrInst.getSuccessor(i: 1));
623
624 // If this is a series of conditions that are or'd or and'd together, emit
625 // this as a sequence of branches instead of setcc's with and/or operations.
626 // As long as jumps are not expensive (exceptions for multi-use logic ops,
627 // unpredictable branches, and vector extracts because those jumps are likely
628 // expensive for any target), this should improve performance.
629 // For example, instead of something like:
630 // cmp A, B
631 // C = seteq
632 // cmp D, E
633 // F = setle
634 // or C, F
635 // jnz foo
636 // Emit:
637 // cmp A, B
638 // je foo
639 // cmp D, E
640 // jle foo
641 using namespace PatternMatch;
642 const Instruction *CondI = dyn_cast<Instruction>(Val: CondVal);
643 if (!TLI->isJumpExpensive() && CondI && CondI->hasOneUse() &&
644 !BrInst.hasMetadata(KindID: LLVMContext::MD_unpredictable)) {
645 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
646 Value *Vec;
647 const Value *BOp0, *BOp1;
648 if (match(V: CondI, P: m_LogicalAnd(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
649 Opcode = Instruction::And;
650 else if (match(V: CondI, P: m_LogicalOr(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
651 Opcode = Instruction::Or;
652
653 if (Opcode && !(match(V: BOp0, P: m_ExtractElt(Val: m_Value(V&: Vec), Idx: m_Value())) &&
654 match(V: BOp1, P: m_ExtractElt(Val: m_Specific(V: Vec), Idx: m_Value())))) {
655 findMergedConditions(Cond: CondI, TBB: Succ0MBB, FBB: Succ1MBB, CurBB: &CurMBB, SwitchBB: &CurMBB, Opc: Opcode,
656 TProb: getEdgeProbability(Src: &CurMBB, Dst: Succ0MBB),
657 FProb: getEdgeProbability(Src: &CurMBB, Dst: Succ1MBB),
658 /*InvertCond=*/false);
659 assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
660
661 // Allow some cases to be rejected.
662 if (shouldEmitAsBranches(Cases: SL->SwitchCases)) {
663 // Emit the branch for this block.
664 emitSwitchCase(CB&: SL->SwitchCases[0], SwitchBB: &CurMBB, MIB&: *CurBuilder);
665 SL->SwitchCases.erase(position: SL->SwitchCases.begin());
666 return true;
667 }
668
669 // Okay, we decided not to do this, remove any inserted MBB's and clear
670 // SwitchCases.
671 for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
672 MF->erase(MBBI: SL->SwitchCases[I].ThisBB);
673
674 SL->SwitchCases.clear();
675 }
676 }
677
678 // Create a CaseBlock record representing this branch.
679 SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
680 ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
681 nullptr, Succ0MBB, Succ1MBB, &CurMBB,
682 CurBuilder->getDebugLoc());
683
684 // Use emitSwitchCase to actually insert the fast branch sequence for this
685 // cond branch.
686 emitSwitchCase(CB, SwitchBB: &CurMBB, MIB&: *CurBuilder);
687 return true;
688}
689
690void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
691 MachineBasicBlock *Dst,
692 BranchProbability Prob) {
693 if (!FuncInfo.BPI) {
694 Src->addSuccessorWithoutProb(Succ: Dst);
695 return;
696 }
697 if (Prob.isUnknown())
698 Prob = getEdgeProbability(Src, Dst);
699 Src->addSuccessor(Succ: Dst, Prob);
700}
701
702BranchProbability
703IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
704 const MachineBasicBlock *Dst) const {
705 const BasicBlock *SrcBB = Src->getBasicBlock();
706 const BasicBlock *DstBB = Dst->getBasicBlock();
707 if (!FuncInfo.BPI) {
708 // If BPI is not available, set the default probability as 1 / N, where N is
709 // the number of successors.
710 auto SuccSize = std::max<uint32_t>(a: succ_size(BB: SrcBB), b: 1);
711 return BranchProbability(1, SuccSize);
712 }
713 return FuncInfo.BPI->getEdgeProbability(Src: SrcBB, Dst: DstBB);
714}
715
716bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
717 using namespace SwitchCG;
718 // Extract cases from the switch.
719 const SwitchInst &SI = cast<SwitchInst>(Val: U);
720 BranchProbabilityInfo *BPI = FuncInfo.BPI;
721 CaseClusterVector Clusters;
722 Clusters.reserve(n: SI.getNumCases());
723 for (const auto &I : SI.cases()) {
724 MachineBasicBlock *Succ = &getMBB(BB: *I.getCaseSuccessor());
725 assert(Succ && "Could not find successor mbb in mapping");
726 const ConstantInt *CaseVal = I.getCaseValue();
727 BranchProbability Prob =
728 BPI ? BPI->getEdgeProbability(Src: SI.getParent(), IndexInSuccessors: I.getSuccessorIndex())
729 : BranchProbability(1, SI.getNumCases() + 1);
730 Clusters.push_back(x: CaseCluster::range(Low: CaseVal, High: CaseVal, MBB: Succ, Prob));
731 }
732
733 MachineBasicBlock *DefaultMBB = &getMBB(BB: *SI.getDefaultDest());
734
735 // Cluster adjacent cases with the same destination. We do this at all
736 // optimization levels because it's cheap to do and will make codegen faster
737 // if there are many clusters.
738 sortAndRangeify(Clusters);
739
740 MachineBasicBlock *SwitchMBB = &getMBB(BB: *SI.getParent());
741
742 // If there is only the default destination, jump there directly.
743 if (Clusters.empty()) {
744 SwitchMBB->addSuccessor(Succ: DefaultMBB);
745 if (DefaultMBB != SwitchMBB->getNextNode())
746 MIB.buildBr(Dest&: *DefaultMBB);
747 return true;
748 }
749
750 SL->findJumpTables(Clusters, SI: &SI, SL: std::nullopt, DefaultMBB, PSI: nullptr, BFI: nullptr);
751 SL->findBitTestClusters(Clusters, SI: &SI);
752
753 LLVM_DEBUG({
754 dbgs() << "Case clusters: ";
755 for (const CaseCluster &C : Clusters) {
756 if (C.Kind == CC_JumpTable)
757 dbgs() << "JT:";
758 if (C.Kind == CC_BitTests)
759 dbgs() << "BT:";
760
761 C.Low->getValue().print(dbgs(), true);
762 if (C.Low != C.High) {
763 dbgs() << '-';
764 C.High->getValue().print(dbgs(), true);
765 }
766 dbgs() << ' ';
767 }
768 dbgs() << '\n';
769 });
770
771 assert(!Clusters.empty());
772 SwitchWorkList WorkList;
773 CaseClusterIt First = Clusters.begin();
774 CaseClusterIt Last = Clusters.end() - 1;
775 auto DefaultProb = getEdgeProbability(Src: SwitchMBB, Dst: DefaultMBB);
776 WorkList.push_back(Elt: {.MBB: SwitchMBB, .FirstCluster: First, .LastCluster: Last, .GE: nullptr, .LT: nullptr, .DefaultProb: DefaultProb});
777
778 while (!WorkList.empty()) {
779 SwitchWorkListItem W = WorkList.pop_back_val();
780
781 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
782 // For optimized builds, lower large range as a balanced binary tree.
783 if (NumClusters > 3 &&
784 MF->getTarget().getOptLevel() != CodeGenOptLevel::None &&
785 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
786 splitWorkItem(WorkList, W, Cond: SI.getCondition(), SwitchMBB, MIB);
787 continue;
788 }
789
790 if (!lowerSwitchWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
791 return false;
792 }
793 return true;
794}
795
796void IRTranslator::splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
797 const SwitchCG::SwitchWorkListItem &W,
798 Value *Cond, MachineBasicBlock *SwitchMBB,
799 MachineIRBuilder &MIB) {
800 using namespace SwitchCG;
801 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
802 "Clusters not sorted?");
803 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
804
805 auto [LastLeft, FirstRight, LeftProb, RightProb] =
806 SL->computeSplitWorkItemInfo(W);
807
808 // Use the first element on the right as pivot since we will make less-than
809 // comparisons against it.
810 CaseClusterIt PivotCluster = FirstRight;
811 assert(PivotCluster > W.FirstCluster);
812 assert(PivotCluster <= W.LastCluster);
813
814 CaseClusterIt FirstLeft = W.FirstCluster;
815 CaseClusterIt LastRight = W.LastCluster;
816
817 const ConstantInt *Pivot = PivotCluster->Low;
818
819 // New blocks will be inserted immediately after the current one.
820 MachineFunction::iterator BBI(W.MBB);
821 ++BBI;
822
823 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
824 // we can branch to its destination directly if it's squeezed exactly in
825 // between the known lower bound and Pivot - 1.
826 MachineBasicBlock *LeftMBB;
827 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
828 FirstLeft->Low == W.GE &&
829 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
830 LeftMBB = FirstLeft->MBB;
831 } else {
832 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
833 FuncInfo.MF->insert(MBBI: BBI, MBB: LeftMBB);
834 WorkList.push_back(
835 Elt: {.MBB: LeftMBB, .FirstCluster: FirstLeft, .LastCluster: LastLeft, .GE: W.GE, .LT: Pivot, .DefaultProb: W.DefaultProb / 2});
836 }
837
838 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
839 // single cluster, RHS.Low == Pivot, and we can branch to its destination
840 // directly if RHS.High equals the current upper bound.
841 MachineBasicBlock *RightMBB;
842 if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT &&
843 (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
844 RightMBB = FirstRight->MBB;
845 } else {
846 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
847 FuncInfo.MF->insert(MBBI: BBI, MBB: RightMBB);
848 WorkList.push_back(
849 Elt: {.MBB: RightMBB, .FirstCluster: FirstRight, .LastCluster: LastRight, .GE: Pivot, .LT: W.LT, .DefaultProb: W.DefaultProb / 2});
850 }
851
852 // Create the CaseBlock record that will be used to lower the branch.
853 CaseBlock CB(ICmpInst::Predicate::ICMP_SLT, false, Cond, Pivot, nullptr,
854 LeftMBB, RightMBB, W.MBB, MIB.getDebugLoc(), LeftProb,
855 RightProb);
856
857 if (W.MBB == SwitchMBB)
858 emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
859 else
860 SL->SwitchCases.push_back(x: CB);
861}
862
863void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
864 MachineBasicBlock *MBB) {
865 // Emit the code for the jump table
866 assert(JT.Reg && "Should lower JT Header first!");
867 MachineIRBuilder MIB(*MBB->getParent());
868 MIB.setMBB(*MBB);
869 MIB.setDebugLoc(CurBuilder->getDebugLoc());
870
871 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
872 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
873
874 auto Table = MIB.buildJumpTable(PtrTy, JTI: JT.JTI);
875 MIB.buildBrJT(TablePtr: Table.getReg(Idx: 0), JTI: JT.JTI, IndexReg: JT.Reg);
876}
877
878bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
879 SwitchCG::JumpTableHeader &JTH,
880 MachineBasicBlock *HeaderBB) {
881 MachineIRBuilder MIB(*HeaderBB->getParent());
882 MIB.setMBB(*HeaderBB);
883 MIB.setDebugLoc(CurBuilder->getDebugLoc());
884
885 const Value &SValue = *JTH.SValue;
886 // Subtract the lowest switch case value from the value being switched on.
887 const LLT SwitchTy = getLLTForType(Ty&: *SValue.getType(), DL: *DL);
888 Register SwitchOpReg = getOrCreateVReg(Val: SValue);
889 auto FirstCst = MIB.buildConstant(Res: SwitchTy, Val: JTH.First);
890 auto Sub = MIB.buildSub(Dst: {SwitchTy}, Src0: SwitchOpReg, Src1: FirstCst);
891
892 // This value may be smaller or larger than the target's pointer type, and
893 // therefore require extension or truncating.
894 auto *PtrIRTy = PointerType::getUnqual(C&: SValue.getContext());
895 const LLT PtrScalarTy = LLT::integer(SizeInBits: DL->getTypeSizeInBits(Ty: PtrIRTy));
896 Sub = MIB.buildZExtOrTrunc(Res: PtrScalarTy, Op: Sub);
897
898 JT.Reg = Sub.getReg(Idx: 0);
899
900 if (JTH.FallthroughUnreachable) {
901 if (JT.MBB != HeaderBB->getNextNode())
902 MIB.buildBr(Dest&: *JT.MBB);
903 return true;
904 }
905
906 // Emit the range check for the jump table, and branch to the default block
907 // for the switch statement if the value being switched on exceeds the
908 // largest case in the switch.
909 auto Cst = getOrCreateVReg(
910 Val: *ConstantInt::get(Ty: SValue.getType(), V: JTH.Last - JTH.First));
911 Cst = MIB.buildZExtOrTrunc(Res: PtrScalarTy, Op: Cst).getReg(Idx: 0);
912 auto Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_UGT, Res: LLT::integer(SizeInBits: 1), Op0: Sub, Op1: Cst);
913
914 auto BrCond = MIB.buildBrCond(Tst: Cmp.getReg(Idx: 0), Dest&: *JT.Default);
915
916 // Avoid emitting unnecessary branches to the next block.
917 if (JT.MBB != HeaderBB->getNextNode())
918 BrCond = MIB.buildBr(Dest&: *JT.MBB);
919 return true;
920}
921
922void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
923 MachineBasicBlock *SwitchBB,
924 MachineIRBuilder &MIB) {
925 Register CondLHS = getOrCreateVReg(Val: *CB.CmpLHS);
926 Register Cond;
927 DebugLoc OldDbgLoc = MIB.getDebugLoc();
928 MIB.setDebugLoc(CB.DbgLoc);
929 MIB.setMBB(*CB.ThisBB);
930
931 if (CB.PredInfo.NoCmp) {
932 // Branch or fall through to TrueBB.
933 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
934 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
935 NewPred: CB.ThisBB);
936 CB.ThisBB->normalizeSuccProbs();
937 if (CB.TrueBB != CB.ThisBB->getNextNode())
938 MIB.buildBr(Dest&: *CB.TrueBB);
939 MIB.setDebugLoc(OldDbgLoc);
940 return;
941 }
942
943 const LLT i1Ty = LLT::integer(SizeInBits: 1);
944 // Build the compare.
945 if (!CB.CmpMHS) {
946 const auto *CI = dyn_cast<ConstantInt>(Val: CB.CmpRHS);
947 // For conditional branch lowering, we might try to do something silly like
948 // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
949 // just re-use the existing condition vreg.
950 if (MRI->getType(Reg: CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
951 CB.PredInfo.Pred == CmpInst::ICMP_EQ) {
952 Cond = CondLHS;
953 } else {
954 Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
955 if (CmpInst::isFPPredicate(P: CB.PredInfo.Pred))
956 Cond =
957 MIB.buildFCmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
958 else
959 Cond =
960 MIB.buildICmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
961 }
962 } else {
963 assert(CB.PredInfo.Pred == CmpInst::ICMP_SLE &&
964 "Can only handle SLE ranges");
965
966 const APInt& Low = cast<ConstantInt>(Val: CB.CmpLHS)->getValue();
967 const APInt& High = cast<ConstantInt>(Val: CB.CmpRHS)->getValue();
968
969 Register CmpOpReg = getOrCreateVReg(Val: *CB.CmpMHS);
970 if (cast<ConstantInt>(Val: CB.CmpLHS)->isMinValue(IsSigned: true)) {
971 Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
972 Cond =
973 MIB.buildICmp(Pred: CmpInst::ICMP_SLE, Res: i1Ty, Op0: CmpOpReg, Op1: CondRHS).getReg(Idx: 0);
974 } else {
975 const LLT CmpTy = MRI->getType(Reg: CmpOpReg);
976 auto Sub = MIB.buildSub(Dst: {CmpTy}, Src0: CmpOpReg, Src1: CondLHS);
977 auto Diff = MIB.buildConstant(Res: CmpTy, Val: High - Low);
978 Cond = MIB.buildICmp(Pred: CmpInst::ICMP_ULE, Res: i1Ty, Op0: Sub, Op1: Diff).getReg(Idx: 0);
979 }
980 }
981
982 // Update successor info
983 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
984
985 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
986 NewPred: CB.ThisBB);
987
988 // TrueBB and FalseBB are always different unless the incoming IR is
989 // degenerate. This only happens when running llc on weird IR.
990 if (CB.TrueBB != CB.FalseBB)
991 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.FalseBB, Prob: CB.FalseProb);
992 CB.ThisBB->normalizeSuccProbs();
993
994 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
995 NewPred: CB.ThisBB);
996
997 MIB.buildBrCond(Tst: Cond, Dest&: *CB.TrueBB);
998 MIB.buildBr(Dest&: *CB.FalseBB);
999 MIB.setDebugLoc(OldDbgLoc);
1000}
1001
1002bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
1003 MachineBasicBlock *SwitchMBB,
1004 MachineBasicBlock *CurMBB,
1005 MachineBasicBlock *DefaultMBB,
1006 MachineIRBuilder &MIB,
1007 MachineFunction::iterator BBI,
1008 BranchProbability UnhandledProbs,
1009 SwitchCG::CaseClusterIt I,
1010 MachineBasicBlock *Fallthrough,
1011 bool FallthroughUnreachable) {
1012 using namespace SwitchCG;
1013 MachineFunction *CurMF = SwitchMBB->getParent();
1014 // FIXME: Optimize away range check based on pivot comparisons.
1015 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
1016 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
1017 BranchProbability DefaultProb = W.DefaultProb;
1018
1019 // The jump block hasn't been inserted yet; insert it here.
1020 MachineBasicBlock *JumpMBB = JT->MBB;
1021 CurMF->insert(MBBI: BBI, MBB: JumpMBB);
1022
1023 // Since the jump table block is separate from the switch block, we need
1024 // to keep track of it as a machine predecessor to the default block,
1025 // otherwise we lose the phi edges.
1026 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1027 NewPred: CurMBB);
1028 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1029 NewPred: JumpMBB);
1030
1031 auto JumpProb = I->Prob;
1032 auto FallthroughProb = UnhandledProbs;
1033
1034 // If the default statement is a target of the jump table, we evenly
1035 // distribute the default probability to successors of CurMBB. Also
1036 // update the probability on the edge from JumpMBB to Fallthrough.
1037 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
1038 SE = JumpMBB->succ_end();
1039 SI != SE; ++SI) {
1040 if (*SI == DefaultMBB) {
1041 JumpProb += DefaultProb / 2;
1042 FallthroughProb -= DefaultProb / 2;
1043 JumpMBB->setSuccProbability(I: SI, Prob: DefaultProb / 2);
1044 JumpMBB->normalizeSuccProbs();
1045 } else {
1046 // Also record edges from the jump table block to it's successors.
1047 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
1048 NewPred: JumpMBB);
1049 }
1050 }
1051
1052 if (FallthroughUnreachable)
1053 JTH->FallthroughUnreachable = true;
1054
1055 if (!JTH->FallthroughUnreachable)
1056 addSuccessorWithProb(Src: CurMBB, Dst: Fallthrough, Prob: FallthroughProb);
1057 addSuccessorWithProb(Src: CurMBB, Dst: JumpMBB, Prob: JumpProb);
1058 CurMBB->normalizeSuccProbs();
1059
1060 // The jump table header will be inserted in our current block, do the
1061 // range check, and fall through to our fallthrough block.
1062 JTH->HeaderBB = CurMBB;
1063 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
1064
1065 // If we're in the right place, emit the jump table header right now.
1066 if (CurMBB == SwitchMBB) {
1067 if (!emitJumpTableHeader(JT&: *JT, JTH&: *JTH, HeaderBB: CurMBB))
1068 return false;
1069 JTH->Emitted = true;
1070 }
1071 return true;
1072}
1073bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
1074 Value *Cond,
1075 MachineBasicBlock *Fallthrough,
1076 bool FallthroughUnreachable,
1077 BranchProbability UnhandledProbs,
1078 MachineBasicBlock *CurMBB,
1079 MachineIRBuilder &MIB,
1080 MachineBasicBlock *SwitchMBB) {
1081 using namespace SwitchCG;
1082 const Value *RHS, *LHS, *MHS;
1083 CmpInst::Predicate Pred;
1084 if (I->Low == I->High) {
1085 // Check Cond == I->Low.
1086 Pred = CmpInst::ICMP_EQ;
1087 LHS = Cond;
1088 RHS = I->Low;
1089 MHS = nullptr;
1090 } else {
1091 // Check I->Low <= Cond <= I->High.
1092 Pred = CmpInst::ICMP_SLE;
1093 LHS = I->Low;
1094 MHS = Cond;
1095 RHS = I->High;
1096 }
1097
1098 // If Fallthrough is unreachable, fold away the comparison.
1099 // The false probability is the sum of all unhandled cases.
1100 CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
1101 CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
1102
1103 emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
1104 return true;
1105}
1106
1107void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B,
1108 MachineBasicBlock *SwitchBB) {
1109 MachineIRBuilder &MIB = *CurBuilder;
1110 MIB.setMBB(*SwitchBB);
1111
1112 // Subtract the minimum value.
1113 Register SwitchOpReg = getOrCreateVReg(Val: *B.SValue);
1114
1115 LLT SwitchOpTy = MRI->getType(Reg: SwitchOpReg);
1116 Register MinValReg = MIB.buildConstant(Res: SwitchOpTy, Val: B.First).getReg(Idx: 0);
1117 auto RangeSub = MIB.buildSub(Dst: SwitchOpTy, Src0: SwitchOpReg, Src1: MinValReg);
1118
1119 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
1120 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1121
1122 LLT MaskTy = SwitchOpTy;
1123 if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
1124 !llvm::has_single_bit<uint32_t>(Value: MaskTy.getSizeInBits()))
1125 MaskTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
1126 else {
1127 // Ensure that the type will fit the mask value.
1128 for (const SwitchCG::BitTestCase &Case : B.Cases) {
1129 if (!isUIntN(N: SwitchOpTy.getSizeInBits(), x: Case.Mask)) {
1130 // Switch table case range are encoded into series of masks.
1131 // Just use pointer type, it's guaranteed to fit.
1132 MaskTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
1133 break;
1134 }
1135 }
1136 }
1137 Register SubReg = RangeSub.getReg(Idx: 0);
1138 if (SwitchOpTy != MaskTy)
1139 SubReg = MIB.buildZExtOrTrunc(Res: MaskTy, Op: SubReg).getReg(Idx: 0);
1140
1141 B.RegVT = getMVTForLLT(Ty: MaskTy);
1142 B.Reg = SubReg;
1143
1144 MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1145
1146 if (!B.FallthroughUnreachable)
1147 addSuccessorWithProb(Src: SwitchBB, Dst: B.Default, Prob: B.DefaultProb);
1148 addSuccessorWithProb(Src: SwitchBB, Dst: MBB, Prob: B.Prob);
1149
1150 SwitchBB->normalizeSuccProbs();
1151
1152 if (!B.FallthroughUnreachable) {
1153 // Conditional branch to the default block.
1154 auto RangeCst = MIB.buildConstant(Res: SwitchOpTy, Val: B.Range);
1155 auto RangeCmp = MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: LLT::integer(SizeInBits: 1),
1156 Op0: RangeSub, Op1: RangeCst);
1157 MIB.buildBrCond(Tst: RangeCmp, Dest&: *B.Default);
1158 }
1159
1160 // Avoid emitting unnecessary branches to the next block.
1161 if (MBB != SwitchBB->getNextNode())
1162 MIB.buildBr(Dest&: *MBB);
1163}
1164
1165void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1166 MachineBasicBlock *NextMBB,
1167 BranchProbability BranchProbToNext,
1168 Register Reg, SwitchCG::BitTestCase &B,
1169 MachineBasicBlock *SwitchBB) {
1170 MachineIRBuilder &MIB = *CurBuilder;
1171 MIB.setMBB(*SwitchBB);
1172
1173 LLT SwitchTy = getLLTForMVT(Ty: BB.RegVT);
1174 Register Cmp;
1175 unsigned PopCount = llvm::popcount(Value: B.Mask);
1176 if (PopCount == 1) {
1177 // Testing for a single bit; just compare the shift count with what it
1178 // would need to be to shift a 1 bit in that position.
1179 auto MaskTrailingZeros =
1180 MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_zero(Val: B.Mask));
1181 Cmp = MIB.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: LLT::integer(SizeInBits: 1), Op0: Reg,
1182 Op1: MaskTrailingZeros)
1183 .getReg(Idx: 0);
1184 } else if (PopCount == BB.Range) {
1185 // There is only one zero bit in the range, test for it directly.
1186 auto MaskTrailingOnes =
1187 MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_one(Value: B.Mask));
1188 Cmp =
1189 MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::integer(SizeInBits: 1), Op0: Reg, Op1: MaskTrailingOnes)
1190 .getReg(Idx: 0);
1191 } else {
1192 // Make desired shift.
1193 auto CstOne = MIB.buildConstant(Res: SwitchTy, Val: 1);
1194 auto SwitchVal = MIB.buildShl(Dst: SwitchTy, Src0: CstOne, Src1: Reg);
1195
1196 // Emit bit tests and jumps.
1197 auto CstMask = MIB.buildConstant(Res: SwitchTy, Val: B.Mask);
1198 auto AndOp = MIB.buildAnd(Dst: SwitchTy, Src0: SwitchVal, Src1: CstMask);
1199 auto CstZero = MIB.buildConstant(Res: SwitchTy, Val: 0);
1200 Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::integer(SizeInBits: 1), Op0: AndOp, Op1: CstZero)
1201 .getReg(Idx: 0);
1202 }
1203
1204 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1205 addSuccessorWithProb(Src: SwitchBB, Dst: B.TargetBB, Prob: B.ExtraProb);
1206 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1207 addSuccessorWithProb(Src: SwitchBB, Dst: NextMBB, Prob: BranchProbToNext);
1208 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1209 // one as they are relative probabilities (and thus work more like weights),
1210 // and hence we need to normalize them to let the sum of them become one.
1211 SwitchBB->normalizeSuccProbs();
1212
1213 // Record the fact that the IR edge from the header to the bit test target
1214 // will go through our new block. Neeeded for PHIs to have nodes added.
1215 addMachineCFGPred(Edge: {BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1216 NewPred: SwitchBB);
1217
1218 MIB.buildBrCond(Tst: Cmp, Dest&: *B.TargetBB);
1219
1220 // Avoid emitting unnecessary branches to the next block.
1221 if (NextMBB != SwitchBB->getNextNode())
1222 MIB.buildBr(Dest&: *NextMBB);
1223}
1224
1225bool IRTranslator::lowerBitTestWorkItem(
1226 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
1227 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1228 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
1229 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1230 SwitchCG::CaseClusterIt I, MachineBasicBlock *Fallthrough,
1231 bool FallthroughUnreachable) {
1232 using namespace SwitchCG;
1233 MachineFunction *CurMF = SwitchMBB->getParent();
1234 // FIXME: Optimize away range check based on pivot comparisons.
1235 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
1236 // The bit test blocks haven't been inserted yet; insert them here.
1237 for (BitTestCase &BTC : BTB->Cases)
1238 CurMF->insert(MBBI: BBI, MBB: BTC.ThisBB);
1239
1240 // Fill in fields of the BitTestBlock.
1241 BTB->Parent = CurMBB;
1242 BTB->Default = Fallthrough;
1243
1244 BTB->DefaultProb = UnhandledProbs;
1245 // If the cases in bit test don't form a contiguous range, we evenly
1246 // distribute the probability on the edge to Fallthrough to two
1247 // successors of CurMBB.
1248 if (!BTB->ContiguousRange) {
1249 BTB->Prob += DefaultProb / 2;
1250 BTB->DefaultProb -= DefaultProb / 2;
1251 }
1252
1253 if (FallthroughUnreachable)
1254 BTB->FallthroughUnreachable = true;
1255
1256 // If we're in the right place, emit the bit test header right now.
1257 if (CurMBB == SwitchMBB) {
1258 emitBitTestHeader(B&: *BTB, SwitchBB: SwitchMBB);
1259 BTB->Emitted = true;
1260 }
1261 return true;
1262}
1263
1264bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
1265 Value *Cond,
1266 MachineBasicBlock *SwitchMBB,
1267 MachineBasicBlock *DefaultMBB,
1268 MachineIRBuilder &MIB) {
1269 using namespace SwitchCG;
1270 MachineFunction *CurMF = FuncInfo.MF;
1271 MachineBasicBlock *NextMBB = nullptr;
1272 MachineFunction::iterator BBI(W.MBB);
1273 if (++BBI != FuncInfo.MF->end())
1274 NextMBB = &*BBI;
1275
1276 if (EnableOpts) {
1277 // Here, we order cases by probability so the most likely case will be
1278 // checked first. However, two clusters can have the same probability in
1279 // which case their relative ordering is non-deterministic. So we use Low
1280 // as a tie-breaker as clusters are guaranteed to never overlap.
1281 llvm::sort(Start: W.FirstCluster, End: W.LastCluster + 1,
1282 Comp: [](const CaseCluster &a, const CaseCluster &b) {
1283 return a.Prob != b.Prob
1284 ? a.Prob > b.Prob
1285 : a.Low->getValue().slt(RHS: b.Low->getValue());
1286 });
1287
1288 // Rearrange the case blocks so that the last one falls through if possible
1289 // without changing the order of probabilities.
1290 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
1291 --I;
1292 if (I->Prob > W.LastCluster->Prob)
1293 break;
1294 if (I->Kind == CC_Range && I->MBB == NextMBB) {
1295 std::swap(a&: *I, b&: *W.LastCluster);
1296 break;
1297 }
1298 }
1299 }
1300
1301 // Compute total probability.
1302 BranchProbability DefaultProb = W.DefaultProb;
1303 BranchProbability UnhandledProbs = DefaultProb;
1304 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
1305 UnhandledProbs += I->Prob;
1306
1307 MachineBasicBlock *CurMBB = W.MBB;
1308 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
1309 bool FallthroughUnreachable = false;
1310 MachineBasicBlock *Fallthrough;
1311 if (I == W.LastCluster) {
1312 // For the last cluster, fall through to the default destination.
1313 Fallthrough = DefaultMBB;
1314 FallthroughUnreachable = isa<UnreachableInst>(
1315 Val: DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
1316 } else {
1317 Fallthrough = CurMF->CreateMachineBasicBlock(BB: CurMBB->getBasicBlock());
1318 CurMF->insert(MBBI: BBI, MBB: Fallthrough);
1319 }
1320 UnhandledProbs -= I->Prob;
1321
1322 switch (I->Kind) {
1323 case CC_BitTests: {
1324 if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1325 DefaultProb, UnhandledProbs, I, Fallthrough,
1326 FallthroughUnreachable)) {
1327 LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
1328 return false;
1329 }
1330 break;
1331 }
1332
1333 case CC_JumpTable: {
1334 if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1335 UnhandledProbs, I, Fallthrough,
1336 FallthroughUnreachable)) {
1337 LLVM_DEBUG(dbgs() << "Failed to lower jump table");
1338 return false;
1339 }
1340 break;
1341 }
1342 case CC_Range: {
1343 if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
1344 FallthroughUnreachable, UnhandledProbs,
1345 CurMBB, MIB, SwitchMBB)) {
1346 LLVM_DEBUG(dbgs() << "Failed to lower switch range");
1347 return false;
1348 }
1349 break;
1350 }
1351 }
1352 CurMBB = Fallthrough;
1353 }
1354
1355 return true;
1356}
1357
1358bool IRTranslator::translateIndirectBr(const User &U,
1359 MachineIRBuilder &MIRBuilder) {
1360 const IndirectBrInst &BrInst = cast<IndirectBrInst>(Val: U);
1361
1362 const Register Tgt = getOrCreateVReg(Val: *BrInst.getAddress());
1363 MIRBuilder.buildBrIndirect(Tgt);
1364
1365 // Link successors.
1366 SmallPtrSet<const BasicBlock *, 32> AddedSuccessors;
1367 MachineBasicBlock &CurBB = MIRBuilder.getMBB();
1368 for (const BasicBlock *Succ : successors(I: &BrInst)) {
1369 // It's legal for indirectbr instructions to have duplicate blocks in the
1370 // destination list. We don't allow this in MIR. Skip anything that's
1371 // already a successor.
1372 if (!AddedSuccessors.insert(Ptr: Succ).second)
1373 continue;
1374 CurBB.addSuccessor(Succ: &getMBB(BB: *Succ));
1375 }
1376
1377 return true;
1378}
1379
1380static bool isSwiftError(const Value *V) {
1381 if (auto Arg = dyn_cast<Argument>(Val: V))
1382 return Arg->hasSwiftErrorAttr();
1383 if (auto AI = dyn_cast<AllocaInst>(Val: V))
1384 return AI->isSwiftError();
1385 return false;
1386}
1387
1388bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
1389 const LoadInst &LI = cast<LoadInst>(Val: U);
1390 TypeSize StoreSize = DL->getTypeStoreSize(Ty: LI.getType());
1391 if (StoreSize.isZero())
1392 return true;
1393
1394 ArrayRef<Register> Regs = getOrCreateVRegs(Val: LI);
1395 Register Base = getOrCreateVReg(Val: *LI.getPointerOperand());
1396 AAMDNodes AAInfo = LI.getAAMetadata();
1397
1398 const Value *Ptr = LI.getPointerOperand();
1399
1400 if (CLI->supportSwiftError() && isSwiftError(V: Ptr)) {
1401 assert(Regs.size() == 1 && "swifterror should be single pointer");
1402 Register VReg =
1403 SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);
1404 MIRBuilder.buildCopy(Res: Regs[0], Op: VReg);
1405 return true;
1406 }
1407
1408 MachineMemOperand::Flags Flags =
1409 TLI->getLoadMemOperandFlags(LI, DL: *DL, AC, LibInfo, OptLevel);
1410 if (AA && !(Flags & MachineMemOperand::MOInvariant)) {
1411 if (AA->pointsToConstantMemory(
1412 Loc: MemoryLocation(Ptr, LocationSize::precise(Value: StoreSize), AAInfo))) {
1413 Flags |= MachineMemOperand::MOInvariant;
1414 }
1415 }
1416
1417 // Fast-path the common single-register load.
1418 if (Regs.size() == 1) {
1419 auto *MMO = MF->getMachineMemOperand(
1420 PtrInfo: MachinePointerInfo(LI.getPointerOperand()), f: Flags,
1421 MemTy: MRI->getType(Reg: Regs[0]), base_alignment: getMemOpAlign(I: LI), AAInfo,
1422 Ranges: LI.getMetadata(KindID: LLVMContext::MD_range), SSID: LI.getSyncScopeID(),
1423 Ordering: LI.getOrdering());
1424 MIRBuilder.buildLoad(Res: Regs[0], Addr: Base, MMO&: *MMO);
1425 return true;
1426 }
1427
1428 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: LI);
1429 Type *OffsetIRTy = DL->getIndexType(PtrTy: Ptr->getType());
1430 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1431 for (unsigned i = 0; i < Regs.size(); ++i) {
1432 Register Addr;
1433 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i]);
1434
1435 MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i]);
1436 Align BaseAlign = getMemOpAlign(I: LI);
1437 auto *MMO = MF->getMachineMemOperand(PtrInfo: Ptr, f: Flags, MemTy: MRI->getType(Reg: Regs[i]),
1438 base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]),
1439 AAInfo, Ranges: nullptr, SSID: LI.getSyncScopeID(),
1440 Ordering: LI.getOrdering());
1441 MIRBuilder.buildLoad(Res: Regs[i], Addr, MMO&: *MMO);
1442 }
1443
1444 return true;
1445}
1446
1447bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
1448 const StoreInst &SI = cast<StoreInst>(Val: U);
1449 if (DL->getTypeStoreSize(Ty: SI.getValueOperand()->getType()).isZero())
1450 return true;
1451
1452 ArrayRef<Register> Vals = getOrCreateVRegs(Val: *SI.getValueOperand());
1453 Register Base = getOrCreateVReg(Val: *SI.getPointerOperand());
1454
1455 if (CLI->supportSwiftError() && isSwiftError(V: SI.getPointerOperand())) {
1456 assert(Vals.size() == 1 && "swifterror should be single pointer");
1457
1458 Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
1459 SI.getPointerOperand());
1460 MIRBuilder.buildCopy(Res: VReg, Op: Vals[0]);
1461 return true;
1462 }
1463
1464 MachineMemOperand::Flags Flags = TLI->getStoreMemOperandFlags(SI, DL: *DL);
1465 // Fast-path the common single-register store.
1466 if (Vals.size() == 1) {
1467 auto *MMO = MF->getMachineMemOperand(
1468 PtrInfo: MachinePointerInfo(SI.getPointerOperand()), f: Flags,
1469 MemTy: MRI->getType(Reg: Vals[0]), base_alignment: getMemOpAlign(I: SI), AAInfo: SI.getAAMetadata(), Ranges: nullptr,
1470 SSID: SI.getSyncScopeID(), Ordering: SI.getOrdering());
1471 MIRBuilder.buildStore(Val: Vals[0], Addr: Base, MMO&: *MMO);
1472 return true;
1473 }
1474
1475 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *SI.getValueOperand());
1476 Type *OffsetIRTy = DL->getIndexType(PtrTy: SI.getPointerOperandType());
1477 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1478 for (unsigned i = 0; i < Vals.size(); ++i) {
1479 Register Addr;
1480 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i]);
1481
1482 MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i]);
1483 Align BaseAlign = getMemOpAlign(I: SI);
1484 auto *MMO = MF->getMachineMemOperand(PtrInfo: Ptr, f: Flags, MemTy: MRI->getType(Reg: Vals[i]),
1485 base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]),
1486 AAInfo: SI.getAAMetadata(), Ranges: nullptr,
1487 SSID: SI.getSyncScopeID(), Ordering: SI.getOrdering());
1488 MIRBuilder.buildStore(Val: Vals[i], Addr, MMO&: *MMO);
1489 }
1490 return true;
1491}
1492
1493static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
1494 const Value *Src = U.getOperand(i: 0);
1495 Type *Int32Ty = Type::getInt32Ty(C&: U.getContext());
1496
1497 // getIndexedOffsetInType is designed for GEPs, so the first index is the
1498 // usual array element rather than looking into the actual aggregate.
1499 SmallVector<Value *, 1> Indices;
1500 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: 0));
1501
1502 if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: &U)) {
1503 for (auto Idx : EVI->indices())
1504 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
1505 } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(Val: &U)) {
1506 for (auto Idx : IVI->indices())
1507 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
1508 } else {
1509 llvm::append_range(C&: Indices, R: drop_begin(RangeOrContainer: U.operands()));
1510 }
1511
1512 return static_cast<uint64_t>(
1513 DL.getIndexedOffsetInType(ElemTy: Src->getType(), Indices));
1514}
1515
1516bool IRTranslator::translateExtractValue(const User &U,
1517 MachineIRBuilder &MIRBuilder) {
1518 const Value *Src = U.getOperand(i: 0);
1519 uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
1520 ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
1521 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *Src);
1522 unsigned Idx = llvm::lower_bound(Range&: Offsets, Value&: Offset) - Offsets.begin();
1523 auto &DstRegs = allocateVRegs(Val: U);
1524
1525 for (unsigned i = 0; i < DstRegs.size(); ++i)
1526 DstRegs[i] = SrcRegs[Idx++];
1527
1528 return true;
1529}
1530
1531bool IRTranslator::translateInsertValue(const User &U,
1532 MachineIRBuilder &MIRBuilder) {
1533 const Value *Src = U.getOperand(i: 0);
1534 uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
1535 auto &DstRegs = allocateVRegs(Val: U);
1536 ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(V: U);
1537 ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
1538 ArrayRef<Register> InsertedRegs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
1539 auto *InsertedIt = InsertedRegs.begin();
1540
1541 for (unsigned i = 0; i < DstRegs.size(); ++i) {
1542 if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
1543 DstRegs[i] = *InsertedIt++;
1544 else
1545 DstRegs[i] = SrcRegs[i];
1546 }
1547
1548 return true;
1549}
1550
1551bool IRTranslator::translateSelect(const User &U,
1552 MachineIRBuilder &MIRBuilder) {
1553 Register Tst = getOrCreateVReg(Val: *U.getOperand(i: 0));
1554 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: U);
1555 ArrayRef<Register> Op0Regs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
1556 ArrayRef<Register> Op1Regs = getOrCreateVRegs(Val: *U.getOperand(i: 2));
1557
1558 uint32_t Flags = 0;
1559 if (const SelectInst *SI = dyn_cast<SelectInst>(Val: &U))
1560 Flags = MachineInstr::copyFlagsFromInstruction(I: *SI);
1561
1562 for (unsigned i = 0; i < ResRegs.size(); ++i) {
1563 MIRBuilder.buildSelect(Res: ResRegs[i], Tst, Op0: Op0Regs[i], Op1: Op1Regs[i], Flags);
1564 }
1565
1566 return true;
1567}
1568
1569bool IRTranslator::translateCopy(const User &U, const Value &V,
1570 MachineIRBuilder &MIRBuilder) {
1571 Register Src = getOrCreateVReg(Val: V);
1572 auto &Regs = *VMap.getVRegs(V: U);
1573 if (Regs.empty()) {
1574 Regs.push_back(Elt: Src);
1575 VMap.getOffsets(V: U)->push_back(Elt: 0);
1576 } else {
1577 // If we already assigned a vreg for this instruction, we can't change that.
1578 // Emit a copy to satisfy the users we already emitted.
1579 MIRBuilder.buildCopy(Res: Regs[0], Op: Src);
1580 }
1581 return true;
1582}
1583
1584bool IRTranslator::translateBitCast(const User &U,
1585 MachineIRBuilder &MIRBuilder) {
1586 Type *SrcTy = U.getOperand(i: 0)->getType();
1587 Type *DstTy = U.getType();
1588
1589 // If we're bitcasting to the source type, we can reuse the source vreg.
1590 if (getLLTForType(Ty&: *SrcTy, DL: *DL) == getLLTForType(Ty&: *DstTy, DL: *DL)) {
1591 // If the source is a ConstantInt then it was probably created by
1592 // ConstantHoisting and we should leave it alone.
1593 if (isa<ConstantInt>(Val: U.getOperand(i: 0)))
1594 return translateCast(Opcode: TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
1595 MIRBuilder);
1596 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
1597 }
1598
1599 // Only the scalar byte<->ptr crossing is redirected to G_INTTOPTR/G_PTRTOINT,
1600 // which is the well-typed MIR shape for that boundary. Vector byte<->ptr
1601 // (e.g. <N x b32> -> ptr produced by mixed-type load coalescing) and other
1602 // legacy ptr/non-ptr IR bitcasts (AMDGPU iN<->p3 kernarg packing, etc.)
1603 // keep their historical G_BITCAST lowering — G_INTTOPTR has no vector-src
1604 // -> scalar-ptr form, and downstream passes already handle G_BITCAST.
1605 if (DstTy->isPointerTy() && SrcTy->isByteTy())
1606 return translateCast(Opcode: TargetOpcode::G_INTTOPTR, U, MIRBuilder);
1607 if (SrcTy->isPointerTy() && DstTy->isByteTy())
1608 return translateCast(Opcode: TargetOpcode::G_PTRTOINT, U, MIRBuilder);
1609
1610 return translateCast(Opcode: TargetOpcode::G_BITCAST, U, MIRBuilder);
1611}
1612
1613bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1614 MachineIRBuilder &MIRBuilder) {
1615 if (!mayTranslateUserTypes(U))
1616 return false;
1617
1618 uint32_t Flags = 0;
1619 if (const Instruction *I = dyn_cast<Instruction>(Val: &U))
1620 Flags = MachineInstr::copyFlagsFromInstruction(I: *I);
1621
1622 Register Op = getOrCreateVReg(Val: *U.getOperand(i: 0));
1623 Register Res = getOrCreateVReg(Val: U);
1624 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op}, Flags);
1625 return true;
1626}
1627
1628bool IRTranslator::translateGetElementPtr(const User &U,
1629 MachineIRBuilder &MIRBuilder) {
1630 Value &Op0 = *U.getOperand(i: 0);
1631 Register BaseReg = getOrCreateVReg(Val: Op0);
1632 Type *PtrIRTy = Op0.getType();
1633 LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1634 Type *OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
1635 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1636
1637 uint32_t PtrAddFlags = 0;
1638 // Each PtrAdd generated to implement the GEP inherits its nuw, nusw, inbounds
1639 // flags.
1640 if (const Instruction *I = dyn_cast<Instruction>(Val: &U))
1641 PtrAddFlags = MachineInstr::copyFlagsFromInstruction(I: *I);
1642
1643 auto PtrAddFlagsWithConst = [&](int64_t Offset) {
1644 // For nusw/inbounds GEP with an offset that is nonnegative when interpreted
1645 // as signed, assume there is no unsigned overflow.
1646 if (Offset >= 0 && (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap))
1647 return PtrAddFlags | MachineInstr::MIFlag::NoUWrap;
1648 return PtrAddFlags;
1649 };
1650
1651 // Normalize Vector GEP - all scalar operands should be converted to the
1652 // splat vector.
1653 unsigned VectorWidth = 0;
1654
1655 // True if we should use a splat vector; using VectorWidth alone is not
1656 // sufficient.
1657 bool WantSplatVector = false;
1658 if (auto *VT = dyn_cast<VectorType>(Val: U.getType())) {
1659 VectorWidth = cast<FixedVectorType>(Val: VT)->getNumElements();
1660 // We don't produce 1 x N vectors; those are treated as scalars.
1661 WantSplatVector = VectorWidth > 1;
1662 }
1663
1664 if (cast<GEPOperator>(Val: U).hasAllZeroIndices())
1665 return translateCopy(U, V: Op0, MIRBuilder);
1666
1667 // We might need to splat the base pointer into a vector if the offsets
1668 // are vectors.
1669 if (WantSplatVector && !PtrTy.isVector()) {
1670 BaseReg = MIRBuilder
1671 .buildSplatBuildVector(Res: LLT::fixed_vector(NumElements: VectorWidth, ScalarTy: PtrTy),
1672 Src: BaseReg)
1673 .getReg(Idx: 0);
1674 PtrIRTy = FixedVectorType::get(ElementType: PtrIRTy, NumElts: VectorWidth);
1675 PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1676 OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
1677 OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1678 }
1679
1680 int64_t Offset = 0;
1681 for (gep_type_iterator GTI = gep_type_begin(GEP: &U), E = gep_type_end(GEP: &U);
1682 GTI != E; ++GTI) {
1683 const Value *Idx = GTI.getOperand();
1684 if (StructType *StTy = GTI.getStructTypeOrNull()) {
1685 unsigned Field = cast<Constant>(Val: Idx)->getUniqueInteger().getZExtValue();
1686 Offset += DL->getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
1687 continue;
1688 } else {
1689 uint64_t ElementSize = GTI.getSequentialElementStride(DL: *DL);
1690
1691 // If this is a scalar constant or a splat vector of constants,
1692 // handle it quickly.
1693 if (const auto *CI = dyn_cast<ConstantInt>(Val: Idx)) {
1694 if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {
1695 Offset += ElementSize * *Val;
1696 continue;
1697 }
1698 }
1699
1700 if (Offset != 0) {
1701 auto OffsetMIB = MIRBuilder.buildConstant(Res: {OffsetTy}, Val: Offset);
1702 BaseReg = MIRBuilder
1703 .buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0),
1704 Flags: PtrAddFlagsWithConst(Offset))
1705 .getReg(Idx: 0);
1706 Offset = 0;
1707 }
1708
1709 Register IdxReg = getOrCreateVReg(Val: *Idx);
1710 LLT IdxTy = MRI->getType(Reg: IdxReg);
1711 if (IdxTy != OffsetTy) {
1712 if (!IdxTy.isVector() && WantSplatVector) {
1713 IdxReg = MIRBuilder
1714 .buildSplatBuildVector(Res: OffsetTy.changeElementType(NewEltTy: IdxTy),
1715 Src: IdxReg)
1716 .getReg(Idx: 0);
1717 }
1718
1719 IdxReg = MIRBuilder.buildSExtOrTrunc(Res: OffsetTy, Op: IdxReg).getReg(Idx: 0);
1720 }
1721
1722 // N = N + Idx * ElementSize;
1723 // Avoid doing it for ElementSize of 1.
1724 Register GepOffsetReg;
1725 if (ElementSize != 1) {
1726 auto ElementSizeMIB = MIRBuilder.buildConstant(
1727 Res: getLLTForType(Ty&: *OffsetIRTy, DL: *DL), Val: ElementSize);
1728
1729 // The multiplication is NUW if the GEP is NUW and NSW if the GEP is
1730 // NUSW.
1731 uint32_t ScaleFlags = PtrAddFlags & MachineInstr::MIFlag::NoUWrap;
1732 if (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap)
1733 ScaleFlags |= MachineInstr::MIFlag::NoSWrap;
1734
1735 GepOffsetReg =
1736 MIRBuilder.buildMul(Dst: OffsetTy, Src0: IdxReg, Src1: ElementSizeMIB, Flags: ScaleFlags)
1737 .getReg(Idx: 0);
1738 } else {
1739 GepOffsetReg = IdxReg;
1740 }
1741
1742 BaseReg =
1743 MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: GepOffsetReg, Flags: PtrAddFlags)
1744 .getReg(Idx: 0);
1745 }
1746 }
1747
1748 if (Offset != 0) {
1749 auto OffsetMIB =
1750 MIRBuilder.buildConstant(Res: OffsetTy, Val: Offset);
1751
1752 MIRBuilder.buildPtrAdd(Res: getOrCreateVReg(Val: U), Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0),
1753 Flags: PtrAddFlagsWithConst(Offset));
1754 return true;
1755 }
1756
1757 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: U), Op: BaseReg);
1758 return true;
1759}
1760
1761bool IRTranslator::translateMemFunc(const CallInst &CI,
1762 MachineIRBuilder &MIRBuilder,
1763 unsigned Opcode) {
1764 const Value *SrcPtr = CI.getArgOperand(i: 1);
1765 // If the source is undef, then just emit a nop.
1766 if (isa<UndefValue>(Val: SrcPtr))
1767 return true;
1768
1769 SmallVector<Register, 3> SrcRegs;
1770
1771 unsigned MinPtrSize = UINT_MAX;
1772 for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(x: AI) != AE; ++AI) {
1773 Register SrcReg = getOrCreateVReg(Val: **AI);
1774 LLT SrcTy = MRI->getType(Reg: SrcReg);
1775 if (SrcTy.isPointer())
1776 MinPtrSize = std::min<unsigned>(a: SrcTy.getSizeInBits(), b: MinPtrSize);
1777 SrcRegs.push_back(Elt: SrcReg);
1778 }
1779
1780 LLT SizeTy = LLT::integer(SizeInBits: MinPtrSize);
1781
1782 // The size operand should be the minimum of the pointer sizes.
1783 Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
1784 if (MRI->getType(Reg: SizeOpReg) != SizeTy)
1785 SizeOpReg = MIRBuilder.buildZExtOrTrunc(Res: SizeTy, Op: SizeOpReg).getReg(Idx: 0);
1786
1787 auto ICall = MIRBuilder.buildInstr(Opcode);
1788 for (Register SrcReg : SrcRegs)
1789 ICall.addUse(RegNo: SrcReg);
1790
1791 Align DstAlign;
1792 Align SrcAlign;
1793 unsigned IsVol =
1794 cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1))->getZExtValue();
1795
1796 ConstantInt *CopySize = nullptr;
1797
1798 if (auto *MCI = dyn_cast<MemCpyInst>(Val: &CI)) {
1799 DstAlign = MCI->getDestAlign().valueOrOne();
1800 SrcAlign = MCI->getSourceAlign().valueOrOne();
1801 CopySize = dyn_cast<ConstantInt>(Val: MCI->getArgOperand(i: 2));
1802 } else if (auto *MMI = dyn_cast<MemMoveInst>(Val: &CI)) {
1803 DstAlign = MMI->getDestAlign().valueOrOne();
1804 SrcAlign = MMI->getSourceAlign().valueOrOne();
1805 CopySize = dyn_cast<ConstantInt>(Val: MMI->getArgOperand(i: 2));
1806 } else {
1807 auto *MSI = cast<MemSetInst>(Val: &CI);
1808 DstAlign = MSI->getDestAlign().valueOrOne();
1809 }
1810
1811 if (Opcode != TargetOpcode::G_MEMCPY_INLINE &&
1812 Opcode != TargetOpcode::G_MEMSET_INLINE) {
1813 // We need to propagate the tail call flag from the IR inst as an argument.
1814 // Otherwise, we have to pessimize and assume later that we cannot tail call
1815 // any memory intrinsics.
1816 ICall.addImm(Val: CI.isTailCall() ? 1 : 0);
1817 }
1818
1819 // Create mem operands to store the alignment and volatile info.
1820 MachineMemOperand::Flags LoadFlags = MachineMemOperand::MOLoad;
1821 MachineMemOperand::Flags StoreFlags = MachineMemOperand::MOStore;
1822 if (IsVol) {
1823 LoadFlags |= MachineMemOperand::MOVolatile;
1824 StoreFlags |= MachineMemOperand::MOVolatile;
1825 }
1826
1827 AAMDNodes AAInfo = CI.getAAMetadata();
1828 if (AA && CopySize &&
1829 AA->pointsToConstantMemory(Loc: MemoryLocation(
1830 SrcPtr, LocationSize::precise(Value: CopySize->getZExtValue()), AAInfo))) {
1831 LoadFlags |= MachineMemOperand::MOInvariant;
1832
1833 // FIXME: pointsToConstantMemory probably does not imply dereferenceable,
1834 // but the previous usage implied it did. Probably should check
1835 // isDereferenceableAndAlignedPointer.
1836 LoadFlags |= MachineMemOperand::MODereferenceable;
1837 }
1838
1839 ICall.addMemOperand(
1840 MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(CI.getArgOperand(i: 0)),
1841 F: StoreFlags, Size: 1, BaseAlignment: DstAlign, AAInfo));
1842 if (Opcode != TargetOpcode::G_MEMSET &&
1843 Opcode != TargetOpcode::G_MEMSET_INLINE)
1844 ICall.addMemOperand(MMO: MF->getMachineMemOperand(
1845 PtrInfo: MachinePointerInfo(SrcPtr), F: LoadFlags, Size: 1, BaseAlignment: SrcAlign, AAInfo));
1846
1847 return true;
1848}
1849
1850bool IRTranslator::translateTrap(const CallInst &CI,
1851 MachineIRBuilder &MIRBuilder,
1852 unsigned Opcode) {
1853 StringRef TrapFuncName =
1854 CI.getAttributes().getFnAttr(Kind: "trap-func-name").getValueAsString();
1855 if (TrapFuncName.empty()) {
1856 if (Opcode == TargetOpcode::G_UBSANTRAP) {
1857 uint64_t Code = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 0))->getZExtValue();
1858 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: ArrayRef<llvm::SrcOp>{Code});
1859 } else {
1860 MIRBuilder.buildInstr(Opcode);
1861 }
1862 return true;
1863 }
1864
1865 CallLowering::CallLoweringInfo Info;
1866 if (Opcode == TargetOpcode::G_UBSANTRAP)
1867 Info.OrigArgs.push_back(Elt: {getOrCreateVRegs(Val: *CI.getArgOperand(i: 0)),
1868 CI.getArgOperand(i: 0)->getType(), 0});
1869
1870 Info.Callee = MachineOperand::CreateES(SymName: TrapFuncName.data());
1871 Info.CB = &CI;
1872 Info.OrigRet = {Register(), Type::getVoidTy(C&: CI.getContext()), 0};
1873 return CLI->lowerCall(MIRBuilder, Info);
1874}
1875
1876bool IRTranslator::translateVectorInterleave2Intrinsic(
1877 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
1878 assert(CI.getIntrinsicID() == Intrinsic::vector_interleave2 &&
1879 "This function can only be called on the interleave2 intrinsic!");
1880 // Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG).
1881 Register Op0 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
1882 Register Op1 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1));
1883 Register Res = getOrCreateVReg(Val: CI);
1884
1885 LLT OpTy = MRI->getType(Reg: Op0);
1886 MIRBuilder.buildShuffleVector(Res, Src1: Op0, Src2: Op1,
1887 Mask: createInterleaveMask(VF: OpTy.getNumElements(), NumVecs: 2));
1888
1889 return true;
1890}
1891
1892bool IRTranslator::translateVectorDeinterleave2Intrinsic(
1893 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
1894 assert(CI.getIntrinsicID() == Intrinsic::vector_deinterleave2 &&
1895 "This function can only be called on the deinterleave2 intrinsic!");
1896 // Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to
1897 // SelectionDAG).
1898 Register Op = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
1899 auto Undef = MIRBuilder.buildUndef(Res: MRI->getType(Reg: Op));
1900 ArrayRef<Register> Res = getOrCreateVRegs(Val: CI);
1901
1902 LLT ResTy = MRI->getType(Reg: Res[0]);
1903 MIRBuilder.buildShuffleVector(Res: Res[0], Src1: Op, Src2: Undef,
1904 Mask: createStrideMask(Start: 0, Stride: 2, VF: ResTy.getNumElements()));
1905 MIRBuilder.buildShuffleVector(Res: Res[1], Src1: Op, Src2: Undef,
1906 Mask: createStrideMask(Start: 1, Stride: 2, VF: ResTy.getNumElements()));
1907
1908 return true;
1909}
1910
1911void IRTranslator::getStackGuard(Register DstReg,
1912 MachineIRBuilder &MIRBuilder) {
1913 Value *Global =
1914 TLI->getSDagStackGuard(M: *MF->getFunction().getParent(), Libcalls: *Libcalls);
1915 if (!Global) {
1916 LLVMContext &Ctx = MIRBuilder.getContext();
1917 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
1918 MIRBuilder.buildUndef(Res: DstReg);
1919 return;
1920 }
1921
1922 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1923 MRI->setRegClass(Reg: DstReg, RC: TRI->getPointerRegClass());
1924 auto MIB =
1925 MIRBuilder.buildInstr(Opc: TargetOpcode::LOAD_STACK_GUARD, DstOps: {DstReg}, SrcOps: {});
1926
1927 unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
1928 LLT PtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL->getPointerSizeInBits(AS: AddrSpace));
1929
1930 MachinePointerInfo MPInfo(Global);
1931 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
1932 MachineMemOperand::MODereferenceable;
1933 MachineMemOperand *MemRef = MF->getMachineMemOperand(
1934 PtrInfo: MPInfo, f: Flags, MemTy: PtrTy, base_alignment: DL->getPointerABIAlignment(AS: AddrSpace));
1935 MIB.setMemRefs({MemRef});
1936}
1937
1938bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1939 MachineIRBuilder &MIRBuilder) {
1940 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: CI);
1941 MIRBuilder.buildInstr(
1942 Opc: Op, DstOps: {ResRegs[0], ResRegs[1]},
1943 SrcOps: {getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)), getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1))});
1944
1945 return true;
1946}
1947
1948bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
1949 MachineIRBuilder &MIRBuilder) {
1950 Register Dst = getOrCreateVReg(Val: CI);
1951 Register Src0 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
1952 Register Src1 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1));
1953 uint64_t Scale = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
1954 MIRBuilder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: { Src0, Src1, Scale });
1955 return true;
1956}
1957
1958unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1959 switch (ID) {
1960 default:
1961 break;
1962 case Intrinsic::acos:
1963 return TargetOpcode::G_FACOS;
1964 case Intrinsic::asin:
1965 return TargetOpcode::G_FASIN;
1966 case Intrinsic::atan:
1967 return TargetOpcode::G_FATAN;
1968 case Intrinsic::atan2:
1969 return TargetOpcode::G_FATAN2;
1970 case Intrinsic::bswap:
1971 return TargetOpcode::G_BSWAP;
1972 case Intrinsic::bitreverse:
1973 return TargetOpcode::G_BITREVERSE;
1974 case Intrinsic::fshl:
1975 return TargetOpcode::G_FSHL;
1976 case Intrinsic::fshr:
1977 return TargetOpcode::G_FSHR;
1978 case Intrinsic::ceil:
1979 return TargetOpcode::G_FCEIL;
1980 case Intrinsic::cos:
1981 return TargetOpcode::G_FCOS;
1982 case Intrinsic::cosh:
1983 return TargetOpcode::G_FCOSH;
1984 case Intrinsic::ctpop:
1985 return TargetOpcode::G_CTPOP;
1986 case Intrinsic::exp:
1987 return TargetOpcode::G_FEXP;
1988 case Intrinsic::exp2:
1989 return TargetOpcode::G_FEXP2;
1990 case Intrinsic::exp10:
1991 return TargetOpcode::G_FEXP10;
1992 case Intrinsic::fabs:
1993 return TargetOpcode::G_FABS;
1994 case Intrinsic::copysign:
1995 return TargetOpcode::G_FCOPYSIGN;
1996 case Intrinsic::minnum:
1997 return TargetOpcode::G_FMINNUM;
1998 case Intrinsic::maxnum:
1999 return TargetOpcode::G_FMAXNUM;
2000 case Intrinsic::minimum:
2001 return TargetOpcode::G_FMINIMUM;
2002 case Intrinsic::maximum:
2003 return TargetOpcode::G_FMAXIMUM;
2004 case Intrinsic::minimumnum:
2005 return TargetOpcode::G_FMINIMUMNUM;
2006 case Intrinsic::maximumnum:
2007 return TargetOpcode::G_FMAXIMUMNUM;
2008 case Intrinsic::canonicalize:
2009 return TargetOpcode::G_FCANONICALIZE;
2010 case Intrinsic::floor:
2011 return TargetOpcode::G_FFLOOR;
2012 case Intrinsic::fma:
2013 return TargetOpcode::G_FMA;
2014 case Intrinsic::log:
2015 return TargetOpcode::G_FLOG;
2016 case Intrinsic::log2:
2017 return TargetOpcode::G_FLOG2;
2018 case Intrinsic::log10:
2019 return TargetOpcode::G_FLOG10;
2020 case Intrinsic::ldexp:
2021 return TargetOpcode::G_FLDEXP;
2022 case Intrinsic::nearbyint:
2023 return TargetOpcode::G_FNEARBYINT;
2024 case Intrinsic::pow:
2025 return TargetOpcode::G_FPOW;
2026 case Intrinsic::powi:
2027 return TargetOpcode::G_FPOWI;
2028 case Intrinsic::rint:
2029 return TargetOpcode::G_FRINT;
2030 case Intrinsic::round:
2031 return TargetOpcode::G_INTRINSIC_ROUND;
2032 case Intrinsic::roundeven:
2033 return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
2034 case Intrinsic::sin:
2035 return TargetOpcode::G_FSIN;
2036 case Intrinsic::sinh:
2037 return TargetOpcode::G_FSINH;
2038 case Intrinsic::sqrt:
2039 return TargetOpcode::G_FSQRT;
2040 case Intrinsic::tan:
2041 return TargetOpcode::G_FTAN;
2042 case Intrinsic::tanh:
2043 return TargetOpcode::G_FTANH;
2044 case Intrinsic::trunc:
2045 return TargetOpcode::G_INTRINSIC_TRUNC;
2046 case Intrinsic::readcyclecounter:
2047 return TargetOpcode::G_READCYCLECOUNTER;
2048 case Intrinsic::readsteadycounter:
2049 return TargetOpcode::G_READSTEADYCOUNTER;
2050 case Intrinsic::ptrmask:
2051 return TargetOpcode::G_PTRMASK;
2052 case Intrinsic::lrint:
2053 return TargetOpcode::G_INTRINSIC_LRINT;
2054 case Intrinsic::llrint:
2055 return TargetOpcode::G_INTRINSIC_LLRINT;
2056 // FADD/FMUL require checking the FMF, so are handled elsewhere.
2057 case Intrinsic::vector_reduce_fmin:
2058 return TargetOpcode::G_VECREDUCE_FMIN;
2059 case Intrinsic::vector_reduce_fmax:
2060 return TargetOpcode::G_VECREDUCE_FMAX;
2061 case Intrinsic::vector_reduce_fminimum:
2062 return TargetOpcode::G_VECREDUCE_FMINIMUM;
2063 case Intrinsic::vector_reduce_fmaximum:
2064 return TargetOpcode::G_VECREDUCE_FMAXIMUM;
2065 case Intrinsic::vector_reduce_add:
2066 return TargetOpcode::G_VECREDUCE_ADD;
2067 case Intrinsic::vector_reduce_mul:
2068 return TargetOpcode::G_VECREDUCE_MUL;
2069 case Intrinsic::vector_reduce_and:
2070 return TargetOpcode::G_VECREDUCE_AND;
2071 case Intrinsic::vector_reduce_or:
2072 return TargetOpcode::G_VECREDUCE_OR;
2073 case Intrinsic::vector_reduce_xor:
2074 return TargetOpcode::G_VECREDUCE_XOR;
2075 case Intrinsic::vector_reduce_smax:
2076 return TargetOpcode::G_VECREDUCE_SMAX;
2077 case Intrinsic::vector_reduce_smin:
2078 return TargetOpcode::G_VECREDUCE_SMIN;
2079 case Intrinsic::vector_reduce_umax:
2080 return TargetOpcode::G_VECREDUCE_UMAX;
2081 case Intrinsic::vector_reduce_umin:
2082 return TargetOpcode::G_VECREDUCE_UMIN;
2083 case Intrinsic::experimental_vector_compress:
2084 return TargetOpcode::G_VECTOR_COMPRESS;
2085 case Intrinsic::lround:
2086 return TargetOpcode::G_LROUND;
2087 case Intrinsic::llround:
2088 return TargetOpcode::G_LLROUND;
2089 case Intrinsic::get_fpenv:
2090 return TargetOpcode::G_GET_FPENV;
2091 case Intrinsic::get_fpmode:
2092 return TargetOpcode::G_GET_FPMODE;
2093 }
2094 return Intrinsic::not_intrinsic;
2095}
2096
2097bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
2098 Intrinsic::ID ID,
2099 MachineIRBuilder &MIRBuilder) {
2100
2101 unsigned Op = getSimpleIntrinsicOpcode(ID);
2102
2103 // Is this a simple intrinsic?
2104 if (Op == Intrinsic::not_intrinsic)
2105 return false;
2106
2107 // Yes. Let's translate it.
2108 SmallVector<llvm::SrcOp, 4> VRegs;
2109 for (const auto &Arg : CI.args())
2110 VRegs.push_back(Elt: getOrCreateVReg(Val: *Arg));
2111
2112 MIRBuilder.buildInstr(Opc: Op, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: VRegs,
2113 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2114 return true;
2115}
2116
2117// TODO: Include ConstainedOps.def when all strict instructions are defined.
2118static unsigned getConstrainedOpcode(Intrinsic::ID ID) {
2119 switch (ID) {
2120 case Intrinsic::experimental_constrained_fadd:
2121 return TargetOpcode::G_STRICT_FADD;
2122 case Intrinsic::experimental_constrained_fsub:
2123 return TargetOpcode::G_STRICT_FSUB;
2124 case Intrinsic::experimental_constrained_fmul:
2125 return TargetOpcode::G_STRICT_FMUL;
2126 case Intrinsic::experimental_constrained_fdiv:
2127 return TargetOpcode::G_STRICT_FDIV;
2128 case Intrinsic::experimental_constrained_frem:
2129 return TargetOpcode::G_STRICT_FREM;
2130 case Intrinsic::experimental_constrained_fma:
2131 return TargetOpcode::G_STRICT_FMA;
2132 case Intrinsic::experimental_constrained_sqrt:
2133 return TargetOpcode::G_STRICT_FSQRT;
2134 case Intrinsic::experimental_constrained_ldexp:
2135 return TargetOpcode::G_STRICT_FLDEXP;
2136 case Intrinsic::experimental_constrained_fcmp:
2137 return TargetOpcode::G_STRICT_FCMP;
2138 case Intrinsic::experimental_constrained_fcmps:
2139 return TargetOpcode::G_STRICT_FCMPS;
2140 default:
2141 return 0;
2142 }
2143}
2144
2145bool IRTranslator::translateConstrainedFPIntrinsic(
2146 const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
2147 fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
2148
2149 unsigned Opcode = getConstrainedOpcode(ID: FPI.getIntrinsicID());
2150 if (!Opcode)
2151 return false;
2152
2153 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: FPI);
2154 if (EB == fp::ExceptionBehavior::ebIgnore)
2155 Flags |= MachineInstr::NoFPExcept;
2156
2157 if (Opcode == TargetOpcode::G_STRICT_FCMP ||
2158 Opcode == TargetOpcode::G_STRICT_FCMPS) {
2159 auto *FPCmp = cast<ConstrainedFPCmpIntrinsic>(Val: &FPI);
2160 Register Operand0 = getOrCreateVReg(Val: *FPCmp->getArgOperand(i: 0));
2161 Register Operand1 = getOrCreateVReg(Val: *FPCmp->getArgOperand(i: 1));
2162 Register Result = getOrCreateVReg(Val: FPI);
2163 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Result}, SrcOps: {}, Flags)
2164 .addPredicate(Pred: FPCmp->getPredicate())
2165 .addUse(RegNo: Operand0)
2166 .addUse(RegNo: Operand1);
2167 return true;
2168 }
2169
2170 SmallVector<llvm::SrcOp, 4> VRegs;
2171 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
2172 VRegs.push_back(Elt: getOrCreateVReg(Val: *FPI.getArgOperand(i: I)));
2173
2174 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: FPI)}, SrcOps: VRegs, Flags);
2175 return true;
2176}
2177
2178std::optional<MCRegister> IRTranslator::getArgPhysReg(Argument &Arg) {
2179 auto VRegs = getOrCreateVRegs(Val: Arg);
2180 if (VRegs.size() != 1)
2181 return std::nullopt;
2182
2183 // Arguments are lowered as a copy of a livein physical register.
2184 auto *VRegDef = MF->getRegInfo().getVRegDef(Reg: VRegs[0]);
2185 if (!VRegDef || !VRegDef->isCopy())
2186 return std::nullopt;
2187 return VRegDef->getOperand(i: 1).getReg().asMCReg();
2188}
2189
2190bool IRTranslator::translateIfEntryValueArgument(bool isDeclare, Value *Val,
2191 const DILocalVariable *Var,
2192 const DIExpression *Expr,
2193 const DebugLoc &DL,
2194 MachineIRBuilder &MIRBuilder) {
2195 auto *Arg = dyn_cast<Argument>(Val);
2196 if (!Arg)
2197 return false;
2198
2199 if (!Expr->isEntryValue())
2200 return false;
2201
2202 std::optional<MCRegister> PhysReg = getArgPhysReg(Arg&: *Arg);
2203 if (!PhysReg) {
2204 LLVM_DEBUG(dbgs() << "Dropping dbg." << (isDeclare ? "declare" : "value")
2205 << ": expression is entry_value but "
2206 << "couldn't find a physical register\n");
2207 LLVM_DEBUG(dbgs() << *Var << "\n");
2208 return true;
2209 }
2210
2211 if (isDeclare) {
2212 // Append an op deref to account for the fact that this is a dbg_declare.
2213 Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
2214 MF->setVariableDbgInfo(Var, Expr, Reg: *PhysReg, Loc: DL);
2215 } else {
2216 MIRBuilder.buildDirectDbgValue(Reg: *PhysReg, Variable: Var, Expr);
2217 }
2218
2219 return true;
2220}
2221
2222static unsigned getConvOpcode(Intrinsic::ID ID) {
2223 switch (ID) {
2224 default:
2225 llvm_unreachable("Unexpected intrinsic");
2226 case Intrinsic::experimental_convergence_anchor:
2227 return TargetOpcode::CONVERGENCECTRL_ANCHOR;
2228 case Intrinsic::experimental_convergence_entry:
2229 return TargetOpcode::CONVERGENCECTRL_ENTRY;
2230 case Intrinsic::experimental_convergence_loop:
2231 return TargetOpcode::CONVERGENCECTRL_LOOP;
2232 }
2233}
2234
2235bool IRTranslator::translateConvergenceControlIntrinsic(
2236 const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
2237 MachineInstrBuilder MIB = MIRBuilder.buildInstr(Opcode: getConvOpcode(ID));
2238 Register OutputReg = getOrCreateConvergenceTokenVReg(Token: CI);
2239 MIB.addDef(RegNo: OutputReg);
2240
2241 if (ID == Intrinsic::experimental_convergence_loop) {
2242 auto Bundle = CI.getOperandBundle(ID: LLVMContext::OB_convergencectrl);
2243 assert(Bundle && "Expected a convergence control token.");
2244 Register InputReg =
2245 getOrCreateConvergenceTokenVReg(Token: *Bundle->Inputs[0].get());
2246 MIB.addUse(RegNo: InputReg);
2247 }
2248
2249 return true;
2250}
2251
2252bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
2253 MachineIRBuilder &MIRBuilder) {
2254 if (auto *MI = dyn_cast<AnyMemIntrinsic>(Val: &CI)) {
2255 if (ORE->enabled()) {
2256 if (MemoryOpRemark::canHandle(I: MI, TLI: *LibInfo)) {
2257 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2258 R.visit(I: MI);
2259 }
2260 }
2261 }
2262
2263 // If this is a simple intrinsic (that is, we just need to add a def of
2264 // a vreg, and uses for each arg operand, then translate it.
2265 if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
2266 return true;
2267
2268 switch (ID) {
2269 default:
2270 break;
2271 case Intrinsic::lifetime_start:
2272 case Intrinsic::lifetime_end: {
2273 // No stack colouring in O0, discard region information.
2274 if (MF->getTarget().getOptLevel() == CodeGenOptLevel::None ||
2275 MF->getFunction().hasOptNone())
2276 return true;
2277
2278 unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
2279 : TargetOpcode::LIFETIME_END;
2280
2281 const AllocaInst *AI = dyn_cast<AllocaInst>(Val: CI.getArgOperand(i: 0));
2282 if (!AI || !AI->isStaticAlloca())
2283 return true;
2284
2285 MIRBuilder.buildInstr(Opcode: Op).addFrameIndex(Idx: getOrCreateFrameIndex(AI: *AI));
2286 return true;
2287 }
2288 case Intrinsic::fake_use: {
2289 SmallVector<llvm::SrcOp, 4> VRegs;
2290 for (const auto &Arg : CI.args())
2291 llvm::append_range(C&: VRegs, R: getOrCreateVRegs(Val: *Arg));
2292 MIRBuilder.buildInstr(Opc: TargetOpcode::FAKE_USE, DstOps: {}, SrcOps: VRegs);
2293 MF->setHasFakeUses(true);
2294 return true;
2295 }
2296 case Intrinsic::dbg_declare: {
2297 const DbgDeclareInst &DI = cast<DbgDeclareInst>(Val: CI);
2298 assert(DI.getVariable() && "Missing variable");
2299 translateDbgDeclareRecord(Address: DI.getAddress(), HasArgList: DI.hasArgList(), Variable: DI.getVariable(),
2300 Expression: DI.getExpression(), DL: DI.getDebugLoc(), MIRBuilder);
2301 return true;
2302 }
2303 case Intrinsic::dbg_label: {
2304 const DbgLabelInst &DI = cast<DbgLabelInst>(Val: CI);
2305 assert(DI.getLabel() && "Missing label");
2306
2307 assert(DI.getLabel()->isValidLocationForIntrinsic(
2308 MIRBuilder.getDebugLoc()) &&
2309 "Expected inlined-at fields to agree");
2310
2311 MIRBuilder.buildDbgLabel(Label: DI.getLabel());
2312 return true;
2313 }
2314 case Intrinsic::vaend:
2315 // No target I know of cares about va_end. Certainly no in-tree target
2316 // does. Simplest intrinsic ever!
2317 return true;
2318 case Intrinsic::vastart: {
2319 Value *Ptr = CI.getArgOperand(i: 0);
2320 unsigned ListSize = TLI->getVaListSizeInBits(DL: *DL) / 8;
2321 Align Alignment = getKnownAlignment(V: Ptr, DL: *DL);
2322
2323 MIRBuilder.buildInstr(Opc: TargetOpcode::G_VASTART, DstOps: {}, SrcOps: {getOrCreateVReg(Val: *Ptr)})
2324 .addMemOperand(MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Ptr),
2325 F: MachineMemOperand::MOStore,
2326 Size: ListSize, BaseAlignment: Alignment));
2327 return true;
2328 }
2329 case Intrinsic::dbg_assign:
2330 // A dbg.assign is a dbg.value with more information about stack locations,
2331 // typically produced during optimisation of variables with leaked
2332 // addresses. We can treat it like a normal dbg_value intrinsic here; to
2333 // benefit from the full analysis of stack/SSA locations, GlobalISel would
2334 // need to register for and use the AssignmentTrackingAnalysis pass.
2335 [[fallthrough]];
2336 case Intrinsic::dbg_value: {
2337 // This form of DBG_VALUE is target-independent.
2338 const DbgValueInst &DI = cast<DbgValueInst>(Val: CI);
2339 translateDbgValueRecord(V: DI.getValue(), HasArgList: DI.hasArgList(), Variable: DI.getVariable(),
2340 Expression: DI.getExpression(), DL: DI.getDebugLoc(), MIRBuilder);
2341 return true;
2342 }
2343 case Intrinsic::uadd_with_overflow:
2344 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UADDO, MIRBuilder);
2345 case Intrinsic::sadd_with_overflow:
2346 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SADDO, MIRBuilder);
2347 case Intrinsic::usub_with_overflow:
2348 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_USUBO, MIRBuilder);
2349 case Intrinsic::ssub_with_overflow:
2350 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SSUBO, MIRBuilder);
2351 case Intrinsic::umul_with_overflow:
2352 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UMULO, MIRBuilder);
2353 case Intrinsic::smul_with_overflow:
2354 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SMULO, MIRBuilder);
2355 case Intrinsic::uadd_sat:
2356 return translateBinaryOp(Opcode: TargetOpcode::G_UADDSAT, U: CI, MIRBuilder);
2357 case Intrinsic::sadd_sat:
2358 return translateBinaryOp(Opcode: TargetOpcode::G_SADDSAT, U: CI, MIRBuilder);
2359 case Intrinsic::usub_sat:
2360 return translateBinaryOp(Opcode: TargetOpcode::G_USUBSAT, U: CI, MIRBuilder);
2361 case Intrinsic::ssub_sat:
2362 return translateBinaryOp(Opcode: TargetOpcode::G_SSUBSAT, U: CI, MIRBuilder);
2363 case Intrinsic::ushl_sat:
2364 return translateBinaryOp(Opcode: TargetOpcode::G_USHLSAT, U: CI, MIRBuilder);
2365 case Intrinsic::sshl_sat:
2366 return translateBinaryOp(Opcode: TargetOpcode::G_SSHLSAT, U: CI, MIRBuilder);
2367 case Intrinsic::umin:
2368 return translateBinaryOp(Opcode: TargetOpcode::G_UMIN, U: CI, MIRBuilder);
2369 case Intrinsic::umax:
2370 return translateBinaryOp(Opcode: TargetOpcode::G_UMAX, U: CI, MIRBuilder);
2371 case Intrinsic::smin:
2372 return translateBinaryOp(Opcode: TargetOpcode::G_SMIN, U: CI, MIRBuilder);
2373 case Intrinsic::smax:
2374 return translateBinaryOp(Opcode: TargetOpcode::G_SMAX, U: CI, MIRBuilder);
2375 case Intrinsic::abs:
2376 // TODO: Preserve "int min is poison" arg in GMIR?
2377 return translateUnaryOp(Opcode: TargetOpcode::G_ABS, U: CI, MIRBuilder);
2378 case Intrinsic::smul_fix:
2379 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIX, CI, MIRBuilder);
2380 case Intrinsic::umul_fix:
2381 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIX, CI, MIRBuilder);
2382 case Intrinsic::smul_fix_sat:
2383 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
2384 case Intrinsic::umul_fix_sat:
2385 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
2386 case Intrinsic::sdiv_fix:
2387 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
2388 case Intrinsic::udiv_fix:
2389 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
2390 case Intrinsic::sdiv_fix_sat:
2391 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
2392 case Intrinsic::udiv_fix_sat:
2393 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
2394 case Intrinsic::fmuladd: {
2395 const TargetMachine &TM = MF->getTarget();
2396 Register Dst = getOrCreateVReg(Val: CI);
2397 Register Op0 = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
2398 Register Op1 = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
2399 Register Op2 = getOrCreateVReg(Val: *CI.getArgOperand(i: 2));
2400 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
2401 TLI->isFMAFasterThanFMulAndFAdd(MF: *MF,
2402 TLI->getValueType(DL: *DL, Ty: CI.getType()))) {
2403 // TODO: Revisit this to see if we should move this part of the
2404 // lowering to the combiner.
2405 MIRBuilder.buildFMA(Dst, Src0: Op0, Src1: Op1, Src2: Op2,
2406 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2407 } else {
2408 LLT Ty = getLLTForType(Ty&: *CI.getType(), DL: *DL);
2409 auto FMul = MIRBuilder.buildFMul(
2410 Dst: Ty, Src0: Op0, Src1: Op1, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2411 MIRBuilder.buildFAdd(Dst, Src0: FMul, Src1: Op2,
2412 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2413 }
2414 return true;
2415 }
2416 case Intrinsic::frexp: {
2417 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
2418 MIRBuilder.buildFFrexp(Fract: VRegs[0], Exp: VRegs[1],
2419 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
2420 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2421 return true;
2422 }
2423 case Intrinsic::modf: {
2424 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
2425 MIRBuilder.buildModf(Fract: VRegs[0], Int: VRegs[1],
2426 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
2427 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2428 return true;
2429 }
2430 case Intrinsic::sincos: {
2431 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
2432 MIRBuilder.buildFSincos(Sin: VRegs[0], Cos: VRegs[1],
2433 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
2434 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2435 return true;
2436 }
2437 case Intrinsic::fptosi_sat:
2438 MIRBuilder.buildFPTOSI_SAT(Dst: getOrCreateVReg(Val: CI),
2439 Src0: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
2440 return true;
2441 case Intrinsic::fptoui_sat:
2442 MIRBuilder.buildFPTOUI_SAT(Dst: getOrCreateVReg(Val: CI),
2443 Src0: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
2444 return true;
2445 case Intrinsic::memcpy_inline:
2446 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY_INLINE);
2447 case Intrinsic::memcpy:
2448 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY);
2449 case Intrinsic::memmove:
2450 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMMOVE);
2451 case Intrinsic::memset:
2452 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMSET);
2453 case Intrinsic::memset_inline:
2454 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMSET_INLINE);
2455 case Intrinsic::eh_typeid_for: {
2456 GlobalValue *GV = ExtractTypeInfo(V: CI.getArgOperand(i: 0));
2457 Register Reg = getOrCreateVReg(Val: CI);
2458 unsigned TypeID = MF->getTypeIDFor(TI: GV);
2459 MIRBuilder.buildConstant(Res: Reg, Val: TypeID);
2460 return true;
2461 }
2462 case Intrinsic::objectsize:
2463 llvm_unreachable("llvm.objectsize.* should have been lowered already");
2464
2465 case Intrinsic::is_constant:
2466 llvm_unreachable("llvm.is.constant.* should have been lowered already");
2467
2468 case Intrinsic::stackguard:
2469 getStackGuard(DstReg: getOrCreateVReg(Val: CI), MIRBuilder);
2470 return true;
2471 case Intrinsic::stackprotector: {
2472 LLT PtrTy = getLLTForType(Ty&: *CI.getArgOperand(i: 0)->getType(), DL: *DL);
2473 Register GuardVal;
2474 if (TLI->useLoadStackGuardNode(M: *CI.getModule())) {
2475 GuardVal = MRI->createGenericVirtualRegister(Ty: PtrTy);
2476 getStackGuard(DstReg: GuardVal, MIRBuilder);
2477 } else
2478 GuardVal = getOrCreateVReg(Val: *CI.getArgOperand(i: 0)); // The guard's value.
2479
2480 AllocaInst *Slot = cast<AllocaInst>(Val: CI.getArgOperand(i: 1));
2481 int FI = getOrCreateFrameIndex(AI: *Slot);
2482 MF->getFrameInfo().setStackProtectorIndex(FI);
2483
2484 MIRBuilder.buildStore(
2485 Val: GuardVal, Addr: getOrCreateVReg(Val: *Slot),
2486 MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI),
2487 f: MachineMemOperand::MOStore |
2488 MachineMemOperand::MOVolatile,
2489 MemTy: PtrTy, base_alignment: Align(8)));
2490 return true;
2491 }
2492 case Intrinsic::stacksave: {
2493 MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKSAVE, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {});
2494 return true;
2495 }
2496 case Intrinsic::stackrestore: {
2497 MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKRESTORE, DstOps: {},
2498 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
2499 return true;
2500 }
2501 case Intrinsic::cttz:
2502 case Intrinsic::ctlz: {
2503 ConstantInt *Cst = cast<ConstantInt>(Val: CI.getArgOperand(i: 1));
2504 bool isTrailing = ID == Intrinsic::cttz;
2505 unsigned Opcode = isTrailing ? Cst->isZero()
2506 ? TargetOpcode::G_CTTZ
2507 : TargetOpcode::G_CTTZ_ZERO_POISON
2508 : Cst->isZero() ? TargetOpcode::G_CTLZ
2509 : TargetOpcode::G_CTLZ_ZERO_POISON;
2510 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: CI)},
2511 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
2512 return true;
2513 }
2514 case Intrinsic::invariant_start: {
2515 MIRBuilder.buildUndef(Res: getOrCreateVReg(Val: CI));
2516 return true;
2517 }
2518 case Intrinsic::invariant_end:
2519 return true;
2520 case Intrinsic::expect:
2521 case Intrinsic::expect_with_probability:
2522 case Intrinsic::annotation:
2523 case Intrinsic::ptr_annotation:
2524 case Intrinsic::launder_invariant_group:
2525 case Intrinsic::strip_invariant_group:
2526 case Intrinsic::threadlocal_address: {
2527 // Drop the intrinsic, but forward the value.
2528 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: CI),
2529 Op: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
2530 return true;
2531 }
2532 case Intrinsic::assume:
2533 case Intrinsic::experimental_noalias_scope_decl:
2534 case Intrinsic::var_annotation:
2535 case Intrinsic::sideeffect:
2536 // Discard annotate attributes, assumptions, and artificial side-effects.
2537 return true;
2538 case Intrinsic::read_volatile_register:
2539 case Intrinsic::read_register: {
2540 Value *Arg = CI.getArgOperand(i: 0);
2541 MIRBuilder
2542 .buildInstr(Opc: TargetOpcode::G_READ_REGISTER, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {})
2543 .addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()));
2544 return true;
2545 }
2546 case Intrinsic::write_register: {
2547 Value *Arg = CI.getArgOperand(i: 0);
2548 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_WRITE_REGISTER)
2549 .addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()))
2550 .addUse(RegNo: getOrCreateVReg(Val: *CI.getArgOperand(i: 1)));
2551 return true;
2552 }
2553 case Intrinsic::localescape: {
2554 MachineBasicBlock &EntryMBB = MF->front();
2555 StringRef EscapedName = GlobalValue::dropLLVMManglingEscape(Name: MF->getName());
2556
2557 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
2558 // is the same on all targets.
2559 for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
2560 Value *Arg = CI.getArgOperand(i: Idx)->stripPointerCasts();
2561 if (isa<ConstantPointerNull>(Val: Arg))
2562 continue; // Skip null pointers. They represent a hole in index space.
2563
2564 int FI = getOrCreateFrameIndex(AI: *cast<AllocaInst>(Val: Arg));
2565 MCSymbol *FrameAllocSym =
2566 MF->getContext().getOrCreateFrameAllocSymbol(FuncName: EscapedName, Idx);
2567
2568 // This should be inserted at the start of the entry block.
2569 auto LocalEscape =
2570 MIRBuilder.buildInstrNoInsert(Opcode: TargetOpcode::LOCAL_ESCAPE)
2571 .addSym(Sym: FrameAllocSym)
2572 .addFrameIndex(Idx: FI);
2573
2574 EntryMBB.insert(I: EntryMBB.begin(), MI: LocalEscape);
2575 }
2576
2577 return true;
2578 }
2579 case Intrinsic::vector_reduce_fadd:
2580 case Intrinsic::vector_reduce_fmul: {
2581 // Need to check for the reassoc flag to decide whether we want a
2582 // sequential reduction opcode or not.
2583 Register Dst = getOrCreateVReg(Val: CI);
2584 Register ScalarSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
2585 Register VecSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
2586 unsigned Opc = 0;
2587 if (!CI.hasAllowReassoc()) {
2588 // The sequential ordering case.
2589 Opc = ID == Intrinsic::vector_reduce_fadd
2590 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
2591 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
2592 if (!MRI->getType(Reg: VecSrc).isVector())
2593 Opc = ID == Intrinsic::vector_reduce_fadd ? TargetOpcode::G_FADD
2594 : TargetOpcode::G_FMUL;
2595 MIRBuilder.buildInstr(Opc, DstOps: {Dst}, SrcOps: {ScalarSrc, VecSrc},
2596 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2597 return true;
2598 }
2599 // We split the operation into a separate G_FADD/G_FMUL + the reduce,
2600 // since the associativity doesn't matter.
2601 unsigned ScalarOpc;
2602 if (ID == Intrinsic::vector_reduce_fadd) {
2603 Opc = TargetOpcode::G_VECREDUCE_FADD;
2604 ScalarOpc = TargetOpcode::G_FADD;
2605 } else {
2606 Opc = TargetOpcode::G_VECREDUCE_FMUL;
2607 ScalarOpc = TargetOpcode::G_FMUL;
2608 }
2609 LLT DstTy = MRI->getType(Reg: Dst);
2610 auto Rdx = MIRBuilder.buildInstr(
2611 Opc, DstOps: {DstTy}, SrcOps: {VecSrc}, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2612 MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {Dst}, SrcOps: {ScalarSrc, Rdx},
2613 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2614
2615 return true;
2616 }
2617 case Intrinsic::trap:
2618 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_TRAP);
2619 case Intrinsic::debugtrap:
2620 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_DEBUGTRAP);
2621 case Intrinsic::ubsantrap:
2622 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_UBSANTRAP);
2623 case Intrinsic::allow_runtime_check:
2624 case Intrinsic::allow_ubsan_check:
2625 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: CI),
2626 Op: getOrCreateVReg(Val: *ConstantInt::getTrue(Ty: CI.getType())));
2627 return true;
2628 case Intrinsic::amdgcn_cs_chain:
2629 case Intrinsic::amdgcn_call_whole_wave:
2630 return translateCallBase(CB: CI, MIRBuilder);
2631 case Intrinsic::fptrunc_round: {
2632 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: CI);
2633
2634 // Convert the metadata argument to a constant integer
2635 Metadata *MD = cast<MetadataAsValue>(Val: CI.getArgOperand(i: 1))->getMetadata();
2636 std::optional<RoundingMode> RoundMode =
2637 convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
2638
2639 // Add the Rounding mode as an integer
2640 MIRBuilder
2641 .buildInstr(Opc: TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
2642 DstOps: {getOrCreateVReg(Val: CI)},
2643 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))}, Flags)
2644 .addImm(Val: (int)*RoundMode);
2645
2646 return true;
2647 }
2648 case Intrinsic::is_fpclass: {
2649 Value *FpValue = CI.getOperand(i_nocapture: 0);
2650 ConstantInt *TestMaskValue = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1));
2651
2652 MIRBuilder
2653 .buildInstr(Opc: TargetOpcode::G_IS_FPCLASS, DstOps: {getOrCreateVReg(Val: CI)},
2654 SrcOps: {getOrCreateVReg(Val: *FpValue)})
2655 .addImm(Val: TestMaskValue->getZExtValue());
2656
2657 return true;
2658 }
2659 case Intrinsic::set_fpenv: {
2660 Value *FPEnv = CI.getOperand(i_nocapture: 0);
2661 MIRBuilder.buildSetFPEnv(Src: getOrCreateVReg(Val: *FPEnv));
2662 return true;
2663 }
2664 case Intrinsic::reset_fpenv:
2665 MIRBuilder.buildResetFPEnv();
2666 return true;
2667 case Intrinsic::set_fpmode: {
2668 Value *FPState = CI.getOperand(i_nocapture: 0);
2669 MIRBuilder.buildSetFPMode(Src: getOrCreateVReg(Val: *FPState));
2670 return true;
2671 }
2672 case Intrinsic::reset_fpmode:
2673 MIRBuilder.buildResetFPMode();
2674 return true;
2675 case Intrinsic::get_rounding:
2676 MIRBuilder.buildGetRounding(Dst: getOrCreateVReg(Val: CI));
2677 return true;
2678 case Intrinsic::set_rounding:
2679 MIRBuilder.buildSetRounding(Src: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)));
2680 return true;
2681 case Intrinsic::vscale: {
2682 MIRBuilder.buildVScale(Res: getOrCreateVReg(Val: CI), MinElts: 1);
2683 return true;
2684 }
2685 case Intrinsic::scmp:
2686 MIRBuilder.buildSCmp(Res: getOrCreateVReg(Val: CI),
2687 Op0: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)),
2688 Op1: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1)));
2689 return true;
2690 case Intrinsic::ucmp:
2691 MIRBuilder.buildUCmp(Res: getOrCreateVReg(Val: CI),
2692 Op0: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)),
2693 Op1: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1)));
2694 return true;
2695 case Intrinsic::vector_extract:
2696 return translateExtractVector(U: CI, MIRBuilder);
2697 case Intrinsic::vector_insert:
2698 return translateInsertVector(U: CI, MIRBuilder);
2699 case Intrinsic::stepvector: {
2700 MIRBuilder.buildStepVector(Res: getOrCreateVReg(Val: CI), Step: 1);
2701 return true;
2702 }
2703 case Intrinsic::prefetch: {
2704 Value *Addr = CI.getOperand(i_nocapture: 0);
2705 unsigned RW = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1))->getZExtValue();
2706 unsigned Locality = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
2707 unsigned CacheType = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 3))->getZExtValue();
2708
2709 auto Flags = RW ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
2710 auto &MMO = *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Addr), f: Flags,
2711 MemTy: LLT(), base_alignment: Align());
2712
2713 MIRBuilder.buildPrefetch(Addr: getOrCreateVReg(Val: *Addr), RW, Locality, CacheType,
2714 MMO);
2715
2716 return true;
2717 }
2718
2719 case Intrinsic::vector_interleave2:
2720 case Intrinsic::vector_deinterleave2: {
2721 // Both intrinsics have at least one operand.
2722 Value *Op0 = CI.getOperand(i_nocapture: 0);
2723 LLT ResTy = getLLTForType(Ty&: *Op0->getType(), DL: MIRBuilder.getDataLayout());
2724 if (!ResTy.isFixedVector())
2725 return false;
2726
2727 if (CI.getIntrinsicID() == Intrinsic::vector_interleave2)
2728 return translateVectorInterleave2Intrinsic(CI, MIRBuilder);
2729
2730 return translateVectorDeinterleave2Intrinsic(CI, MIRBuilder);
2731 }
2732
2733#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
2734 case Intrinsic::INTRINSIC:
2735#include "llvm/IR/ConstrainedOps.def"
2736 return translateConstrainedFPIntrinsic(FPI: cast<ConstrainedFPIntrinsic>(Val: CI),
2737 MIRBuilder);
2738 case Intrinsic::experimental_convergence_anchor:
2739 case Intrinsic::experimental_convergence_entry:
2740 case Intrinsic::experimental_convergence_loop:
2741 return translateConvergenceControlIntrinsic(CI, ID, MIRBuilder);
2742 case Intrinsic::reloc_none: {
2743 Metadata *MD = cast<MetadataAsValue>(Val: CI.getArgOperand(i: 0))->getMetadata();
2744 StringRef SymbolName = cast<MDString>(Val: MD)->getString();
2745 MIRBuilder.buildInstr(Opcode: TargetOpcode::RELOC_NONE)
2746 .addExternalSymbol(FnName: SymbolName.data());
2747 return true;
2748 }
2749 }
2750 return false;
2751}
2752
2753bool IRTranslator::translateInlineAsm(const CallBase &CB,
2754 MachineIRBuilder &MIRBuilder) {
2755 if (!mayTranslateUserTypes(U: CB))
2756 return false;
2757
2758 const InlineAsmLowering *ALI = MF->getSubtarget().getInlineAsmLowering();
2759
2760 if (!ALI) {
2761 LLVM_DEBUG(
2762 dbgs() << "Inline asm lowering is not supported for this target yet\n");
2763 return false;
2764 }
2765
2766 return ALI->lowerInlineAsm(
2767 MIRBuilder, CB, GetOrCreateVRegs: [&](const Value &Val) { return getOrCreateVRegs(Val); });
2768}
2769
2770bool IRTranslator::translateCallBase(const CallBase &CB,
2771 MachineIRBuilder &MIRBuilder) {
2772 ArrayRef<Register> Res = getOrCreateVRegs(Val: CB);
2773
2774 SmallVector<ArrayRef<Register>, 8> Args;
2775 Register SwiftInVReg = 0;
2776 Register SwiftErrorVReg = 0;
2777 for (const auto &Arg : CB.args()) {
2778 if (CLI->supportSwiftError() && isSwiftError(V: Arg)) {
2779 assert(SwiftInVReg == 0 && "Expected only one swift error argument");
2780 LLT Ty = getLLTForType(Ty&: *Arg->getType(), DL: *DL);
2781 SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
2782 MIRBuilder.buildCopy(Res: SwiftInVReg, Op: SwiftError.getOrCreateVRegUseAt(
2783 &CB, &MIRBuilder.getMBB(), Arg));
2784 Args.emplace_back(Args: ArrayRef(SwiftInVReg));
2785 SwiftErrorVReg =
2786 SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
2787 continue;
2788 }
2789 Args.push_back(Elt: getOrCreateVRegs(Val: *Arg));
2790 }
2791
2792 if (auto *CI = dyn_cast<CallInst>(Val: &CB)) {
2793 if (ORE->enabled()) {
2794 if (MemoryOpRemark::canHandle(I: CI, TLI: *LibInfo)) {
2795 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2796 R.visit(I: CI);
2797 }
2798 }
2799 }
2800
2801 std::optional<CallLowering::PtrAuthInfo> PAI;
2802 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_ptrauth)) {
2803 // Functions should never be ptrauth-called directly.
2804 assert(!CB.getCalledFunction() && "invalid direct ptrauth call");
2805
2806 const Value *Key = Bundle->Inputs[0];
2807 const Value *Discriminator = Bundle->Inputs[1];
2808
2809 // Look through ptrauth constants to try to eliminate the matching bundle
2810 // and turn this into a direct call with no ptrauth.
2811 // CallLowering will use the raw pointer if it doesn't find the PAI.
2812 const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(Val: CB.getCalledOperand());
2813 if (!CalleeCPA || !isa<Function>(Val: CalleeCPA->getPointer()) ||
2814 !CalleeCPA->isKnownCompatibleWith(Key, Discriminator, DL: *DL)) {
2815 // If we can't make it direct, package the bundle into PAI.
2816 Register DiscReg = getOrCreateVReg(Val: *Discriminator);
2817 PAI = CallLowering::PtrAuthInfo{.Key: cast<ConstantInt>(Val: Key)->getZExtValue(),
2818 .Discriminator: DiscReg};
2819 }
2820 }
2821
2822 Register ConvergenceCtrlToken = 0;
2823 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
2824 const auto &Token = *Bundle->Inputs[0].get();
2825 ConvergenceCtrlToken = getOrCreateConvergenceTokenVReg(Token);
2826 }
2827
2828 // We don't set HasCalls on MFI here yet because call lowering may decide to
2829 // optimize into tail calls. Instead, we defer that to selection where a final
2830 // scan is done to check if any instructions are calls.
2831 bool Success = CLI->lowerCall(
2832 MIRBuilder, Call: CB, ResRegs: Res, ArgRegs: Args, SwiftErrorVReg, PAI, ConvergenceCtrlToken,
2833 GetCalleeReg: [&]() { return getOrCreateVReg(Val: *CB.getCalledOperand()); });
2834
2835 // Check if we just inserted a tail call.
2836 if (Success) {
2837 assert(!HasTailCall && "Can't tail call return twice from block?");
2838 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
2839 HasTailCall = TII->isTailCall(Inst: *std::prev(x: MIRBuilder.getInsertPt()));
2840 }
2841
2842 return Success;
2843}
2844
2845bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
2846 if (!mayTranslateUserTypes(U))
2847 return false;
2848
2849 const CallInst &CI = cast<CallInst>(Val: U);
2850 const Function *F = CI.getCalledFunction();
2851
2852 // FIXME: support Windows dllimport function calls and calls through
2853 // weak symbols.
2854 if (F && (F->hasDLLImportStorageClass() ||
2855 (MF->getTarget().getTargetTriple().isOSWindows() &&
2856 F->hasExternalWeakLinkage())))
2857 return false;
2858
2859 // FIXME: support control flow guard targets.
2860 if (CI.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
2861 return false;
2862
2863 // FIXME: support statepoints and related.
2864 if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(Val: U))
2865 return false;
2866
2867 if (CI.isInlineAsm())
2868 return translateInlineAsm(CB: CI, MIRBuilder);
2869
2870 Intrinsic::ID ID = F ? F->getIntrinsicID() : Intrinsic::not_intrinsic;
2871 if (!F || ID == Intrinsic::not_intrinsic) {
2872 if (translateCallBase(CB: CI, MIRBuilder)) {
2873 diagnoseDontCall(CI);
2874 return true;
2875 }
2876 return false;
2877 }
2878
2879 assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
2880
2881 if (!MF->getSubtarget().isIntrinsicSupported(IntrinsicID: ID)) {
2882 const Function &Fn = MF->getFunction();
2883 Fn.getContext().diagnose(
2884 DI: DiagnosticInfoUnsupportedTargetIntrinsic(Fn, ID, CI.getDebugLoc()));
2885 }
2886
2887 if (translateKnownIntrinsic(CI, ID, MIRBuilder))
2888 return true;
2889
2890 SmallVector<TargetLowering::IntrinsicInfo> Infos;
2891 TLI->getTgtMemIntrinsic(Infos, I: CI, MF&: *MF, Intrinsic: ID);
2892
2893 return translateIntrinsic(CB: CI, ID, MIRBuilder, TgtMemIntrinsicInfos: Infos);
2894}
2895
2896/// Translate a call or callbr to an intrinsic.
2897bool IRTranslator::translateIntrinsic(
2898 const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
2899 ArrayRef<TargetLowering::IntrinsicInfo> TgtMemIntrinsicInfos) {
2900 if (!MF->getSubtarget().isIntrinsicSupported(IntrinsicID: ID)) {
2901 const Function &F = MF->getFunction();
2902 F.getContext().diagnose(
2903 DI: DiagnosticInfoUnsupportedTargetIntrinsic(F, ID, CB.getDebugLoc()));
2904 }
2905
2906 ArrayRef<Register> ResultRegs;
2907 if (!CB.getType()->isVoidTy())
2908 ResultRegs = getOrCreateVRegs(Val: CB);
2909
2910 // Ignore the callsite attributes. Backend code is most likely not expecting
2911 // an intrinsic to sometimes have side effects and sometimes not.
2912 MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, Res: ResultRegs);
2913 if (isa<FPMathOperator>(Val: CB))
2914 MIB->copyIRFlags(I: CB);
2915
2916 for (const auto &Arg : enumerate(First: CB.args())) {
2917 // If this is required to be an immediate, don't materialize it in a
2918 // register.
2919 if (CB.paramHasAttr(ArgNo: Arg.index(), Kind: Attribute::ImmArg)) {
2920 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Arg.value())) {
2921 // imm arguments are more convenient than cimm (and realistically
2922 // probably sufficient), so use them.
2923 assert(CI->getBitWidth() <= 64 &&
2924 "large intrinsic immediates not handled");
2925 MIB.addImm(Val: CI->getSExtValue());
2926 } else {
2927 MIB.addFPImm(Val: cast<ConstantFP>(Val: Arg.value()));
2928 }
2929 } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Val: Arg.value())) {
2930 auto *MD = MDVal->getMetadata();
2931 auto *MDN = dyn_cast<MDNode>(Val: MD);
2932 if (!MDN) {
2933 if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(Val: MD))
2934 MDN = MDNode::get(Context&: MF->getFunction().getContext(), MDs: ConstMD);
2935 else // This was probably an MDString.
2936 return false;
2937 }
2938 MIB.addMetadata(MD: MDN);
2939 } else {
2940 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: *Arg.value());
2941 if (VRegs.size() > 1)
2942 return false;
2943 MIB.addUse(RegNo: VRegs[0]);
2944 }
2945 }
2946
2947 // Add MachineMemOperands for each memory access described by the target.
2948 for (const auto &Info : TgtMemIntrinsicInfos) {
2949 Align Alignment = Info.align.value_or(
2950 u: DL->getABITypeAlign(Ty: Info.memVT.getTypeForEVT(Context&: CB.getContext())));
2951 LLT MemTy = Info.memVT.isSimple()
2952 ? getLLTForMVT(Ty: Info.memVT.getSimpleVT())
2953 : LLT::scalar(SizeInBits: Info.memVT.getStoreSizeInBits());
2954
2955 // TODO: We currently just fallback to address space 0 if
2956 // getTgtMemIntrinsic didn't yield anything useful.
2957 MachinePointerInfo MPI;
2958 if (Info.ptrVal) {
2959 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
2960 } else if (Info.fallbackAddressSpace) {
2961 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
2962 }
2963 MIB.addMemOperand(MMO: MF->getMachineMemOperand(
2964 PtrInfo: MPI, f: Info.flags, MemTy, base_alignment: Alignment, AAInfo: CB.getAAMetadata(),
2965 /*Ranges=*/nullptr, SSID: Info.ssid, Ordering: Info.order, FailureOrdering: Info.failureOrder));
2966 }
2967
2968 if (CB.isConvergent()) {
2969 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
2970 auto *Token = Bundle->Inputs[0].get();
2971 Register TokenReg = getOrCreateVReg(Val: *Token);
2972 MIB.addUse(RegNo: TokenReg, Flags: RegState::Implicit);
2973 }
2974 }
2975
2976 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol))
2977 MIB->setDeactivationSymbol(MF&: *MF, DS: Bundle->Inputs[0].get());
2978
2979 return true;
2980}
2981
2982bool IRTranslator::findUnwindDestinations(
2983 const BasicBlock *EHPadBB,
2984 BranchProbability Prob,
2985 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2986 &UnwindDests) {
2987 EHPersonality Personality = classifyEHPersonality(
2988 Pers: EHPadBB->getParent()->getFunction().getPersonalityFn());
2989 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2990 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2991 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2992 bool IsSEH = isAsynchronousEHPersonality(Pers: Personality);
2993
2994 if (IsWasmCXX) {
2995 // Ignore this for now.
2996 return false;
2997 }
2998
2999 while (EHPadBB) {
3000 BasicBlock::const_iterator Pad = EHPadBB->getFirstNonPHIIt();
3001 BasicBlock *NewEHPadBB = nullptr;
3002 if (isa<LandingPadInst>(Val: Pad)) {
3003 // Stop on landingpads. They are not funclets.
3004 UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
3005 break;
3006 }
3007 if (isa<CleanupPadInst>(Val: Pad)) {
3008 // Stop on cleanup pads. Cleanups are always funclet entries for all known
3009 // personalities.
3010 UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
3011 UnwindDests.back().first->setIsEHScopeEntry();
3012 UnwindDests.back().first->setIsEHFuncletEntry();
3013 break;
3014 }
3015 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: Pad)) {
3016 // Add the catchpad handlers to the possible destinations.
3017 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
3018 UnwindDests.emplace_back(Args: &getMBB(BB: *CatchPadBB), Args&: Prob);
3019 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
3020 if (IsMSVCCXX || IsCoreCLR)
3021 UnwindDests.back().first->setIsEHFuncletEntry();
3022 if (!IsSEH)
3023 UnwindDests.back().first->setIsEHScopeEntry();
3024 }
3025 NewEHPadBB = CatchSwitch->getUnwindDest();
3026 } else {
3027 continue;
3028 }
3029
3030 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3031 if (BPI && NewEHPadBB)
3032 Prob *= BPI->getEdgeProbability(Src: EHPadBB, Dst: NewEHPadBB);
3033 EHPadBB = NewEHPadBB;
3034 }
3035 return true;
3036}
3037
3038bool IRTranslator::translateInvoke(const User &U,
3039 MachineIRBuilder &MIRBuilder) {
3040 const InvokeInst &I = cast<InvokeInst>(Val: U);
3041 MCContext &Context = MF->getContext();
3042
3043 const BasicBlock *ReturnBB = I.getSuccessor(i: 0);
3044 const BasicBlock *EHPadBB = I.getSuccessor(i: 1);
3045
3046 const Function *Fn = I.getCalledFunction();
3047
3048 // FIXME: support invoking patchpoint and statepoint intrinsics.
3049 if (Fn && Fn->isIntrinsic())
3050 return false;
3051
3052 // FIXME: support whatever these are.
3053 if (I.hasDeoptState())
3054 return false;
3055
3056 // FIXME: support control flow guard targets.
3057 if (I.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
3058 return false;
3059
3060 // FIXME: support Windows exception handling.
3061 if (!isa<LandingPadInst>(Val: EHPadBB->getFirstNonPHIIt()))
3062 return false;
3063
3064 // FIXME: support Windows dllimport function calls and calls through
3065 // weak symbols.
3066 if (Fn && (Fn->hasDLLImportStorageClass() ||
3067 (MF->getTarget().getTargetTriple().isOSWindows() &&
3068 Fn->hasExternalWeakLinkage())))
3069 return false;
3070
3071 bool LowerInlineAsm = I.isInlineAsm();
3072 bool NeedEHLabel = true;
3073
3074 // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
3075 // the region covered by the try.
3076 MCSymbol *BeginSymbol = nullptr;
3077 if (NeedEHLabel) {
3078 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_INVOKE_REGION_START);
3079 BeginSymbol = Context.createTempSymbol();
3080 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: BeginSymbol);
3081 }
3082
3083 if (LowerInlineAsm) {
3084 if (!translateInlineAsm(CB: I, MIRBuilder))
3085 return false;
3086 } else if (!translateCallBase(CB: I, MIRBuilder))
3087 return false;
3088
3089 MCSymbol *EndSymbol = nullptr;
3090 if (NeedEHLabel) {
3091 EndSymbol = Context.createTempSymbol();
3092 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: EndSymbol);
3093 }
3094
3095 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
3096 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3097 MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
3098 BranchProbability EHPadBBProb =
3099 BPI ? BPI->getEdgeProbability(Src: InvokeMBB->getBasicBlock(), Dst: EHPadBB)
3100 : BranchProbability::getZero();
3101
3102 if (!findUnwindDestinations(EHPadBB, Prob: EHPadBBProb, UnwindDests))
3103 return false;
3104
3105 MachineBasicBlock &EHPadMBB = getMBB(BB: *EHPadBB),
3106 &ReturnMBB = getMBB(BB: *ReturnBB);
3107 // Update successor info.
3108 addSuccessorWithProb(Src: InvokeMBB, Dst: &ReturnMBB);
3109 for (auto &UnwindDest : UnwindDests) {
3110 UnwindDest.first->setIsEHPad();
3111 addSuccessorWithProb(Src: InvokeMBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
3112 }
3113 InvokeMBB->normalizeSuccProbs();
3114
3115 if (NeedEHLabel) {
3116 assert(BeginSymbol && "Expected a begin symbol!");
3117 assert(EndSymbol && "Expected an end symbol!");
3118 MF->addInvoke(LandingPad: &EHPadMBB, BeginLabel: BeginSymbol, EndLabel: EndSymbol);
3119 }
3120
3121 MIRBuilder.buildBr(Dest&: ReturnMBB);
3122 return true;
3123}
3124
3125/// The intrinsics currently supported by callbr are implicit control flow
3126/// intrinsics such as amdgcn.kill.
3127bool IRTranslator::translateCallBr(const User &U,
3128 MachineIRBuilder &MIRBuilder) {
3129 if (!mayTranslateUserTypes(U))
3130 return false; // see translateCall
3131
3132 const CallBrInst &I = cast<CallBrInst>(Val: U);
3133 MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB();
3134
3135 Intrinsic::ID IID = I.getIntrinsicID();
3136 if (I.isInlineAsm()) {
3137 // FIXME: inline asm is not yet supported for callbr in GlobalISel. As soon
3138 // as we add support, we need to handle the indirect asm targets, see
3139 // SelectionDAGBuilder::visitCallBr().
3140 return false;
3141 }
3142 if (!translateIntrinsic(CB: I, ID: IID, MIRBuilder))
3143 return false;
3144
3145 // Retrieve successors.
3146 SmallPtrSet<BasicBlock *, 8> Dests = {I.getDefaultDest()};
3147 MachineBasicBlock *Return = &getMBB(BB: *I.getDefaultDest());
3148
3149 // Update successor info.
3150 addSuccessorWithProb(Src: CallBrMBB, Dst: Return, Prob: BranchProbability::getOne());
3151
3152 // Add indirect targets as successors. For intrinsic callbr, these represent
3153 // implicit control flow (e.g., the "kill" path for amdgcn.kill). We mark them
3154 // with setIsInlineAsmBrIndirectTarget so the machine verifier accepts them as
3155 // valid successors, even though they're not from inline asm.
3156 for (BasicBlock *Dest : I.getIndirectDests()) {
3157 MachineBasicBlock &Target = getMBB(BB: *Dest);
3158 Target.setIsInlineAsmBrIndirectTarget();
3159 Target.setLabelMustBeEmitted();
3160 // Don't add duplicate machine successors.
3161 if (Dests.insert(Ptr: Dest).second)
3162 addSuccessorWithProb(Src: CallBrMBB, Dst: &Target, Prob: BranchProbability::getZero());
3163 }
3164
3165 CallBrMBB->normalizeSuccProbs();
3166
3167 // Drop into default successor.
3168 MIRBuilder.buildBr(Dest&: *Return);
3169
3170 return true;
3171}
3172
3173bool IRTranslator::translateLandingPad(const User &U,
3174 MachineIRBuilder &MIRBuilder) {
3175 const LandingPadInst &LP = cast<LandingPadInst>(Val: U);
3176
3177 MachineBasicBlock &MBB = MIRBuilder.getMBB();
3178
3179 MBB.setIsEHPad();
3180
3181 // If there aren't registers to copy the values into (e.g., during SjLj
3182 // exceptions), then don't bother.
3183 const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
3184 if (TLI->getExceptionPointerRegister(PersonalityFn) == 0 &&
3185 TLI->getExceptionSelectorRegister(PersonalityFn) == 0)
3186 return true;
3187
3188 // If landingpad's return type is token type, we don't create DAG nodes
3189 // for its exception pointer and selector value. The extraction of exception
3190 // pointer or selector value from token type landingpads is not currently
3191 // supported.
3192 if (LP.getType()->isTokenTy())
3193 return true;
3194
3195 // Add a label to mark the beginning of the landing pad. Deletion of the
3196 // landing pad can thus be detected via the MachineModuleInfo.
3197 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL)
3198 .addSym(Sym: MF->addLandingPad(LandingPad: &MBB));
3199
3200 // If the unwinder does not preserve all registers, ensure that the
3201 // function marks the clobbered registers as used.
3202 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
3203 if (auto *RegMask = TRI.getCustomEHPadPreservedMask(MF: *MF))
3204 MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask);
3205
3206 LLT Ty = getLLTForType(Ty&: *LP.getType(), DL: *DL);
3207 Register Undef = MRI->createGenericVirtualRegister(Ty);
3208 MIRBuilder.buildUndef(Res: Undef);
3209
3210 SmallVector<LLT, 2> Tys;
3211 for (Type *Ty : cast<StructType>(Val: LP.getType())->elements())
3212 Tys.push_back(Elt: getLLTForType(Ty&: *Ty, DL: *DL));
3213 assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
3214
3215 // Mark exception register as live in.
3216 Register ExceptionReg = TLI->getExceptionPointerRegister(PersonalityFn);
3217 if (!ExceptionReg)
3218 return false;
3219
3220 MBB.addLiveIn(PhysReg: ExceptionReg);
3221 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: LP);
3222 MIRBuilder.buildCopy(Res: ResRegs[0], Op: ExceptionReg);
3223
3224 Register SelectorReg = TLI->getExceptionSelectorRegister(PersonalityFn);
3225 if (!SelectorReg)
3226 return false;
3227
3228 MBB.addLiveIn(PhysReg: SelectorReg);
3229 Register PtrVReg = MRI->createGenericVirtualRegister(Ty: Tys[0]);
3230 MIRBuilder.buildCopy(Res: PtrVReg, Op: SelectorReg);
3231 MIRBuilder.buildCast(Dst: ResRegs[1], Src: PtrVReg);
3232
3233 return true;
3234}
3235
3236bool IRTranslator::translateAlloca(const User &U,
3237 MachineIRBuilder &MIRBuilder) {
3238 auto &AI = cast<AllocaInst>(Val: U);
3239
3240 if (AI.isSwiftError())
3241 return true;
3242
3243 if (AI.isStaticAlloca()) {
3244 Register Res = getOrCreateVReg(Val: AI);
3245 int FI = getOrCreateFrameIndex(AI);
3246 MIRBuilder.buildFrameIndex(Res, Idx: FI);
3247 return true;
3248 }
3249
3250 // FIXME: support stack probing for Windows.
3251 if (MF->getTarget().getTargetTriple().isOSWindows())
3252 return false;
3253
3254 // Now we're in the harder dynamic case.
3255 Register NumElts = getOrCreateVReg(Val: *AI.getArraySize());
3256 Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
3257 LLT IntPtrTy = getLLTForType(Ty&: *IntPtrIRTy, DL: *DL);
3258 if (MRI->getType(Reg: NumElts) != IntPtrTy) {
3259 Register ExtElts = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
3260 MIRBuilder.buildZExtOrTrunc(Res: ExtElts, Op: NumElts);
3261 NumElts = ExtElts;
3262 }
3263
3264 Type *Ty = AI.getAllocatedType();
3265 TypeSize TySize = DL->getTypeAllocSize(Ty);
3266
3267 Register AllocSize = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
3268 Register TySizeReg;
3269 if (TySize.isScalable()) {
3270 // For scalable types, use vscale * min_value
3271 TySizeReg = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
3272 MIRBuilder.buildVScale(Res: TySizeReg, MinElts: TySize.getKnownMinValue());
3273 } else {
3274 // For fixed types, use a constant
3275 TySizeReg =
3276 getOrCreateVReg(Val: *ConstantInt::get(Ty: IntPtrIRTy, V: TySize.getFixedValue()));
3277 }
3278 MIRBuilder.buildMul(Dst: AllocSize, Src0: NumElts, Src1: TySizeReg);
3279
3280 // Round the size of the allocation up to the stack alignment size
3281 // by add SA-1 to the size. This doesn't overflow because we're computing
3282 // an address inside an alloca.
3283 Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();
3284 auto SAMinusOne = MIRBuilder.buildConstant(Res: IntPtrTy, Val: StackAlign.value() - 1);
3285 auto AllocAdd = MIRBuilder.buildAdd(Dst: IntPtrTy, Src0: AllocSize, Src1: SAMinusOne,
3286 Flags: MachineInstr::NoUWrap);
3287 auto AlignCst =
3288 MIRBuilder.buildConstant(Res: IntPtrTy, Val: ~(uint64_t)(StackAlign.value() - 1));
3289 auto AlignedAlloc = MIRBuilder.buildAnd(Dst: IntPtrTy, Src0: AllocAdd, Src1: AlignCst);
3290
3291 Align Alignment = AI.getAlign();
3292 if (Alignment <= StackAlign)
3293 Alignment = Align(1);
3294 MIRBuilder.buildDynStackAlloc(Res: getOrCreateVReg(Val: AI), Size: AlignedAlloc, Alignment);
3295
3296 MF->getFrameInfo().CreateVariableSizedObject(Alignment, Alloca: &AI);
3297 assert(MF->getFrameInfo().hasVarSizedObjects());
3298 return true;
3299}
3300
3301bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
3302 // FIXME: We may need more info about the type. Because of how LLT works,
3303 // we're completely discarding the i64/double distinction here (amongst
3304 // others). Fortunately the ABIs I know of where that matters don't use va_arg
3305 // anyway but that's not guaranteed.
3306 MIRBuilder.buildInstr(Opc: TargetOpcode::G_VAARG, DstOps: {getOrCreateVReg(Val: U)},
3307 SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
3308 DL->getABITypeAlign(Ty: U.getType()).value()});
3309 return true;
3310}
3311
3312bool IRTranslator::translateUnreachable(const User &U,
3313 MachineIRBuilder &MIRBuilder) {
3314 auto &UI = cast<UnreachableInst>(Val: U);
3315 if (!UI.shouldLowerToTrap(TrapUnreachable: MF->getTarget().Options.TrapUnreachable,
3316 NoTrapAfterNoreturn: MF->getTarget().Options.NoTrapAfterNoreturn))
3317 return true;
3318
3319 MIRBuilder.buildTrap();
3320 return true;
3321}
3322
3323bool IRTranslator::translateInsertElement(const User &U,
3324 MachineIRBuilder &MIRBuilder) {
3325 // If it is a <1 x Ty> vector, use the scalar as it is
3326 // not a legal vector type in LLT.
3327 if (auto *FVT = dyn_cast<FixedVectorType>(Val: U.getType());
3328 FVT && FVT->getNumElements() == 1)
3329 return translateCopy(U, V: *U.getOperand(i: 1), MIRBuilder);
3330
3331 Register Res = getOrCreateVReg(Val: U);
3332 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
3333 Register Elt = getOrCreateVReg(Val: *U.getOperand(i: 1));
3334 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
3335 Register Idx;
3336 if (auto *CI = dyn_cast<ConstantInt>(Val: U.getOperand(i: 2))) {
3337 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3338 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
3339 auto *NewIdxCI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
3340 Idx = getOrCreateVReg(Val: *NewIdxCI);
3341 }
3342 }
3343 if (!Idx)
3344 Idx = getOrCreateVReg(Val: *U.getOperand(i: 2));
3345 if (MRI->getType(Reg: Idx).getSizeInBits() != PreferredVecIdxWidth) {
3346 const LLT VecIdxTy =
3347 MRI->getType(Reg: Idx).changeElementSize(NewEltSize: PreferredVecIdxWidth);
3348 Idx = MIRBuilder.buildZExtOrTrunc(Res: VecIdxTy, Op: Idx).getReg(Idx: 0);
3349 }
3350 MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
3351 return true;
3352}
3353
3354bool IRTranslator::translateInsertVector(const User &U,
3355 MachineIRBuilder &MIRBuilder) {
3356 Register Dst = getOrCreateVReg(Val: U);
3357 Register Vec = getOrCreateVReg(Val: *U.getOperand(i: 0));
3358 Register Elt = getOrCreateVReg(Val: *U.getOperand(i: 1));
3359
3360 ConstantInt *CI = cast<ConstantInt>(Val: U.getOperand(i: 2));
3361 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
3362
3363 // Resize Index to preferred index width.
3364 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3365 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
3366 CI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
3367 }
3368
3369 // If it is a <1 x Ty> vector, we have to use other means.
3370 if (auto *ResultType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 1)->getType());
3371 ResultType && ResultType->getNumElements() == 1) {
3372 if (auto *InputType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType());
3373 InputType && InputType->getNumElements() == 1) {
3374 // We are inserting an illegal fixed vector into an illegal
3375 // fixed vector, use the scalar as it is not a legal vector type
3376 // in LLT.
3377 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
3378 }
3379 if (isa<FixedVectorType>(Val: U.getOperand(i: 0)->getType())) {
3380 // We are inserting an illegal fixed vector into a legal fixed
3381 // vector, use the scalar as it is not a legal vector type in
3382 // LLT.
3383 Register Idx = getOrCreateVReg(Val: *CI);
3384 MIRBuilder.buildInsertVectorElement(Res: Dst, Val: Vec, Elt, Idx);
3385 return true;
3386 }
3387 if (isa<ScalableVectorType>(Val: U.getOperand(i: 0)->getType())) {
3388 // We are inserting an illegal fixed vector into a scalable
3389 // vector, use a scalar element insert.
3390 LLT VecIdxTy = LLT::scalar(SizeInBits: PreferredVecIdxWidth);
3391 Register Idx = getOrCreateVReg(Val: *CI);
3392 auto ScaledIndex = MIRBuilder.buildMul(
3393 Dst: VecIdxTy, Src0: MIRBuilder.buildVScale(Res: VecIdxTy, MinElts: 1), Src1: Idx);
3394 MIRBuilder.buildInsertVectorElement(Res: Dst, Val: Vec, Elt, Idx: ScaledIndex);
3395 return true;
3396 }
3397 }
3398
3399 MIRBuilder.buildInsertSubvector(
3400 Res: getOrCreateVReg(Val: U), Src0: getOrCreateVReg(Val: *U.getOperand(i: 0)),
3401 Src1: getOrCreateVReg(Val: *U.getOperand(i: 1)), Index: CI->getZExtValue());
3402 return true;
3403}
3404
3405bool IRTranslator::translateExtractElement(const User &U,
3406 MachineIRBuilder &MIRBuilder) {
3407 // If it is a <1 x Ty> vector, use the scalar as it is
3408 // not a legal vector type in LLT.
3409 if (const FixedVectorType *FVT =
3410 dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType()))
3411 if (FVT->getNumElements() == 1)
3412 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
3413
3414 Register Res = getOrCreateVReg(Val: U);
3415 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
3416 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
3417 Register Idx;
3418 if (auto *CI = dyn_cast<ConstantInt>(Val: U.getOperand(i: 1))) {
3419 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3420 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
3421 auto *NewIdxCI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
3422 Idx = getOrCreateVReg(Val: *NewIdxCI);
3423 }
3424 }
3425 if (!Idx)
3426 Idx = getOrCreateVReg(Val: *U.getOperand(i: 1));
3427 if (MRI->getType(Reg: Idx).getSizeInBits() != PreferredVecIdxWidth) {
3428 const LLT VecIdxTy =
3429 MRI->getType(Reg: Idx).changeElementSize(NewEltSize: PreferredVecIdxWidth);
3430 Idx = MIRBuilder.buildZExtOrTrunc(Res: VecIdxTy, Op: Idx).getReg(Idx: 0);
3431 }
3432 MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
3433 return true;
3434}
3435
3436bool IRTranslator::translateExtractVector(const User &U,
3437 MachineIRBuilder &MIRBuilder) {
3438 Register Res = getOrCreateVReg(Val: U);
3439 Register Vec = getOrCreateVReg(Val: *U.getOperand(i: 0));
3440 ConstantInt *CI = cast<ConstantInt>(Val: U.getOperand(i: 1));
3441 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
3442
3443 // Resize Index to preferred index width.
3444 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3445 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
3446 CI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
3447 }
3448
3449 // If it is a <1 x Ty> vector, we have to use other means.
3450 if (auto *ResultType = dyn_cast<FixedVectorType>(Val: U.getType());
3451 ResultType && ResultType->getNumElements() == 1) {
3452 if (auto *InputType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType());
3453 InputType && InputType->getNumElements() == 1) {
3454 // We are extracting an illegal fixed vector from an illegal fixed vector,
3455 // use the scalar as it is not a legal vector type in LLT.
3456 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
3457 }
3458 if (isa<FixedVectorType>(Val: U.getOperand(i: 0)->getType())) {
3459 // We are extracting an illegal fixed vector from a legal fixed
3460 // vector, use the scalar as it is not a legal vector type in
3461 // LLT.
3462 Register Idx = getOrCreateVReg(Val: *CI);
3463 MIRBuilder.buildExtractVectorElement(Res, Val: Vec, Idx);
3464 return true;
3465 }
3466 if (isa<ScalableVectorType>(Val: U.getOperand(i: 0)->getType())) {
3467 // We are extracting an illegal fixed vector from a scalable
3468 // vector, use a scalar element extract.
3469 LLT VecIdxTy = LLT::scalar(SizeInBits: PreferredVecIdxWidth);
3470 Register Idx = getOrCreateVReg(Val: *CI);
3471 auto ScaledIndex = MIRBuilder.buildMul(
3472 Dst: VecIdxTy, Src0: MIRBuilder.buildVScale(Res: VecIdxTy, MinElts: 1), Src1: Idx);
3473 MIRBuilder.buildExtractVectorElement(Res, Val: Vec, Idx: ScaledIndex);
3474 return true;
3475 }
3476 }
3477
3478 MIRBuilder.buildExtractSubvector(Res: getOrCreateVReg(Val: U),
3479 Src: getOrCreateVReg(Val: *U.getOperand(i: 0)),
3480 Index: CI->getZExtValue());
3481 return true;
3482}
3483
3484bool IRTranslator::translateShuffleVector(const User &U,
3485 MachineIRBuilder &MIRBuilder) {
3486 // A ShuffleVector that operates on scalable vectors is a splat vector where
3487 // the value of the splat vector is the 0th element of the first operand,
3488 // since the index mask operand is the zeroinitializer (undef and
3489 // poison are treated as zeroinitializer here).
3490 if (U.getOperand(i: 0)->getType()->isScalableTy()) {
3491 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
3492 auto SplatVal = MIRBuilder.buildExtractVectorElementConstant(
3493 Res: MRI->getType(Reg: Val).getElementType(), Val, Idx: 0);
3494 MIRBuilder.buildSplatVector(Res: getOrCreateVReg(Val: U), Val: SplatVal);
3495 return true;
3496 }
3497
3498 ArrayRef<int> Mask;
3499 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: &U))
3500 Mask = SVI->getShuffleMask();
3501 else
3502 Mask = cast<ConstantExpr>(Val: U).getShuffleMask();
3503
3504 // As GISel does not represent <1 x > vectors as a separate type from scalars,
3505 // we transform shuffle_vector with a scalar output to an
3506 // ExtractVectorElement. If the input type is also scalar it becomes a Copy.
3507 unsigned DstElts = cast<FixedVectorType>(Val: U.getType())->getNumElements();
3508 unsigned SrcElts =
3509 cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType())->getNumElements();
3510 if (DstElts == 1) {
3511 unsigned M = Mask[0];
3512 if (SrcElts == 1) {
3513 if (M == 0 || M == 1)
3514 return translateCopy(U, V: *U.getOperand(i: M), MIRBuilder);
3515 MIRBuilder.buildUndef(Res: getOrCreateVReg(Val: U));
3516 } else {
3517 Register Dst = getOrCreateVReg(Val: U);
3518 if (M < SrcElts) {
3519 MIRBuilder.buildExtractVectorElementConstant(
3520 Res: Dst, Val: getOrCreateVReg(Val: *U.getOperand(i: 0)), Idx: M);
3521 } else if (M < SrcElts * 2) {
3522 MIRBuilder.buildExtractVectorElementConstant(
3523 Res: Dst, Val: getOrCreateVReg(Val: *U.getOperand(i: 1)), Idx: M - SrcElts);
3524 } else {
3525 MIRBuilder.buildUndef(Res: Dst);
3526 }
3527 }
3528 return true;
3529 }
3530
3531 // A single element src is transformed to a build_vector.
3532 if (SrcElts == 1) {
3533 SmallVector<Register> Ops;
3534 Register Undef;
3535 for (int M : Mask) {
3536 LLT SrcTy = getLLTForType(Ty&: *U.getOperand(i: 0)->getType(), DL: *DL);
3537 if (M == 0 || M == 1) {
3538 Ops.push_back(Elt: getOrCreateVReg(Val: *U.getOperand(i: M)));
3539 } else {
3540 if (!Undef.isValid()) {
3541 Undef = MRI->createGenericVirtualRegister(Ty: SrcTy);
3542 MIRBuilder.buildUndef(Res: Undef);
3543 }
3544 Ops.push_back(Elt: Undef);
3545 }
3546 }
3547 MIRBuilder.buildBuildVector(Res: getOrCreateVReg(Val: U), Ops);
3548 return true;
3549 }
3550
3551 ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
3552 MIRBuilder
3553 .buildInstr(Opc: TargetOpcode::G_SHUFFLE_VECTOR, DstOps: {getOrCreateVReg(Val: U)},
3554 SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
3555 getOrCreateVReg(Val: *U.getOperand(i: 1))})
3556 .addShuffleMask(Val: MaskAlloc);
3557 return true;
3558}
3559
3560bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
3561 const PHINode &PI = cast<PHINode>(Val: U);
3562
3563 SmallVector<MachineInstr *, 4> Insts;
3564 for (auto Reg : getOrCreateVRegs(Val: PI)) {
3565 auto MIB = MIRBuilder.buildInstr(Opc: TargetOpcode::G_PHI, DstOps: {Reg}, SrcOps: {});
3566 Insts.push_back(Elt: MIB.getInstr());
3567 }
3568
3569 PendingPHIs.emplace_back(Args: &PI, Args: std::move(Insts));
3570 return true;
3571}
3572
3573bool IRTranslator::translateAtomicCmpXchg(const User &U,
3574 MachineIRBuilder &MIRBuilder) {
3575 const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(Val: U);
3576
3577 auto Flags = TLI->getAtomicMemOperandFlags(AI: I, DL: *DL);
3578
3579 auto Res = getOrCreateVRegs(Val: I);
3580 Register OldValRes = Res[0];
3581 Register SuccessRes = Res[1];
3582 Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
3583 Register Cmp = getOrCreateVReg(Val: *I.getCompareOperand());
3584 Register NewVal = getOrCreateVReg(Val: *I.getNewValOperand());
3585
3586 MIRBuilder.buildAtomicCmpXchgWithSuccess(
3587 OldValRes, SuccessRes, Addr, CmpVal: Cmp, NewVal,
3588 MMO&: *MF->getMachineMemOperand(
3589 PtrInfo: MachinePointerInfo(I.getPointerOperand()), f: Flags, MemTy: MRI->getType(Reg: Cmp),
3590 base_alignment: getMemOpAlign(I), AAInfo: I.getAAMetadata(), Ranges: nullptr, SSID: I.getSyncScopeID(),
3591 Ordering: I.getSuccessOrdering(), FailureOrdering: I.getFailureOrdering()));
3592 return true;
3593}
3594
3595bool IRTranslator::translateAtomicRMW(const User &U,
3596 MachineIRBuilder &MIRBuilder) {
3597 if (!mayTranslateUserTypes(U))
3598 return false;
3599
3600 const AtomicRMWInst &I = cast<AtomicRMWInst>(Val: U);
3601 auto Flags = TLI->getAtomicMemOperandFlags(AI: I, DL: *DL);
3602
3603 Register Res = getOrCreateVReg(Val: I);
3604 Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
3605 Register Val = getOrCreateVReg(Val: *I.getValOperand());
3606
3607 unsigned Opcode = 0;
3608 switch (I.getOperation()) {
3609 default:
3610 return false;
3611 case AtomicRMWInst::Xchg:
3612 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
3613 break;
3614 case AtomicRMWInst::Add:
3615 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
3616 break;
3617 case AtomicRMWInst::Sub:
3618 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
3619 break;
3620 case AtomicRMWInst::And:
3621 Opcode = TargetOpcode::G_ATOMICRMW_AND;
3622 break;
3623 case AtomicRMWInst::Nand:
3624 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
3625 break;
3626 case AtomicRMWInst::Or:
3627 Opcode = TargetOpcode::G_ATOMICRMW_OR;
3628 break;
3629 case AtomicRMWInst::Xor:
3630 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
3631 break;
3632 case AtomicRMWInst::Max:
3633 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
3634 break;
3635 case AtomicRMWInst::Min:
3636 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
3637 break;
3638 case AtomicRMWInst::UMax:
3639 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
3640 break;
3641 case AtomicRMWInst::UMin:
3642 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
3643 break;
3644 case AtomicRMWInst::FAdd:
3645 Opcode = TargetOpcode::G_ATOMICRMW_FADD;
3646 break;
3647 case AtomicRMWInst::FSub:
3648 Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
3649 break;
3650 case AtomicRMWInst::FMax:
3651 Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
3652 break;
3653 case AtomicRMWInst::FMin:
3654 Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
3655 break;
3656 case AtomicRMWInst::FMaximum:
3657 Opcode = TargetOpcode::G_ATOMICRMW_FMAXIMUM;
3658 break;
3659 case AtomicRMWInst::FMinimum:
3660 Opcode = TargetOpcode::G_ATOMICRMW_FMINIMUM;
3661 break;
3662 case AtomicRMWInst::FMaximumNum:
3663 Opcode = TargetOpcode::G_ATOMICRMW_FMAXIMUMNUM;
3664 break;
3665 case AtomicRMWInst::FMinimumNum:
3666 Opcode = TargetOpcode::G_ATOMICRMW_FMINIMUMNUM;
3667 break;
3668 case AtomicRMWInst::UIncWrap:
3669 Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
3670 break;
3671 case AtomicRMWInst::UDecWrap:
3672 Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
3673 break;
3674 case AtomicRMWInst::USubCond:
3675 Opcode = TargetOpcode::G_ATOMICRMW_USUB_COND;
3676 break;
3677 case AtomicRMWInst::USubSat:
3678 Opcode = TargetOpcode::G_ATOMICRMW_USUB_SAT;
3679 break;
3680 }
3681
3682 MIRBuilder.buildAtomicRMW(
3683 Opcode, OldValRes: Res, Addr, Val,
3684 MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(I.getPointerOperand()),
3685 f: Flags, MemTy: MRI->getType(Reg: Val), base_alignment: getMemOpAlign(I),
3686 AAInfo: I.getAAMetadata(), Ranges: nullptr, SSID: I.getSyncScopeID(),
3687 Ordering: I.getOrdering()));
3688 return true;
3689}
3690
3691bool IRTranslator::translateFence(const User &U,
3692 MachineIRBuilder &MIRBuilder) {
3693 const FenceInst &Fence = cast<FenceInst>(Val: U);
3694 MIRBuilder.buildFence(Ordering: static_cast<unsigned>(Fence.getOrdering()),
3695 Scope: Fence.getSyncScopeID());
3696 return true;
3697}
3698
3699bool IRTranslator::translateFreeze(const User &U,
3700 MachineIRBuilder &MIRBuilder) {
3701 const ArrayRef<Register> DstRegs = getOrCreateVRegs(Val: U);
3702 const ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *U.getOperand(i: 0));
3703
3704 assert(DstRegs.size() == SrcRegs.size() &&
3705 "Freeze with different source and destination type?");
3706
3707 for (unsigned I = 0; I < DstRegs.size(); ++I) {
3708 MIRBuilder.buildFreeze(Dst: DstRegs[I], Src: SrcRegs[I]);
3709 }
3710
3711 return true;
3712}
3713
3714void IRTranslator::finishPendingPhis() {
3715#ifndef NDEBUG
3716 DILocationVerifier Verifier;
3717 GISelObserverWrapper WrapperObserver(&Verifier);
3718 RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);
3719#endif // ifndef NDEBUG
3720 for (auto &Phi : PendingPHIs) {
3721 const PHINode *PI = Phi.first;
3722 if (PI->getType()->isEmptyTy())
3723 continue;
3724 ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
3725 MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
3726 EntryBuilder->setDebugLoc(PI->getDebugLoc());
3727#ifndef NDEBUG
3728 Verifier.setCurrentInst(PI);
3729#endif // ifndef NDEBUG
3730
3731 SmallPtrSet<const MachineBasicBlock *, 16> SeenPreds;
3732 for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
3733 auto IRPred = PI->getIncomingBlock(i);
3734 ArrayRef<Register> ValRegs = getOrCreateVRegs(Val: *PI->getIncomingValue(i));
3735 for (auto *Pred : getMachinePredBBs(Edge: {IRPred, PI->getParent()})) {
3736 if (SeenPreds.count(Ptr: Pred) || !PhiMBB->isPredecessor(MBB: Pred))
3737 continue;
3738 SeenPreds.insert(Ptr: Pred);
3739 for (unsigned j = 0; j < ValRegs.size(); ++j) {
3740 MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
3741 MIB.addUse(RegNo: ValRegs[j]);
3742 MIB.addMBB(MBB: Pred);
3743 }
3744 }
3745 }
3746 }
3747}
3748
3749void IRTranslator::translateDbgValueRecord(Value *V, bool HasArgList,
3750 const DILocalVariable *Variable,
3751 const DIExpression *Expression,
3752 const DebugLoc &DL,
3753 MachineIRBuilder &MIRBuilder) {
3754 assert(Variable->isValidLocationForIntrinsic(DL) &&
3755 "Expected inlined-at fields to agree");
3756 // Act as if we're handling a debug intrinsic.
3757 MIRBuilder.setDebugLoc(DL);
3758
3759 if (!V || HasArgList) {
3760 // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
3761 // terminate any prior location.
3762 MIRBuilder.buildIndirectDbgValue(Reg: 0, Variable, Expr: Expression);
3763 return;
3764 }
3765
3766 if (const auto *CI = dyn_cast<Constant>(Val: V)) {
3767 MIRBuilder.buildConstDbgValue(C: *CI, Variable, Expr: Expression);
3768 return;
3769 }
3770
3771 if (auto *AI = dyn_cast<AllocaInst>(Val: V);
3772 AI && AI->isStaticAlloca() && Expression->startsWithDeref()) {
3773 // If the value is an alloca and the expression starts with a
3774 // dereference, track a stack slot instead of a register, as registers
3775 // may be clobbered.
3776 auto ExprOperands = Expression->getElements();
3777 auto *ExprDerefRemoved =
3778 DIExpression::get(Context&: AI->getContext(), Elements: ExprOperands.drop_front());
3779 MIRBuilder.buildFIDbgValue(FI: getOrCreateFrameIndex(AI: *AI), Variable,
3780 Expr: ExprDerefRemoved);
3781 return;
3782 }
3783 if (translateIfEntryValueArgument(isDeclare: false, Val: V, Var: Variable, Expr: Expression, DL,
3784 MIRBuilder))
3785 return;
3786 for (Register Reg : getOrCreateVRegs(Val: *V)) {
3787 // FIXME: This does not handle register-indirect values at offset 0. The
3788 // direct/indirect thing shouldn't really be handled by something as
3789 // implicit as reg+noreg vs reg+imm in the first place, but it seems
3790 // pretty baked in right now.
3791 MIRBuilder.buildDirectDbgValue(Reg, Variable, Expr: Expression);
3792 }
3793}
3794
3795void IRTranslator::translateDbgDeclareRecord(Value *Address, bool HasArgList,
3796 const DILocalVariable *Variable,
3797 const DIExpression *Expression,
3798 const DebugLoc &DL,
3799 MachineIRBuilder &MIRBuilder) {
3800 if (!Address || isa<UndefValue>(Val: Address)) {
3801 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *Variable << "\n");
3802 return;
3803 }
3804
3805 assert(Variable->isValidLocationForIntrinsic(DL) &&
3806 "Expected inlined-at fields to agree");
3807 auto AI = dyn_cast<AllocaInst>(Val: Address);
3808 if (AI && AI->isStaticAlloca()) {
3809 // Static allocas are tracked at the MF level, no need for DBG_VALUE
3810 // instructions (in fact, they get ignored if they *do* exist).
3811 MF->setVariableDbgInfo(Var: Variable, Expr: Expression,
3812 Slot: getOrCreateFrameIndex(AI: *AI), Loc: DL);
3813 return;
3814 }
3815
3816 if (translateIfEntryValueArgument(isDeclare: true, Val: Address, Var: Variable,
3817 Expr: Expression, DL,
3818 MIRBuilder))
3819 return;
3820
3821 // A dbg.declare describes the address of a source variable, so lower it
3822 // into an indirect DBG_VALUE.
3823 MIRBuilder.setDebugLoc(DL);
3824 MIRBuilder.buildIndirectDbgValue(Reg: getOrCreateVReg(Val: *Address), Variable,
3825 Expr: Expression);
3826}
3827
3828void IRTranslator::translateDbgInfo(const Instruction &Inst,
3829 MachineIRBuilder &MIRBuilder) {
3830 for (DbgRecord &DR : Inst.getDbgRecordRange()) {
3831 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
3832 MIRBuilder.setDebugLoc(DLR->getDebugLoc());
3833 assert(DLR->getLabel() && "Missing label");
3834 assert(DLR->getLabel()->isValidLocationForIntrinsic(
3835 MIRBuilder.getDebugLoc()) &&
3836 "Expected inlined-at fields to agree");
3837 MIRBuilder.buildDbgLabel(Label: DLR->getLabel());
3838 continue;
3839 }
3840 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
3841 const DILocalVariable *Variable = DVR.getVariable();
3842 const DIExpression *Expression = DVR.getExpression();
3843 Value *V = DVR.getVariableLocationOp(OpIdx: 0);
3844 if (DVR.isDbgDeclare())
3845 translateDbgDeclareRecord(Address: V, HasArgList: DVR.hasArgList(), Variable, Expression,
3846 DL: DVR.getDebugLoc(), MIRBuilder);
3847 else
3848 translateDbgValueRecord(V, HasArgList: DVR.hasArgList(), Variable, Expression,
3849 DL: DVR.getDebugLoc(), MIRBuilder);
3850 }
3851}
3852
3853bool IRTranslator::translate(const Instruction &Inst) {
3854 CurBuilder->setDebugLoc(Inst.getDebugLoc());
3855 CurBuilder->setPCSections(Inst.getMetadata(KindID: LLVMContext::MD_pcsections));
3856 CurBuilder->setMMRAMetadata(Inst.getMetadata(KindID: LLVMContext::MD_mmra));
3857
3858 if (TLI->fallBackToDAGISel(Inst))
3859 return false;
3860
3861 switch (Inst.getOpcode()) {
3862#define HANDLE_INST(NUM, OPCODE, CLASS) \
3863 case Instruction::OPCODE: \
3864 return translate##OPCODE(Inst, *CurBuilder.get());
3865#include "llvm/IR/Instruction.def"
3866 default:
3867 return false;
3868 }
3869}
3870
3871bool IRTranslator::translate(const Constant &C, Register Reg) {
3872 // We only emit constants into the entry block from here. To prevent jumpy
3873 // debug behaviour remove debug line.
3874 if (auto CurrInstDL = CurBuilder->getDL())
3875 EntryBuilder->setDebugLoc(DebugLoc());
3876
3877 if (auto CI = dyn_cast<ConstantInt>(Val: &C)) {
3878 // buildConstant expects a to-be-splatted scalar ConstantInt.
3879 if (isa<VectorType>(Val: CI->getType()))
3880 CI = ConstantInt::get(Context&: CI->getContext(), V: CI->getValue());
3881 EntryBuilder->buildConstant(Res: Reg, Val: *CI);
3882 } else if (auto CB = dyn_cast<ConstantByte>(Val: &C)) {
3883 // Byte constants share G_CONSTANT with integers; the destination Reg's
3884 // LLT (an integer LLT, see getLLTForType) determines vector splatting.
3885 EntryBuilder->buildConstant(Res: Reg, Val: CB->getValue());
3886 } else if (auto CF = dyn_cast<ConstantFP>(Val: &C)) {
3887 // buildFConstant expects a to-be-splatted scalar ConstantFP.
3888 if (isa<VectorType>(Val: CF->getType()))
3889 CF = ConstantFP::get(Context&: CF->getContext(), V: CF->getValue());
3890 EntryBuilder->buildFConstant(Res: Reg, Val: *CF);
3891 } else if (isa<UndefValue>(Val: C))
3892 EntryBuilder->buildUndef(Res: Reg);
3893 else if (isa<ConstantPointerNull>(Val: C))
3894 EntryBuilder->buildConstant(Res: Reg, Val: 0);
3895 else if (auto GV = dyn_cast<GlobalValue>(Val: &C))
3896 EntryBuilder->buildGlobalValue(Res: Reg, GV);
3897 else if (auto CPA = dyn_cast<ConstantPtrAuth>(Val: &C)) {
3898 Register Addr = getOrCreateVReg(Val: *CPA->getPointer());
3899 Register AddrDisc = getOrCreateVReg(Val: *CPA->getAddrDiscriminator());
3900 EntryBuilder->buildConstantPtrAuth(Res: Reg, CPA, Addr, AddrDisc);
3901 } else if (auto CAZ = dyn_cast<ConstantAggregateZero>(Val: &C)) {
3902 Constant &Elt = *CAZ->getElementValue(Idx: 0u);
3903 if (isa<ScalableVectorType>(Val: CAZ->getType())) {
3904 EntryBuilder->buildSplatVector(Res: Reg, Val: getOrCreateVReg(Val: Elt));
3905 return true;
3906 }
3907 // Return the scalar if it is a <1 x Ty> vector.
3908 unsigned NumElts = CAZ->getElementCount().getFixedValue();
3909 if (NumElts == 1)
3910 return translateCopy(U: C, V: Elt, MIRBuilder&: *EntryBuilder);
3911 // All elements are zero so we can just use the first one.
3912 EntryBuilder->buildSplatBuildVector(Res: Reg, Src: getOrCreateVReg(Val: Elt));
3913 } else if (auto CV = dyn_cast<ConstantDataVector>(Val: &C)) {
3914 // Return the scalar if it is a <1 x Ty> vector.
3915 if (CV->getNumElements() == 1)
3916 return translateCopy(U: C, V: *CV->getElementAsConstant(i: 0), MIRBuilder&: *EntryBuilder);
3917 SmallVector<Register, 4> Ops;
3918 for (unsigned i = 0; i < CV->getNumElements(); ++i) {
3919 Constant &Elt = *CV->getElementAsConstant(i);
3920 Ops.push_back(Elt: getOrCreateVReg(Val: Elt));
3921 }
3922 EntryBuilder->buildBuildVector(Res: Reg, Ops);
3923 } else if (auto CE = dyn_cast<ConstantExpr>(Val: &C)) {
3924 switch(CE->getOpcode()) {
3925#define HANDLE_INST(NUM, OPCODE, CLASS) \
3926 case Instruction::OPCODE: \
3927 return translate##OPCODE(*CE, *EntryBuilder.get());
3928#include "llvm/IR/Instruction.def"
3929 default:
3930 return false;
3931 }
3932 } else if (auto CV = dyn_cast<ConstantVector>(Val: &C)) {
3933 if (CV->getNumOperands() == 1)
3934 return translateCopy(U: C, V: *CV->getOperand(i_nocapture: 0), MIRBuilder&: *EntryBuilder);
3935 SmallVector<Register, 4> Ops;
3936 for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
3937 Ops.push_back(Elt: getOrCreateVReg(Val: *CV->getOperand(i_nocapture: i)));
3938 }
3939 EntryBuilder->buildBuildVector(Res: Reg, Ops);
3940 } else if (auto *BA = dyn_cast<BlockAddress>(Val: &C)) {
3941 EntryBuilder->buildBlockAddress(Res: Reg, BA);
3942 } else
3943 return false;
3944
3945 return true;
3946}
3947
3948bool IRTranslator::mayTranslateUserTypes(const User &U) const {
3949 const TargetMachine &TM = TLI->getTargetMachine();
3950 if (LLT::getUseExtended())
3951 return true;
3952
3953 // BF16 cannot currently be represented by default LLT. To avoid miscompiles
3954 // we prevent any instructions using them by default in all targets that do
3955 // not explicitly enable it via LLT::setUseExtended(true).
3956 // SPIRV target is exception.
3957 return TM.getTargetTriple().isSPIRV() ||
3958 (!U.getType()->getScalarType()->isBFloatTy() &&
3959 !any_of(Range: U.operands(), P: [](Value *V) {
3960 return V->getType()->getScalarType()->isBFloatTy();
3961 }));
3962}
3963
3964bool IRTranslator::finalizeBasicBlock(const BasicBlock &BB,
3965 MachineBasicBlock &MBB) {
3966 for (auto &BTB : SL->BitTestCases) {
3967 // Emit header first, if it wasn't already emitted.
3968 if (!BTB.Emitted)
3969 emitBitTestHeader(B&: BTB, SwitchBB: BTB.Parent);
3970
3971 BranchProbability UnhandledProb = BTB.Prob;
3972 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
3973 UnhandledProb -= BTB.Cases[j].ExtraProb;
3974 // Set the current basic block to the mbb we wish to insert the code into
3975 MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
3976 // If all cases cover a contiguous range, it is not necessary to jump to
3977 // the default block after the last bit test fails. This is because the
3978 // range check during bit test header creation has guaranteed that every
3979 // case here doesn't go outside the range. In this case, there is no need
3980 // to perform the last bit test, as it will always be true. Instead, make
3981 // the second-to-last bit-test fall through to the target of the last bit
3982 // test, and delete the last bit test.
3983
3984 MachineBasicBlock *NextMBB;
3985 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3986 // Second-to-last bit-test with contiguous range: fall through to the
3987 // target of the final bit test.
3988 NextMBB = BTB.Cases[j + 1].TargetBB;
3989 } else if (j + 1 == ej) {
3990 // For the last bit test, fall through to Default.
3991 NextMBB = BTB.Default;
3992 } else {
3993 // Otherwise, fall through to the next bit test.
3994 NextMBB = BTB.Cases[j + 1].ThisBB;
3995 }
3996
3997 emitBitTestCase(BB&: BTB, NextMBB, BranchProbToNext: UnhandledProb, Reg: BTB.Reg, B&: BTB.Cases[j], SwitchBB: MBB);
3998
3999 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
4000 // We need to record the replacement phi edge here that normally
4001 // happens in emitBitTestCase before we delete the case, otherwise the
4002 // phi edge will be lost.
4003 addMachineCFGPred(Edge: {BTB.Parent->getBasicBlock(),
4004 BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
4005 NewPred: MBB);
4006 // Since we're not going to use the final bit test, remove it.
4007 BTB.Cases.pop_back();
4008 break;
4009 }
4010 }
4011 // This is "default" BB. We have two jumps to it. From "header" BB and from
4012 // last "case" BB, unless the latter was skipped.
4013 CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
4014 BTB.Default->getBasicBlock()};
4015 addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Parent);
4016 if (!BTB.ContiguousRange) {
4017 addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Cases.back().ThisBB);
4018 }
4019 }
4020 SL->BitTestCases.clear();
4021
4022 for (auto &JTCase : SL->JTCases) {
4023 // Emit header first, if it wasn't already emitted.
4024 if (!JTCase.first.Emitted)
4025 emitJumpTableHeader(JT&: JTCase.second, JTH&: JTCase.first, HeaderBB: JTCase.first.HeaderBB);
4026
4027 emitJumpTable(JT&: JTCase.second, MBB: JTCase.second.MBB);
4028 }
4029 SL->JTCases.clear();
4030
4031 for (auto &SwCase : SL->SwitchCases)
4032 emitSwitchCase(CB&: SwCase, SwitchBB: &CurBuilder->getMBB(), MIB&: *CurBuilder);
4033 SL->SwitchCases.clear();
4034
4035 // Check if we need to generate stack-protector guard checks.
4036 StackProtector &SP = getAnalysis<StackProtector>();
4037 if (SP.shouldEmitSDCheck(BB)) {
4038 bool FunctionBasedInstrumentation =
4039 TLI->getSSPStackGuardCheck(M: *MF->getFunction().getParent(), Libcalls: *Libcalls);
4040 SPDescriptor.initialize(BB: &BB, MBB: &MBB, FunctionBasedInstrumentation);
4041 }
4042 // Handle stack protector.
4043 if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
4044 LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
4045 return false;
4046 } else if (SPDescriptor.shouldEmitStackProtector()) {
4047 MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
4048 MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
4049
4050 // Find the split point to split the parent mbb. At the same time copy all
4051 // physical registers used in the tail of parent mbb into virtual registers
4052 // before the split point and back into physical registers after the split
4053 // point. This prevents us needing to deal with Live-ins and many other
4054 // register allocation issues caused by us splitting the parent mbb. The
4055 // register allocator will clean up said virtual copies later on.
4056 MachineBasicBlock::iterator SplitPoint = findSplitPointForStackProtector(
4057 BB: ParentMBB, TII: *MF->getSubtarget().getInstrInfo());
4058
4059 // Splice the terminator of ParentMBB into SuccessMBB.
4060 SuccessMBB->splice(Where: SuccessMBB->end(), Other: ParentMBB, From: SplitPoint,
4061 To: ParentMBB->end());
4062
4063 // Add compare/jump on neq/jump to the parent BB.
4064 if (!emitSPDescriptorParent(SPD&: SPDescriptor, ParentBB: ParentMBB))
4065 return false;
4066
4067 // CodeGen Failure MBB if we have not codegened it yet.
4068 MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
4069 if (FailureMBB->empty()) {
4070 if (!emitSPDescriptorFailure(SPD&: SPDescriptor, FailureBB: FailureMBB))
4071 return false;
4072 }
4073
4074 // Clear the Per-BB State.
4075 SPDescriptor.resetPerBBState();
4076 }
4077 return true;
4078}
4079
4080bool IRTranslator::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
4081 MachineBasicBlock *ParentBB) {
4082 CurBuilder->setInsertPt(MBB&: *ParentBB, II: ParentBB->end());
4083 // First create the loads to the guard/stack slot for the comparison.
4084 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
4085 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
4086 LLT PtrMemTy = getLLTForMVT(Ty: TLI->getPointerMemTy(DL: *DL));
4087
4088 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
4089 int FI = MFI.getStackProtectorIndex();
4090
4091 Register Guard;
4092 Register StackSlotPtr = CurBuilder->buildFrameIndex(Res: PtrTy, Idx: FI).getReg(Idx: 0);
4093 const Module &M = *ParentBB->getParent()->getFunction().getParent();
4094 Align Align = DL->getPrefTypeAlign(Ty: PointerType::getUnqual(C&: M.getContext()));
4095
4096 // Generate code to load the content of the guard slot.
4097 Register GuardVal =
4098 CurBuilder
4099 ->buildLoad(Res: PtrMemTy, Addr: StackSlotPtr,
4100 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
4101 MMOFlags: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile)
4102 .getReg(Idx: 0);
4103
4104 // Retrieve guard check function, nullptr if instrumentation is inlined.
4105 if (const Function *GuardCheckFn = TLI->getSSPStackGuardCheck(M, Libcalls: *Libcalls)) {
4106 // This path is currently untestable on GlobalISel, since the only platform
4107 // that needs this seems to be Windows, and we fall back on that currently.
4108 // The code still lives here in case that changes.
4109 // Silence warning about unused variable until the code below that uses
4110 // 'GuardCheckFn' is enabled.
4111 (void)GuardCheckFn;
4112 return false;
4113#if 0
4114 // The target provides a guard check function to validate the guard value.
4115 // Generate a call to that function with the content of the guard slot as
4116 // argument.
4117 FunctionType *FnTy = GuardCheckFn->getFunctionType();
4118 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
4119 ISD::ArgFlagsTy Flags;
4120 if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
4121 Flags.setInReg();
4122 CallLowering::ArgInfo GuardArgInfo(
4123 {GuardVal, FnTy->getParamType(0), {Flags}});
4124
4125 CallLowering::CallLoweringInfo Info;
4126 Info.OrigArgs.push_back(GuardArgInfo);
4127 Info.CallConv = GuardCheckFn->getCallingConv();
4128 Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
4129 Info.OrigRet = {Register(), FnTy->getReturnType()};
4130 if (!CLI->lowerCall(MIRBuilder, Info)) {
4131 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
4132 return false;
4133 }
4134 return true;
4135#endif
4136 }
4137
4138 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
4139 // Otherwise, emit a volatile load to retrieve the stack guard value.
4140 if (TLI->useLoadStackGuardNode(M: *ParentBB->getBasicBlock()->getModule())) {
4141 Guard =
4142 MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: PtrTy.getSizeInBits()));
4143 getStackGuard(DstReg: Guard, MIRBuilder&: *CurBuilder);
4144 } else {
4145 // TODO: test using android subtarget when we support @llvm.thread.pointer.
4146 const Value *IRGuard = TLI->getSDagStackGuard(M, Libcalls: *Libcalls);
4147 Register GuardPtr = getOrCreateVReg(Val: *IRGuard);
4148
4149 Guard = CurBuilder
4150 ->buildLoad(Res: PtrMemTy, Addr: GuardPtr,
4151 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
4152 MMOFlags: MachineMemOperand::MOLoad |
4153 MachineMemOperand::MOVolatile)
4154 .getReg(Idx: 0);
4155 }
4156
4157 // Perform the comparison.
4158 auto Cmp =
4159 CurBuilder->buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::integer(SizeInBits: 1), Op0: Guard, Op1: GuardVal);
4160 // If the guard/stackslot do not equal, branch to failure MBB.
4161 CurBuilder->buildBrCond(Tst: Cmp, Dest&: *SPD.getFailureMBB());
4162 // Otherwise branch to success MBB.
4163 CurBuilder->buildBr(Dest&: *SPD.getSuccessMBB());
4164 return true;
4165}
4166
4167bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
4168 MachineBasicBlock *FailureBB) {
4169 const RTLIB::LibcallImpl LibcallImpl =
4170 Libcalls->getLibcallImpl(Call: RTLIB::STACKPROTECTOR_CHECK_FAIL);
4171 if (LibcallImpl == RTLIB::Unsupported)
4172 return false;
4173
4174 CurBuilder->setInsertPt(MBB&: *FailureBB, II: FailureBB->end());
4175
4176 CallLowering::CallLoweringInfo Info;
4177 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: LibcallImpl);
4178
4179 StringRef LibcallName =
4180 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LibcallImpl);
4181 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
4182 Info.OrigRet = {Register(), Type::getVoidTy(C&: MF->getFunction().getContext()),
4183 0};
4184 if (!CLI->lowerCall(MIRBuilder&: *CurBuilder, Info)) {
4185 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
4186 return false;
4187 }
4188
4189 // Emit a trap instruction if we are required to do so.
4190 const TargetOptions &TargetOpts = TLI->getTargetMachine().Options;
4191 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
4192 CurBuilder->buildInstr(Opcode: TargetOpcode::G_TRAP);
4193
4194 return true;
4195}
4196
4197void IRTranslator::finalizeFunction() {
4198 // Release the memory used by the different maps we
4199 // needed during the translation.
4200 PendingPHIs.clear();
4201 VMap.reset();
4202 FrameIndices.clear();
4203 MachinePreds.clear();
4204 // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
4205 // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
4206 // destroying it twice (in ~IRTranslator() and ~LLVMContext())
4207 EntryBuilder.reset();
4208 CurBuilder.reset();
4209 FuncInfo.clear();
4210 SPDescriptor.resetPerFunctionState();
4211}
4212
4213/// Returns true if a BasicBlock \p BB within a variadic function contains a
4214/// variadic musttail call.
4215static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
4216 if (!IsVarArg)
4217 return false;
4218
4219 // Walk the block backwards, because tail calls usually only appear at the end
4220 // of a block.
4221 return llvm::any_of(Range: llvm::reverse(C: BB), P: [](const Instruction &I) {
4222 const auto *CI = dyn_cast<CallInst>(Val: &I);
4223 return CI && CI->isMustTailCall();
4224 });
4225}
4226
4227bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
4228 MF = &CurMF;
4229 const Function &F = MF->getFunction();
4230 ORE = std::make_unique<OptimizationRemarkEmitter>(args: &F);
4231 CLI = MF->getSubtarget().getCallLowering();
4232
4233 if (CLI->fallBackToDAGISel(MF: *MF)) {
4234 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
4235 F.getSubprogram(), &F.getEntryBlock());
4236 R << "unable to lower function: "
4237 << ore::NV("Prototype", F.getFunctionType());
4238
4239 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
4240 return false;
4241 }
4242
4243 GISelCSEAnalysisWrapper &Wrapper =
4244 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
4245 // Set the CSEConfig and run the analysis.
4246 GISelCSEInfo *CSEInfo = nullptr;
4247 TPC = &getAnalysis<TargetPassConfig>();
4248
4249 bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
4250 ? EnableCSEInIRTranslator
4251 : TPC->isGISelCSEEnabled();
4252
4253 const TargetSubtargetInfo &Subtarget = MF->getSubtarget();
4254 TLI = Subtarget.getTargetLowering();
4255
4256 if (EnableCSE) {
4257 EntryBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
4258 CSEInfo = &Wrapper.get(CSEOpt: TPC->getCSEConfig());
4259 EntryBuilder->setCSEInfo(CSEInfo);
4260 CurBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
4261 CurBuilder->setCSEInfo(CSEInfo);
4262 } else {
4263 EntryBuilder = std::make_unique<MachineIRBuilder>();
4264 CurBuilder = std::make_unique<MachineIRBuilder>();
4265 }
4266 CLI = Subtarget.getCallLowering();
4267 CurBuilder->setMF(*MF);
4268 EntryBuilder->setMF(*MF);
4269 MRI = &MF->getRegInfo();
4270 DL = &F.getDataLayout();
4271 const TargetMachine &TM = MF->getTarget();
4272 EnableOpts = OptLevel != CodeGenOptLevel::None && !skipFunction(F);
4273 FuncInfo.MF = MF;
4274 if (EnableOpts) {
4275 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
4276 FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
4277 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
4278 F&: MF->getFunction());
4279 } else {
4280 AA = nullptr;
4281 FuncInfo.BPI = nullptr;
4282 AC = nullptr;
4283 }
4284 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
4285 Libcalls = &getAnalysis<LibcallLoweringInfoWrapper>().getLibcallLowering(
4286 M: *F.getParent(), Subtarget);
4287
4288 FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(MF&: *MF);
4289
4290 SL = std::make_unique<GISelSwitchLowering>(args: this, args&: FuncInfo);
4291 SL->init(tli: *TLI, tm: TM, dl: *DL);
4292
4293 assert(PendingPHIs.empty() && "stale PHIs");
4294
4295 // Targets which want to use big endian can enable it using
4296 // enableBigEndian()
4297 if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
4298 // Currently we don't properly handle big endian code.
4299 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
4300 F.getSubprogram(), &F.getEntryBlock());
4301 R << "unable to translate in big endian mode";
4302 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
4303 return false;
4304 }
4305
4306 // Release the per-function state when we return, whether we succeeded or not.
4307 llvm::scope_exit FinalizeOnReturn([this]() { finalizeFunction(); });
4308
4309 // Setup a separate basic-block for the arguments and constants
4310 MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
4311 MF->push_back(MBB: EntryBB);
4312 EntryBuilder->setMBB(*EntryBB);
4313
4314 DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHIIt()->getDebugLoc();
4315 SwiftError.setFunction(CurMF);
4316 SwiftError.createEntriesInEntryBlock(DbgLoc);
4317
4318 bool IsVarArg = F.isVarArg();
4319 bool HasMustTailInVarArgFn = false;
4320
4321 // Create all blocks, in IR order, to preserve the layout.
4322 FuncInfo.MBBMap.resize(N: F.getMaxBlockNumber());
4323 for (const BasicBlock &BB: F) {
4324 auto *&MBB = FuncInfo.MBBMap[BB.getNumber()];
4325
4326 MBB = MF->CreateMachineBasicBlock(BB: &BB);
4327 MF->push_back(MBB);
4328
4329 // Only mark the block if the BlockAddress actually has users. The
4330 // hasAddressTaken flag may be stale if the BlockAddress was optimized away
4331 // but the constant still exists in the uniquing table.
4332 if (BB.hasAddressTaken()) {
4333 if (BlockAddress *BA = BlockAddress::lookup(BB: &BB))
4334 if (!BA->hasZeroLiveUses())
4335 MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
4336 }
4337
4338 if (!HasMustTailInVarArgFn)
4339 HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
4340 }
4341
4342 MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
4343
4344 // Make our arguments/constants entry block fallthrough to the IR entry block.
4345 EntryBB->addSuccessor(Succ: &getMBB(BB: F.front()));
4346
4347 // Lower the actual args into this basic block.
4348 SmallVector<ArrayRef<Register>, 8> VRegArgs;
4349 for (const Argument &Arg: F.args()) {
4350 if (DL->getTypeStoreSize(Ty: Arg.getType()).isZero())
4351 continue; // Don't handle zero sized types.
4352 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: Arg);
4353 VRegArgs.push_back(Elt: VRegs);
4354
4355 if (CLI->supportSwiftError() && Arg.hasSwiftErrorAttr()) {
4356 assert(VRegs.size() == 1 && "Too many vregs for Swift error");
4357 SwiftError.setCurrentVReg(MBB: EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
4358 }
4359 }
4360
4361 if (!CLI->lowerFormalArguments(MIRBuilder&: *EntryBuilder, F, VRegs: VRegArgs, FLI&: FuncInfo)) {
4362 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
4363 F.getSubprogram(), &F.getEntryBlock());
4364 R << "unable to lower arguments: "
4365 << ore::NV("Prototype", F.getFunctionType());
4366 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
4367 return false;
4368 }
4369
4370 // Need to visit defs before uses when translating instructions.
4371 GISelObserverWrapper WrapperObserver;
4372 if (EnableCSE && CSEInfo)
4373 WrapperObserver.addObserver(O: CSEInfo);
4374 {
4375 ReversePostOrderTraversal<const Function *> RPOT(&F);
4376#ifndef NDEBUG
4377 DILocationVerifier Verifier;
4378 WrapperObserver.addObserver(&Verifier);
4379#endif // ifndef NDEBUG
4380 RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);
4381 for (const BasicBlock *BB : RPOT) {
4382 MachineBasicBlock &MBB = getMBB(BB: *BB);
4383 // Set the insertion point of all the following translations to
4384 // the end of this basic block.
4385 CurBuilder->setMBB(MBB);
4386 HasTailCall = false;
4387 for (const Instruction &Inst : *BB) {
4388 // If we translated a tail call in the last step, then we know
4389 // everything after the call is either a return, or something that is
4390 // handled by the call itself. (E.g. a lifetime marker or assume
4391 // intrinsic.) In this case, we should stop translating the block and
4392 // move on.
4393 if (HasTailCall)
4394 break;
4395#ifndef NDEBUG
4396 Verifier.setCurrentInst(&Inst);
4397#endif // ifndef NDEBUG
4398
4399 // Translate any debug-info attached to the instruction.
4400 translateDbgInfo(Inst, MIRBuilder&: *CurBuilder);
4401
4402 if (translate(Inst))
4403 continue;
4404
4405 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
4406 Inst.getDebugLoc(), BB);
4407 R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
4408
4409 if (ORE->allowExtraAnalysis(PassName: "gisel-irtranslator")) {
4410 std::string InstStrStorage;
4411 raw_string_ostream InstStr(InstStrStorage);
4412 InstStr << Inst;
4413
4414 R << ": '" << InstStrStorage << "'";
4415 }
4416
4417 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
4418 return false;
4419 }
4420
4421 if (!finalizeBasicBlock(BB: *BB, MBB)) {
4422 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
4423 BB->getTerminator()->getDebugLoc(), BB);
4424 R << "unable to translate basic block";
4425 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
4426 return false;
4427 }
4428 }
4429#ifndef NDEBUG
4430 WrapperObserver.removeObserver(&Verifier);
4431#endif
4432 }
4433
4434 finishPendingPhis();
4435
4436 SwiftError.propagateVRegs();
4437
4438 // Merge the argument lowering and constants block with its single
4439 // successor, the LLVM-IR entry block. We want the basic block to
4440 // be maximal.
4441 assert(EntryBB->succ_size() == 1 &&
4442 "Custom BB used for lowering should have only one successor");
4443 // Get the successor of the current entry block.
4444 MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
4445 assert(NewEntryBB.pred_size() == 1 &&
4446 "LLVM-IR entry block has a predecessor!?");
4447 // Move all the instruction from the current entry block to the
4448 // new entry block.
4449 NewEntryBB.splice(Where: NewEntryBB.begin(), Other: EntryBB, From: EntryBB->begin(),
4450 To: EntryBB->end());
4451
4452 // Update the live-in information for the new entry block.
4453 for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
4454 NewEntryBB.addLiveIn(RegMaskPair: LiveIn);
4455 NewEntryBB.sortUniqueLiveIns();
4456
4457 // Get rid of the now empty basic block.
4458 EntryBB->removeSuccessor(Succ: &NewEntryBB);
4459 MF->remove(MBBI: EntryBB);
4460 MF->deleteMachineBasicBlock(MBB: EntryBB);
4461
4462 assert(&MF->front() == &NewEntryBB &&
4463 "New entry wasn't next in the list of basic block!");
4464
4465 // Initialize stack protector information.
4466 StackProtector &SP = getAnalysis<StackProtector>();
4467 SP.copyToMachineFrameInfo(MFI&: MF->getFrameInfo());
4468
4469 return false;
4470}
4471