1//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the IRTranslator class.
10//===----------------------------------------------------------------------===//
11
12#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
13#include "llvm/ADT/PostOrderIterator.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/Analysis/AliasAnalysis.h"
18#include "llvm/Analysis/AssumptionCache.h"
19#include "llvm/Analysis/BranchProbabilityInfo.h"
20#include "llvm/Analysis/Loads.h"
21#include "llvm/Analysis/OptimizationRemarkEmitter.h"
22#include "llvm/Analysis/ValueTracking.h"
23#include "llvm/Analysis/VectorUtils.h"
24#include "llvm/CodeGen/Analysis.h"
25#include "llvm/CodeGen/CodeGenCommonISel.h"
26#include "llvm/CodeGen/FunctionLoweringInfo.h"
27#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
28#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
29#include "llvm/CodeGen/GlobalISel/CallLowering.h"
30#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
31#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
32#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
33#include "llvm/CodeGen/LowLevelTypeUtils.h"
34#include "llvm/CodeGen/MachineBasicBlock.h"
35#include "llvm/CodeGen/MachineFrameInfo.h"
36#include "llvm/CodeGen/MachineFunction.h"
37#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
38#include "llvm/CodeGen/MachineInstrBuilder.h"
39#include "llvm/CodeGen/MachineMemOperand.h"
40#include "llvm/CodeGen/MachineModuleInfo.h"
41#include "llvm/CodeGen/MachineOperand.h"
42#include "llvm/CodeGen/MachinePassManager.h"
43#include "llvm/CodeGen/MachineRegisterInfo.h"
44#include "llvm/CodeGen/StackProtector.h"
45#include "llvm/CodeGen/SwiftErrorValueTracking.h"
46#include "llvm/CodeGen/SwitchLoweringUtils.h"
47#include "llvm/CodeGen/TargetFrameLowering.h"
48#include "llvm/CodeGen/TargetInstrInfo.h"
49#include "llvm/CodeGen/TargetLowering.h"
50#include "llvm/CodeGen/TargetOpcodes.h"
51#include "llvm/CodeGen/TargetPassConfig.h"
52#include "llvm/CodeGen/TargetRegisterInfo.h"
53#include "llvm/CodeGen/TargetSubtargetInfo.h"
54#include "llvm/CodeGenTypes/LowLevelType.h"
55#include "llvm/IR/Analysis.h"
56#include "llvm/IR/BasicBlock.h"
57#include "llvm/IR/CFG.h"
58#include "llvm/IR/Constant.h"
59#include "llvm/IR/Constants.h"
60#include "llvm/IR/DataLayout.h"
61#include "llvm/IR/DerivedTypes.h"
62#include "llvm/IR/DiagnosticInfo.h"
63#include "llvm/IR/Function.h"
64#include "llvm/IR/GetElementPtrTypeIterator.h"
65#include "llvm/IR/InlineAsm.h"
66#include "llvm/IR/InstrTypes.h"
67#include "llvm/IR/Instructions.h"
68#include "llvm/IR/IntrinsicInst.h"
69#include "llvm/IR/Intrinsics.h"
70#include "llvm/IR/IntrinsicsAMDGPU.h"
71#include "llvm/IR/LLVMContext.h"
72#include "llvm/IR/Metadata.h"
73#include "llvm/IR/Module.h"
74#include "llvm/IR/PatternMatch.h"
75#include "llvm/IR/Statepoint.h"
76#include "llvm/IR/Type.h"
77#include "llvm/IR/User.h"
78#include "llvm/IR/Value.h"
79#include "llvm/InitializePasses.h"
80#include "llvm/MC/MCContext.h"
81#include "llvm/Pass.h"
82#include "llvm/Support/Casting.h"
83#include "llvm/Support/CodeGen.h"
84#include "llvm/Support/Debug.h"
85#include "llvm/Support/ErrorHandling.h"
86#include "llvm/Support/MathExtras.h"
87#include "llvm/Support/raw_ostream.h"
88#include "llvm/Target/TargetMachine.h"
89#include "llvm/Transforms/Utils/Local.h"
90#include "llvm/Transforms/Utils/MemoryOpRemark.h"
91#include <algorithm>
92#include <cassert>
93#include <cstdint>
94#include <iterator>
95#include <optional>
96#include <string>
97#include <utility>
98#include <vector>
99
100#define DEBUG_TYPE "irtranslator"
101
102using namespace llvm;
103
104static cl::opt<bool>
105 EnableCSEInIRTranslator("enable-cse-in-irtranslator",
106 cl::desc("Should enable CSE in irtranslator"),
107 cl::Optional, cl::init(Val: false));
108
109namespace llvm {
110
111class IRTranslatorImpl {
112 /// Interface used to lower the everything related to calls.
113 const CallLowering *CLI = nullptr;
114
115 SSPLayoutInfo *SPInfo = nullptr;
116
117 /// This class contains the mapping between the Values to vreg related data.
118 class ValueToVRegInfo {
119 public:
120 ValueToVRegInfo() = default;
121
122 using VRegListT = SmallVector<Register, 1>;
123 using OffsetListT = SmallVector<uint64_t, 1>;
124
125 using const_vreg_iterator =
126 DenseMap<const Value *, VRegListT *>::const_iterator;
127 using const_offset_iterator =
128 DenseMap<const Value *, OffsetListT *>::const_iterator;
129
130 inline const_vreg_iterator vregs_end() const { return ValToVRegs.end(); }
131
132 VRegListT *getVRegs(const Value &V) {
133 auto [It, Inserted] = ValToVRegs.try_emplace(Key: &V);
134 if (!Inserted)
135 return It->second;
136
137 // We placement new using our fast allocator since we never try to free
138 // the vectors until translation is finished.
139 It->second = new (VRegAlloc.Allocate()) VRegListT();
140 return It->second;
141 }
142
143 OffsetListT *getOffsets(const Value &V) {
144 assert(V.getType()->isAggregateType() &&
145 "Offsets are for aggregate values");
146 auto [It, Inserted] = TypeToOffsets.try_emplace(Key: V.getType());
147 if (!Inserted)
148 return It->second;
149
150 It->second = new (OffsetAlloc.Allocate()) OffsetListT();
151 return It->second;
152 }
153
154 const_vreg_iterator findVRegs(const Value &V) const {
155 return ValToVRegs.find(Val: &V);
156 }
157
158 bool contains(const Value &V) const { return ValToVRegs.contains(Val: &V); }
159
160 void reserveVRegs(unsigned NumValues) { ValToVRegs.reserve(NumEntries: NumValues); }
161
162 void reset() {
163 ValToVRegs.clear();
164 TypeToOffsets.clear();
165 VRegAlloc.DestroyAll();
166 OffsetAlloc.DestroyAll();
167 }
168
169 private:
170 SpecificBumpPtrAllocator<VRegListT> VRegAlloc;
171 SpecificBumpPtrAllocator<OffsetListT> OffsetAlloc;
172
173 // We store pointers to vectors here since references may be invalidated
174 // while we hold them if we stored the vectors directly.
175 DenseMap<const Value *, VRegListT *> ValToVRegs;
176 DenseMap<const Type *, OffsetListT *> TypeToOffsets;
177 };
178
179 /// Mapping of the values of the current LLVM IR function to the related
180 /// virtual registers and offsets.
181 ValueToVRegInfo VMap;
182
183 // One BasicBlock can be translated to multiple MachineBasicBlocks. For such
184 // BasicBlocks translated to multiple MachineBasicBlocks, MachinePreds retains
185 // a mapping between the edges arriving at the BasicBlock to the corresponding
186 // created MachineBasicBlocks. Some BasicBlocks that get translated to a
187 // single MachineBasicBlock may also end up in this Map.
188 using CFGEdge = std::pair<const BasicBlock *, const BasicBlock *>;
189 DenseMap<CFGEdge, SmallVector<MachineBasicBlock *, 1>> MachinePreds;
190
191 // List of stubbed PHI instructions, for values and basic blocks to be filled
192 // in once all MachineBasicBlocks have been created.
193 SmallVector<std::pair<const PHINode *, SmallVector<MachineInstr *, 1>>, 4>
194 PendingPHIs;
195
196 /// Record of what frame index has been allocated to specified allocas for
197 /// this function.
198 DenseMap<const AllocaInst *, int> FrameIndices;
199
200 SwiftErrorValueTracking SwiftError;
201
202 /// \name Methods for translating form LLVM IR to MachineInstr.
203 /// \see ::translate for general information on the translate methods.
204 /// @{
205
206 /// Translate \p Inst into its corresponding MachineInstr instruction(s).
207 /// Insert the newly translated instruction(s) right where the CurBuilder
208 /// is set.
209 ///
210 /// The general algorithm is:
211 /// 1. Look for a virtual register for each operand or
212 /// create one.
213 /// 2 Update the VMap accordingly.
214 /// 2.alt. For constant arguments, if they are compile time constants,
215 /// produce an immediate in the right operand and do not touch
216 /// ValToReg. Actually we will go with a virtual register for each
217 /// constants because it may be expensive to actually materialize the
218 /// constant. Moreover, if the constant spans on several instructions,
219 /// CSE may not catch them.
220 /// => Update ValToVReg and remember that we saw a constant in Constants.
221 /// We will materialize all the constants in finalize.
222 /// Note: we would need to do something so that we can recognize such operand
223 /// as constants.
224 /// 3. Create the generic instruction.
225 ///
226 /// \return true if the translation succeeded.
227 bool translate(const Instruction &Inst);
228
229 /// Materialize \p C into virtual-register \p Reg. The generic instructions
230 /// performing this materialization will be inserted into the entry block of
231 /// the function.
232 ///
233 /// \return true if the materialization succeeded.
234 bool translate(const Constant &C, Register Reg);
235
236 /// Examine any debug-info attached to the instruction (in the form of
237 /// DbgRecords) and translate it.
238 void translateDbgInfo(const Instruction &Inst, MachineIRBuilder &MIRBuilder);
239
240 /// Translate a debug-info record of a dbg.value into a DBG_* instruction.
241 /// Pass in all the contents of the record, rather than relying on how it's
242 /// stored.
243 void translateDbgValueRecord(Value *V, bool HasArgList,
244 const DILocalVariable *Variable,
245 const DIExpression *Expression,
246 const DebugLoc &DL,
247 MachineIRBuilder &MIRBuilder);
248
249 /// Translate a debug-info record of a dbg.declare into an indirect DBG_*
250 /// instruction. Pass in all the contents of the record, rather than relying
251 /// on how it's stored.
252 void translateDbgDeclareRecord(Value *Address, bool HasArgList,
253 const DILocalVariable *Variable,
254 const DIExpression *Expression,
255 const DebugLoc &DL,
256 MachineIRBuilder &MIRBuilder);
257
258 // Translate U as a copy of V.
259 bool translateCopy(const User &U, const Value &V,
260 MachineIRBuilder &MIRBuilder);
261 bool translateCopy(const User &U, Register Src, MachineIRBuilder &MIRBuilder);
262
263 /// Translate an LLVM bitcast into generic IR. Either a COPY or a G_BITCAST is
264 /// emitted.
265 bool translateBitCast(const User &U, MachineIRBuilder &MIRBuilder);
266
267 /// Translate an LLVM load instruction into generic IR.
268 bool translateLoad(const User &U, MachineIRBuilder &MIRBuilder);
269
270 /// Translate an LLVM store instruction into generic IR.
271 bool translateStore(const User &U, MachineIRBuilder &MIRBuilder);
272
273 /// Translate an LLVM string intrinsic (memcpy, memset, ...).
274 bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder,
275 unsigned Opcode);
276
277 /// Translate an LLVM trap intrinsic (trap, debugtrap, ubsantrap).
278 bool translateTrap(const CallInst &U, MachineIRBuilder &MIRBuilder,
279 unsigned Opcode);
280
281 // Translate @llvm.vector.interleave2 and
282 // @llvm.vector.deinterleave2 intrinsics for fixed-width vector
283 // types into vector shuffles.
284 bool translateVectorInterleave2Intrinsic(const CallInst &CI,
285 MachineIRBuilder &MIRBuilder);
286 bool translateVectorDeinterleave2Intrinsic(const CallInst &CI,
287 MachineIRBuilder &MIRBuilder);
288
289 void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder);
290
291 bool translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
292 MachineIRBuilder &MIRBuilder);
293 bool translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
294 MachineIRBuilder &MIRBuilder);
295
296 /// Helper function for translateSimpleIntrinsic.
297 /// \return The generic opcode for \p IntrinsicID if \p IntrinsicID is a
298 /// simple intrinsic (ceil, fabs, etc.). Otherwise, returns
299 /// Intrinsic::not_intrinsic.
300 unsigned getSimpleIntrinsicOpcode(Intrinsic::ID ID);
301
302 /// Translates the intrinsics defined in getSimpleIntrinsicOpcode.
303 /// \return true if the translation succeeded.
304 bool translateSimpleIntrinsic(const CallInst &CI, Intrinsic::ID ID,
305 MachineIRBuilder &MIRBuilder);
306
307 bool translateConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI,
308 MachineIRBuilder &MIRBuilder);
309
310 bool translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
311 MachineIRBuilder &MIRBuilder);
312
313 /// Returns the single livein physical register Arg was lowered to, if
314 /// possible.
315 std::optional<MCRegister> getArgPhysReg(Argument &Arg);
316
317 /// If debug-info targets an Argument and its expression is an EntryValue,
318 /// lower it as either an entry in the MF debug table (dbg.declare), or a
319 /// DBG_VALUE targeting the corresponding livein register for that Argument
320 /// (dbg.value).
321 bool translateIfEntryValueArgument(bool isDeclare, Value *Arg,
322 const DILocalVariable *Var,
323 const DIExpression *Expr,
324 const DebugLoc &DL,
325 MachineIRBuilder &MIRBuilder);
326
327 bool translateInlineAsm(const CallBase &CB, MachineIRBuilder &MIRBuilder);
328
329 /// Common code for translating normal calls or invokes.
330 bool translateCallBase(const CallBase &CB, MachineIRBuilder &MIRBuilder);
331
332 /// Translate call instruction.
333 /// \pre \p U is a call instruction.
334 bool translateCall(const User &U, MachineIRBuilder &MIRBuilder);
335
336 bool translateIntrinsic(
337 const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
338 ArrayRef<TargetLowering::IntrinsicInfo> TgtMemIntrinsicInfos = {});
339
340 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
341 /// many places it could ultimately go. In the IR, we have a single unwind
342 /// destination, but in the machine CFG, we enumerate all the possible blocks.
343 /// This function skips over imaginary basic blocks that hold catchswitch
344 /// instructions, and finds all the "real" machine
345 /// basic block destinations. As those destinations may not be successors of
346 /// EHPadBB, here we also calculate the edge probability to those
347 /// destinations. The passed-in Prob is the edge probability to EHPadBB.
348 bool findUnwindDestinations(
349 const BasicBlock *EHPadBB, BranchProbability Prob,
350 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
351 &UnwindDests);
352
353 bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder);
354
355 bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder);
356
357 bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder);
358
359 /// Translate one of LLVM's cast instructions into MachineInstrs, with the
360 /// given generic Opcode.
361 bool translateCast(unsigned Opcode, const User &U,
362 MachineIRBuilder &MIRBuilder);
363
364 /// Translate a phi instruction.
365 bool translatePHI(const User &U, MachineIRBuilder &MIRBuilder);
366
367 /// Translate a comparison (icmp or fcmp) instruction or constant.
368 bool translateCompare(const User &U, MachineIRBuilder &MIRBuilder);
369
370 /// Translate an integer compare instruction (or constant).
371 bool translateICmp(const User &U, MachineIRBuilder &MIRBuilder) {
372 return translateCompare(U, MIRBuilder);
373 }
374
375 /// Translate a floating-point compare instruction (or constant).
376 bool translateFCmp(const User &U, MachineIRBuilder &MIRBuilder) {
377 return translateCompare(U, MIRBuilder);
378 }
379
380 /// Add remaining operands onto phis we've translated. Executed after all
381 /// MachineBasicBlocks for the function have been created.
382 void finishPendingPhis();
383
384 /// Translate \p Inst into a unary operation \p Opcode.
385 /// \pre \p U is a unary operation.
386 bool translateUnaryOp(unsigned Opcode, const User &U,
387 MachineIRBuilder &MIRBuilder);
388
389 /// Translate \p Inst into a binary operation \p Opcode.
390 /// \pre \p U is a binary operation.
391 bool translateBinaryOp(unsigned Opcode, const User &U,
392 MachineIRBuilder &MIRBuilder);
393
394 /// If the set of cases should be emitted as a series of branches, return
395 /// true. If we should emit this as a bunch of and/or'd together conditions,
396 /// return false.
397 bool shouldEmitAsBranches(const std::vector<SwitchCG::CaseBlock> &Cases);
398 /// Helper method for findMergedConditions.
399 /// This function emits a branch and is used at the leaves of an OR or an
400 /// AND operator tree.
401 void emitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB,
402 MachineBasicBlock *FBB,
403 MachineBasicBlock *CurBB,
404 MachineBasicBlock *SwitchBB,
405 BranchProbability TProb,
406 BranchProbability FProb, bool InvertCond);
407 /// Used during condbr translation to find trees of conditions that can be
408 /// optimized.
409 void findMergedConditions(const Value *Cond, MachineBasicBlock *TBB,
410 MachineBasicBlock *FBB, MachineBasicBlock *CurBB,
411 MachineBasicBlock *SwitchBB,
412 Instruction::BinaryOps Opc, BranchProbability TProb,
413 BranchProbability FProb, bool InvertCond);
414
415 /// Translate branch (br) instruction.
416 /// \pre \p U is a branch instruction.
417 bool translateUncondBr(const User &U, MachineIRBuilder &MIRBuilder);
418 bool translateCondBr(const User &U, MachineIRBuilder &MIRBuilder);
419
420 // Begin switch lowering functions.
421 bool emitJumpTableHeader(SwitchCG::JumpTable &JT,
422 SwitchCG::JumpTableHeader &JTH,
423 MachineBasicBlock *HeaderBB);
424 void emitJumpTable(SwitchCG::JumpTable &JT, MachineBasicBlock *MBB);
425
426 void emitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB,
427 MachineIRBuilder &MIB);
428
429 /// Generate for the BitTest header block, which precedes each sequence of
430 /// BitTestCases.
431 void emitBitTestHeader(SwitchCG::BitTestBlock &BTB,
432 MachineBasicBlock *SwitchMBB);
433 /// Generate code to produces one "bit test" for a given BitTestCase \p B.
434 void emitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB,
435 BranchProbability BranchProbToNext, Register Reg,
436 SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB);
437
438 void splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
439 const SwitchCG::SwitchWorkListItem &W, Value *Cond,
440 MachineBasicBlock *SwitchMBB, MachineIRBuilder &MIB);
441
442 bool lowerJumpTableWorkItem(
443 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
444 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
445 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
446 BranchProbability UnhandledProbs, SwitchCG::CaseClusterIt I,
447 MachineBasicBlock *Fallthrough, bool FallthroughUnreachable);
448
449 bool lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I, Value *Cond,
450 MachineBasicBlock *Fallthrough,
451 bool FallthroughUnreachable,
452 BranchProbability UnhandledProbs,
453 MachineBasicBlock *CurMBB,
454 MachineIRBuilder &MIB,
455 MachineBasicBlock *SwitchMBB);
456
457 bool lowerBitTestWorkItem(
458 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
459 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
460 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
461 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
462 SwitchCG::CaseClusterIt I, MachineBasicBlock *Fallthrough,
463 bool FallthroughUnreachable);
464
465 bool lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W, Value *Cond,
466 MachineBasicBlock *SwitchMBB,
467 MachineBasicBlock *DefaultMBB,
468 MachineIRBuilder &MIB);
469
470 bool translateSwitch(const User &U, MachineIRBuilder &MIRBuilder);
471 // End switch lowering section.
472
473 bool translateIndirectBr(const User &U, MachineIRBuilder &MIRBuilder);
474
475 bool translateExtractValue(const User &U, MachineIRBuilder &MIRBuilder);
476
477 bool translateInsertValue(const User &U, MachineIRBuilder &MIRBuilder);
478
479 bool translateSelect(const User &U, MachineIRBuilder &MIRBuilder);
480
481 bool translateGetElementPtr(const User &U, MachineIRBuilder &MIRBuilder);
482
483 bool translateAlloca(const User &U, MachineIRBuilder &MIRBuilder);
484
485 /// Translate return (ret) instruction.
486 /// The target needs to implement CallLowering::lowerReturn for
487 /// this to succeed.
488 /// \pre \p U is a return instruction.
489 bool translateRet(const User &U, MachineIRBuilder &MIRBuilder);
490
491 bool translateFNeg(const User &U, MachineIRBuilder &MIRBuilder);
492
493 bool translateAdd(const User &U, MachineIRBuilder &MIRBuilder) {
494 return translateBinaryOp(Opcode: TargetOpcode::G_ADD, U, MIRBuilder);
495 }
496 bool translateSub(const User &U, MachineIRBuilder &MIRBuilder) {
497 return translateBinaryOp(Opcode: TargetOpcode::G_SUB, U, MIRBuilder);
498 }
499 bool translateAnd(const User &U, MachineIRBuilder &MIRBuilder) {
500 return translateBinaryOp(Opcode: TargetOpcode::G_AND, U, MIRBuilder);
501 }
502 bool translateMul(const User &U, MachineIRBuilder &MIRBuilder) {
503 return translateBinaryOp(Opcode: TargetOpcode::G_MUL, U, MIRBuilder);
504 }
505 bool translateOr(const User &U, MachineIRBuilder &MIRBuilder) {
506 return translateBinaryOp(Opcode: TargetOpcode::G_OR, U, MIRBuilder);
507 }
508 bool translateXor(const User &U, MachineIRBuilder &MIRBuilder) {
509 return translateBinaryOp(Opcode: TargetOpcode::G_XOR, U, MIRBuilder);
510 }
511
512 bool translateUDiv(const User &U, MachineIRBuilder &MIRBuilder) {
513 return translateBinaryOp(Opcode: TargetOpcode::G_UDIV, U, MIRBuilder);
514 }
515 bool translateSDiv(const User &U, MachineIRBuilder &MIRBuilder) {
516 return translateBinaryOp(Opcode: TargetOpcode::G_SDIV, U, MIRBuilder);
517 }
518 bool translateURem(const User &U, MachineIRBuilder &MIRBuilder) {
519 return translateBinaryOp(Opcode: TargetOpcode::G_UREM, U, MIRBuilder);
520 }
521 bool translateSRem(const User &U, MachineIRBuilder &MIRBuilder) {
522 return translateBinaryOp(Opcode: TargetOpcode::G_SREM, U, MIRBuilder);
523 }
524 bool translateIntToPtr(const User &U, MachineIRBuilder &MIRBuilder) {
525 return translateCast(Opcode: TargetOpcode::G_INTTOPTR, U, MIRBuilder);
526 }
527 bool translatePtrToInt(const User &U, MachineIRBuilder &MIRBuilder) {
528 return translateCast(Opcode: TargetOpcode::G_PTRTOINT, U, MIRBuilder);
529 }
530 bool translatePtrToAddr(const User &U, MachineIRBuilder &MIRBuilder) {
531 // FIXME: this is not correct for pointers with addr width != pointer width
532 return translatePtrToInt(U, MIRBuilder);
533 }
534 bool translateTrunc(const User &U, MachineIRBuilder &MIRBuilder) {
535 return translateCast(Opcode: TargetOpcode::G_TRUNC, U, MIRBuilder);
536 }
537 bool translateFPTrunc(const User &U, MachineIRBuilder &MIRBuilder) {
538 return translateCast(Opcode: TargetOpcode::G_FPTRUNC, U, MIRBuilder);
539 }
540 bool translateFPExt(const User &U, MachineIRBuilder &MIRBuilder) {
541 return translateCast(Opcode: TargetOpcode::G_FPEXT, U, MIRBuilder);
542 }
543 bool translateFPToUI(const User &U, MachineIRBuilder &MIRBuilder) {
544 return translateCast(Opcode: TargetOpcode::G_FPTOUI, U, MIRBuilder);
545 }
546 bool translateFPToSI(const User &U, MachineIRBuilder &MIRBuilder) {
547 return translateCast(Opcode: TargetOpcode::G_FPTOSI, U, MIRBuilder);
548 }
549 bool translateUIToFP(const User &U, MachineIRBuilder &MIRBuilder) {
550 return translateCast(Opcode: TargetOpcode::G_UITOFP, U, MIRBuilder);
551 }
552 bool translateSIToFP(const User &U, MachineIRBuilder &MIRBuilder) {
553 return translateCast(Opcode: TargetOpcode::G_SITOFP, U, MIRBuilder);
554 }
555 bool translateUnreachable(const User &U, MachineIRBuilder &MIRBuilder);
556
557 bool translateSExt(const User &U, MachineIRBuilder &MIRBuilder) {
558 return translateCast(Opcode: TargetOpcode::G_SEXT, U, MIRBuilder);
559 }
560
561 bool translateZExt(const User &U, MachineIRBuilder &MIRBuilder) {
562 return translateCast(Opcode: TargetOpcode::G_ZEXT, U, MIRBuilder);
563 }
564
565 bool translateShl(const User &U, MachineIRBuilder &MIRBuilder) {
566 return translateBinaryOp(Opcode: TargetOpcode::G_SHL, U, MIRBuilder);
567 }
568 bool translateLShr(const User &U, MachineIRBuilder &MIRBuilder) {
569 return translateBinaryOp(Opcode: TargetOpcode::G_LSHR, U, MIRBuilder);
570 }
571 bool translateAShr(const User &U, MachineIRBuilder &MIRBuilder) {
572 return translateBinaryOp(Opcode: TargetOpcode::G_ASHR, U, MIRBuilder);
573 }
574
575 bool translateFAdd(const User &U, MachineIRBuilder &MIRBuilder) {
576 return translateBinaryOp(Opcode: TargetOpcode::G_FADD, U, MIRBuilder);
577 }
578 bool translateFSub(const User &U, MachineIRBuilder &MIRBuilder) {
579 return translateBinaryOp(Opcode: TargetOpcode::G_FSUB, U, MIRBuilder);
580 }
581 bool translateFMul(const User &U, MachineIRBuilder &MIRBuilder) {
582 return translateBinaryOp(Opcode: TargetOpcode::G_FMUL, U, MIRBuilder);
583 }
584 bool translateFDiv(const User &U, MachineIRBuilder &MIRBuilder) {
585 return translateBinaryOp(Opcode: TargetOpcode::G_FDIV, U, MIRBuilder);
586 }
587 bool translateFRem(const User &U, MachineIRBuilder &MIRBuilder) {
588 return translateBinaryOp(Opcode: TargetOpcode::G_FREM, U, MIRBuilder);
589 }
590
591 bool translateVAArg(const User &U, MachineIRBuilder &MIRBuilder);
592
593 bool translateInsertElement(const User &U, MachineIRBuilder &MIRBuilder);
594 bool translateInsertVector(const User &U, MachineIRBuilder &MIRBuilder);
595
596 bool translateExtractElement(const User &U, MachineIRBuilder &MIRBuilder);
597 bool translateExtractVector(const User &U, MachineIRBuilder &MIRBuilder);
598
599 bool translateShuffleVector(const User &U, MachineIRBuilder &MIRBuilder);
600
601 bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder);
602 bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder);
603 bool translateFence(const User &U, MachineIRBuilder &MIRBuilder);
604 bool translateFreeze(const User &U, MachineIRBuilder &MIRBuilder);
605
606 // Stubs to keep the compiler happy while we implement the rest of the
607 // translation.
608 bool translateResume(const User &U, MachineIRBuilder &MIRBuilder) {
609 return false;
610 }
611 bool translateCleanupRet(const User &U, MachineIRBuilder &MIRBuilder) {
612 return false;
613 }
614 bool translateCatchRet(const User &U, MachineIRBuilder &MIRBuilder) {
615 return false;
616 }
617 bool translateCatchSwitch(const User &U, MachineIRBuilder &MIRBuilder) {
618 return false;
619 }
620 bool translateAddrSpaceCast(const User &U, MachineIRBuilder &MIRBuilder) {
621 return translateCast(Opcode: TargetOpcode::G_ADDRSPACE_CAST, U, MIRBuilder);
622 }
623 bool translateCleanupPad(const User &U, MachineIRBuilder &MIRBuilder) {
624 return false;
625 }
626 bool translateCatchPad(const User &U, MachineIRBuilder &MIRBuilder) {
627 return false;
628 }
629 bool translateUserOp1(const User &U, MachineIRBuilder &MIRBuilder) {
630 return false;
631 }
632 bool translateUserOp2(const User &U, MachineIRBuilder &MIRBuilder) {
633 return false;
634 }
635
636 bool translateConvergenceControlIntrinsic(const CallInst &CI,
637 Intrinsic::ID ID,
638 MachineIRBuilder &MIRBuilder);
639
640 /// @}
641
642 // Builder for machine instruction a la IRBuilder.
643 // I.e., compared to regular MIBuilder, this one also inserts the instruction
644 // in the current block, it can creates block, etc., basically a kind of
645 // IRBuilder, but for Machine IR.
646 // CSEMIRBuilder CurBuilder;
647 std::unique_ptr<MachineIRBuilder> CurBuilder;
648
649 // Builder set to the entry block (just after ABI lowering instructions). Used
650 // as a convenient location for Constants.
651 // CSEMIRBuilder EntryBuilder;
652 std::unique_ptr<MachineIRBuilder> EntryBuilder;
653
654 // The MachineFunction currently being translated.
655 MachineFunction *MF = nullptr;
656
657 /// MachineRegisterInfo used to create virtual registers.
658 MachineRegisterInfo *MRI = nullptr;
659
660 const DataLayout *DL = nullptr;
661
662 CodeGenOptLevel OptLevel;
663
664 /// Current optimization remark emitter. Used to report failures.
665 std::unique_ptr<OptimizationRemarkEmitter> ORE;
666
667 AAResults *AA = nullptr;
668 AssumptionCache *AC = nullptr;
669 const TargetLibraryInfo *LibInfo = nullptr;
670 const LibcallLoweringInfo *Libcalls = nullptr;
671 const TargetLowering *TLI = nullptr;
672 FunctionLoweringInfo FuncInfo;
673
674 // True when either the Target Machine specifies no optimizations or the
675 // function has the optnone attribute.
676 bool EnableOpts = false;
677
678 /// True when the block contains a tail call. This allows the IRTranslator to
679 /// stop translating such blocks early.
680 bool HasTailCall = false;
681
682 StackProtectorDescriptor SPDescriptor;
683
684 bool mayTranslateUserTypes(const User &U) const;
685
686 /// Switch analysis and optimization.
687 class GISelSwitchLowering : public SwitchCG::SwitchLowering {
688 public:
689 GISelSwitchLowering(IRTranslatorImpl *irt, FunctionLoweringInfo &funcinfo)
690 : SwitchLowering(funcinfo), IRT(irt) {
691 assert(irt && "irt is null!");
692 }
693
694 void addSuccessorWithProb(
695 MachineBasicBlock *Src, MachineBasicBlock *Dst,
696 BranchProbability Prob = BranchProbability::getUnknown()) override {
697 IRT->addSuccessorWithProb(Src, Dst, Prob);
698 }
699
700 ~GISelSwitchLowering() override = default;
701
702 private:
703 IRTranslatorImpl *IRT;
704 };
705
706 std::unique_ptr<GISelSwitchLowering> SL;
707
708 // * Insert all the code needed to materialize the constants
709 // at the proper place. E.g., Entry block or dominator block
710 // of each constant depending on how fancy we want to be.
711 // * Clear the different maps.
712 void finalizeFunction();
713
714 // Processing steps done per block. E.g. emitting jump tables, stack
715 // protectors etc. Returns true if no errors, false if there was a problem
716 // that caused an abort.
717 bool finalizeBasicBlock(const BasicBlock &BB, MachineBasicBlock &MBB);
718
719 /// Codegen a new tail for a stack protector check ParentMBB which has had its
720 /// tail spliced into a stack protector check success bb.
721 ///
722 /// For a high level explanation of how this fits into the stack protector
723 /// generation see the comment on the declaration of class
724 /// StackProtectorDescriptor.
725 ///
726 /// \return true if there were no problems.
727 bool emitSPDescriptorParent(StackProtectorDescriptor &SPD,
728 MachineBasicBlock *ParentBB);
729
730 /// Codegen the failure basic block for a stack protector check.
731 ///
732 /// A failure stack protector machine basic block consists simply of a call to
733 /// __stack_chk_fail().
734 ///
735 /// For a high level explanation of how this fits into the stack protector
736 /// generation see the comment on the declaration of class
737 /// StackProtectorDescriptor.
738 ///
739 /// \return true if there were no problems.
740 bool emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
741 MachineBasicBlock *FailureBB);
742
743 /// Get the VRegs that represent \p Val.
744 /// Non-aggregate types have just one corresponding VReg and the list can be
745 /// used as a single "unsigned". Aggregates get flattened. If such VRegs do
746 /// not exist, they are created.
747 ArrayRef<Register> getOrCreateVRegs(const Value &Val);
748
749 Register getOrCreateVReg(const Value &Val) {
750 auto Regs = getOrCreateVRegs(Val);
751 if (Regs.empty())
752 return 0;
753 assert(Regs.size() == 1 &&
754 "attempt to get single VReg for aggregate or void");
755 return Regs[0];
756 }
757
758 Register getOrCreateConvergenceTokenVReg(const Value &Token) {
759 assert(Token.getType()->isTokenTy());
760 auto &Regs = *VMap.getVRegs(V: Token);
761 if (!Regs.empty()) {
762 assert(Regs.size() == 1 &&
763 "Expected a single register for convergence tokens.");
764 return Regs[0];
765 }
766
767 auto Reg = MRI->createGenericVirtualRegister(Ty: LLT::token());
768 Regs.push_back(Elt: Reg);
769 return Reg;
770 }
771
772 /// Allocate empty vregs for \p Val. For aggregate values, also populate
773 /// their offsets.
774 ValueToVRegInfo::VRegListT &allocateVRegs(const Value &Val);
775
776 /// Get the frame index that represents \p Val.
777 /// If such VReg does not exist, it is created.
778 int getOrCreateFrameIndex(const AllocaInst &AI);
779
780 /// Get the alignment of the given memory operation instruction. This will
781 /// either be the explicitly specified value or the ABI-required alignment for
782 /// the type being accessed (according to the Module's DataLayout).
783 Align getMemOpAlign(const Instruction &I);
784
785 /// Get the MachineBasicBlock that represents \p BB. Specifically, the block
786 /// returned will be the head of the translated block (suitable for branch
787 /// destinations).
788 MachineBasicBlock &getMBB(const BasicBlock &BB);
789
790 /// Record \p NewPred as a Machine predecessor to `Edge.second`, corresponding
791 /// to `Edge.first` at the IR level. This is used when IRTranslation creates
792 /// multiple MachineBasicBlocks for a given IR block and the CFG is no longer
793 /// represented simply by the IR-level CFG.
794 void addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred);
795
796 /// Returns the Machine IR predecessors for the given IR CFG edge. Usually
797 /// this is just the single MachineBasicBlock corresponding to the predecessor
798 /// in the IR. More complex lowering can result in multiple MachineBasicBlocks
799 /// preceding the original though (e.g. switch instructions).
800 SmallVector<MachineBasicBlock *, 1> getMachinePredBBs(CFGEdge Edge) {
801 auto RemappedEdge = MachinePreds.find(Val: Edge);
802 if (RemappedEdge != MachinePreds.end())
803 return RemappedEdge->second;
804 return SmallVector<MachineBasicBlock *, 4>(1, &getMBB(BB: *Edge.first));
805 }
806
807 /// Return branch probability calculated by BranchProbabilityInfo for IR
808 /// blocks.
809 BranchProbability getEdgeProbability(const MachineBasicBlock *Src,
810 const MachineBasicBlock *Dst) const;
811
812 void addSuccessorWithProb(
813 MachineBasicBlock *Src, MachineBasicBlock *Dst,
814 BranchProbability Prob = BranchProbability::getUnknown());
815
816public:
817 IRTranslatorImpl(CodeGenOptLevel OptLevel = CodeGenOptLevel::None)
818 : OptLevel(OptLevel) {}
819
820 // Algo:
821 // CallLowering = MF.subtarget.getCallLowering()
822 // F = MF.getParent()
823 // MIRBuilder.reset(MF)
824 // getMBB(F.getEntryBB())
825 // CallLowering->translateArguments(MIRBuilder, F, ValToVReg)
826 // for each bb in F
827 // getMBB(bb)
828 // for each inst in bb
829 // if (!translate(MIRBuilder, inst, ValToVReg, ConstantToSequence))
830 // reportFatalUsageError("Don't know how to translate input");
831 // finalize()
832 bool runOnMachineFunction(MachineFunction &MF,
833 function_ref<GISelCSEInfo *()> GetCSEInfo,
834 bool ShouldSkipOpts,
835 function_ref<AAResults *()> GetAAResults,
836 function_ref<BranchProbabilityInfo *()> GetBPI,
837 function_ref<AssumptionCache *()> GetAC,
838 TargetLibraryInfo *LibraryInfo,
839 const LibcallLoweringInfo *LibcallInfo,
840 SSPLayoutInfo *StackProtectorInfo);
841};
842
843} // namespace llvm
844
845char IRTranslatorLegacy::ID = 0;
846
847INITIALIZE_PASS_BEGIN(IRTranslatorLegacy, DEBUG_TYPE,
848 "IRTranslator LLVM IR -> MI", false, false)
849INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
850INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
851INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
852INITIALIZE_PASS_DEPENDENCY(StackProtector)
853INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
854INITIALIZE_PASS_END(IRTranslatorLegacy, DEBUG_TYPE,
855 "IRTranslator LLVM IR -> MI", false, false)
856
857static void reportTranslationError(MachineFunction &MF,
858 OptimizationRemarkEmitter &ORE,
859 OptimizationRemarkMissed &R) {
860 MF.getProperties().setFailedISel();
861 bool IsGlobalISelAbortEnabled =
862 MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
863
864 // Print the function name explicitly if we don't have a debug location (which
865 // makes the diagnostic less useful) or if we're going to emit a raw error.
866 if (!R.getLocation().isValid() || IsGlobalISelAbortEnabled)
867 R << (" (in function: " + MF.getName() + ")").str();
868
869 if (IsGlobalISelAbortEnabled)
870 report_fatal_error(reason: Twine(R.getMsg()));
871 else
872 ORE.emit(OptDiag&: R);
873}
874
875IRTranslatorLegacy::IRTranslatorLegacy(CodeGenOptLevel OptLevel)
876 : MachineFunctionPass(ID), OptLevel(OptLevel),
877 Impl(std::make_unique<IRTranslatorImpl>(args&: OptLevel)) {}
878
879IRTranslatorLegacy::~IRTranslatorLegacy() = default;
880
881#ifndef NDEBUG
882namespace {
883/// Verify that every instruction created has the same DILocation as the
884/// instruction being translated.
885class DILocationVerifier : public GISelChangeObserver {
886 const Instruction *CurrInst = nullptr;
887
888public:
889 DILocationVerifier() = default;
890 ~DILocationVerifier() override = default;
891
892 const Instruction *getCurrentInst() const { return CurrInst; }
893 void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
894
895 void erasingInstr(MachineInstr &MI) override {}
896 void changingInstr(MachineInstr &MI) override {}
897 void changedInstr(MachineInstr &MI) override {}
898
899 void createdInstr(MachineInstr &MI) override {
900 assert(getCurrentInst() && "Inserted instruction without a current MI");
901
902 // Only print the check message if we're actually checking it.
903#ifndef NDEBUG
904 LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
905 << " was copied to " << MI);
906#endif
907 // We allow insts in the entry block to have no debug loc because
908 // they could have originated from constants, and we don't want a jumpy
909 // debug experience.
910 assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
911 (MI.getParent()->isEntryBlock() && !MI.getDebugLoc()) ||
912 (MI.isDebugInstr())) &&
913 "Line info was not transferred to all instructions");
914 }
915};
916} // namespace
917#endif // ifndef NDEBUG
918
919void IRTranslatorLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
920 AU.addRequired<StackProtector>();
921 AU.addRequired<TargetPassConfig>();
922 AU.addRequired<GISelCSEAnalysisWrapperPass>();
923 if (OptLevel != CodeGenOptLevel::None) {
924 AU.addRequired<AssumptionCacheTracker>();
925 AU.addRequired<BranchProbabilityInfoWrapperPass>();
926 AU.addRequired<AAResultsWrapperPass>();
927 }
928 AU.addRequired<TargetLibraryInfoWrapperPass>();
929 AU.addPreserved<TargetLibraryInfoWrapperPass>();
930 AU.addRequired<LibcallLoweringInfoWrapper>();
931
932 getSelectionDAGFallbackAnalysisUsage(AU);
933 MachineFunctionPass::getAnalysisUsage(AU);
934}
935
936IRTranslatorImpl::ValueToVRegInfo::VRegListT &
937IRTranslatorImpl::allocateVRegs(const Value &Val) {
938 auto VRegsIt = VMap.findVRegs(V: Val);
939 if (VRegsIt != VMap.vregs_end())
940 return *VRegsIt->second;
941 auto *Regs = VMap.getVRegs(V: Val);
942 if (!Val.getType()->isAggregateType()) {
943 Regs->push_back(Elt: 0);
944 return *Regs;
945 }
946
947 auto *Offsets = VMap.getOffsets(V: Val);
948 SmallVector<LLT, 4> SplitTys;
949 computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueLLTs&: SplitTys,
950 FixedOffsets: Offsets->empty() ? Offsets : nullptr);
951 for (unsigned i = 0; i < SplitTys.size(); ++i)
952 Regs->push_back(Elt: 0);
953 return *Regs;
954}
955
956ArrayRef<Register> IRTranslatorImpl::getOrCreateVRegs(const Value &Val) {
957 auto VRegsIt = VMap.findVRegs(V: Val);
958 if (VRegsIt != VMap.vregs_end())
959 return *VRegsIt->second;
960
961 if (Val.getType()->isVoidTy())
962 return *VMap.getVRegs(V: Val);
963
964 // Create entry for this type.
965 auto *VRegs = VMap.getVRegs(V: Val);
966
967 if (!Val.getType()->isTokenTy())
968 assert(Val.getType()->isSized() &&
969 "Don't know how to create an empty vreg");
970
971 // Fast-path values that lower to a single vreg.
972 if (!Val.getType()->isAggregateType()) {
973 LLT Ty = getLLTForType(Ty&: *Val.getType(), DL: *DL);
974 VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty));
975 if (isa<Constant>(Val)) {
976 bool Success = translate(C: cast<Constant>(Val), Reg: VRegs->front());
977 if (!Success) {
978 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
979 MF->getFunction().getSubprogram(),
980 &MF->getFunction().getEntryBlock());
981 R << "unable to translate constant: " << ore::NV("Type", Val.getType());
982 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
983 }
984 }
985 return *VRegs;
986 }
987
988 SmallVector<LLT, 4> SplitTys;
989 auto *Offsets = VMap.getOffsets(V: Val);
990 computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueLLTs&: SplitTys,
991 FixedOffsets: Offsets->empty() ? Offsets : nullptr);
992
993 if (!isa<Constant>(Val)) {
994 for (auto Ty : SplitTys)
995 VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty));
996 return *VRegs;
997 }
998
999 // UndefValue, ConstantAggregateZero
1000 auto &C = cast<Constant>(Val);
1001 unsigned Idx = 0;
1002 while (auto Elt = C.getAggregateElement(Elt: Idx++)) {
1003 auto EltRegs = getOrCreateVRegs(Val: *Elt);
1004 llvm::append_range(C&: *VRegs, R&: EltRegs);
1005 }
1006
1007 return *VRegs;
1008}
1009
1010int IRTranslatorImpl::getOrCreateFrameIndex(const AllocaInst &AI) {
1011 auto [MapEntry, Inserted] = FrameIndices.try_emplace(Key: &AI);
1012 if (!Inserted)
1013 return MapEntry->second;
1014
1015 TypeSize TySize = AI.getAllocationSize(DL: *DL).value_or(u: TypeSize::getZero());
1016 uint64_t Size = TySize.getKnownMinValue();
1017
1018 // Always allocate at least one byte.
1019 Size = std::max<uint64_t>(a: Size, b: 1u);
1020
1021 int &FI = MapEntry->second;
1022 FI = MF->getFrameInfo().CreateStackObject(Size, Alignment: AI.getAlign(), isSpillSlot: false, Alloca: &AI);
1023
1024 // Scalable vectors and structures that contain scalable vectors may
1025 // need a special StackID to distinguish them from other (fixed size)
1026 // stack objects.
1027 if (TySize.isScalable()) {
1028 auto StackID =
1029 MF->getSubtarget().getFrameLowering()->getStackIDForScalableVectors();
1030 MF->getFrameInfo().setStackID(ObjectIdx: FI, ID: StackID);
1031 }
1032
1033 return FI;
1034}
1035
1036Align IRTranslatorImpl::getMemOpAlign(const Instruction &I) {
1037 if (const StoreInst *SI = dyn_cast<StoreInst>(Val: &I))
1038 return SI->getAlign();
1039 if (const LoadInst *LI = dyn_cast<LoadInst>(Val: &I))
1040 return LI->getAlign();
1041 if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(Val: &I))
1042 return AI->getAlign();
1043 if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(Val: &I))
1044 return AI->getAlign();
1045
1046 OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
1047 R << "unable to translate memop: " << ore::NV("Opcode", &I);
1048 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
1049 return Align(1);
1050}
1051
1052MachineBasicBlock &IRTranslatorImpl::getMBB(const BasicBlock &BB) {
1053 MachineBasicBlock *MBB = FuncInfo.getMBB(BB: &BB);
1054 assert(MBB && "BasicBlock was not encountered before");
1055 return *MBB;
1056}
1057
1058void IRTranslatorImpl::addMachineCFGPred(CFGEdge Edge,
1059 MachineBasicBlock *NewPred) {
1060 assert(NewPred && "new predecessor must be a real MachineBasicBlock");
1061 MachinePreds[Edge].push_back(Elt: NewPred);
1062}
1063
1064bool IRTranslatorImpl::translateBinaryOp(unsigned Opcode, const User &U,
1065 MachineIRBuilder &MIRBuilder) {
1066 if (!mayTranslateUserTypes(U))
1067 return false;
1068
1069 // Get or create a virtual register for each value.
1070 // Unless the value is a Constant => loadimm cst?
1071 // or inline constant each time?
1072 // Creation of a virtual register needs to have a size.
1073 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
1074 Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
1075 Register Res = getOrCreateVReg(Val: U);
1076 uint32_t Flags = 0;
1077 if (isa<Instruction>(Val: U)) {
1078 const Instruction &I = cast<Instruction>(Val: U);
1079 Flags = MachineInstr::copyFlagsFromInstruction(I);
1080 }
1081
1082 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0, Op1}, Flags);
1083 return true;
1084}
1085
1086bool IRTranslatorImpl::translateUnaryOp(unsigned Opcode, const User &U,
1087 MachineIRBuilder &MIRBuilder) {
1088 if (!mayTranslateUserTypes(U))
1089 return false;
1090
1091 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
1092 Register Res = getOrCreateVReg(Val: U);
1093 uint32_t Flags = 0;
1094 if (isa<Instruction>(Val: U)) {
1095 const Instruction &I = cast<Instruction>(Val: U);
1096 Flags = MachineInstr::copyFlagsFromInstruction(I);
1097 }
1098 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0}, Flags);
1099 return true;
1100}
1101
1102bool IRTranslatorImpl::translateFNeg(const User &U,
1103 MachineIRBuilder &MIRBuilder) {
1104 return translateUnaryOp(Opcode: TargetOpcode::G_FNEG, U, MIRBuilder);
1105}
1106
1107bool IRTranslatorImpl::translateCompare(const User &U,
1108 MachineIRBuilder &MIRBuilder) {
1109 if (!mayTranslateUserTypes(U))
1110 return false;
1111
1112 auto *CI = cast<CmpInst>(Val: &U);
1113 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
1114 Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
1115 Register Res = getOrCreateVReg(Val: U);
1116 CmpInst::Predicate Pred = CI->getPredicate();
1117 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: *CI);
1118 if (CmpInst::isIntPredicate(P: Pred))
1119 MIRBuilder.buildICmp(Pred, Res, Op0, Op1, Flags);
1120 else if (Pred == CmpInst::FCMP_FALSE)
1121 MIRBuilder.buildCopy(
1122 Res, Op: getOrCreateVReg(Val: *Constant::getNullValue(Ty: U.getType())));
1123 else if (Pred == CmpInst::FCMP_TRUE)
1124 MIRBuilder.buildCopy(
1125 Res, Op: getOrCreateVReg(Val: *Constant::getAllOnesValue(Ty: U.getType())));
1126 else
1127 MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
1128
1129 return true;
1130}
1131
1132bool IRTranslatorImpl::translateRet(const User &U,
1133 MachineIRBuilder &MIRBuilder) {
1134 const ReturnInst &RI = cast<ReturnInst>(Val: U);
1135 const Value *Ret = RI.getReturnValue();
1136 if (Ret && DL->getTypeStoreSize(Ty: Ret->getType()).isZero())
1137 Ret = nullptr;
1138
1139 ArrayRef<Register> VRegs;
1140 if (Ret)
1141 VRegs = getOrCreateVRegs(Val: *Ret);
1142
1143 Register SwiftErrorVReg = 0;
1144 if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
1145 SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
1146 &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
1147 }
1148
1149 // The target may mess up with the insertion point, but
1150 // this is not important as a return is the last instruction
1151 // of the block anyway.
1152 return CLI->lowerReturn(MIRBuilder, Val: Ret, VRegs, FLI&: FuncInfo, SwiftErrorVReg);
1153}
1154
1155void IRTranslatorImpl::emitBranchForMergedCondition(
1156 const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
1157 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
1158 BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
1159 // If the leaf of the tree is a comparison, merge the condition into
1160 // the caseblock.
1161 if (const CmpInst *BOp = dyn_cast<CmpInst>(Val: Cond)) {
1162 CmpInst::Predicate Condition;
1163 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Val: Cond)) {
1164 Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
1165 } else {
1166 const FCmpInst *FC = cast<FCmpInst>(Val: Cond);
1167 Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
1168 }
1169
1170 SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(i_nocapture: 0),
1171 BOp->getOperand(i_nocapture: 1), nullptr, TBB, FBB, CurBB,
1172 CurBuilder->getDebugLoc(), TProb, FProb);
1173 SL->SwitchCases.push_back(x: CB);
1174 return;
1175 }
1176
1177 // Create a CaseBlock record representing this branch.
1178 CmpInst::Predicate Pred = InvertCond ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ;
1179 SwitchCG::CaseBlock CB(
1180 Pred, false, Cond, ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
1181 nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
1182 SL->SwitchCases.push_back(x: CB);
1183}
1184
1185static bool isValInBlock(const Value *V, const BasicBlock *BB) {
1186 if (const Instruction *I = dyn_cast<Instruction>(Val: V))
1187 return I->getParent() == BB;
1188 return true;
1189}
1190
1191void IRTranslatorImpl::findMergedConditions(
1192 const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
1193 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
1194 Instruction::BinaryOps Opc, BranchProbability TProb,
1195 BranchProbability FProb, bool InvertCond) {
1196 using namespace PatternMatch;
1197 assert((Opc == Instruction::And || Opc == Instruction::Or) &&
1198 "Expected Opc to be AND/OR");
1199 // Skip over not part of the tree and remember to invert op and operands at
1200 // next level.
1201 Value *NotCond;
1202 if (match(V: Cond, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: NotCond)))) &&
1203 isValInBlock(V: NotCond, BB: CurBB->getBasicBlock())) {
1204 findMergedConditions(Cond: NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
1205 InvertCond: !InvertCond);
1206 return;
1207 }
1208
1209 const Instruction *BOp = dyn_cast<Instruction>(Val: Cond);
1210 const Value *BOpOp0, *BOpOp1;
1211 // Compute the effective opcode for Cond, taking into account whether it needs
1212 // to be inverted, e.g.
1213 // and (not (or A, B)), C
1214 // gets lowered as
1215 // and (and (not A, not B), C)
1216 Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
1217 if (BOp) {
1218 BOpc = match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
1219 ? Instruction::And
1220 : (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
1221 ? Instruction::Or
1222 : (Instruction::BinaryOps)0);
1223 if (InvertCond) {
1224 if (BOpc == Instruction::And)
1225 BOpc = Instruction::Or;
1226 else if (BOpc == Instruction::Or)
1227 BOpc = Instruction::And;
1228 }
1229 }
1230
1231 // If this node is not part of the or/and tree, emit it as a branch.
1232 // Note that all nodes in the tree should have same opcode.
1233 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
1234 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
1235 !isValInBlock(V: BOpOp0, BB: CurBB->getBasicBlock()) ||
1236 !isValInBlock(V: BOpOp1, BB: CurBB->getBasicBlock())) {
1237 emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
1238 InvertCond);
1239 return;
1240 }
1241
1242 // Create TmpBB after CurBB.
1243 MachineFunction::iterator BBI(CurBB);
1244 MachineBasicBlock *TmpBB =
1245 MF->CreateMachineBasicBlock(BB: CurBB->getBasicBlock());
1246 CurBB->getParent()->insert(MBBI: ++BBI, MBB: TmpBB);
1247
1248 if (Opc == Instruction::Or) {
1249 // Codegen X | Y as:
1250 // BB1:
1251 // jmp_if_X TBB
1252 // jmp TmpBB
1253 // TmpBB:
1254 // jmp_if_Y TBB
1255 // jmp FBB
1256 //
1257
1258 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1259 // The requirement is that
1260 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1261 // = TrueProb for original BB.
1262 // Assuming the original probabilities are A and B, one choice is to set
1263 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1264 // A/(1+B) and 2B/(1+B). This choice assumes that
1265 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1266 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1267 // TmpBB, but the math is more complicated.
1268
1269 auto NewTrueProb = TProb / 2;
1270 auto NewFalseProb = TProb / 2 + FProb;
1271 // Emit the LHS condition.
1272 findMergedConditions(Cond: BOpOp0, TBB, FBB: TmpBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
1273 FProb: NewFalseProb, InvertCond);
1274
1275 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1276 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1277 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
1278 // Emit the RHS condition into TmpBB.
1279 findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
1280 FProb: Probs[1], InvertCond);
1281 } else {
1282 assert(Opc == Instruction::And && "Unknown merge op!");
1283 // Codegen X & Y as:
1284 // BB1:
1285 // jmp_if_X TmpBB
1286 // jmp FBB
1287 // TmpBB:
1288 // jmp_if_Y TBB
1289 // jmp FBB
1290 //
1291 // This requires creation of TmpBB after CurBB.
1292
1293 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1294 // The requirement is that
1295 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1296 // = FalseProb for original BB.
1297 // Assuming the original probabilities are A and B, one choice is to set
1298 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1299 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1300 // TrueProb for BB1 * FalseProb for TmpBB.
1301
1302 auto NewTrueProb = TProb + FProb / 2;
1303 auto NewFalseProb = FProb / 2;
1304 // Emit the LHS condition.
1305 findMergedConditions(Cond: BOpOp0, TBB: TmpBB, FBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
1306 FProb: NewFalseProb, InvertCond);
1307
1308 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1309 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1310 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
1311 // Emit the RHS condition into TmpBB.
1312 findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
1313 FProb: Probs[1], InvertCond);
1314 }
1315}
1316
1317bool IRTranslatorImpl::shouldEmitAsBranches(
1318 const std::vector<SwitchCG::CaseBlock> &Cases) {
1319 // For multiple cases, it's better to emit as branches.
1320 if (Cases.size() != 2)
1321 return true;
1322
1323 // If this is two comparisons of the same values or'd or and'd together, they
1324 // will get folded into a single comparison, so don't emit two blocks.
1325 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1326 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1327 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1328 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1329 return false;
1330 }
1331
1332 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1333 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1334 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1335 Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
1336 isa<Constant>(Val: Cases[0].CmpRHS) &&
1337 cast<Constant>(Val: Cases[0].CmpRHS)->isNullValue()) {
1338 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
1339 Cases[0].TrueBB == Cases[1].ThisBB)
1340 return false;
1341 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
1342 Cases[0].FalseBB == Cases[1].ThisBB)
1343 return false;
1344 }
1345
1346 return true;
1347}
1348
1349bool IRTranslatorImpl::translateUncondBr(const User &U,
1350 MachineIRBuilder &MIRBuilder) {
1351 const UncondBrInst &BrInst = cast<UncondBrInst>(Val: U);
1352 auto &CurMBB = MIRBuilder.getMBB();
1353 auto *Succ0MBB = &getMBB(BB: *BrInst.getSuccessor(i: 0));
1354
1355 // If the unconditional target is the layout successor, fallthrough.
1356 if (OptLevel == CodeGenOptLevel::None || !CurMBB.isLayoutSuccessor(MBB: Succ0MBB))
1357 MIRBuilder.buildBr(Dest&: *Succ0MBB);
1358
1359 // Link successors.
1360 for (const BasicBlock *Succ : successors(I: &BrInst))
1361 CurMBB.addSuccessor(Succ: &getMBB(BB: *Succ));
1362 return true;
1363}
1364
1365bool IRTranslatorImpl::translateCondBr(const User &U,
1366 MachineIRBuilder &MIRBuilder) {
1367 const CondBrInst &BrInst = cast<CondBrInst>(Val: U);
1368 auto &CurMBB = MIRBuilder.getMBB();
1369 auto *Succ0MBB = &getMBB(BB: *BrInst.getSuccessor(i: 0));
1370
1371 // If this condition is one of the special cases we handle, do special stuff
1372 // now.
1373 const Value *CondVal = BrInst.getCondition();
1374 MachineBasicBlock *Succ1MBB = &getMBB(BB: *BrInst.getSuccessor(i: 1));
1375
1376 // If this is a series of conditions that are or'd or and'd together, emit
1377 // this as a sequence of branches instead of setcc's with and/or operations.
1378 // As long as jumps are not expensive (exceptions for multi-use logic ops,
1379 // unpredictable branches, and vector extracts because those jumps are likely
1380 // expensive for any target), this should improve performance.
1381 // For example, instead of something like:
1382 // cmp A, B
1383 // C = seteq
1384 // cmp D, E
1385 // F = setle
1386 // or C, F
1387 // jnz foo
1388 // Emit:
1389 // cmp A, B
1390 // je foo
1391 // cmp D, E
1392 // jle foo
1393 using namespace PatternMatch;
1394 const Instruction *CondI = dyn_cast<Instruction>(Val: CondVal);
1395 if (!TLI->isJumpExpensive() && CondI && CondI->hasOneUse() &&
1396 !BrInst.hasMetadata(KindID: LLVMContext::MD_unpredictable)) {
1397 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
1398 Value *Vec;
1399 const Value *BOp0, *BOp1;
1400 if (match(V: CondI, P: m_LogicalAnd(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
1401 Opcode = Instruction::And;
1402 else if (match(V: CondI, P: m_LogicalOr(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
1403 Opcode = Instruction::Or;
1404
1405 if (Opcode && !(match(V: BOp0, P: m_ExtractElt(Val: m_Value(V&: Vec), Idx: m_Value())) &&
1406 match(V: BOp1, P: m_ExtractElt(Val: m_Specific(V: Vec), Idx: m_Value())))) {
1407 findMergedConditions(Cond: CondI, TBB: Succ0MBB, FBB: Succ1MBB, CurBB: &CurMBB, SwitchBB: &CurMBB, Opc: Opcode,
1408 TProb: getEdgeProbability(Src: &CurMBB, Dst: Succ0MBB),
1409 FProb: getEdgeProbability(Src: &CurMBB, Dst: Succ1MBB),
1410 /*InvertCond=*/false);
1411 assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
1412
1413 // Allow some cases to be rejected.
1414 if (shouldEmitAsBranches(Cases: SL->SwitchCases)) {
1415 // Emit the branch for this block.
1416 emitSwitchCase(CB&: SL->SwitchCases[0], SwitchBB: &CurMBB, MIB&: *CurBuilder);
1417 SL->SwitchCases.erase(position: SL->SwitchCases.begin());
1418 return true;
1419 }
1420
1421 // Okay, we decided not to do this, remove any inserted MBB's and clear
1422 // SwitchCases.
1423 for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
1424 MF->erase(MBBI: SL->SwitchCases[I].ThisBB);
1425
1426 SL->SwitchCases.clear();
1427 }
1428 }
1429
1430 // Create a CaseBlock record representing this branch.
1431 SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
1432 ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
1433 nullptr, Succ0MBB, Succ1MBB, &CurMBB,
1434 CurBuilder->getDebugLoc());
1435
1436 // Use emitSwitchCase to actually insert the fast branch sequence for this
1437 // cond branch.
1438 emitSwitchCase(CB, SwitchBB: &CurMBB, MIB&: *CurBuilder);
1439 return true;
1440}
1441
1442void IRTranslatorImpl::addSuccessorWithProb(MachineBasicBlock *Src,
1443 MachineBasicBlock *Dst,
1444 BranchProbability Prob) {
1445 if (!FuncInfo.BPI) {
1446 Src->addSuccessorWithoutProb(Succ: Dst);
1447 return;
1448 }
1449 if (Prob.isUnknown())
1450 Prob = getEdgeProbability(Src, Dst);
1451 Src->addSuccessor(Succ: Dst, Prob);
1452}
1453
1454BranchProbability
1455IRTranslatorImpl::getEdgeProbability(const MachineBasicBlock *Src,
1456 const MachineBasicBlock *Dst) const {
1457 const BasicBlock *SrcBB = Src->getBasicBlock();
1458 const BasicBlock *DstBB = Dst->getBasicBlock();
1459 if (!FuncInfo.BPI) {
1460 // If BPI is not available, set the default probability as 1 / N, where N is
1461 // the number of successors.
1462 auto SuccSize = std::max<uint32_t>(a: succ_size(BB: SrcBB), b: 1);
1463 return BranchProbability(1, SuccSize);
1464 }
1465 return FuncInfo.BPI->getEdgeProbability(Src: SrcBB, Dst: DstBB);
1466}
1467
1468bool IRTranslatorImpl::translateSwitch(const User &U, MachineIRBuilder &MIB) {
1469 using namespace SwitchCG;
1470 // Extract cases from the switch.
1471 const SwitchInst &SI = cast<SwitchInst>(Val: U);
1472 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1473 CaseClusterVector Clusters;
1474 Clusters.reserve(n: SI.getNumCases());
1475 for (const auto &I : SI.cases()) {
1476 MachineBasicBlock *Succ = &getMBB(BB: *I.getCaseSuccessor());
1477 assert(Succ && "Could not find successor mbb in mapping");
1478 const ConstantInt *CaseVal = I.getCaseValue();
1479 BranchProbability Prob =
1480 BPI ? BPI->getEdgeProbability(Src: SI.getParent(), IndexInSuccessors: I.getSuccessorIndex())
1481 : BranchProbability(1, SI.getNumCases() + 1);
1482 Clusters.push_back(x: CaseCluster::range(Low: CaseVal, High: CaseVal, MBB: Succ, Prob));
1483 }
1484
1485 MachineBasicBlock *DefaultMBB = &getMBB(BB: *SI.getDefaultDest());
1486
1487 // Cluster adjacent cases with the same destination. We do this at all
1488 // optimization levels because it's cheap to do and will make codegen faster
1489 // if there are many clusters.
1490 sortAndRangeify(Clusters);
1491
1492 MachineBasicBlock *SwitchMBB = &getMBB(BB: *SI.getParent());
1493
1494 // If there is only the default destination, jump there directly.
1495 if (Clusters.empty()) {
1496 SwitchMBB->addSuccessor(Succ: DefaultMBB);
1497 if (DefaultMBB != SwitchMBB->getNextNode())
1498 MIB.buildBr(Dest&: *DefaultMBB);
1499 return true;
1500 }
1501
1502 SL->findJumpTables(Clusters, SI: &SI, SL: std::nullopt, DefaultMBB, PSI: nullptr, BFI: nullptr);
1503 SL->findBitTestClusters(Clusters, SI: &SI);
1504
1505 LLVM_DEBUG({
1506 dbgs() << "Case clusters: ";
1507 for (const CaseCluster &C : Clusters) {
1508 if (C.Kind == CC_JumpTable)
1509 dbgs() << "JT:";
1510 if (C.Kind == CC_BitTests)
1511 dbgs() << "BT:";
1512
1513 C.Low->getValue().print(dbgs(), true);
1514 if (C.Low != C.High) {
1515 dbgs() << '-';
1516 C.High->getValue().print(dbgs(), true);
1517 }
1518 dbgs() << ' ';
1519 }
1520 dbgs() << '\n';
1521 });
1522
1523 assert(!Clusters.empty());
1524 SwitchWorkList WorkList;
1525 CaseClusterIt First = Clusters.begin();
1526 CaseClusterIt Last = Clusters.end() - 1;
1527 auto DefaultProb = getEdgeProbability(Src: SwitchMBB, Dst: DefaultMBB);
1528 WorkList.push_back(Elt: {.MBB: SwitchMBB, .FirstCluster: First, .LastCluster: Last, .GE: nullptr, .LT: nullptr, .DefaultProb: DefaultProb});
1529
1530 while (!WorkList.empty()) {
1531 SwitchWorkListItem W = WorkList.pop_back_val();
1532
1533 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
1534 // For optimized builds, lower large range as a balanced binary tree.
1535 if (NumClusters > 3 &&
1536 MF->getTarget().getOptLevel() != CodeGenOptLevel::None &&
1537 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
1538 splitWorkItem(WorkList, W, Cond: SI.getCondition(), SwitchMBB, MIB);
1539 continue;
1540 }
1541
1542 if (!lowerSwitchWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
1543 return false;
1544 }
1545 return true;
1546}
1547
1548void IRTranslatorImpl::splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
1549 const SwitchCG::SwitchWorkListItem &W,
1550 Value *Cond, MachineBasicBlock *SwitchMBB,
1551 MachineIRBuilder &MIB) {
1552 using namespace SwitchCG;
1553 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
1554 "Clusters not sorted?");
1555 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
1556
1557 auto [LastLeft, FirstRight, LeftProb, RightProb] =
1558 SL->computeSplitWorkItemInfo(W);
1559
1560 // Use the first element on the right as pivot since we will make less-than
1561 // comparisons against it.
1562 CaseClusterIt PivotCluster = FirstRight;
1563 assert(PivotCluster > W.FirstCluster);
1564 assert(PivotCluster <= W.LastCluster);
1565
1566 CaseClusterIt FirstLeft = W.FirstCluster;
1567 CaseClusterIt LastRight = W.LastCluster;
1568
1569 const ConstantInt *Pivot = PivotCluster->Low;
1570
1571 // New blocks will be inserted immediately after the current one.
1572 MachineFunction::iterator BBI(W.MBB);
1573 ++BBI;
1574
1575 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
1576 // we can branch to its destination directly if it's squeezed exactly in
1577 // between the known lower bound and Pivot - 1.
1578 MachineBasicBlock *LeftMBB;
1579 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
1580 FirstLeft->Low == W.GE &&
1581 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
1582 LeftMBB = FirstLeft->MBB;
1583 } else {
1584 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
1585 FuncInfo.MF->insert(MBBI: BBI, MBB: LeftMBB);
1586 WorkList.push_back(
1587 Elt: {.MBB: LeftMBB, .FirstCluster: FirstLeft, .LastCluster: LastLeft, .GE: W.GE, .LT: Pivot, .DefaultProb: W.DefaultProb / 2});
1588 }
1589
1590 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
1591 // single cluster, RHS.Low == Pivot, and we can branch to its destination
1592 // directly if RHS.High equals the current upper bound.
1593 MachineBasicBlock *RightMBB;
1594 if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT &&
1595 (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
1596 RightMBB = FirstRight->MBB;
1597 } else {
1598 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
1599 FuncInfo.MF->insert(MBBI: BBI, MBB: RightMBB);
1600 WorkList.push_back(
1601 Elt: {.MBB: RightMBB, .FirstCluster: FirstRight, .LastCluster: LastRight, .GE: Pivot, .LT: W.LT, .DefaultProb: W.DefaultProb / 2});
1602 }
1603
1604 // Create the CaseBlock record that will be used to lower the branch.
1605 CaseBlock CB(ICmpInst::Predicate::ICMP_SLT, false, Cond, Pivot, nullptr,
1606 LeftMBB, RightMBB, W.MBB, MIB.getDebugLoc(), LeftProb,
1607 RightProb);
1608
1609 if (W.MBB == SwitchMBB)
1610 emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
1611 else
1612 SL->SwitchCases.push_back(x: CB);
1613}
1614
1615void IRTranslatorImpl::emitJumpTable(SwitchCG::JumpTable &JT,
1616 MachineBasicBlock *MBB) {
1617 // Emit the code for the jump table
1618 assert(JT.Reg && "Should lower JT Header first!");
1619 MachineIRBuilder MIB(*MBB->getParent());
1620 MIB.setMBB(*MBB);
1621 MIB.setDebugLoc(CurBuilder->getDebugLoc());
1622
1623 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
1624 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1625
1626 auto Table = MIB.buildJumpTable(PtrTy, JTI: JT.JTI);
1627 MIB.buildBrJT(TablePtr: Table.getReg(Idx: 0), JTI: JT.JTI, IndexReg: JT.Reg);
1628}
1629
1630bool IRTranslatorImpl::emitJumpTableHeader(SwitchCG::JumpTable &JT,
1631 SwitchCG::JumpTableHeader &JTH,
1632 MachineBasicBlock *HeaderBB) {
1633 MachineIRBuilder MIB(*HeaderBB->getParent());
1634 MIB.setMBB(*HeaderBB);
1635 MIB.setDebugLoc(CurBuilder->getDebugLoc());
1636
1637 const Value &SValue = *JTH.SValue;
1638 // Subtract the lowest switch case value from the value being switched on.
1639 const LLT SwitchTy = getLLTForType(Ty&: *SValue.getType(), DL: *DL);
1640 Register SwitchOpReg = getOrCreateVReg(Val: SValue);
1641 auto FirstCst = MIB.buildConstant(Res: SwitchTy, Val: JTH.First);
1642 auto Sub = MIB.buildSub(Dst: {SwitchTy}, Src0: SwitchOpReg, Src1: FirstCst);
1643
1644 // This value may be smaller or larger than the target's pointer type, and
1645 // therefore require extension or truncating.
1646 auto *PtrIRTy = PointerType::getUnqual(C&: SValue.getContext());
1647 const LLT PtrScalarTy = LLT::integer(SizeInBits: DL->getTypeSizeInBits(Ty: PtrIRTy));
1648 auto Index = MIB.buildZExtOrTrunc(Res: PtrScalarTy, Op: Sub);
1649
1650 JT.Reg = Index.getReg(Idx: 0);
1651
1652 if (JTH.FallthroughUnreachable) {
1653 if (JT.MBB != HeaderBB->getNextNode())
1654 MIB.buildBr(Dest&: *JT.MBB);
1655 return true;
1656 }
1657
1658 // Emit the range check for the jump table, and branch to the default block
1659 // for the switch statement if the value being switched on exceeds the
1660 // largest case in the switch.
1661 auto Cst = getOrCreateVReg(
1662 Val: *ConstantInt::get(Ty: SValue.getType(), V: JTH.Last - JTH.First));
1663 auto Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_UGT, Res: LLT::integer(SizeInBits: 1), Op0: Sub, Op1: Cst);
1664
1665 auto BrCond = MIB.buildBrCond(Tst: Cmp.getReg(Idx: 0), Dest&: *JT.Default);
1666
1667 // Avoid emitting unnecessary branches to the next block.
1668 if (JT.MBB != HeaderBB->getNextNode())
1669 BrCond = MIB.buildBr(Dest&: *JT.MBB);
1670 return true;
1671}
1672
1673void IRTranslatorImpl::emitSwitchCase(SwitchCG::CaseBlock &CB,
1674 MachineBasicBlock *SwitchBB,
1675 MachineIRBuilder &MIB) {
1676 Register CondLHS = getOrCreateVReg(Val: *CB.CmpLHS);
1677 Register Cond;
1678 DebugLoc OldDbgLoc = MIB.getDebugLoc();
1679 MIB.setDebugLoc(CB.DbgLoc);
1680 MIB.setMBB(*CB.ThisBB);
1681
1682 if (CB.PredInfo.NoCmp) {
1683 // Branch or fall through to TrueBB.
1684 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
1685 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
1686 NewPred: CB.ThisBB);
1687 CB.ThisBB->normalizeSuccProbs();
1688 if (CB.TrueBB != CB.ThisBB->getNextNode())
1689 MIB.buildBr(Dest&: *CB.TrueBB);
1690 MIB.setDebugLoc(OldDbgLoc);
1691 return;
1692 }
1693
1694 const LLT i1Ty = LLT::integer(SizeInBits: 1);
1695 // Build the compare.
1696 if (!CB.CmpMHS) {
1697 const auto *CI = dyn_cast<ConstantInt>(Val: CB.CmpRHS);
1698 // For conditional branch lowering, we might try to do something silly like
1699 // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
1700 // just re-use the existing condition vreg.
1701 if (MRI->getType(Reg: CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
1702 CB.PredInfo.Pred == CmpInst::ICMP_EQ) {
1703 Cond = CondLHS;
1704 } else {
1705 Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
1706 if (CmpInst::isFPPredicate(P: CB.PredInfo.Pred))
1707 Cond =
1708 MIB.buildFCmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
1709 else
1710 Cond =
1711 MIB.buildICmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
1712 }
1713 } else {
1714 assert(CB.PredInfo.Pred == CmpInst::ICMP_SLE &&
1715 "Can only handle SLE ranges");
1716
1717 const APInt& Low = cast<ConstantInt>(Val: CB.CmpLHS)->getValue();
1718 const APInt& High = cast<ConstantInt>(Val: CB.CmpRHS)->getValue();
1719
1720 Register CmpOpReg = getOrCreateVReg(Val: *CB.CmpMHS);
1721 if (cast<ConstantInt>(Val: CB.CmpLHS)->isMinValue(IsSigned: true)) {
1722 Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
1723 Cond =
1724 MIB.buildICmp(Pred: CmpInst::ICMP_SLE, Res: i1Ty, Op0: CmpOpReg, Op1: CondRHS).getReg(Idx: 0);
1725 } else {
1726 const LLT CmpTy = MRI->getType(Reg: CmpOpReg);
1727 auto Sub = MIB.buildSub(Dst: {CmpTy}, Src0: CmpOpReg, Src1: CondLHS);
1728 auto Diff = MIB.buildConstant(Res: CmpTy, Val: High - Low);
1729 Cond = MIB.buildICmp(Pred: CmpInst::ICMP_ULE, Res: i1Ty, Op0: Sub, Op1: Diff).getReg(Idx: 0);
1730 }
1731 }
1732
1733 // Update successor info
1734 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
1735
1736 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
1737 NewPred: CB.ThisBB);
1738
1739 // TrueBB and FalseBB are always different unless the incoming IR is
1740 // degenerate. This only happens when running llc on weird IR.
1741 if (CB.TrueBB != CB.FalseBB)
1742 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.FalseBB, Prob: CB.FalseProb);
1743 CB.ThisBB->normalizeSuccProbs();
1744
1745 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
1746 NewPred: CB.ThisBB);
1747
1748 MIB.buildBrCond(Tst: Cond, Dest&: *CB.TrueBB);
1749 MIB.buildBr(Dest&: *CB.FalseBB);
1750 MIB.setDebugLoc(OldDbgLoc);
1751}
1752
1753bool IRTranslatorImpl::lowerJumpTableWorkItem(
1754 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
1755 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1756 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
1757 BranchProbability UnhandledProbs, SwitchCG::CaseClusterIt I,
1758 MachineBasicBlock *Fallthrough, bool FallthroughUnreachable) {
1759 using namespace SwitchCG;
1760 MachineFunction *CurMF = SwitchMBB->getParent();
1761 // FIXME: Optimize away range check based on pivot comparisons.
1762 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
1763 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
1764 BranchProbability DefaultProb = W.DefaultProb;
1765
1766 // The jump block hasn't been inserted yet; insert it here.
1767 MachineBasicBlock *JumpMBB = JT->MBB;
1768 CurMF->insert(MBBI: BBI, MBB: JumpMBB);
1769
1770 // Since the jump table block is separate from the switch block, we need
1771 // to keep track of it as a machine predecessor to the default block,
1772 // otherwise we lose the phi edges.
1773 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1774 NewPred: CurMBB);
1775 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1776 NewPred: JumpMBB);
1777
1778 auto JumpProb = I->Prob;
1779 auto FallthroughProb = UnhandledProbs;
1780
1781 // If the default statement is a target of the jump table, we evenly
1782 // distribute the default probability to successors of CurMBB. Also
1783 // update the probability on the edge from JumpMBB to Fallthrough.
1784 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
1785 SE = JumpMBB->succ_end();
1786 SI != SE; ++SI) {
1787 if (*SI == DefaultMBB) {
1788 JumpProb += DefaultProb / 2;
1789 FallthroughProb -= DefaultProb / 2;
1790 JumpMBB->setSuccProbability(I: SI, Prob: DefaultProb / 2);
1791 JumpMBB->normalizeSuccProbs();
1792 } else {
1793 // Also record edges from the jump table block to it's successors.
1794 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
1795 NewPred: JumpMBB);
1796 }
1797 }
1798
1799 if (FallthroughUnreachable)
1800 JTH->FallthroughUnreachable = true;
1801
1802 if (!JTH->FallthroughUnreachable)
1803 addSuccessorWithProb(Src: CurMBB, Dst: Fallthrough, Prob: FallthroughProb);
1804 addSuccessorWithProb(Src: CurMBB, Dst: JumpMBB, Prob: JumpProb);
1805 CurMBB->normalizeSuccProbs();
1806
1807 // The jump table header will be inserted in our current block, do the
1808 // range check, and fall through to our fallthrough block.
1809 JTH->HeaderBB = CurMBB;
1810 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
1811
1812 // If we're in the right place, emit the jump table header right now.
1813 if (CurMBB == SwitchMBB) {
1814 if (!emitJumpTableHeader(JT&: *JT, JTH&: *JTH, HeaderBB: CurMBB))
1815 return false;
1816 JTH->Emitted = true;
1817 }
1818 return true;
1819}
1820bool IRTranslatorImpl::lowerSwitchRangeWorkItem(
1821 SwitchCG::CaseClusterIt I, Value *Cond, MachineBasicBlock *Fallthrough,
1822 bool FallthroughUnreachable, BranchProbability UnhandledProbs,
1823 MachineBasicBlock *CurMBB, MachineIRBuilder &MIB,
1824 MachineBasicBlock *SwitchMBB) {
1825 using namespace SwitchCG;
1826 const Value *RHS, *LHS, *MHS;
1827 CmpInst::Predicate Pred;
1828 if (I->Low == I->High) {
1829 // Check Cond == I->Low.
1830 Pred = CmpInst::ICMP_EQ;
1831 LHS = Cond;
1832 RHS = I->Low;
1833 MHS = nullptr;
1834 } else {
1835 // Check I->Low <= Cond <= I->High.
1836 Pred = CmpInst::ICMP_SLE;
1837 LHS = I->Low;
1838 MHS = Cond;
1839 RHS = I->High;
1840 }
1841
1842 // If Fallthrough is unreachable, fold away the comparison.
1843 // The false probability is the sum of all unhandled cases.
1844 CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
1845 CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
1846
1847 emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
1848 return true;
1849}
1850
1851void IRTranslatorImpl::emitBitTestHeader(SwitchCG::BitTestBlock &B,
1852 MachineBasicBlock *SwitchBB) {
1853 MachineIRBuilder &MIB = *CurBuilder;
1854 MIB.setMBB(*SwitchBB);
1855
1856 // Subtract the minimum value.
1857 Register SwitchOpReg = getOrCreateVReg(Val: *B.SValue);
1858
1859 LLT SwitchOpTy = MRI->getType(Reg: SwitchOpReg);
1860 Register MinValReg = MIB.buildConstant(Res: SwitchOpTy, Val: B.First).getReg(Idx: 0);
1861 auto RangeSub = MIB.buildSub(Dst: SwitchOpTy, Src0: SwitchOpReg, Src1: MinValReg);
1862
1863 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
1864 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1865
1866 LLT MaskTy = SwitchOpTy;
1867 if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
1868 !llvm::has_single_bit<uint32_t>(Value: MaskTy.getSizeInBits()))
1869 MaskTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
1870 else {
1871 // Ensure that the type will fit the mask value.
1872 for (const SwitchCG::BitTestCase &Case : B.Cases) {
1873 if (!isUIntN(N: SwitchOpTy.getSizeInBits(), x: Case.Mask)) {
1874 // Switch table case range are encoded into series of masks.
1875 // Just use pointer type, it's guaranteed to fit.
1876 MaskTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
1877 break;
1878 }
1879 }
1880 }
1881 Register SubReg = RangeSub.getReg(Idx: 0);
1882 if (SwitchOpTy != MaskTy)
1883 SubReg = MIB.buildZExtOrTrunc(Res: MaskTy, Op: SubReg).getReg(Idx: 0);
1884
1885 B.RegVT = getMVTForLLT(Ty: MaskTy);
1886 B.Reg = SubReg;
1887
1888 MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1889
1890 if (!B.FallthroughUnreachable)
1891 addSuccessorWithProb(Src: SwitchBB, Dst: B.Default, Prob: B.DefaultProb);
1892 addSuccessorWithProb(Src: SwitchBB, Dst: MBB, Prob: B.Prob);
1893
1894 SwitchBB->normalizeSuccProbs();
1895
1896 if (!B.FallthroughUnreachable) {
1897 // Conditional branch to the default block.
1898 auto RangeCst = MIB.buildConstant(Res: SwitchOpTy, Val: B.Range);
1899 auto RangeCmp = MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: LLT::integer(SizeInBits: 1),
1900 Op0: RangeSub, Op1: RangeCst);
1901 MIB.buildBrCond(Tst: RangeCmp, Dest&: *B.Default);
1902 }
1903
1904 // Avoid emitting unnecessary branches to the next block.
1905 if (MBB != SwitchBB->getNextNode())
1906 MIB.buildBr(Dest&: *MBB);
1907}
1908
1909void IRTranslatorImpl::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1910 MachineBasicBlock *NextMBB,
1911 BranchProbability BranchProbToNext,
1912 Register Reg, SwitchCG::BitTestCase &B,
1913 MachineBasicBlock *SwitchBB) {
1914 MachineIRBuilder &MIB = *CurBuilder;
1915 MIB.setMBB(*SwitchBB);
1916
1917 LLT SwitchTy = getLLTForMVT(Ty: BB.RegVT);
1918 Register Cmp;
1919 unsigned PopCount = llvm::popcount(Value: B.Mask);
1920 if (PopCount == 1) {
1921 // Testing for a single bit; just compare the shift count with what it
1922 // would need to be to shift a 1 bit in that position.
1923 auto MaskTrailingZeros =
1924 MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_zero(Val: B.Mask));
1925 Cmp = MIB.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: LLT::integer(SizeInBits: 1), Op0: Reg,
1926 Op1: MaskTrailingZeros)
1927 .getReg(Idx: 0);
1928 } else if (PopCount == BB.Range) {
1929 // There is only one zero bit in the range, test for it directly.
1930 auto MaskTrailingOnes =
1931 MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_one(Value: B.Mask));
1932 Cmp =
1933 MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::integer(SizeInBits: 1), Op0: Reg, Op1: MaskTrailingOnes)
1934 .getReg(Idx: 0);
1935 } else {
1936 // Make desired shift.
1937 auto CstOne = MIB.buildConstant(Res: SwitchTy, Val: 1);
1938 auto SwitchVal = MIB.buildShl(Dst: SwitchTy, Src0: CstOne, Src1: Reg);
1939
1940 // Emit bit tests and jumps.
1941 auto CstMask = MIB.buildConstant(Res: SwitchTy, Val: B.Mask);
1942 auto AndOp = MIB.buildAnd(Dst: SwitchTy, Src0: SwitchVal, Src1: CstMask);
1943 auto CstZero = MIB.buildConstant(Res: SwitchTy, Val: 0);
1944 Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::integer(SizeInBits: 1), Op0: AndOp, Op1: CstZero)
1945 .getReg(Idx: 0);
1946 }
1947
1948 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1949 addSuccessorWithProb(Src: SwitchBB, Dst: B.TargetBB, Prob: B.ExtraProb);
1950 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1951 addSuccessorWithProb(Src: SwitchBB, Dst: NextMBB, Prob: BranchProbToNext);
1952 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1953 // one as they are relative probabilities (and thus work more like weights),
1954 // and hence we need to normalize them to let the sum of them become one.
1955 SwitchBB->normalizeSuccProbs();
1956
1957 // Record the fact that the IR edge from the header to the bit test target
1958 // will go through our new block. Neeeded for PHIs to have nodes added.
1959 addMachineCFGPred(Edge: {BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1960 NewPred: SwitchBB);
1961
1962 MIB.buildBrCond(Tst: Cmp, Dest&: *B.TargetBB);
1963
1964 // Avoid emitting unnecessary branches to the next block.
1965 if (NextMBB != SwitchBB->getNextNode())
1966 MIB.buildBr(Dest&: *NextMBB);
1967}
1968
1969bool IRTranslatorImpl::lowerBitTestWorkItem(
1970 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
1971 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1972 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
1973 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1974 SwitchCG::CaseClusterIt I, MachineBasicBlock *Fallthrough,
1975 bool FallthroughUnreachable) {
1976 using namespace SwitchCG;
1977 MachineFunction *CurMF = SwitchMBB->getParent();
1978 // FIXME: Optimize away range check based on pivot comparisons.
1979 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
1980 // The bit test blocks haven't been inserted yet; insert them here.
1981 for (BitTestCase &BTC : BTB->Cases)
1982 CurMF->insert(MBBI: BBI, MBB: BTC.ThisBB);
1983
1984 // Fill in fields of the BitTestBlock.
1985 BTB->Parent = CurMBB;
1986 BTB->Default = Fallthrough;
1987
1988 BTB->DefaultProb = UnhandledProbs;
1989 // If the cases in bit test don't form a contiguous range, we evenly
1990 // distribute the probability on the edge to Fallthrough to two
1991 // successors of CurMBB.
1992 if (!BTB->ContiguousRange) {
1993 BTB->Prob += DefaultProb / 2;
1994 BTB->DefaultProb -= DefaultProb / 2;
1995 }
1996
1997 if (FallthroughUnreachable)
1998 BTB->FallthroughUnreachable = true;
1999
2000 // If we're in the right place, emit the bit test header right now.
2001 if (CurMBB == SwitchMBB) {
2002 emitBitTestHeader(B&: *BTB, SwitchBB: SwitchMBB);
2003 BTB->Emitted = true;
2004 }
2005 return true;
2006}
2007
2008bool IRTranslatorImpl::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
2009 Value *Cond,
2010 MachineBasicBlock *SwitchMBB,
2011 MachineBasicBlock *DefaultMBB,
2012 MachineIRBuilder &MIB) {
2013 using namespace SwitchCG;
2014 MachineFunction *CurMF = FuncInfo.MF;
2015 MachineBasicBlock *NextMBB = nullptr;
2016 MachineFunction::iterator BBI(W.MBB);
2017 if (++BBI != FuncInfo.MF->end())
2018 NextMBB = &*BBI;
2019
2020 if (EnableOpts) {
2021 // Here, we order cases by probability so the most likely case will be
2022 // checked first. However, two clusters can have the same probability in
2023 // which case their relative ordering is non-deterministic. So we use Low
2024 // as a tie-breaker as clusters are guaranteed to never overlap.
2025 llvm::sort(Start: W.FirstCluster, End: W.LastCluster + 1,
2026 Comp: [](const CaseCluster &a, const CaseCluster &b) {
2027 return a.Prob != b.Prob
2028 ? a.Prob > b.Prob
2029 : a.Low->getValue().slt(RHS: b.Low->getValue());
2030 });
2031
2032 // Rearrange the case blocks so that the last one falls through if possible
2033 // without changing the order of probabilities.
2034 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
2035 --I;
2036 if (I->Prob > W.LastCluster->Prob)
2037 break;
2038 if (I->Kind == CC_Range && I->MBB == NextMBB) {
2039 std::swap(a&: *I, b&: *W.LastCluster);
2040 break;
2041 }
2042 }
2043 }
2044
2045 // Compute total probability.
2046 BranchProbability DefaultProb = W.DefaultProb;
2047 BranchProbability UnhandledProbs = DefaultProb;
2048 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
2049 UnhandledProbs += I->Prob;
2050
2051 MachineBasicBlock *CurMBB = W.MBB;
2052 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
2053 bool FallthroughUnreachable = false;
2054 MachineBasicBlock *Fallthrough;
2055 if (I == W.LastCluster) {
2056 // For the last cluster, fall through to the default destination.
2057 Fallthrough = DefaultMBB;
2058 FallthroughUnreachable = isa<UnreachableInst>(
2059 Val: DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
2060 } else {
2061 Fallthrough = CurMF->CreateMachineBasicBlock(BB: CurMBB->getBasicBlock());
2062 CurMF->insert(MBBI: BBI, MBB: Fallthrough);
2063 }
2064 UnhandledProbs -= I->Prob;
2065
2066 switch (I->Kind) {
2067 case CC_BitTests: {
2068 if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
2069 DefaultProb, UnhandledProbs, I, Fallthrough,
2070 FallthroughUnreachable)) {
2071 LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
2072 return false;
2073 }
2074 break;
2075 }
2076
2077 case CC_JumpTable: {
2078 if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
2079 UnhandledProbs, I, Fallthrough,
2080 FallthroughUnreachable)) {
2081 LLVM_DEBUG(dbgs() << "Failed to lower jump table");
2082 return false;
2083 }
2084 break;
2085 }
2086 case CC_Range: {
2087 if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
2088 FallthroughUnreachable, UnhandledProbs,
2089 CurMBB, MIB, SwitchMBB)) {
2090 LLVM_DEBUG(dbgs() << "Failed to lower switch range");
2091 return false;
2092 }
2093 break;
2094 }
2095 }
2096 CurMBB = Fallthrough;
2097 }
2098
2099 return true;
2100}
2101
2102bool IRTranslatorImpl::translateIndirectBr(const User &U,
2103 MachineIRBuilder &MIRBuilder) {
2104 const IndirectBrInst &BrInst = cast<IndirectBrInst>(Val: U);
2105
2106 const Register Tgt = getOrCreateVReg(Val: *BrInst.getAddress());
2107 MIRBuilder.buildBrIndirect(Tgt);
2108
2109 // Link successors.
2110 SmallPtrSet<const BasicBlock *, 32> AddedSuccessors;
2111 MachineBasicBlock &CurBB = MIRBuilder.getMBB();
2112 for (const BasicBlock *Succ : successors(I: &BrInst)) {
2113 // It's legal for indirectbr instructions to have duplicate blocks in the
2114 // destination list. We don't allow this in MIR. Skip anything that's
2115 // already a successor.
2116 if (!AddedSuccessors.insert(Ptr: Succ).second)
2117 continue;
2118 CurBB.addSuccessor(Succ: &getMBB(BB: *Succ));
2119 }
2120
2121 return true;
2122}
2123
2124static bool isSwiftError(const Value *V) {
2125 if (auto Arg = dyn_cast<Argument>(Val: V))
2126 return Arg->hasSwiftErrorAttr();
2127 if (auto AI = dyn_cast<AllocaInst>(Val: V))
2128 return AI->isSwiftError();
2129 return false;
2130}
2131
2132bool IRTranslatorImpl::translateLoad(const User &U,
2133 MachineIRBuilder &MIRBuilder) {
2134 const LoadInst &LI = cast<LoadInst>(Val: U);
2135 TypeSize StoreSize = DL->getTypeStoreSize(Ty: LI.getType());
2136 if (StoreSize.isZero())
2137 return true;
2138
2139 ArrayRef<Register> Regs = getOrCreateVRegs(Val: LI);
2140 Register Base = getOrCreateVReg(Val: *LI.getPointerOperand());
2141 AAMDNodes AAInfo = LI.getAAMetadata();
2142
2143 const Value *Ptr = LI.getPointerOperand();
2144
2145 if (CLI->supportSwiftError() && isSwiftError(V: Ptr)) {
2146 assert(Regs.size() == 1 && "swifterror should be single pointer");
2147 Register VReg =
2148 SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);
2149 MIRBuilder.buildCopy(Res: Regs[0], Op: VReg);
2150 return true;
2151 }
2152
2153 MachineMemOperand::Flags Flags =
2154 TLI->getLoadMemOperandFlags(LI, DL: *DL, AC, LibInfo, OptLevel);
2155 if (AA && !(Flags & MachineMemOperand::MOInvariant)) {
2156 if (AA->pointsToConstantMemory(
2157 Loc: MemoryLocation(Ptr, LocationSize::precise(Value: StoreSize), AAInfo))) {
2158 Flags |= MachineMemOperand::MOInvariant;
2159 }
2160 }
2161
2162 // Fast-path the common single-register load.
2163 if (Regs.size() == 1) {
2164 auto *MMO = MF->getMachineMemOperand(
2165 PtrInfo: MachinePointerInfo(LI.getPointerOperand()), F: Flags,
2166 MemTy: MRI->getType(Reg: Regs[0]), BaseAlignment: getMemOpAlign(I: LI),
2167 Metadata: MMOMetadata(AAInfo, LI.getMetadata(KindID: LLVMContext::MD_range)),
2168 SSID: LI.getSyncScopeID(), Ordering: LI.getOrdering());
2169 MIRBuilder.buildLoad(Res: Regs[0], Addr: Base, MMO&: *MMO);
2170 return true;
2171 }
2172
2173 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: LI);
2174 Type *OffsetIRTy = DL->getIndexType(PtrTy: Ptr->getType());
2175 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2176 for (unsigned i = 0; i < Regs.size(); ++i) {
2177 Register Addr;
2178 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i]);
2179
2180 MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i]);
2181 Align BaseAlign = getMemOpAlign(I: LI);
2182 auto *MMO =
2183 MF->getMachineMemOperand(PtrInfo: Ptr, F: Flags, MemTy: MRI->getType(Reg: Regs[i]),
2184 BaseAlignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]), Metadata: AAInfo,
2185 SSID: LI.getSyncScopeID(), Ordering: LI.getOrdering());
2186 MIRBuilder.buildLoad(Res: Regs[i], Addr, MMO&: *MMO);
2187 }
2188
2189 return true;
2190}
2191
2192bool IRTranslatorImpl::translateStore(const User &U,
2193 MachineIRBuilder &MIRBuilder) {
2194 const StoreInst &SI = cast<StoreInst>(Val: U);
2195 if (DL->getTypeStoreSize(Ty: SI.getValueOperand()->getType()).isZero())
2196 return true;
2197
2198 ArrayRef<Register> Vals = getOrCreateVRegs(Val: *SI.getValueOperand());
2199 Register Base = getOrCreateVReg(Val: *SI.getPointerOperand());
2200
2201 if (CLI->supportSwiftError() && isSwiftError(V: SI.getPointerOperand())) {
2202 assert(Vals.size() == 1 && "swifterror should be single pointer");
2203
2204 Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
2205 SI.getPointerOperand());
2206 MIRBuilder.buildCopy(Res: VReg, Op: Vals[0]);
2207 return true;
2208 }
2209
2210 MachineMemOperand::Flags Flags = TLI->getStoreMemOperandFlags(SI, DL: *DL);
2211 // Fast-path the common single-register store.
2212 if (Vals.size() == 1) {
2213 auto *MMO = MF->getMachineMemOperand(
2214 PtrInfo: MachinePointerInfo(SI.getPointerOperand()), F: Flags,
2215 MemTy: MRI->getType(Reg: Vals[0]), BaseAlignment: getMemOpAlign(I: SI), Metadata: SI.getAAMetadata(),
2216 SSID: SI.getSyncScopeID(), Ordering: SI.getOrdering());
2217 MIRBuilder.buildStore(Val: Vals[0], Addr: Base, MMO&: *MMO);
2218 return true;
2219 }
2220
2221 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *SI.getValueOperand());
2222 Type *OffsetIRTy = DL->getIndexType(PtrTy: SI.getPointerOperandType());
2223 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2224 for (unsigned i = 0; i < Vals.size(); ++i) {
2225 Register Addr;
2226 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i]);
2227
2228 MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i]);
2229 Align BaseAlign = getMemOpAlign(I: SI);
2230 auto *MMO = MF->getMachineMemOperand(PtrInfo: Ptr, F: Flags, MemTy: MRI->getType(Reg: Vals[i]),
2231 BaseAlignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]),
2232 Metadata: SI.getAAMetadata(),
2233 SSID: SI.getSyncScopeID(), Ordering: SI.getOrdering());
2234 MIRBuilder.buildStore(Val: Vals[i], Addr, MMO&: *MMO);
2235 }
2236 return true;
2237}
2238
2239static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
2240 const Value *Src = U.getOperand(i: 0);
2241 Type *Int32Ty = Type::getInt32Ty(C&: U.getContext());
2242
2243 // getIndexedOffsetInType is designed for GEPs, so the first index is the
2244 // usual array element rather than looking into the actual aggregate.
2245 SmallVector<Value *, 1> Indices;
2246 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: 0));
2247
2248 if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: &U)) {
2249 for (auto Idx : EVI->indices())
2250 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
2251 } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(Val: &U)) {
2252 for (auto Idx : IVI->indices())
2253 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
2254 } else {
2255 llvm::append_range(C&: Indices, R: drop_begin(RangeOrContainer: U.operands()));
2256 }
2257
2258 return static_cast<uint64_t>(
2259 DL.getIndexedOffsetInType(ElemTy: Src->getType(), Indices));
2260}
2261
2262bool IRTranslatorImpl::translateExtractValue(const User &U,
2263 MachineIRBuilder &MIRBuilder) {
2264 const Value *Src = U.getOperand(i: 0);
2265 uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
2266 ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
2267 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *Src);
2268 unsigned Idx = llvm::lower_bound(Range&: Offsets, Value&: Offset) - Offsets.begin();
2269 auto &DstRegs = allocateVRegs(Val: U);
2270
2271 for (unsigned i = 0; i < DstRegs.size(); ++i)
2272 DstRegs[i] = SrcRegs[Idx++];
2273
2274 return true;
2275}
2276
2277bool IRTranslatorImpl::translateInsertValue(const User &U,
2278 MachineIRBuilder &MIRBuilder) {
2279 const Value *Src = U.getOperand(i: 0);
2280 uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
2281 auto &DstRegs = allocateVRegs(Val: U);
2282 ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(V: U);
2283 ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
2284 ArrayRef<Register> InsertedRegs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
2285 auto *InsertedIt = InsertedRegs.begin();
2286
2287 for (unsigned i = 0; i < DstRegs.size(); ++i) {
2288 if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
2289 DstRegs[i] = *InsertedIt++;
2290 else
2291 DstRegs[i] = SrcRegs[i];
2292 }
2293
2294 return true;
2295}
2296
2297bool IRTranslatorImpl::translateSelect(const User &U,
2298 MachineIRBuilder &MIRBuilder) {
2299 Register Tst = getOrCreateVReg(Val: *U.getOperand(i: 0));
2300 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: U);
2301 ArrayRef<Register> Op0Regs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
2302 ArrayRef<Register> Op1Regs = getOrCreateVRegs(Val: *U.getOperand(i: 2));
2303
2304 uint32_t Flags = 0;
2305 if (const SelectInst *SI = dyn_cast<SelectInst>(Val: &U))
2306 Flags = MachineInstr::copyFlagsFromInstruction(I: *SI);
2307
2308 for (unsigned i = 0; i < ResRegs.size(); ++i) {
2309 MIRBuilder.buildSelect(Res: ResRegs[i], Tst, Op0: Op0Regs[i], Op1: Op1Regs[i], Flags);
2310 }
2311
2312 return true;
2313}
2314
2315bool IRTranslatorImpl::translateCopy(const User &U, const Value &V,
2316 MachineIRBuilder &MIRBuilder) {
2317 return translateCopy(U, Src: getOrCreateVReg(Val: V), MIRBuilder);
2318}
2319
2320bool IRTranslatorImpl::translateCopy(const User &U, Register Src,
2321 MachineIRBuilder &MIRBuilder) {
2322 auto &Regs = *VMap.getVRegs(V: U);
2323 if (Regs.empty()) {
2324 Regs.push_back(Elt: Src);
2325 } else {
2326 // If we already assigned a vreg for this instruction, we can't change that.
2327 // Emit a copy to satisfy the users we already emitted.
2328 MIRBuilder.buildCopy(Res: Regs[0], Op: Src);
2329 }
2330 return true;
2331}
2332
2333bool IRTranslatorImpl::translateBitCast(const User &U,
2334 MachineIRBuilder &MIRBuilder) {
2335 Type *SrcTy = U.getOperand(i: 0)->getType();
2336 Type *DstTy = U.getType();
2337
2338 // If we're bitcasting to the source type, we can reuse the source vreg.
2339 if (getLLTForType(Ty&: *SrcTy, DL: *DL) == getLLTForType(Ty&: *DstTy, DL: *DL)) {
2340 // If the source is a ConstantInt then it was probably created by
2341 // ConstantHoisting and we should leave it alone.
2342 if (isa<ConstantInt>(Val: U.getOperand(i: 0)))
2343 return translateCast(Opcode: TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
2344 MIRBuilder);
2345 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
2346 }
2347
2348 // Only the scalar byte<->ptr crossing is redirected to G_INTTOPTR/G_PTRTOINT,
2349 // which is the well-typed MIR shape for that boundary. Vector byte<->ptr
2350 // (e.g. <N x b32> -> ptr produced by mixed-type load coalescing) and other
2351 // legacy ptr/non-ptr IR bitcasts (AMDGPU iN<->p3 kernarg packing, etc.)
2352 // keep their historical G_BITCAST lowering — G_INTTOPTR has no vector-src
2353 // -> scalar-ptr form, and downstream passes already handle G_BITCAST.
2354 if (DstTy->isPointerTy() && SrcTy->isByteTy())
2355 return translateCast(Opcode: TargetOpcode::G_INTTOPTR, U, MIRBuilder);
2356 if (SrcTy->isPointerTy() && DstTy->isByteTy())
2357 return translateCast(Opcode: TargetOpcode::G_PTRTOINT, U, MIRBuilder);
2358
2359 return translateCast(Opcode: TargetOpcode::G_BITCAST, U, MIRBuilder);
2360}
2361
2362bool IRTranslatorImpl::translateCast(unsigned Opcode, const User &U,
2363 MachineIRBuilder &MIRBuilder) {
2364 if (!mayTranslateUserTypes(U))
2365 return false;
2366
2367 uint32_t Flags = 0;
2368 if (const Instruction *I = dyn_cast<Instruction>(Val: &U))
2369 Flags = MachineInstr::copyFlagsFromInstruction(I: *I);
2370
2371 Register Op = getOrCreateVReg(Val: *U.getOperand(i: 0));
2372 Register Res = getOrCreateVReg(Val: U);
2373 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op}, Flags);
2374 return true;
2375}
2376
2377bool IRTranslatorImpl::translateGetElementPtr(const User &U,
2378 MachineIRBuilder &MIRBuilder) {
2379 Value &Op0 = *U.getOperand(i: 0);
2380 Register BaseReg = getOrCreateVReg(Val: Op0);
2381 Type *PtrIRTy = Op0.getType();
2382 LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
2383 Type *OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
2384 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2385
2386 uint32_t PtrAddFlags = 0;
2387 // Each PtrAdd generated to implement the GEP inherits its nuw, nusw, inbounds
2388 // flags.
2389 if (const Instruction *I = dyn_cast<Instruction>(Val: &U))
2390 PtrAddFlags = MachineInstr::copyFlagsFromInstruction(I: *I);
2391
2392 auto PtrAddFlagsWithConst = [&](int64_t Offset) {
2393 // For nusw/inbounds GEP with an offset that is nonnegative when interpreted
2394 // as signed, assume there is no unsigned overflow.
2395 if (Offset >= 0 && (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap))
2396 return PtrAddFlags | MachineInstr::MIFlag::NoUWrap;
2397 return PtrAddFlags;
2398 };
2399
2400 // Normalize Vector GEP - all scalar operands should be converted to the
2401 // splat vector.
2402 unsigned VectorWidth = 0;
2403
2404 // True if we should use a splat vector; using VectorWidth alone is not
2405 // sufficient.
2406 bool WantSplatVector = false;
2407 if (auto *VT = dyn_cast<VectorType>(Val: U.getType())) {
2408 VectorWidth = cast<FixedVectorType>(Val: VT)->getNumElements();
2409 // We don't produce 1 x N vectors; those are treated as scalars.
2410 WantSplatVector = VectorWidth > 1;
2411 }
2412
2413 if (cast<GEPOperator>(Val: U).hasAllZeroIndices())
2414 return translateCopy(U, Src: BaseReg, MIRBuilder);
2415
2416 // We might need to splat the base pointer into a vector if the offsets
2417 // are vectors.
2418 if (WantSplatVector && !PtrTy.isVector()) {
2419 BaseReg = MIRBuilder
2420 .buildSplatBuildVector(Res: LLT::fixed_vector(NumElements: VectorWidth, ScalarTy: PtrTy),
2421 Src: BaseReg)
2422 .getReg(Idx: 0);
2423 PtrIRTy = FixedVectorType::get(ElementType: PtrIRTy, NumElts: VectorWidth);
2424 PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
2425 OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
2426 OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2427 }
2428
2429 int64_t Offset = 0;
2430 for (gep_type_iterator GTI = gep_type_begin(GEP: &U), E = gep_type_end(GEP: &U);
2431 GTI != E; ++GTI) {
2432 const Value *Idx = GTI.getOperand();
2433 if (StructType *StTy = GTI.getStructTypeOrNull()) {
2434 unsigned Field = cast<Constant>(Val: Idx)->getUniqueInteger().getZExtValue();
2435 Offset += DL->getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
2436 continue;
2437 } else {
2438 uint64_t ElementSize = GTI.getSequentialElementStride(DL: *DL);
2439
2440 // If this is a scalar constant or a splat vector of constants,
2441 // handle it quickly.
2442 if (const auto *CI = dyn_cast<ConstantInt>(Val: Idx)) {
2443 if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {
2444 Offset += ElementSize * *Val;
2445 continue;
2446 }
2447 }
2448
2449 if (Offset != 0) {
2450 auto OffsetMIB = MIRBuilder.buildConstant(Res: {OffsetTy}, Val: Offset);
2451 BaseReg = MIRBuilder
2452 .buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0),
2453 Flags: PtrAddFlagsWithConst(Offset))
2454 .getReg(Idx: 0);
2455 Offset = 0;
2456 }
2457
2458 Register IdxReg = getOrCreateVReg(Val: *Idx);
2459 LLT IdxTy = MRI->getType(Reg: IdxReg);
2460 if (IdxTy != OffsetTy) {
2461 if (!IdxTy.isVector() && WantSplatVector) {
2462 IdxReg = MIRBuilder
2463 .buildSplatBuildVector(Res: OffsetTy.changeElementType(NewEltTy: IdxTy),
2464 Src: IdxReg)
2465 .getReg(Idx: 0);
2466 }
2467
2468 IdxReg = MIRBuilder.buildSExtOrTrunc(Res: OffsetTy, Op: IdxReg).getReg(Idx: 0);
2469 }
2470
2471 // N = N + Idx * ElementSize;
2472 // Avoid doing it for ElementSize of 1.
2473 Register GepOffsetReg;
2474 if (ElementSize != 1) {
2475 auto ElementSizeMIB = MIRBuilder.buildConstant(
2476 Res: getLLTForType(Ty&: *OffsetIRTy, DL: *DL), Val: ElementSize);
2477
2478 // The multiplication is NUW if the GEP is NUW and NSW if the GEP is
2479 // NUSW.
2480 uint32_t ScaleFlags = PtrAddFlags & MachineInstr::MIFlag::NoUWrap;
2481 if (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap)
2482 ScaleFlags |= MachineInstr::MIFlag::NoSWrap;
2483
2484 GepOffsetReg =
2485 MIRBuilder.buildMul(Dst: OffsetTy, Src0: IdxReg, Src1: ElementSizeMIB, Flags: ScaleFlags)
2486 .getReg(Idx: 0);
2487 } else {
2488 GepOffsetReg = IdxReg;
2489 }
2490
2491 BaseReg =
2492 MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: GepOffsetReg, Flags: PtrAddFlags)
2493 .getReg(Idx: 0);
2494 }
2495 }
2496
2497 if (Offset != 0) {
2498 auto OffsetMIB =
2499 MIRBuilder.buildConstant(Res: OffsetTy, Val: Offset);
2500
2501 MIRBuilder.buildPtrAdd(Res: getOrCreateVReg(Val: U), Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0),
2502 Flags: PtrAddFlagsWithConst(Offset));
2503 return true;
2504 }
2505
2506 return translateCopy(U, Src: BaseReg, MIRBuilder);
2507}
2508
2509bool IRTranslatorImpl::translateMemFunc(const CallInst &CI,
2510 MachineIRBuilder &MIRBuilder,
2511 unsigned Opcode) {
2512 const Value *SrcPtr = CI.getArgOperand(i: 1);
2513 // If the source is undef, then just emit a nop.
2514 if (isa<UndefValue>(Val: SrcPtr))
2515 return true;
2516
2517 SmallVector<Register, 3> SrcRegs;
2518
2519 unsigned MinPtrSize = UINT_MAX;
2520 for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(x: AI) != AE; ++AI) {
2521 Register SrcReg = getOrCreateVReg(Val: **AI);
2522 LLT SrcTy = MRI->getType(Reg: SrcReg);
2523 if (SrcTy.isPointer())
2524 MinPtrSize = std::min<unsigned>(a: SrcTy.getSizeInBits(), b: MinPtrSize);
2525 SrcRegs.push_back(Elt: SrcReg);
2526 }
2527
2528 LLT SizeTy = LLT::integer(SizeInBits: MinPtrSize);
2529
2530 // The size operand should be the minimum of the pointer sizes.
2531 Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
2532 if (MRI->getType(Reg: SizeOpReg) != SizeTy)
2533 SizeOpReg = MIRBuilder.buildZExtOrTrunc(Res: SizeTy, Op: SizeOpReg).getReg(Idx: 0);
2534
2535 auto ICall = MIRBuilder.buildInstr(Opcode);
2536 for (Register SrcReg : SrcRegs)
2537 ICall.addUse(RegNo: SrcReg);
2538
2539 Align DstAlign;
2540 Align SrcAlign;
2541 unsigned IsVol =
2542 cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1))->getZExtValue();
2543
2544 ConstantInt *CopySize = nullptr;
2545
2546 if (auto *MCI = dyn_cast<MemCpyInst>(Val: &CI)) {
2547 DstAlign = MCI->getDestAlign().valueOrOne();
2548 SrcAlign = MCI->getSourceAlign().valueOrOne();
2549 CopySize = dyn_cast<ConstantInt>(Val: MCI->getArgOperand(i: 2));
2550 } else if (auto *MMI = dyn_cast<MemMoveInst>(Val: &CI)) {
2551 DstAlign = MMI->getDestAlign().valueOrOne();
2552 SrcAlign = MMI->getSourceAlign().valueOrOne();
2553 CopySize = dyn_cast<ConstantInt>(Val: MMI->getArgOperand(i: 2));
2554 } else {
2555 auto *MSI = cast<MemSetInst>(Val: &CI);
2556 DstAlign = MSI->getDestAlign().valueOrOne();
2557 }
2558
2559 if (Opcode != TargetOpcode::G_MEMCPY_INLINE &&
2560 Opcode != TargetOpcode::G_MEMSET_INLINE) {
2561 // We need to propagate the tail call flag from the IR inst as an argument.
2562 // Otherwise, we have to pessimize and assume later that we cannot tail call
2563 // any memory intrinsics.
2564 ICall.addImm(Val: CI.isTailCall() ? 1 : 0);
2565 }
2566
2567 // Create mem operands to store the alignment and volatile info.
2568 MachineMemOperand::Flags LoadFlags = MachineMemOperand::MOLoad;
2569 MachineMemOperand::Flags StoreFlags = MachineMemOperand::MOStore;
2570 if (IsVol) {
2571 LoadFlags |= MachineMemOperand::MOVolatile;
2572 StoreFlags |= MachineMemOperand::MOVolatile;
2573 }
2574
2575 AAMDNodes AAInfo = CI.getAAMetadata();
2576 if (AA && CopySize &&
2577 AA->pointsToConstantMemory(Loc: MemoryLocation(
2578 SrcPtr, LocationSize::precise(Value: CopySize->getZExtValue()), AAInfo))) {
2579 LoadFlags |= MachineMemOperand::MOInvariant;
2580
2581 // FIXME: pointsToConstantMemory probably does not imply dereferenceable,
2582 // but the previous usage implied it did. Probably should check
2583 // isDereferenceableAndAlignedPointer.
2584 LoadFlags |= MachineMemOperand::MODereferenceable;
2585 }
2586
2587 ICall.addMemOperand(
2588 MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(CI.getArgOperand(i: 0)),
2589 F: StoreFlags, Size: 1, BaseAlignment: DstAlign, Metadata: AAInfo));
2590 if (Opcode != TargetOpcode::G_MEMSET &&
2591 Opcode != TargetOpcode::G_MEMSET_INLINE)
2592 ICall.addMemOperand(MMO: MF->getMachineMemOperand(
2593 PtrInfo: MachinePointerInfo(SrcPtr), F: LoadFlags, Size: 1, BaseAlignment: SrcAlign, Metadata: AAInfo));
2594
2595 return true;
2596}
2597
2598bool IRTranslatorImpl::translateTrap(const CallInst &CI,
2599 MachineIRBuilder &MIRBuilder,
2600 unsigned Opcode) {
2601 StringRef TrapFuncName =
2602 CI.getAttributes().getFnAttr(Kind: "trap-func-name").getValueAsString();
2603 if (TrapFuncName.empty()) {
2604 if (Opcode == TargetOpcode::G_UBSANTRAP) {
2605 uint64_t Code = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 0))->getZExtValue();
2606 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: ArrayRef<llvm::SrcOp>{Code});
2607 } else {
2608 MIRBuilder.buildInstr(Opcode);
2609 }
2610 return true;
2611 }
2612
2613 CallLowering::CallLoweringInfo Info;
2614 if (Opcode == TargetOpcode::G_UBSANTRAP)
2615 Info.OrigArgs.push_back(Elt: {getOrCreateVRegs(Val: *CI.getArgOperand(i: 0)),
2616 CI.getArgOperand(i: 0)->getType(), 0});
2617
2618 Info.Callee = MachineOperand::CreateES(SymName: TrapFuncName.data());
2619 Info.CB = &CI;
2620 Info.OrigRet = {Register(), Type::getVoidTy(C&: CI.getContext()), 0};
2621 return CLI->lowerCall(MIRBuilder, Info);
2622}
2623
2624bool IRTranslatorImpl::translateVectorInterleave2Intrinsic(
2625 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
2626 assert(CI.getIntrinsicID() == Intrinsic::vector_interleave2 &&
2627 "This function can only be called on the interleave2 intrinsic!");
2628 // Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG).
2629 Register Op0 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
2630 Register Op1 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1));
2631 Register Res = getOrCreateVReg(Val: CI);
2632
2633 LLT OpTy = MRI->getType(Reg: Op0);
2634 MIRBuilder.buildShuffleVector(Res, Src1: Op0, Src2: Op1,
2635 Mask: createInterleaveMask(VF: OpTy.getNumElements(), NumVecs: 2));
2636
2637 return true;
2638}
2639
2640bool IRTranslatorImpl::translateVectorDeinterleave2Intrinsic(
2641 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
2642 assert(CI.getIntrinsicID() == Intrinsic::vector_deinterleave2 &&
2643 "This function can only be called on the deinterleave2 intrinsic!");
2644 // Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to
2645 // SelectionDAG).
2646 Register Op = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
2647 auto Undef = MIRBuilder.buildUndef(Res: MRI->getType(Reg: Op));
2648 ArrayRef<Register> Res = getOrCreateVRegs(Val: CI);
2649
2650 LLT ResTy = MRI->getType(Reg: Res[0]);
2651 if (ResTy.isScalar()) {
2652 MIRBuilder.buildExtractVectorElementConstant(Res: Res[0], Val: Op, Idx: 0);
2653 MIRBuilder.buildExtractVectorElementConstant(Res: Res[1], Val: Op, Idx: 1);
2654
2655 return true;
2656 }
2657
2658 assert(ResTy.isVector() && "Expected vector result type");
2659 MIRBuilder.buildShuffleVector(Res: Res[0], Src1: Op, Src2: Undef,
2660 Mask: createStrideMask(Start: 0, Stride: 2, VF: ResTy.getNumElements()));
2661 MIRBuilder.buildShuffleVector(Res: Res[1], Src1: Op, Src2: Undef,
2662 Mask: createStrideMask(Start: 1, Stride: 2, VF: ResTy.getNumElements()));
2663
2664 return true;
2665}
2666
2667void IRTranslatorImpl::getStackGuard(Register DstReg,
2668 MachineIRBuilder &MIRBuilder) {
2669 Value *Global =
2670 TLI->getSDagStackGuard(M: *MF->getFunction().getParent(), Libcalls: *Libcalls);
2671 if (!Global) {
2672 LLVMContext &Ctx = MIRBuilder.getContext();
2673 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
2674 MIRBuilder.buildUndef(Res: DstReg);
2675 return;
2676 }
2677
2678 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
2679 MRI->setRegClass(Reg: DstReg,
2680 RC: TII.getRegClass(MCID: TII.get(Opcode: TargetOpcode::LOAD_STACK_GUARD), OpNum: 0));
2681 auto MIB =
2682 MIRBuilder.buildInstr(Opc: TargetOpcode::LOAD_STACK_GUARD, DstOps: {DstReg}, SrcOps: {});
2683
2684 unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
2685 LLT PtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL->getPointerSizeInBits(AS: AddrSpace));
2686
2687 MachinePointerInfo MPInfo(Global);
2688 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2689 MachineMemOperand::MODereferenceable;
2690 MachineMemOperand *MemRef = MF->getMachineMemOperand(
2691 PtrInfo: MPInfo, F: Flags, MemTy: PtrTy, BaseAlignment: DL->getPointerABIAlignment(AS: AddrSpace));
2692 MIB.setMemRefs({MemRef});
2693}
2694
2695bool IRTranslatorImpl::translateOverflowIntrinsic(
2696 const CallInst &CI, unsigned Op, MachineIRBuilder &MIRBuilder) {
2697 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: CI);
2698 MIRBuilder.buildInstr(
2699 Opc: Op, DstOps: {ResRegs[0], ResRegs[1]},
2700 SrcOps: {getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)), getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1))});
2701
2702 return true;
2703}
2704
2705bool IRTranslatorImpl::translateFixedPointIntrinsic(
2706 unsigned Op, const CallInst &CI, MachineIRBuilder &MIRBuilder) {
2707 Register Dst = getOrCreateVReg(Val: CI);
2708 Register Src0 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
2709 Register Src1 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1));
2710 uint64_t Scale = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
2711 MIRBuilder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: { Src0, Src1, Scale });
2712 return true;
2713}
2714
2715unsigned IRTranslatorImpl::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
2716 switch (ID) {
2717 default:
2718 break;
2719 case Intrinsic::acos:
2720 return TargetOpcode::G_FACOS;
2721 case Intrinsic::asin:
2722 return TargetOpcode::G_FASIN;
2723 case Intrinsic::atan:
2724 return TargetOpcode::G_FATAN;
2725 case Intrinsic::atan2:
2726 return TargetOpcode::G_FATAN2;
2727 case Intrinsic::bswap:
2728 return TargetOpcode::G_BSWAP;
2729 case Intrinsic::bitreverse:
2730 return TargetOpcode::G_BITREVERSE;
2731 case Intrinsic::clmul:
2732 return TargetOpcode::G_CLMUL;
2733 case Intrinsic::fshl:
2734 return TargetOpcode::G_FSHL;
2735 case Intrinsic::fshr:
2736 return TargetOpcode::G_FSHR;
2737 case Intrinsic::ceil:
2738 return TargetOpcode::G_FCEIL;
2739 case Intrinsic::cos:
2740 return TargetOpcode::G_FCOS;
2741 case Intrinsic::cosh:
2742 return TargetOpcode::G_FCOSH;
2743 case Intrinsic::ctpop:
2744 return TargetOpcode::G_CTPOP;
2745 case Intrinsic::smulh:
2746 return TargetOpcode::G_SMULH;
2747 case Intrinsic::umulh:
2748 return TargetOpcode::G_UMULH;
2749 case Intrinsic::exp:
2750 return TargetOpcode::G_FEXP;
2751 case Intrinsic::exp2:
2752 return TargetOpcode::G_FEXP2;
2753 case Intrinsic::exp10:
2754 return TargetOpcode::G_FEXP10;
2755 case Intrinsic::fabs:
2756 return TargetOpcode::G_FABS;
2757 case Intrinsic::copysign:
2758 return TargetOpcode::G_FCOPYSIGN;
2759 case Intrinsic::minnum:
2760 return TargetOpcode::G_FMINNUM;
2761 case Intrinsic::maxnum:
2762 return TargetOpcode::G_FMAXNUM;
2763 case Intrinsic::minimum:
2764 return TargetOpcode::G_FMINIMUM;
2765 case Intrinsic::maximum:
2766 return TargetOpcode::G_FMAXIMUM;
2767 case Intrinsic::minimumnum:
2768 return TargetOpcode::G_FMINIMUMNUM;
2769 case Intrinsic::maximumnum:
2770 return TargetOpcode::G_FMAXIMUMNUM;
2771 case Intrinsic::canonicalize:
2772 return TargetOpcode::G_FCANONICALIZE;
2773 case Intrinsic::floor:
2774 return TargetOpcode::G_FFLOOR;
2775 case Intrinsic::fma:
2776 return TargetOpcode::G_FMA;
2777 case Intrinsic::log:
2778 return TargetOpcode::G_FLOG;
2779 case Intrinsic::log2:
2780 return TargetOpcode::G_FLOG2;
2781 case Intrinsic::log10:
2782 return TargetOpcode::G_FLOG10;
2783 case Intrinsic::ldexp:
2784 return TargetOpcode::G_FLDEXP;
2785 case Intrinsic::nearbyint:
2786 return TargetOpcode::G_FNEARBYINT;
2787 case Intrinsic::pow:
2788 return TargetOpcode::G_FPOW;
2789 case Intrinsic::powi:
2790 return TargetOpcode::G_FPOWI;
2791 case Intrinsic::rint:
2792 return TargetOpcode::G_FRINT;
2793 case Intrinsic::round:
2794 return TargetOpcode::G_INTRINSIC_ROUND;
2795 case Intrinsic::roundeven:
2796 return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
2797 case Intrinsic::sin:
2798 return TargetOpcode::G_FSIN;
2799 case Intrinsic::sinh:
2800 return TargetOpcode::G_FSINH;
2801 case Intrinsic::sqrt:
2802 return TargetOpcode::G_FSQRT;
2803 case Intrinsic::tan:
2804 return TargetOpcode::G_FTAN;
2805 case Intrinsic::tanh:
2806 return TargetOpcode::G_FTANH;
2807 case Intrinsic::trunc:
2808 return TargetOpcode::G_INTRINSIC_TRUNC;
2809 case Intrinsic::readcyclecounter:
2810 return TargetOpcode::G_READCYCLECOUNTER;
2811 case Intrinsic::readsteadycounter:
2812 return TargetOpcode::G_READSTEADYCOUNTER;
2813 case Intrinsic::ptrmask:
2814 return TargetOpcode::G_PTRMASK;
2815 case Intrinsic::lrint:
2816 return TargetOpcode::G_INTRINSIC_LRINT;
2817 case Intrinsic::llrint:
2818 return TargetOpcode::G_INTRINSIC_LLRINT;
2819 // FADD/FMUL require checking the FMF, so are handled elsewhere.
2820 case Intrinsic::vector_reduce_fmin:
2821 return TargetOpcode::G_VECREDUCE_FMIN;
2822 case Intrinsic::vector_reduce_fmax:
2823 return TargetOpcode::G_VECREDUCE_FMAX;
2824 case Intrinsic::vector_reduce_fminimum:
2825 return TargetOpcode::G_VECREDUCE_FMINIMUM;
2826 case Intrinsic::vector_reduce_fmaximum:
2827 return TargetOpcode::G_VECREDUCE_FMAXIMUM;
2828 case Intrinsic::vector_reduce_fminimumnum:
2829 return TargetOpcode::G_VECREDUCE_FMINIMUMNUM;
2830 case Intrinsic::vector_reduce_fmaximumnum:
2831 return TargetOpcode::G_VECREDUCE_FMAXIMUMNUM;
2832 case Intrinsic::vector_reduce_add:
2833 return TargetOpcode::G_VECREDUCE_ADD;
2834 case Intrinsic::vector_reduce_mul:
2835 return TargetOpcode::G_VECREDUCE_MUL;
2836 case Intrinsic::vector_reduce_and:
2837 return TargetOpcode::G_VECREDUCE_AND;
2838 case Intrinsic::vector_reduce_or:
2839 return TargetOpcode::G_VECREDUCE_OR;
2840 case Intrinsic::vector_reduce_xor:
2841 return TargetOpcode::G_VECREDUCE_XOR;
2842 case Intrinsic::vector_reduce_smax:
2843 return TargetOpcode::G_VECREDUCE_SMAX;
2844 case Intrinsic::vector_reduce_smin:
2845 return TargetOpcode::G_VECREDUCE_SMIN;
2846 case Intrinsic::vector_reduce_umax:
2847 return TargetOpcode::G_VECREDUCE_UMAX;
2848 case Intrinsic::vector_reduce_umin:
2849 return TargetOpcode::G_VECREDUCE_UMIN;
2850 case Intrinsic::experimental_vector_compress:
2851 return TargetOpcode::G_VECTOR_COMPRESS;
2852 case Intrinsic::lround:
2853 return TargetOpcode::G_LROUND;
2854 case Intrinsic::llround:
2855 return TargetOpcode::G_LLROUND;
2856 case Intrinsic::get_fpenv:
2857 return TargetOpcode::G_GET_FPENV;
2858 case Intrinsic::get_fpmode:
2859 return TargetOpcode::G_GET_FPMODE;
2860 }
2861 return Intrinsic::not_intrinsic;
2862}
2863
2864bool IRTranslatorImpl::translateSimpleIntrinsic(const CallInst &CI,
2865 Intrinsic::ID ID,
2866 MachineIRBuilder &MIRBuilder) {
2867
2868 unsigned Op = getSimpleIntrinsicOpcode(ID);
2869
2870 // Is this a simple intrinsic?
2871 if (Op == Intrinsic::not_intrinsic)
2872 return false;
2873
2874 // Yes. Let's translate it.
2875 SmallVector<llvm::SrcOp, 4> VRegs;
2876 for (const auto &Arg : CI.args())
2877 VRegs.push_back(Elt: getOrCreateVReg(Val: *Arg));
2878
2879 MIRBuilder.buildInstr(Opc: Op, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: VRegs,
2880 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2881 return true;
2882}
2883
2884// TODO: Include ConstainedOps.def when all strict instructions are defined.
2885static unsigned getConstrainedOpcode(Intrinsic::ID ID) {
2886 switch (ID) {
2887 case Intrinsic::experimental_constrained_fadd:
2888 return TargetOpcode::G_STRICT_FADD;
2889 case Intrinsic::experimental_constrained_fsub:
2890 return TargetOpcode::G_STRICT_FSUB;
2891 case Intrinsic::experimental_constrained_fmul:
2892 return TargetOpcode::G_STRICT_FMUL;
2893 case Intrinsic::experimental_constrained_fdiv:
2894 return TargetOpcode::G_STRICT_FDIV;
2895 case Intrinsic::experimental_constrained_frem:
2896 return TargetOpcode::G_STRICT_FREM;
2897 case Intrinsic::experimental_constrained_fma:
2898 return TargetOpcode::G_STRICT_FMA;
2899 case Intrinsic::experimental_constrained_sqrt:
2900 return TargetOpcode::G_STRICT_FSQRT;
2901 case Intrinsic::experimental_constrained_ldexp:
2902 return TargetOpcode::G_STRICT_FLDEXP;
2903 case Intrinsic::experimental_constrained_fcmp:
2904 return TargetOpcode::G_STRICT_FCMP;
2905 case Intrinsic::experimental_constrained_fcmps:
2906 return TargetOpcode::G_STRICT_FCMPS;
2907 default:
2908 return 0;
2909 }
2910}
2911
2912bool IRTranslatorImpl::translateConstrainedFPIntrinsic(
2913 const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
2914 fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
2915
2916 unsigned Opcode = getConstrainedOpcode(ID: FPI.getIntrinsicID());
2917 if (!Opcode)
2918 return false;
2919
2920 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: FPI);
2921 if (EB == fp::ExceptionBehavior::ebIgnore)
2922 Flags |= MachineInstr::NoFPExcept;
2923
2924 if (Opcode == TargetOpcode::G_STRICT_FCMP ||
2925 Opcode == TargetOpcode::G_STRICT_FCMPS) {
2926 auto *FPCmp = cast<ConstrainedFPCmpIntrinsic>(Val: &FPI);
2927 Register Operand0 = getOrCreateVReg(Val: *FPCmp->getArgOperand(i: 0));
2928 Register Operand1 = getOrCreateVReg(Val: *FPCmp->getArgOperand(i: 1));
2929 Register Result = getOrCreateVReg(Val: FPI);
2930 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Result}, SrcOps: {}, Flags)
2931 .addPredicate(Pred: FPCmp->getPredicate())
2932 .addUse(RegNo: Operand0)
2933 .addUse(RegNo: Operand1);
2934 return true;
2935 }
2936
2937 SmallVector<llvm::SrcOp, 4> VRegs;
2938 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
2939 VRegs.push_back(Elt: getOrCreateVReg(Val: *FPI.getArgOperand(i: I)));
2940
2941 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: FPI)}, SrcOps: VRegs, Flags);
2942 return true;
2943}
2944
2945std::optional<MCRegister> IRTranslatorImpl::getArgPhysReg(Argument &Arg) {
2946 auto VRegs = getOrCreateVRegs(Val: Arg);
2947 if (VRegs.size() != 1)
2948 return std::nullopt;
2949
2950 // Arguments are lowered as a copy of a livein physical register.
2951 auto *VRegDef = MF->getRegInfo().getVRegDef(Reg: VRegs[0]);
2952 if (!VRegDef || !VRegDef->isCopy())
2953 return std::nullopt;
2954 return VRegDef->getOperand(i: 1).getReg().asMCReg();
2955}
2956
2957bool IRTranslatorImpl::translateIfEntryValueArgument(
2958 bool isDeclare, Value *Val, const DILocalVariable *Var,
2959 const DIExpression *Expr, const DebugLoc &DL,
2960 MachineIRBuilder &MIRBuilder) {
2961 auto *Arg = dyn_cast<Argument>(Val);
2962 if (!Arg)
2963 return false;
2964
2965 if (!Expr->isEntryValue())
2966 return false;
2967
2968 std::optional<MCRegister> PhysReg = getArgPhysReg(Arg&: *Arg);
2969 if (!PhysReg) {
2970 LLVM_DEBUG(dbgs() << "Dropping dbg." << (isDeclare ? "declare" : "value")
2971 << ": expression is entry_value but "
2972 << "couldn't find a physical register\n");
2973 LLVM_DEBUG(dbgs() << *Var << "\n");
2974 return true;
2975 }
2976
2977 if (isDeclare) {
2978 // Append an op deref to account for the fact that this is a dbg_declare.
2979 Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
2980 MF->setVariableDbgInfo(Var, Expr, Reg: *PhysReg, Loc: DL);
2981 } else {
2982 MIRBuilder.buildDirectDbgValue(Reg: *PhysReg, Variable: Var, Expr);
2983 }
2984
2985 return true;
2986}
2987
2988static unsigned getConvOpcode(Intrinsic::ID ID) {
2989 switch (ID) {
2990 default:
2991 llvm_unreachable("Unexpected intrinsic");
2992 case Intrinsic::experimental_convergence_anchor:
2993 return TargetOpcode::CONVERGENCECTRL_ANCHOR;
2994 case Intrinsic::experimental_convergence_entry:
2995 return TargetOpcode::CONVERGENCECTRL_ENTRY;
2996 case Intrinsic::experimental_convergence_loop:
2997 return TargetOpcode::CONVERGENCECTRL_LOOP;
2998 }
2999}
3000
3001bool IRTranslatorImpl::translateConvergenceControlIntrinsic(
3002 const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
3003 MachineInstrBuilder MIB = MIRBuilder.buildInstr(Opcode: getConvOpcode(ID));
3004 Register OutputReg = getOrCreateConvergenceTokenVReg(Token: CI);
3005 MIB.addDef(RegNo: OutputReg);
3006
3007 if (ID == Intrinsic::experimental_convergence_loop) {
3008 auto Bundle = CI.getOperandBundle(ID: LLVMContext::OB_convergencectrl);
3009 assert(Bundle && "Expected a convergence control token.");
3010 Register InputReg =
3011 getOrCreateConvergenceTokenVReg(Token: *Bundle->Inputs[0].get());
3012 MIB.addUse(RegNo: InputReg);
3013 }
3014
3015 return true;
3016}
3017
3018bool IRTranslatorImpl::translateKnownIntrinsic(const CallInst &CI,
3019 Intrinsic::ID ID,
3020 MachineIRBuilder &MIRBuilder) {
3021 if (auto *MI = dyn_cast<AnyMemIntrinsic>(Val: &CI)) {
3022 if (ORE->enabled()) {
3023 if (MemoryOpRemark::canHandle(I: MI, TLI: *LibInfo)) {
3024 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
3025 R.visit(I: MI);
3026 }
3027 }
3028 }
3029
3030 // If this is a simple intrinsic (that is, we just need to add a def of
3031 // a vreg, and uses for each arg operand, then translate it.
3032 if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
3033 return true;
3034
3035 switch (ID) {
3036 default:
3037 break;
3038 case Intrinsic::lifetime_start:
3039 case Intrinsic::lifetime_end: {
3040 // No stack colouring in O0, discard region information.
3041 if (MF->getTarget().getOptLevel() == CodeGenOptLevel::None ||
3042 MF->getFunction().hasOptNone())
3043 return true;
3044
3045 unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
3046 : TargetOpcode::LIFETIME_END;
3047
3048 const AllocaInst *AI = dyn_cast<AllocaInst>(Val: CI.getArgOperand(i: 0));
3049 if (!AI || !AI->isStaticAlloca())
3050 return true;
3051
3052 MIRBuilder.buildInstr(Opcode: Op).addFrameIndex(Idx: getOrCreateFrameIndex(AI: *AI));
3053 return true;
3054 }
3055 case Intrinsic::fake_use: {
3056 SmallVector<llvm::SrcOp, 4> VRegs;
3057 for (const auto &Arg : CI.args())
3058 llvm::append_range(C&: VRegs, R: getOrCreateVRegs(Val: *Arg));
3059 MIRBuilder.buildInstr(Opc: TargetOpcode::FAKE_USE, DstOps: {}, SrcOps: VRegs);
3060 MF->setHasFakeUses(true);
3061 return true;
3062 }
3063 case Intrinsic::dbg_declare: {
3064 const DbgDeclareInst &DI = cast<DbgDeclareInst>(Val: CI);
3065 assert(DI.getVariable() && "Missing variable");
3066 translateDbgDeclareRecord(Address: DI.getAddress(), HasArgList: DI.hasArgList(), Variable: DI.getVariable(),
3067 Expression: DI.getExpression(), DL: DI.getDebugLoc(), MIRBuilder);
3068 return true;
3069 }
3070 case Intrinsic::dbg_label: {
3071 const DbgLabelInst &DI = cast<DbgLabelInst>(Val: CI);
3072 assert(DI.getLabel() && "Missing label");
3073
3074 assert(DI.getLabel()->isValidLocationForIntrinsic(
3075 MIRBuilder.getDebugLoc()) &&
3076 "Expected inlined-at fields to agree");
3077
3078 MIRBuilder.buildDbgLabel(Label: DI.getLabel());
3079 return true;
3080 }
3081 case Intrinsic::vaend:
3082 // No target I know of cares about va_end. Certainly no in-tree target
3083 // does. Simplest intrinsic ever!
3084 return true;
3085 case Intrinsic::vastart: {
3086 Value *Ptr = CI.getArgOperand(i: 0);
3087 unsigned ListSize = TLI->getVaListSizeInBits(DL: *DL) / 8;
3088 Align Alignment = getKnownAlignment(V: Ptr, DL: *DL);
3089
3090 MIRBuilder.buildInstr(Opc: TargetOpcode::G_VASTART, DstOps: {}, SrcOps: {getOrCreateVReg(Val: *Ptr)})
3091 .addMemOperand(MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Ptr),
3092 F: MachineMemOperand::MOStore,
3093 Size: ListSize, BaseAlignment: Alignment));
3094 return true;
3095 }
3096 case Intrinsic::dbg_assign:
3097 // A dbg.assign is a dbg.value with more information about stack locations,
3098 // typically produced during optimisation of variables with leaked
3099 // addresses. We can treat it like a normal dbg_value intrinsic here; to
3100 // benefit from the full analysis of stack/SSA locations, GlobalISel would
3101 // need to register for and use the AssignmentTrackingAnalysis pass.
3102 [[fallthrough]];
3103 case Intrinsic::dbg_value: {
3104 // This form of DBG_VALUE is target-independent.
3105 const DbgValueInst &DI = cast<DbgValueInst>(Val: CI);
3106 translateDbgValueRecord(V: DI.getValue(), HasArgList: DI.hasArgList(), Variable: DI.getVariable(),
3107 Expression: DI.getExpression(), DL: DI.getDebugLoc(), MIRBuilder);
3108 return true;
3109 }
3110 case Intrinsic::uadd_with_overflow:
3111 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UADDO, MIRBuilder);
3112 case Intrinsic::sadd_with_overflow:
3113 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SADDO, MIRBuilder);
3114 case Intrinsic::usub_with_overflow:
3115 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_USUBO, MIRBuilder);
3116 case Intrinsic::ssub_with_overflow:
3117 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SSUBO, MIRBuilder);
3118 case Intrinsic::umul_with_overflow:
3119 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UMULO, MIRBuilder);
3120 case Intrinsic::smul_with_overflow:
3121 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SMULO, MIRBuilder);
3122 case Intrinsic::uadd_sat:
3123 return translateBinaryOp(Opcode: TargetOpcode::G_UADDSAT, U: CI, MIRBuilder);
3124 case Intrinsic::sadd_sat:
3125 return translateBinaryOp(Opcode: TargetOpcode::G_SADDSAT, U: CI, MIRBuilder);
3126 case Intrinsic::usub_sat:
3127 return translateBinaryOp(Opcode: TargetOpcode::G_USUBSAT, U: CI, MIRBuilder);
3128 case Intrinsic::ssub_sat:
3129 return translateBinaryOp(Opcode: TargetOpcode::G_SSUBSAT, U: CI, MIRBuilder);
3130 case Intrinsic::ushl_sat:
3131 return translateBinaryOp(Opcode: TargetOpcode::G_USHLSAT, U: CI, MIRBuilder);
3132 case Intrinsic::sshl_sat:
3133 return translateBinaryOp(Opcode: TargetOpcode::G_SSHLSAT, U: CI, MIRBuilder);
3134 case Intrinsic::umin:
3135 return translateBinaryOp(Opcode: TargetOpcode::G_UMIN, U: CI, MIRBuilder);
3136 case Intrinsic::umax:
3137 return translateBinaryOp(Opcode: TargetOpcode::G_UMAX, U: CI, MIRBuilder);
3138 case Intrinsic::smin:
3139 return translateBinaryOp(Opcode: TargetOpcode::G_SMIN, U: CI, MIRBuilder);
3140 case Intrinsic::smax:
3141 return translateBinaryOp(Opcode: TargetOpcode::G_SMAX, U: CI, MIRBuilder);
3142 case Intrinsic::abs:
3143 // TODO: Preserve "int min is poison" arg in GMIR?
3144 return translateUnaryOp(Opcode: TargetOpcode::G_ABS, U: CI, MIRBuilder);
3145 case Intrinsic::smul_fix:
3146 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIX, CI, MIRBuilder);
3147 case Intrinsic::umul_fix:
3148 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIX, CI, MIRBuilder);
3149 case Intrinsic::smul_fix_sat:
3150 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
3151 case Intrinsic::umul_fix_sat:
3152 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
3153 case Intrinsic::sdiv_fix:
3154 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
3155 case Intrinsic::udiv_fix:
3156 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
3157 case Intrinsic::sdiv_fix_sat:
3158 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
3159 case Intrinsic::udiv_fix_sat:
3160 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
3161 case Intrinsic::fmuladd: {
3162 Register Dst = getOrCreateVReg(Val: CI);
3163 Register Op0 = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
3164 Register Op1 = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
3165 Register Op2 = getOrCreateVReg(Val: *CI.getArgOperand(i: 2));
3166 if (TLI->isFMAFasterThanFMulAndFAdd(MF: *MF,
3167 TLI->getValueType(DL: *DL, Ty: CI.getType()))) {
3168 // TODO: Revisit this to see if we should move this part of the
3169 // lowering to the combiner.
3170 MIRBuilder.buildFMA(Dst, Src0: Op0, Src1: Op1, Src2: Op2,
3171 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3172 } else {
3173 LLT Ty = getLLTForType(Ty&: *CI.getType(), DL: *DL);
3174 auto FMul = MIRBuilder.buildFMul(
3175 Dst: Ty, Src0: Op0, Src1: Op1, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3176 MIRBuilder.buildFAdd(Dst, Src0: FMul, Src1: Op2,
3177 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3178 }
3179 return true;
3180 }
3181 case Intrinsic::frexp: {
3182 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
3183 MIRBuilder.buildFFrexp(Fract: VRegs[0], Exp: VRegs[1],
3184 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
3185 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3186 return true;
3187 }
3188 case Intrinsic::modf: {
3189 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
3190 MIRBuilder.buildModf(Fract: VRegs[0], Int: VRegs[1],
3191 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
3192 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3193 return true;
3194 }
3195 case Intrinsic::sincos: {
3196 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
3197 MIRBuilder.buildFSincos(Sin: VRegs[0], Cos: VRegs[1],
3198 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
3199 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3200 return true;
3201 }
3202 case Intrinsic::fptosi_sat:
3203 MIRBuilder.buildFPTOSI_SAT(Dst: getOrCreateVReg(Val: CI),
3204 Src0: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
3205 return true;
3206 case Intrinsic::fptoui_sat:
3207 MIRBuilder.buildFPTOUI_SAT(Dst: getOrCreateVReg(Val: CI),
3208 Src0: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
3209 return true;
3210 case Intrinsic::memcpy_inline:
3211 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY_INLINE);
3212 case Intrinsic::memcpy:
3213 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY);
3214 case Intrinsic::memmove:
3215 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMMOVE);
3216 case Intrinsic::memset:
3217 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMSET);
3218 case Intrinsic::memset_inline:
3219 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMSET_INLINE);
3220 case Intrinsic::eh_typeid_for: {
3221 GlobalValue *GV = ExtractTypeInfo(V: CI.getArgOperand(i: 0));
3222 Register Reg = getOrCreateVReg(Val: CI);
3223 unsigned TypeID = MF->getTypeIDFor(TI: GV);
3224 MIRBuilder.buildConstant(Res: Reg, Val: TypeID);
3225 return true;
3226 }
3227 case Intrinsic::objectsize:
3228 llvm_unreachable("llvm.objectsize.* should have been lowered already");
3229
3230 case Intrinsic::is_constant:
3231 llvm_unreachable("llvm.is.constant.* should have been lowered already");
3232
3233 case Intrinsic::stackguard:
3234 getStackGuard(DstReg: getOrCreateVReg(Val: CI), MIRBuilder);
3235 return true;
3236 case Intrinsic::stackprotector: {
3237 LLT PtrTy = getLLTForType(Ty&: *CI.getArgOperand(i: 0)->getType(), DL: *DL);
3238 Register GuardVal;
3239 if (TLI->useLoadStackGuardNode(M: *CI.getModule())) {
3240 GuardVal = MRI->createGenericVirtualRegister(Ty: PtrTy);
3241 getStackGuard(DstReg: GuardVal, MIRBuilder);
3242 } else
3243 GuardVal = getOrCreateVReg(Val: *CI.getArgOperand(i: 0)); // The guard's value.
3244
3245 AllocaInst *Slot = cast<AllocaInst>(Val: CI.getArgOperand(i: 1));
3246 int FI = getOrCreateFrameIndex(AI: *Slot);
3247 MF->getFrameInfo().setStackProtectorIndex(FI);
3248
3249 MIRBuilder.buildStore(
3250 Val: GuardVal, Addr: getOrCreateVReg(Val: *Slot),
3251 MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI),
3252 F: MachineMemOperand::MOStore |
3253 MachineMemOperand::MOVolatile,
3254 MemTy: PtrTy, BaseAlignment: Align(8)));
3255 return true;
3256 }
3257 case Intrinsic::stacksave: {
3258 MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKSAVE, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {});
3259 return true;
3260 }
3261 case Intrinsic::stackrestore: {
3262 MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKRESTORE, DstOps: {},
3263 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
3264 return true;
3265 }
3266 case Intrinsic::cttz:
3267 case Intrinsic::ctlz: {
3268 ConstantInt *Cst = cast<ConstantInt>(Val: CI.getArgOperand(i: 1));
3269 bool isTrailing = ID == Intrinsic::cttz;
3270 unsigned Opcode = isTrailing ? Cst->isZero()
3271 ? TargetOpcode::G_CTTZ
3272 : TargetOpcode::G_CTTZ_ZERO_POISON
3273 : Cst->isZero() ? TargetOpcode::G_CTLZ
3274 : TargetOpcode::G_CTLZ_ZERO_POISON;
3275 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: CI)},
3276 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
3277 return true;
3278 }
3279 case Intrinsic::invariant_start: {
3280 MIRBuilder.buildUndef(Res: getOrCreateVReg(Val: CI));
3281 return true;
3282 }
3283 case Intrinsic::invariant_end:
3284 return true;
3285 case Intrinsic::expect:
3286 case Intrinsic::expect_with_probability:
3287 case Intrinsic::annotation:
3288 case Intrinsic::ptr_annotation:
3289 case Intrinsic::launder_invariant_group:
3290 case Intrinsic::strip_invariant_group:
3291 case Intrinsic::threadlocal_address: {
3292 // Drop the intrinsic, but forward the value.
3293 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: CI),
3294 Op: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
3295 return true;
3296 }
3297 case Intrinsic::assume:
3298 case Intrinsic::experimental_noalias_scope_decl:
3299 case Intrinsic::var_annotation:
3300 case Intrinsic::sideeffect:
3301 // Discard annotate attributes, assumptions, and artificial side-effects.
3302 return true;
3303 case Intrinsic::read_volatile_register:
3304 case Intrinsic::read_register: {
3305 Value *Arg = CI.getArgOperand(i: 0);
3306 MIRBuilder
3307 .buildInstr(Opc: TargetOpcode::G_READ_REGISTER, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {})
3308 .addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()));
3309 return true;
3310 }
3311 case Intrinsic::write_register: {
3312 Value *Arg = CI.getArgOperand(i: 0);
3313 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_WRITE_REGISTER)
3314 .addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()))
3315 .addUse(RegNo: getOrCreateVReg(Val: *CI.getArgOperand(i: 1)));
3316 return true;
3317 }
3318 case Intrinsic::localescape: {
3319 MachineBasicBlock &EntryMBB = MF->front();
3320 StringRef EscapedName = GlobalValue::dropLLVMManglingEscape(Name: MF->getName());
3321
3322 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
3323 // is the same on all targets.
3324 for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
3325 Value *Arg = CI.getArgOperand(i: Idx)->stripPointerCasts();
3326 if (isa<ConstantPointerNull>(Val: Arg))
3327 continue; // Skip null pointers. They represent a hole in index space.
3328
3329 int FI = getOrCreateFrameIndex(AI: *cast<AllocaInst>(Val: Arg));
3330 MCSymbol *FrameAllocSym =
3331 MF->getContext().getOrCreateFrameAllocSymbol(FuncName: EscapedName, Idx);
3332
3333 // This should be inserted at the start of the entry block.
3334 auto LocalEscape =
3335 MIRBuilder.buildInstrNoInsert(Opcode: TargetOpcode::LOCAL_ESCAPE)
3336 .addSym(Sym: FrameAllocSym)
3337 .addFrameIndex(Idx: FI);
3338
3339 EntryMBB.insert(I: EntryMBB.begin(), MI: LocalEscape);
3340 }
3341
3342 return true;
3343 }
3344 case Intrinsic::vector_reduce_fadd:
3345 case Intrinsic::vector_reduce_fmul: {
3346 // Need to check for the reassoc flag to decide whether we want a
3347 // sequential reduction opcode or not.
3348 Register Dst = getOrCreateVReg(Val: CI);
3349 Register ScalarSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
3350 Register VecSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
3351 unsigned Opc = 0;
3352 if (!CI.hasAllowReassoc()) {
3353 // The sequential ordering case.
3354 Opc = ID == Intrinsic::vector_reduce_fadd
3355 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
3356 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
3357 if (!MRI->getType(Reg: VecSrc).isVector())
3358 Opc = ID == Intrinsic::vector_reduce_fadd ? TargetOpcode::G_FADD
3359 : TargetOpcode::G_FMUL;
3360 MIRBuilder.buildInstr(Opc, DstOps: {Dst}, SrcOps: {ScalarSrc, VecSrc},
3361 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3362 return true;
3363 }
3364 // We split the operation into a separate G_FADD/G_FMUL + the reduce,
3365 // since the associativity doesn't matter.
3366 unsigned ScalarOpc;
3367 if (ID == Intrinsic::vector_reduce_fadd) {
3368 Opc = TargetOpcode::G_VECREDUCE_FADD;
3369 ScalarOpc = TargetOpcode::G_FADD;
3370 } else {
3371 Opc = TargetOpcode::G_VECREDUCE_FMUL;
3372 ScalarOpc = TargetOpcode::G_FMUL;
3373 }
3374 LLT DstTy = MRI->getType(Reg: Dst);
3375 auto Rdx = MIRBuilder.buildInstr(
3376 Opc, DstOps: {DstTy}, SrcOps: {VecSrc}, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3377 MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {Dst}, SrcOps: {ScalarSrc, Rdx},
3378 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3379
3380 return true;
3381 }
3382 case Intrinsic::trap:
3383 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_TRAP);
3384 case Intrinsic::debugtrap:
3385 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_DEBUGTRAP);
3386 case Intrinsic::ubsantrap:
3387 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_UBSANTRAP);
3388 case Intrinsic::allow_runtime_check:
3389 case Intrinsic::allow_ubsan_check:
3390 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: CI),
3391 Op: getOrCreateVReg(Val: *ConstantInt::getTrue(Ty: CI.getType())));
3392 return true;
3393 case Intrinsic::amdgcn_cs_chain:
3394 case Intrinsic::amdgcn_call_whole_wave:
3395 return translateCallBase(CB: CI, MIRBuilder);
3396 case Intrinsic::fptrunc_round: {
3397 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: CI);
3398
3399 // Convert the metadata argument to a constant integer
3400 Metadata *MD = cast<MetadataAsValue>(Val: CI.getArgOperand(i: 1))->getMetadata();
3401 std::optional<RoundingMode> RoundMode =
3402 convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
3403
3404 // Add the Rounding mode as an integer
3405 MIRBuilder
3406 .buildInstr(Opc: TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
3407 DstOps: {getOrCreateVReg(Val: CI)},
3408 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))}, Flags)
3409 .addImm(Val: (int)*RoundMode);
3410
3411 return true;
3412 }
3413 case Intrinsic::is_fpclass: {
3414 Value *FpValue = CI.getOperand(i_nocapture: 0);
3415 ConstantInt *TestMaskValue = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1));
3416
3417 MIRBuilder
3418 .buildInstr(Opc: TargetOpcode::G_IS_FPCLASS, DstOps: {getOrCreateVReg(Val: CI)},
3419 SrcOps: {getOrCreateVReg(Val: *FpValue)})
3420 .addImm(Val: TestMaskValue->getZExtValue());
3421
3422 return true;
3423 }
3424 case Intrinsic::set_fpenv: {
3425 Value *FPEnv = CI.getOperand(i_nocapture: 0);
3426 MIRBuilder.buildSetFPEnv(Src: getOrCreateVReg(Val: *FPEnv));
3427 return true;
3428 }
3429 case Intrinsic::reset_fpenv:
3430 MIRBuilder.buildResetFPEnv();
3431 return true;
3432 case Intrinsic::set_fpmode: {
3433 Value *FPState = CI.getOperand(i_nocapture: 0);
3434 MIRBuilder.buildSetFPMode(Src: getOrCreateVReg(Val: *FPState));
3435 return true;
3436 }
3437 case Intrinsic::reset_fpmode:
3438 MIRBuilder.buildResetFPMode();
3439 return true;
3440 case Intrinsic::get_rounding:
3441 MIRBuilder.buildGetRounding(Dst: getOrCreateVReg(Val: CI));
3442 return true;
3443 case Intrinsic::set_rounding:
3444 MIRBuilder.buildSetRounding(Src: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)));
3445 return true;
3446 case Intrinsic::vscale: {
3447 MIRBuilder.buildVScale(Res: getOrCreateVReg(Val: CI), MinElts: 1);
3448 return true;
3449 }
3450 case Intrinsic::scmp:
3451 MIRBuilder.buildSCmp(Res: getOrCreateVReg(Val: CI),
3452 Op0: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)),
3453 Op1: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1)));
3454 return true;
3455 case Intrinsic::ucmp:
3456 MIRBuilder.buildUCmp(Res: getOrCreateVReg(Val: CI),
3457 Op0: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)),
3458 Op1: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1)));
3459 return true;
3460 case Intrinsic::vector_extract:
3461 return translateExtractVector(U: CI, MIRBuilder);
3462 case Intrinsic::vector_insert:
3463 return translateInsertVector(U: CI, MIRBuilder);
3464 case Intrinsic::stepvector: {
3465 MIRBuilder.buildStepVector(Res: getOrCreateVReg(Val: CI), Step: 1);
3466 return true;
3467 }
3468 case Intrinsic::prefetch: {
3469 Value *Addr = CI.getOperand(i_nocapture: 0);
3470 unsigned RW = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1))->getZExtValue();
3471 unsigned Locality = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
3472 unsigned CacheType = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 3))->getZExtValue();
3473
3474 auto Flags = RW ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
3475 auto &MMO = *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Addr), F: Flags,
3476 MemTy: LLT(), BaseAlignment: Align());
3477
3478 MIRBuilder.buildPrefetch(Addr: getOrCreateVReg(Val: *Addr), RW, Locality, CacheType,
3479 MMO);
3480
3481 return true;
3482 }
3483
3484 case Intrinsic::speculative_load: {
3485 // Only the pointer operand is needed at codegen; the remaining arguments
3486 // carry IR-level semantics only.
3487 const Value *Ptr = CI.getArgOperand(i: 0);
3488 Register Dst = getOrCreateVReg(Val: CI);
3489 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad;
3490 Flags |= TLI->getTargetMMOFlags(I: CI);
3491 if (CI.hasMetadata(KindID: LLVMContext::MD_nontemporal))
3492 Flags |= MachineMemOperand::MONonTemporal;
3493 if (CI.hasMetadata(KindID: LLVMContext::MD_invariant_load))
3494 Flags |= MachineMemOperand::MOInvariant;
3495 auto *MMO = MF->getMachineMemOperand(
3496 PtrInfo: MachinePointerInfo(Ptr), F: Flags, MemTy: MRI->getType(Reg: Dst),
3497 BaseAlignment: CI.getParamAlign(ArgNo: 0).valueOrOne(), Metadata: MMOMetadata(CI.getAAMetadata()));
3498 MIRBuilder.buildLoad(Res: Dst, Addr: getOrCreateVReg(Val: *Ptr), MMO&: *MMO);
3499 return true;
3500 }
3501
3502 case Intrinsic::vector_interleave2:
3503 case Intrinsic::vector_deinterleave2: {
3504 // Both intrinsics have at least one operand.
3505 Value *Op0 = CI.getOperand(i_nocapture: 0);
3506 LLT ResTy = getLLTForType(Ty&: *Op0->getType(), DL: MIRBuilder.getDataLayout());
3507 if (!ResTy.isFixedVector())
3508 return false;
3509
3510 if (CI.getIntrinsicID() == Intrinsic::vector_interleave2)
3511 return translateVectorInterleave2Intrinsic(CI, MIRBuilder);
3512
3513 return translateVectorDeinterleave2Intrinsic(CI, MIRBuilder);
3514 }
3515
3516#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
3517 case Intrinsic::INTRINSIC:
3518#include "llvm/IR/ConstrainedOps.def"
3519 return translateConstrainedFPIntrinsic(FPI: cast<ConstrainedFPIntrinsic>(Val: CI),
3520 MIRBuilder);
3521 case Intrinsic::experimental_convergence_anchor:
3522 case Intrinsic::experimental_convergence_entry:
3523 case Intrinsic::experimental_convergence_loop:
3524 return translateConvergenceControlIntrinsic(CI, ID, MIRBuilder);
3525 case Intrinsic::reloc_none: {
3526 Metadata *MD = cast<MetadataAsValue>(Val: CI.getArgOperand(i: 0))->getMetadata();
3527 StringRef SymbolName = cast<MDString>(Val: MD)->getString();
3528 MIRBuilder.buildInstr(Opcode: TargetOpcode::RELOC_NONE)
3529 .addExternalSymbol(FnName: SymbolName.data());
3530 return true;
3531 }
3532 }
3533 return false;
3534}
3535
3536bool IRTranslatorImpl::translateInlineAsm(const CallBase &CB,
3537 MachineIRBuilder &MIRBuilder) {
3538 if (!mayTranslateUserTypes(U: CB))
3539 return false;
3540
3541 const InlineAsmLowering *ALI = MF->getSubtarget().getInlineAsmLowering();
3542
3543 if (!ALI) {
3544 LLVM_DEBUG(
3545 dbgs() << "Inline asm lowering is not supported for this target yet\n");
3546 return false;
3547 }
3548
3549 return ALI->lowerInlineAsm(
3550 MIRBuilder, CB, GetOrCreateVRegs: [&](const Value &Val) { return getOrCreateVRegs(Val); });
3551}
3552
3553bool IRTranslatorImpl::translateCallBase(const CallBase &CB,
3554 MachineIRBuilder &MIRBuilder) {
3555 ArrayRef<Register> Res = getOrCreateVRegs(Val: CB);
3556
3557 SmallVector<ArrayRef<Register>, 8> Args;
3558 Register SwiftInVReg = 0;
3559 Register SwiftErrorVReg = 0;
3560 for (const auto &Arg : CB.args()) {
3561 if (CLI->supportSwiftError() && isSwiftError(V: Arg)) {
3562 assert(SwiftInVReg == 0 && "Expected only one swift error argument");
3563 LLT Ty = getLLTForType(Ty&: *Arg->getType(), DL: *DL);
3564 SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
3565 MIRBuilder.buildCopy(Res: SwiftInVReg, Op: SwiftError.getOrCreateVRegUseAt(
3566 &CB, &MIRBuilder.getMBB(), Arg));
3567 Args.emplace_back(Args: ArrayRef(SwiftInVReg));
3568 SwiftErrorVReg =
3569 SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
3570 continue;
3571 }
3572 Args.push_back(Elt: getOrCreateVRegs(Val: *Arg));
3573 }
3574
3575 if (auto *CI = dyn_cast<CallInst>(Val: &CB)) {
3576 if (ORE->enabled()) {
3577 if (MemoryOpRemark::canHandle(I: CI, TLI: *LibInfo)) {
3578 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
3579 R.visit(I: CI);
3580 }
3581 }
3582 }
3583
3584 std::optional<CallLowering::PtrAuthInfo> PAI;
3585 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_ptrauth)) {
3586 // Functions should never be ptrauth-called directly.
3587 assert(!CB.getCalledFunction() && "invalid direct ptrauth call");
3588
3589 const Value *Key = Bundle->Inputs[0];
3590 const Value *Discriminator = Bundle->Inputs[1];
3591
3592 // Look through ptrauth constants to try to eliminate the matching bundle
3593 // and turn this into a direct call with no ptrauth.
3594 // CallLowering will use the raw pointer if it doesn't find the PAI.
3595 const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(Val: CB.getCalledOperand());
3596 if (!CalleeCPA || !isa<Function>(Val: CalleeCPA->getPointer()) ||
3597 !CalleeCPA->isKnownCompatibleWith(Key, Discriminator, DL: *DL)) {
3598 // If we can't make it direct, package the bundle into PAI.
3599 Register DiscReg = getOrCreateVReg(Val: *Discriminator);
3600 PAI = CallLowering::PtrAuthInfo{.Key: cast<ConstantInt>(Val: Key)->getZExtValue(),
3601 .Discriminator: DiscReg};
3602 }
3603 }
3604
3605 Register ConvergenceCtrlToken = 0;
3606 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
3607 const auto &Token = *Bundle->Inputs[0].get();
3608 ConvergenceCtrlToken = getOrCreateConvergenceTokenVReg(Token);
3609 }
3610
3611 // We don't set HasCalls on MFI here yet because call lowering may decide to
3612 // optimize into tail calls. Instead, we defer that to selection where a final
3613 // scan is done to check if any instructions are calls.
3614 bool Success = CLI->lowerCall(
3615 MIRBuilder, Call: CB, ResRegs: Res, ArgRegs: Args, SwiftErrorVReg, PAI, ConvergenceCtrlToken,
3616 GetCalleeReg: [&]() { return getOrCreateVReg(Val: *CB.getCalledOperand()); });
3617
3618 // Check if we just inserted a tail call.
3619 if (Success) {
3620 assert(!HasTailCall && "Can't tail call return twice from block?");
3621 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
3622 HasTailCall = TII->isTailCall(Inst: *std::prev(x: MIRBuilder.getInsertPt()));
3623 }
3624
3625 return Success;
3626}
3627
3628bool IRTranslatorImpl::translateCall(const User &U,
3629 MachineIRBuilder &MIRBuilder) {
3630 if (!mayTranslateUserTypes(U))
3631 return false;
3632
3633 const CallInst &CI = cast<CallInst>(Val: U);
3634 const Function *F = CI.getCalledFunction();
3635
3636 // FIXME: support Windows dllimport function calls and calls through
3637 // weak symbols.
3638 if (F && (F->hasDLLImportStorageClass() ||
3639 (MF->getTarget().getTargetTriple().isOSWindows() &&
3640 F->hasExternalWeakLinkage())))
3641 return false;
3642
3643 // FIXME: support control flow guard targets.
3644 if (CI.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
3645 return false;
3646
3647 // FIXME: support statepoints and related.
3648 if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(Val: U))
3649 return false;
3650
3651 if (CI.isInlineAsm())
3652 return translateInlineAsm(CB: CI, MIRBuilder);
3653
3654 Intrinsic::ID ID = F ? F->getIntrinsicID() : Intrinsic::not_intrinsic;
3655 if (!F || ID == Intrinsic::not_intrinsic) {
3656 if (translateCallBase(CB: CI, MIRBuilder)) {
3657 diagnoseDontCall(CI);
3658 return true;
3659 }
3660 return false;
3661 }
3662
3663 assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
3664
3665 if (!MF->getSubtarget().isIntrinsicSupported(IntrinsicID: ID)) {
3666 const Function &Fn = MF->getFunction();
3667 Fn.getContext().diagnose(
3668 DI: DiagnosticInfoUnsupportedTargetIntrinsic(Fn, ID, CI.getDebugLoc()));
3669 }
3670
3671 if (translateKnownIntrinsic(CI, ID, MIRBuilder))
3672 return true;
3673
3674 SmallVector<TargetLowering::IntrinsicInfo> Infos;
3675 TLI->getTgtMemIntrinsic(Infos, I: CI, MF&: *MF, Intrinsic: ID);
3676
3677 return translateIntrinsic(CB: CI, ID, MIRBuilder, TgtMemIntrinsicInfos: Infos);
3678}
3679
3680/// Translate a call or callbr to an intrinsic.
3681bool IRTranslatorImpl::translateIntrinsic(
3682 const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
3683 ArrayRef<TargetLowering::IntrinsicInfo> TgtMemIntrinsicInfos) {
3684 if (!MF->getSubtarget().isIntrinsicSupported(IntrinsicID: ID)) {
3685 const Function &F = MF->getFunction();
3686 F.getContext().diagnose(
3687 DI: DiagnosticInfoUnsupportedTargetIntrinsic(F, ID, CB.getDebugLoc()));
3688 }
3689
3690 ArrayRef<Register> ResultRegs;
3691 if (!CB.getType()->isVoidTy())
3692 ResultRegs = getOrCreateVRegs(Val: CB);
3693
3694 // Ignore the callsite attributes. Backend code is most likely not expecting
3695 // an intrinsic to sometimes have side effects and sometimes not.
3696 MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, Res: ResultRegs);
3697 if (isa<FPMathOperator>(Val: CB))
3698 MIB->copyIRFlags(I: CB);
3699
3700 for (const auto &Arg : enumerate(First: CB.args())) {
3701 // If this is required to be an immediate, don't materialize it in a
3702 // register.
3703 if (CB.paramHasAttr(ArgNo: Arg.index(), Kind: Attribute::ImmArg)) {
3704 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Arg.value())) {
3705 // imm arguments are more convenient than cimm (and realistically
3706 // probably sufficient), so use them.
3707 assert(CI->getBitWidth() <= 64 &&
3708 "large intrinsic immediates not handled");
3709 MIB.addImm(Val: CI->getSExtValue());
3710 } else {
3711 MIB.addFPImm(Val: cast<ConstantFP>(Val: Arg.value()));
3712 }
3713 } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Val: Arg.value())) {
3714 auto *MD = MDVal->getMetadata();
3715 auto *MDN = dyn_cast<MDNode>(Val: MD);
3716 if (!MDN) {
3717 if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(Val: MD))
3718 MDN = MDNode::get(Context&: MF->getFunction().getContext(), MDs: ConstMD);
3719 else // This was probably an MDString.
3720 return false;
3721 }
3722 MIB.addMetadata(MD: MDN);
3723 } else {
3724 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: *Arg.value());
3725 if (VRegs.size() > 1)
3726 return false;
3727 MIB.addUse(RegNo: VRegs[0]);
3728 }
3729 }
3730
3731 // Add MachineMemOperands for each memory access described by the target.
3732 for (const auto &Info : TgtMemIntrinsicInfos) {
3733 Align Alignment = Info.align.value_or(
3734 u: DL->getABITypeAlign(Ty: Info.memVT.getTypeForEVT(Context&: CB.getContext())));
3735 LLT MemTy = Info.memVT.isSimple()
3736 ? getLLTForMVT(Ty: Info.memVT.getSimpleVT())
3737 : LLT::scalar(SizeInBits: Info.memVT.getStoreSizeInBits());
3738
3739 // TODO: We currently just fallback to address space 0 if
3740 // getTgtMemIntrinsic didn't yield anything useful.
3741 MachinePointerInfo MPI;
3742 if (Info.ptrVal) {
3743 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
3744 } else if (Info.fallbackAddressSpace) {
3745 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
3746 }
3747 MIB.addMemOperand(MMO: MF->getMachineMemOperand(
3748 PtrInfo: MPI, F: Info.flags, MemTy, BaseAlignment: Alignment, Metadata: CB.getAAMetadata(), SSID: Info.ssid,
3749 Ordering: Info.order, FailureOrdering: Info.failureOrder));
3750 }
3751
3752 if (CB.isConvergent()) {
3753 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
3754 auto *Token = Bundle->Inputs[0].get();
3755 Register TokenReg = getOrCreateVReg(Val: *Token);
3756 MIB.addUse(RegNo: TokenReg, Flags: RegState::Implicit);
3757 }
3758 }
3759
3760 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol))
3761 MIB->setDeactivationSymbol(MF&: *MF, DS: Bundle->Inputs[0].get());
3762
3763 return true;
3764}
3765
3766bool IRTranslatorImpl::findUnwindDestinations(
3767 const BasicBlock *EHPadBB, BranchProbability Prob,
3768 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
3769 &UnwindDests) {
3770 EHPersonality Personality = classifyEHPersonality(
3771 Pers: EHPadBB->getParent()->getFunction().getPersonalityFn());
3772 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
3773 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
3774 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
3775 bool IsSEH = isAsynchronousEHPersonality(Pers: Personality);
3776
3777 if (IsWasmCXX) {
3778 // Ignore this for now.
3779 return false;
3780 }
3781
3782 while (EHPadBB) {
3783 BasicBlock::const_iterator Pad = EHPadBB->getFirstNonPHIIt();
3784 BasicBlock *NewEHPadBB = nullptr;
3785 if (isa<LandingPadInst>(Val: Pad)) {
3786 // Stop on landingpads. They are not funclets.
3787 UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
3788 break;
3789 }
3790 if (isa<CleanupPadInst>(Val: Pad)) {
3791 // Stop on cleanup pads. Cleanups are always funclet entries for all known
3792 // personalities.
3793 UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
3794 UnwindDests.back().first->setIsEHScopeEntry();
3795 UnwindDests.back().first->setIsEHFuncletEntry();
3796 break;
3797 }
3798 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: Pad)) {
3799 // Add the catchpad handlers to the possible destinations.
3800 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
3801 UnwindDests.emplace_back(Args: &getMBB(BB: *CatchPadBB), Args&: Prob);
3802 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
3803 if (IsMSVCCXX || IsCoreCLR)
3804 UnwindDests.back().first->setIsEHFuncletEntry();
3805 if (!IsSEH)
3806 UnwindDests.back().first->setIsEHScopeEntry();
3807 }
3808 NewEHPadBB = CatchSwitch->getUnwindDest();
3809 } else {
3810 continue;
3811 }
3812
3813 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3814 if (BPI && NewEHPadBB)
3815 Prob *= BPI->getEdgeProbability(Src: EHPadBB, Dst: NewEHPadBB);
3816 EHPadBB = NewEHPadBB;
3817 }
3818 return true;
3819}
3820
3821bool IRTranslatorImpl::translateInvoke(const User &U,
3822 MachineIRBuilder &MIRBuilder) {
3823 const InvokeInst &I = cast<InvokeInst>(Val: U);
3824 MCContext &Context = MF->getContext();
3825
3826 const BasicBlock *ReturnBB = I.getSuccessor(i: 0);
3827 const BasicBlock *EHPadBB = I.getSuccessor(i: 1);
3828
3829 const Function *Fn = I.getCalledFunction();
3830
3831 // FIXME: support invoking patchpoint and statepoint intrinsics.
3832 if (Fn && Fn->isIntrinsic())
3833 return false;
3834
3835 // FIXME: support whatever these are.
3836 if (I.hasDeoptState())
3837 return false;
3838
3839 // FIXME: support control flow guard targets.
3840 if (I.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
3841 return false;
3842
3843 // FIXME: support Windows exception handling.
3844 if (!isa<LandingPadInst>(Val: EHPadBB->getFirstNonPHIIt()))
3845 return false;
3846
3847 // FIXME: support Windows dllimport function calls and calls through
3848 // weak symbols.
3849 if (Fn && (Fn->hasDLLImportStorageClass() ||
3850 (MF->getTarget().getTargetTriple().isOSWindows() &&
3851 Fn->hasExternalWeakLinkage())))
3852 return false;
3853
3854 bool LowerInlineAsm = I.isInlineAsm();
3855 bool NeedEHLabel = true;
3856
3857 // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
3858 // the region covered by the try.
3859 MCSymbol *BeginSymbol = nullptr;
3860 if (NeedEHLabel) {
3861 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_INVOKE_REGION_START);
3862 BeginSymbol = Context.createTempSymbol();
3863 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: BeginSymbol);
3864 }
3865
3866 if (LowerInlineAsm) {
3867 if (!translateInlineAsm(CB: I, MIRBuilder))
3868 return false;
3869 } else if (!translateCallBase(CB: I, MIRBuilder))
3870 return false;
3871
3872 MCSymbol *EndSymbol = nullptr;
3873 if (NeedEHLabel) {
3874 EndSymbol = Context.createTempSymbol();
3875 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: EndSymbol);
3876 }
3877
3878 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
3879 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3880 MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
3881 BranchProbability EHPadBBProb =
3882 BPI ? BPI->getEdgeProbability(Src: InvokeMBB->getBasicBlock(), Dst: EHPadBB)
3883 : BranchProbability::getZero();
3884
3885 if (!findUnwindDestinations(EHPadBB, Prob: EHPadBBProb, UnwindDests))
3886 return false;
3887
3888 MachineBasicBlock &EHPadMBB = getMBB(BB: *EHPadBB),
3889 &ReturnMBB = getMBB(BB: *ReturnBB);
3890 // Update successor info.
3891 addSuccessorWithProb(Src: InvokeMBB, Dst: &ReturnMBB);
3892 for (auto &UnwindDest : UnwindDests) {
3893 UnwindDest.first->setIsEHPad();
3894 addSuccessorWithProb(Src: InvokeMBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
3895 }
3896 InvokeMBB->normalizeSuccProbs();
3897
3898 if (NeedEHLabel) {
3899 assert(BeginSymbol && "Expected a begin symbol!");
3900 assert(EndSymbol && "Expected an end symbol!");
3901 MF->addInvoke(LandingPad: &EHPadMBB, BeginLabel: BeginSymbol, EndLabel: EndSymbol);
3902 }
3903
3904 MIRBuilder.buildBr(Dest&: ReturnMBB);
3905 return true;
3906}
3907
3908/// The intrinsics currently supported by callbr are implicit control flow
3909/// intrinsics such as amdgcn.kill.
3910bool IRTranslatorImpl::translateCallBr(const User &U,
3911 MachineIRBuilder &MIRBuilder) {
3912 if (!mayTranslateUserTypes(U))
3913 return false; // see translateCall
3914
3915 const CallBrInst &I = cast<CallBrInst>(Val: U);
3916 MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB();
3917
3918 Intrinsic::ID IID = I.getIntrinsicID();
3919 if (I.isInlineAsm()) {
3920 // FIXME: inline asm is not yet supported for callbr in GlobalISel. As soon
3921 // as we add support, we need to handle the indirect asm targets, see
3922 // SelectionDAGBuilder::visitCallBr().
3923 return false;
3924 }
3925 if (!translateIntrinsic(CB: I, ID: IID, MIRBuilder))
3926 return false;
3927
3928 // Retrieve successors.
3929 SmallPtrSet<BasicBlock *, 8> Dests = {I.getDefaultDest()};
3930 MachineBasicBlock *Return = &getMBB(BB: *I.getDefaultDest());
3931
3932 // Update successor info.
3933 addSuccessorWithProb(Src: CallBrMBB, Dst: Return, Prob: BranchProbability::getOne());
3934
3935 // Add indirect targets as successors. For intrinsic callbr, these represent
3936 // implicit control flow (e.g., the "kill" path for amdgcn.kill). We mark them
3937 // with setIsInlineAsmBrIndirectTarget so the machine verifier accepts them as
3938 // valid successors, even though they're not from inline asm.
3939 for (BasicBlock *Dest : I.getIndirectDests()) {
3940 MachineBasicBlock &Target = getMBB(BB: *Dest);
3941 Target.setIsInlineAsmBrIndirectTarget();
3942 Target.setLabelMustBeEmitted();
3943 // Don't add duplicate machine successors.
3944 if (Dests.insert(Ptr: Dest).second)
3945 addSuccessorWithProb(Src: CallBrMBB, Dst: &Target, Prob: BranchProbability::getZero());
3946 }
3947
3948 CallBrMBB->normalizeSuccProbs();
3949
3950 // Drop into default successor.
3951 MIRBuilder.buildBr(Dest&: *Return);
3952
3953 return true;
3954}
3955
3956bool IRTranslatorImpl::translateLandingPad(const User &U,
3957 MachineIRBuilder &MIRBuilder) {
3958 const LandingPadInst &LP = cast<LandingPadInst>(Val: U);
3959
3960 MachineBasicBlock &MBB = MIRBuilder.getMBB();
3961
3962 MBB.setIsEHPad();
3963
3964 // If there aren't registers to copy the values into (e.g., during SjLj
3965 // exceptions), then don't bother.
3966 const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
3967 if (TLI->getExceptionPointerRegister(EH: FuncInfo.ExceptionModel,
3968 PersonalityFn) == 0 &&
3969 TLI->getExceptionSelectorRegister(EH: FuncInfo.ExceptionModel,
3970 PersonalityFn) == 0)
3971 return true;
3972
3973 // If landingpad's return type is token type, we don't create DAG nodes
3974 // for its exception pointer and selector value. The extraction of exception
3975 // pointer or selector value from token type landingpads is not currently
3976 // supported.
3977 if (LP.getType()->isTokenTy())
3978 return true;
3979
3980 // Add a label to mark the beginning of the landing pad. Deletion of the
3981 // landing pad can thus be detected via the MachineModuleInfo.
3982 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL)
3983 .addSym(Sym: MF->addLandingPad(LandingPad: &MBB));
3984
3985 // If the unwinder does not preserve all registers, ensure that the
3986 // function marks the clobbered registers as used.
3987 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
3988 if (auto *RegMask = TRI.getCustomEHPadPreservedMask(MF: *MF))
3989 MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask);
3990
3991 LLT Ty = getLLTForType(Ty&: *LP.getType(), DL: *DL);
3992 Register Undef = MRI->createGenericVirtualRegister(Ty);
3993 MIRBuilder.buildUndef(Res: Undef);
3994
3995 SmallVector<LLT, 2> Tys;
3996 for (Type *Ty : cast<StructType>(Val: LP.getType())->elements())
3997 Tys.push_back(Elt: getLLTForType(Ty&: *Ty, DL: *DL));
3998 assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
3999
4000 // Mark exception register as live in.
4001 Register ExceptionReg =
4002 TLI->getExceptionPointerRegister(EH: FuncInfo.ExceptionModel, PersonalityFn);
4003 if (!ExceptionReg)
4004 return false;
4005
4006 MBB.addLiveIn(PhysReg: ExceptionReg);
4007 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: LP);
4008 MIRBuilder.buildCopy(Res: ResRegs[0], Op: ExceptionReg);
4009
4010 Register SelectorReg =
4011 TLI->getExceptionSelectorRegister(EH: FuncInfo.ExceptionModel, PersonalityFn);
4012 if (!SelectorReg)
4013 return false;
4014
4015 MBB.addLiveIn(PhysReg: SelectorReg);
4016 Register PtrVReg = MRI->createGenericVirtualRegister(Ty: Tys[0]);
4017 MIRBuilder.buildCopy(Res: PtrVReg, Op: SelectorReg);
4018 MIRBuilder.buildCast(Dst: ResRegs[1], Src: PtrVReg);
4019
4020 return true;
4021}
4022
4023bool IRTranslatorImpl::translateAlloca(const User &U,
4024 MachineIRBuilder &MIRBuilder) {
4025 auto &AI = cast<AllocaInst>(Val: U);
4026
4027 if (AI.isSwiftError())
4028 return true;
4029
4030 if (AI.isStaticAlloca()) {
4031 Register Res = getOrCreateVReg(Val: AI);
4032 int FI = getOrCreateFrameIndex(AI);
4033 MIRBuilder.buildFrameIndex(Res, Idx: FI);
4034 return true;
4035 }
4036
4037 // FIXME: support stack probing for Windows.
4038 if (MF->getTarget().getTargetTriple().isOSWindows())
4039 return false;
4040
4041 // Now we're in the harder dynamic case.
4042 Register NumElts = getOrCreateVReg(Val: *AI.getArraySize());
4043 Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
4044 LLT IntPtrTy = getLLTForType(Ty&: *IntPtrIRTy, DL: *DL);
4045 if (MRI->getType(Reg: NumElts) != IntPtrTy) {
4046 Register ExtElts = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
4047 MIRBuilder.buildZExtOrTrunc(Res: ExtElts, Op: NumElts);
4048 NumElts = ExtElts;
4049 }
4050
4051 TypeSize TySize = AI.getAllocationBaseSize(DL: *DL);
4052
4053 Register AllocSize = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
4054 Register TySizeReg;
4055 if (TySize.isScalable()) {
4056 // For scalable types, use vscale * min_value
4057 TySizeReg = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
4058 MIRBuilder.buildVScale(Res: TySizeReg, MinElts: TySize.getKnownMinValue());
4059 } else {
4060 // For fixed types, use a constant
4061 TySizeReg =
4062 getOrCreateVReg(Val: *ConstantInt::get(Ty: IntPtrIRTy, V: TySize.getFixedValue()));
4063 }
4064 MIRBuilder.buildMul(Dst: AllocSize, Src0: NumElts, Src1: TySizeReg);
4065
4066 // Round the size of the allocation up to the stack alignment size
4067 // by add SA-1 to the size. This doesn't overflow because we're computing
4068 // an address inside an alloca.
4069 Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();
4070 auto SAMinusOne = MIRBuilder.buildConstant(Res: IntPtrTy, Val: StackAlign.value() - 1);
4071 auto AllocAdd = MIRBuilder.buildAdd(Dst: IntPtrTy, Src0: AllocSize, Src1: SAMinusOne,
4072 Flags: MachineInstr::NoUWrap);
4073 auto AlignCst =
4074 MIRBuilder.buildConstant(Res: IntPtrTy, Val: ~(uint64_t)(StackAlign.value() - 1));
4075 auto AlignedAlloc = MIRBuilder.buildAnd(Dst: IntPtrTy, Src0: AllocAdd, Src1: AlignCst);
4076
4077 Align Alignment = AI.getAlign();
4078 if (Alignment <= StackAlign)
4079 Alignment = Align(1);
4080 MIRBuilder.buildDynStackAlloc(Res: getOrCreateVReg(Val: AI), Size: AlignedAlloc, Alignment);
4081
4082 MF->getFrameInfo().CreateVariableSizedObject(Alignment, Alloca: &AI);
4083 assert(MF->getFrameInfo().hasVarSizedObjects());
4084 return true;
4085}
4086
4087bool IRTranslatorImpl::translateVAArg(const User &U,
4088 MachineIRBuilder &MIRBuilder) {
4089 // FIXME: We may need more info about the type. Because of how LLT works,
4090 // we're completely discarding the i64/double distinction here (amongst
4091 // others). Fortunately the ABIs I know of where that matters don't use va_arg
4092 // anyway but that's not guaranteed.
4093 MIRBuilder.buildInstr(Opc: TargetOpcode::G_VAARG, DstOps: {getOrCreateVReg(Val: U)},
4094 SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
4095 DL->getABITypeAlign(Ty: U.getType()).value()});
4096 return true;
4097}
4098
4099bool IRTranslatorImpl::translateUnreachable(const User &U,
4100 MachineIRBuilder &MIRBuilder) {
4101 auto &UI = cast<UnreachableInst>(Val: U);
4102 if (!UI.shouldLowerToTrap(TrapUnreachable: MF->getTarget().Options.TrapUnreachable,
4103 NoTrapAfterNoreturn: MF->getTarget().Options.NoTrapAfterNoreturn))
4104 return true;
4105
4106 MIRBuilder.buildTrap();
4107 return true;
4108}
4109
4110bool IRTranslatorImpl::translateInsertElement(const User &U,
4111 MachineIRBuilder &MIRBuilder) {
4112 // If it is a <1 x Ty> vector, use the scalar as it is
4113 // not a legal vector type in LLT.
4114 if (auto *FVT = dyn_cast<FixedVectorType>(Val: U.getType());
4115 FVT && FVT->getNumElements() == 1)
4116 return translateCopy(U, V: *U.getOperand(i: 1), MIRBuilder);
4117
4118 Register Res = getOrCreateVReg(Val: U);
4119 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
4120 Register Elt = getOrCreateVReg(Val: *U.getOperand(i: 1));
4121 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4122 Register Idx;
4123 if (auto *CI = dyn_cast<ConstantInt>(Val: U.getOperand(i: 2))) {
4124 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4125 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4126 auto *NewIdxCI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4127 Idx = getOrCreateVReg(Val: *NewIdxCI);
4128 }
4129 }
4130 if (!Idx)
4131 Idx = getOrCreateVReg(Val: *U.getOperand(i: 2));
4132 if (MRI->getType(Reg: Idx).getSizeInBits() != PreferredVecIdxWidth) {
4133 const LLT VecIdxTy =
4134 MRI->getType(Reg: Idx).changeElementSize(NewEltSize: PreferredVecIdxWidth);
4135 Idx = MIRBuilder.buildZExtOrTrunc(Res: VecIdxTy, Op: Idx).getReg(Idx: 0);
4136 }
4137 MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
4138 return true;
4139}
4140
4141bool IRTranslatorImpl::translateInsertVector(const User &U,
4142 MachineIRBuilder &MIRBuilder) {
4143 Register Dst = getOrCreateVReg(Val: U);
4144 Register Vec = getOrCreateVReg(Val: *U.getOperand(i: 0));
4145 Register Elt = getOrCreateVReg(Val: *U.getOperand(i: 1));
4146
4147 ConstantInt *CI = cast<ConstantInt>(Val: U.getOperand(i: 2));
4148 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4149
4150 // Resize Index to preferred index width.
4151 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4152 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4153 CI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4154 }
4155
4156 // If it is a <1 x Ty> vector, we have to use other means.
4157 if (auto *ResultType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 1)->getType());
4158 ResultType && ResultType->getNumElements() == 1) {
4159 if (auto *InputType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType());
4160 InputType && InputType->getNumElements() == 1) {
4161 // We are inserting an illegal fixed vector into an illegal
4162 // fixed vector, use the scalar as it is not a legal vector type
4163 // in LLT.
4164 return translateCopy(U, Src: Vec, MIRBuilder);
4165 }
4166 if (isa<FixedVectorType>(Val: U.getOperand(i: 0)->getType())) {
4167 // We are inserting an illegal fixed vector into a legal fixed
4168 // vector, use the scalar as it is not a legal vector type in
4169 // LLT.
4170 Register Idx = getOrCreateVReg(Val: *CI);
4171 MIRBuilder.buildInsertVectorElement(Res: Dst, Val: Vec, Elt, Idx);
4172 return true;
4173 }
4174 if (isa<ScalableVectorType>(Val: U.getOperand(i: 0)->getType())) {
4175 // We are inserting an illegal fixed vector into a scalable
4176 // vector, use a scalar element insert.
4177 LLT VecIdxTy = LLT::integer(SizeInBits: PreferredVecIdxWidth);
4178 Register Idx = getOrCreateVReg(Val: *CI);
4179 auto ScaledIndex = MIRBuilder.buildMul(
4180 Dst: VecIdxTy, Src0: MIRBuilder.buildVScale(Res: VecIdxTy, MinElts: 1), Src1: Idx);
4181 MIRBuilder.buildInsertVectorElement(Res: Dst, Val: Vec, Elt, Idx: ScaledIndex);
4182 return true;
4183 }
4184 }
4185
4186 MIRBuilder.buildInsertSubvector(Res: Dst, Src0: Vec, Src1: Elt, Index: CI->getZExtValue());
4187 return true;
4188}
4189
4190bool IRTranslatorImpl::translateExtractElement(const User &U,
4191 MachineIRBuilder &MIRBuilder) {
4192 // If it is a <1 x Ty> vector, use the scalar as it is
4193 // not a legal vector type in LLT.
4194 if (const FixedVectorType *FVT =
4195 dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType()))
4196 if (FVT->getNumElements() == 1)
4197 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
4198
4199 Register Res = getOrCreateVReg(Val: U);
4200 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
4201 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4202 Register Idx;
4203 if (auto *CI = dyn_cast<ConstantInt>(Val: U.getOperand(i: 1))) {
4204 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4205 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4206 auto *NewIdxCI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4207 Idx = getOrCreateVReg(Val: *NewIdxCI);
4208 }
4209 }
4210 if (!Idx)
4211 Idx = getOrCreateVReg(Val: *U.getOperand(i: 1));
4212 if (MRI->getType(Reg: Idx).getSizeInBits() != PreferredVecIdxWidth) {
4213 const LLT VecIdxTy =
4214 MRI->getType(Reg: Idx).changeElementSize(NewEltSize: PreferredVecIdxWidth);
4215 Idx = MIRBuilder.buildZExtOrTrunc(Res: VecIdxTy, Op: Idx).getReg(Idx: 0);
4216 }
4217 MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
4218 return true;
4219}
4220
4221bool IRTranslatorImpl::translateExtractVector(const User &U,
4222 MachineIRBuilder &MIRBuilder) {
4223 Register Res = getOrCreateVReg(Val: U);
4224 Register Vec = getOrCreateVReg(Val: *U.getOperand(i: 0));
4225 ConstantInt *CI = cast<ConstantInt>(Val: U.getOperand(i: 1));
4226 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4227
4228 // Resize Index to preferred index width.
4229 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4230 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4231 CI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4232 }
4233
4234 // If it is a <1 x Ty> vector, we have to use other means.
4235 if (auto *ResultType = dyn_cast<FixedVectorType>(Val: U.getType());
4236 ResultType && ResultType->getNumElements() == 1) {
4237 if (auto *InputType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType());
4238 InputType && InputType->getNumElements() == 1) {
4239 // We are extracting an illegal fixed vector from an illegal fixed vector,
4240 // use the scalar as it is not a legal vector type in LLT.
4241 return translateCopy(U, Src: Vec, MIRBuilder);
4242 }
4243 if (isa<FixedVectorType>(Val: U.getOperand(i: 0)->getType())) {
4244 // We are extracting an illegal fixed vector from a legal fixed
4245 // vector, use the scalar as it is not a legal vector type in
4246 // LLT.
4247 Register Idx = getOrCreateVReg(Val: *CI);
4248 MIRBuilder.buildExtractVectorElement(Res, Val: Vec, Idx);
4249 return true;
4250 }
4251 if (isa<ScalableVectorType>(Val: U.getOperand(i: 0)->getType())) {
4252 // We are extracting an illegal fixed vector from a scalable
4253 // vector, use a scalar element extract.
4254 LLT VecIdxTy = LLT::integer(SizeInBits: PreferredVecIdxWidth);
4255 Register Idx = getOrCreateVReg(Val: *CI);
4256 auto ScaledIndex = MIRBuilder.buildMul(
4257 Dst: VecIdxTy, Src0: MIRBuilder.buildVScale(Res: VecIdxTy, MinElts: 1), Src1: Idx);
4258 MIRBuilder.buildExtractVectorElement(Res, Val: Vec, Idx: ScaledIndex);
4259 return true;
4260 }
4261 }
4262
4263 MIRBuilder.buildExtractSubvector(Res, Src: Vec, Index: CI->getZExtValue());
4264 return true;
4265}
4266
4267bool IRTranslatorImpl::translateShuffleVector(const User &U,
4268 MachineIRBuilder &MIRBuilder) {
4269 // A ShuffleVector that operates on scalable vectors is a splat vector where
4270 // the value of the splat vector is the 0th element of the first operand,
4271 // since the index mask operand is the zeroinitializer (undef and
4272 // poison are treated as zeroinitializer here).
4273 if (U.getOperand(i: 0)->getType()->isScalableTy()) {
4274 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
4275 auto SplatVal = MIRBuilder.buildExtractVectorElementConstant(
4276 Res: MRI->getType(Reg: Val).getElementType(), Val, Idx: 0);
4277 MIRBuilder.buildSplatVector(Res: getOrCreateVReg(Val: U), Val: SplatVal);
4278 return true;
4279 }
4280
4281 ArrayRef<int> Mask;
4282 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: &U))
4283 Mask = SVI->getShuffleMask();
4284 else
4285 Mask = cast<ConstantExpr>(Val: U).getShuffleMask();
4286
4287 // As GISel does not represent <1 x > vectors as a separate type from scalars,
4288 // we transform shuffle_vector with a scalar output to an
4289 // ExtractVectorElement. If the input type is also scalar it becomes a Copy.
4290 unsigned DstElts = cast<FixedVectorType>(Val: U.getType())->getNumElements();
4291 unsigned SrcElts =
4292 cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType())->getNumElements();
4293 if (DstElts == 1) {
4294 unsigned M = Mask[0];
4295 if (SrcElts == 1) {
4296 if (M == 0 || M == 1)
4297 return translateCopy(U, V: *U.getOperand(i: M), MIRBuilder);
4298 MIRBuilder.buildUndef(Res: getOrCreateVReg(Val: U));
4299 } else {
4300 Register Dst = getOrCreateVReg(Val: U);
4301 if (M < SrcElts) {
4302 MIRBuilder.buildExtractVectorElementConstant(
4303 Res: Dst, Val: getOrCreateVReg(Val: *U.getOperand(i: 0)), Idx: M);
4304 } else if (M < SrcElts * 2) {
4305 MIRBuilder.buildExtractVectorElementConstant(
4306 Res: Dst, Val: getOrCreateVReg(Val: *U.getOperand(i: 1)), Idx: M - SrcElts);
4307 } else {
4308 MIRBuilder.buildUndef(Res: Dst);
4309 }
4310 }
4311 return true;
4312 }
4313
4314 // A single element src is transformed to a build_vector.
4315 if (SrcElts == 1) {
4316 SmallVector<Register> Ops;
4317 Register Undef;
4318 for (int M : Mask) {
4319 LLT SrcTy = getLLTForType(Ty&: *U.getOperand(i: 0)->getType(), DL: *DL);
4320 if (M == 0 || M == 1) {
4321 Ops.push_back(Elt: getOrCreateVReg(Val: *U.getOperand(i: M)));
4322 } else {
4323 if (!Undef.isValid()) {
4324 Undef = MRI->createGenericVirtualRegister(Ty: SrcTy);
4325 MIRBuilder.buildUndef(Res: Undef);
4326 }
4327 Ops.push_back(Elt: Undef);
4328 }
4329 }
4330 MIRBuilder.buildBuildVector(Res: getOrCreateVReg(Val: U), Ops);
4331 return true;
4332 }
4333
4334 ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
4335 MIRBuilder
4336 .buildInstr(Opc: TargetOpcode::G_SHUFFLE_VECTOR, DstOps: {getOrCreateVReg(Val: U)},
4337 SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
4338 getOrCreateVReg(Val: *U.getOperand(i: 1))})
4339 .addShuffleMask(Val: MaskAlloc);
4340 return true;
4341}
4342
4343bool IRTranslatorImpl::translatePHI(const User &U,
4344 MachineIRBuilder &MIRBuilder) {
4345 const PHINode &PI = cast<PHINode>(Val: U);
4346
4347 SmallVector<MachineInstr *, 4> Insts;
4348 for (auto Reg : getOrCreateVRegs(Val: PI)) {
4349 auto MIB = MIRBuilder.buildInstr(Opc: TargetOpcode::G_PHI, DstOps: {Reg}, SrcOps: {});
4350 Insts.push_back(Elt: MIB.getInstr());
4351 }
4352
4353 PendingPHIs.emplace_back(Args: &PI, Args: std::move(Insts));
4354 return true;
4355}
4356
4357bool IRTranslatorImpl::translateAtomicCmpXchg(const User &U,
4358 MachineIRBuilder &MIRBuilder) {
4359 const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(Val: U);
4360
4361 auto Flags = TLI->getAtomicMemOperandFlags(AI: I, DL: *DL);
4362
4363 auto Res = getOrCreateVRegs(Val: I);
4364 Register OldValRes = Res[0];
4365 Register SuccessRes = Res[1];
4366 Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
4367 Register Cmp = getOrCreateVReg(Val: *I.getCompareOperand());
4368 Register NewVal = getOrCreateVReg(Val: *I.getNewValOperand());
4369
4370 MIRBuilder.buildAtomicCmpXchgWithSuccess(
4371 OldValRes, SuccessRes, Addr, CmpVal: Cmp, NewVal,
4372 MMO&: *MF->getMachineMemOperand(
4373 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, MemTy: MRI->getType(Reg: Cmp),
4374 BaseAlignment: getMemOpAlign(I), Metadata: I.getAAMetadata(), SSID: I.getSyncScopeID(),
4375 Ordering: I.getSuccessOrdering(), FailureOrdering: I.getFailureOrdering()));
4376 return true;
4377}
4378
4379bool IRTranslatorImpl::translateAtomicRMW(const User &U,
4380 MachineIRBuilder &MIRBuilder) {
4381 if (!mayTranslateUserTypes(U))
4382 return false;
4383
4384 const AtomicRMWInst &I = cast<AtomicRMWInst>(Val: U);
4385 auto Flags = TLI->getAtomicMemOperandFlags(AI: I, DL: *DL);
4386
4387 Register Res = getOrCreateVReg(Val: I);
4388 Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
4389 Register Val = getOrCreateVReg(Val: *I.getValOperand());
4390
4391 unsigned Opcode = 0;
4392 switch (I.getOperation()) {
4393 default:
4394 return false;
4395 case AtomicRMWInst::Xchg:
4396 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
4397 break;
4398 case AtomicRMWInst::Add:
4399 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
4400 break;
4401 case AtomicRMWInst::Sub:
4402 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
4403 break;
4404 case AtomicRMWInst::And:
4405 Opcode = TargetOpcode::G_ATOMICRMW_AND;
4406 break;
4407 case AtomicRMWInst::Nand:
4408 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
4409 break;
4410 case AtomicRMWInst::Or:
4411 Opcode = TargetOpcode::G_ATOMICRMW_OR;
4412 break;
4413 case AtomicRMWInst::Xor:
4414 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
4415 break;
4416 case AtomicRMWInst::Max:
4417 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
4418 break;
4419 case AtomicRMWInst::Min:
4420 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
4421 break;
4422 case AtomicRMWInst::UMax:
4423 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
4424 break;
4425 case AtomicRMWInst::UMin:
4426 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
4427 break;
4428 case AtomicRMWInst::FAdd:
4429 Opcode = TargetOpcode::G_ATOMICRMW_FADD;
4430 break;
4431 case AtomicRMWInst::FSub:
4432 Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
4433 break;
4434 case AtomicRMWInst::FMax:
4435 Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
4436 break;
4437 case AtomicRMWInst::FMin:
4438 Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
4439 break;
4440 case AtomicRMWInst::FMaximum:
4441 Opcode = TargetOpcode::G_ATOMICRMW_FMAXIMUM;
4442 break;
4443 case AtomicRMWInst::FMinimum:
4444 Opcode = TargetOpcode::G_ATOMICRMW_FMINIMUM;
4445 break;
4446 case AtomicRMWInst::FMaximumNum:
4447 Opcode = TargetOpcode::G_ATOMICRMW_FMAXIMUMNUM;
4448 break;
4449 case AtomicRMWInst::FMinimumNum:
4450 Opcode = TargetOpcode::G_ATOMICRMW_FMINIMUMNUM;
4451 break;
4452 case AtomicRMWInst::UIncWrap:
4453 Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
4454 break;
4455 case AtomicRMWInst::UDecWrap:
4456 Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
4457 break;
4458 case AtomicRMWInst::USubCond:
4459 Opcode = TargetOpcode::G_ATOMICRMW_USUB_COND;
4460 break;
4461 case AtomicRMWInst::USubSat:
4462 Opcode = TargetOpcode::G_ATOMICRMW_USUB_SAT;
4463 break;
4464 }
4465
4466 MIRBuilder.buildAtomicRMW(
4467 Opcode, OldValRes: Res, Addr, Val,
4468 MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(I.getPointerOperand()),
4469 F: Flags, MemTy: MRI->getType(Reg: Val), BaseAlignment: getMemOpAlign(I),
4470 Metadata: I.getAAMetadata(), SSID: I.getSyncScopeID(),
4471 Ordering: I.getOrdering()));
4472 return true;
4473}
4474
4475bool IRTranslatorImpl::translateFence(const User &U,
4476 MachineIRBuilder &MIRBuilder) {
4477 const FenceInst &Fence = cast<FenceInst>(Val: U);
4478 MIRBuilder.buildFence(Ordering: static_cast<unsigned>(Fence.getOrdering()),
4479 Scope: Fence.getSyncScopeID());
4480 return true;
4481}
4482
4483bool IRTranslatorImpl::translateFreeze(const User &U,
4484 MachineIRBuilder &MIRBuilder) {
4485 const ArrayRef<Register> DstRegs = getOrCreateVRegs(Val: U);
4486 const ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *U.getOperand(i: 0));
4487
4488 assert(DstRegs.size() == SrcRegs.size() &&
4489 "Freeze with different source and destination type?");
4490
4491 for (unsigned I = 0; I < DstRegs.size(); ++I) {
4492 MIRBuilder.buildFreeze(Dst: DstRegs[I], Src: SrcRegs[I]);
4493 }
4494
4495 return true;
4496}
4497
4498void IRTranslatorImpl::finishPendingPhis() {
4499#ifndef NDEBUG
4500 DILocationVerifier Verifier;
4501 GISelObserverWrapper WrapperObserver(&Verifier);
4502 RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);
4503#endif // ifndef NDEBUG
4504 for (auto &Phi : PendingPHIs) {
4505 const PHINode *PI = Phi.first;
4506 if (PI->getType()->isEmptyTy())
4507 continue;
4508 ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
4509 MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
4510 EntryBuilder->setDebugLoc(PI->getDebugLoc());
4511#ifndef NDEBUG
4512 Verifier.setCurrentInst(PI);
4513#endif // ifndef NDEBUG
4514
4515 SmallPtrSet<const MachineBasicBlock *, 16> SeenPreds;
4516 for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
4517 auto IRPred = PI->getIncomingBlock(i);
4518 ArrayRef<Register> ValRegs = getOrCreateVRegs(Val: *PI->getIncomingValue(i));
4519 for (auto *Pred : getMachinePredBBs(Edge: {IRPred, PI->getParent()})) {
4520 if (SeenPreds.count(Ptr: Pred) || !PhiMBB->isPredecessor(MBB: Pred))
4521 continue;
4522 SeenPreds.insert(Ptr: Pred);
4523 for (unsigned j = 0; j < ValRegs.size(); ++j) {
4524 MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
4525 MIB.addUse(RegNo: ValRegs[j]);
4526 MIB.addMBB(MBB: Pred);
4527 }
4528 }
4529 }
4530 }
4531}
4532
4533void IRTranslatorImpl::translateDbgValueRecord(Value *V, bool HasArgList,
4534 const DILocalVariable *Variable,
4535 const DIExpression *Expression,
4536 const DebugLoc &DL,
4537 MachineIRBuilder &MIRBuilder) {
4538 assert(Variable->isValidLocationForIntrinsic(DL) &&
4539 "Expected inlined-at fields to agree");
4540 // Act as if we're handling a debug intrinsic.
4541 MIRBuilder.setDebugLoc(DL);
4542
4543 if (!V || HasArgList) {
4544 // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
4545 // terminate any prior location.
4546 MIRBuilder.buildIndirectDbgValue(Reg: 0, Variable, Expr: Expression);
4547 return;
4548 }
4549
4550 if (const auto *CI = dyn_cast<Constant>(Val: V)) {
4551 MIRBuilder.buildConstDbgValue(C: *CI, Variable, Expr: Expression);
4552 return;
4553 }
4554
4555 if (auto *AI = dyn_cast<AllocaInst>(Val: V);
4556 AI && AI->isStaticAlloca() && Expression->startsWithDeref()) {
4557 // If the value is an alloca and the expression starts with a
4558 // dereference, track a stack slot instead of a register, as registers
4559 // may be clobbered.
4560 auto ExprOperands = Expression->getElements();
4561 auto *ExprDerefRemoved =
4562 DIExpression::get(Context&: AI->getContext(), Elements: ExprOperands.drop_front());
4563 MIRBuilder.buildFIDbgValue(FI: getOrCreateFrameIndex(AI: *AI), Variable,
4564 Expr: ExprDerefRemoved);
4565 return;
4566 }
4567 if (translateIfEntryValueArgument(isDeclare: false, Val: V, Var: Variable, Expr: Expression, DL,
4568 MIRBuilder))
4569 return;
4570 for (Register Reg : getOrCreateVRegs(Val: *V)) {
4571 // FIXME: This does not handle register-indirect values at offset 0. The
4572 // direct/indirect thing shouldn't really be handled by something as
4573 // implicit as reg+noreg vs reg+imm in the first place, but it seems
4574 // pretty baked in right now.
4575 MIRBuilder.buildDirectDbgValue(Reg, Variable, Expr: Expression);
4576 }
4577}
4578
4579void IRTranslatorImpl::translateDbgDeclareRecord(
4580 Value *Address, bool HasArgList, const DILocalVariable *Variable,
4581 const DIExpression *Expression, const DebugLoc &DL,
4582 MachineIRBuilder &MIRBuilder) {
4583 if (!Address || isa<UndefValue>(Val: Address)) {
4584 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *Variable << "\n");
4585 return;
4586 }
4587
4588 assert(Variable->isValidLocationForIntrinsic(DL) &&
4589 "Expected inlined-at fields to agree");
4590 auto AI = dyn_cast<AllocaInst>(Val: Address);
4591 if (AI && AI->isStaticAlloca()) {
4592 // Static allocas are tracked at the MF level, no need for DBG_VALUE
4593 // instructions (in fact, they get ignored if they *do* exist).
4594 MF->setVariableDbgInfo(Var: Variable, Expr: Expression,
4595 Slot: getOrCreateFrameIndex(AI: *AI), Loc: DL);
4596 return;
4597 }
4598
4599 if (translateIfEntryValueArgument(isDeclare: true, Val: Address, Var: Variable,
4600 Expr: Expression, DL,
4601 MIRBuilder))
4602 return;
4603
4604 // A dbg.declare describes the address of a source variable, so lower it
4605 // into an indirect DBG_VALUE.
4606 MIRBuilder.setDebugLoc(DL);
4607 MIRBuilder.buildIndirectDbgValue(Reg: getOrCreateVReg(Val: *Address), Variable,
4608 Expr: Expression);
4609}
4610
4611void IRTranslatorImpl::translateDbgInfo(const Instruction &Inst,
4612 MachineIRBuilder &MIRBuilder) {
4613 for (DbgRecord &DR : Inst.getDbgRecordRange()) {
4614 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
4615 MIRBuilder.setDebugLoc(DLR->getDebugLoc());
4616 assert(DLR->getLabel() && "Missing label");
4617 assert(DLR->getLabel()->isValidLocationForIntrinsic(
4618 MIRBuilder.getDebugLoc()) &&
4619 "Expected inlined-at fields to agree");
4620 MIRBuilder.buildDbgLabel(Label: DLR->getLabel());
4621 continue;
4622 }
4623 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
4624 const DILocalVariable *Variable = DVR.getVariable();
4625 const DIExpression *Expression = DVR.getExpression();
4626 Value *V = DVR.getVariableLocationOp(OpIdx: 0);
4627 if (DVR.isDbgDeclare())
4628 translateDbgDeclareRecord(Address: V, HasArgList: DVR.hasArgList(), Variable, Expression,
4629 DL: DVR.getDebugLoc(), MIRBuilder);
4630 else
4631 translateDbgValueRecord(V, HasArgList: DVR.hasArgList(), Variable, Expression,
4632 DL: DVR.getDebugLoc(), MIRBuilder);
4633 }
4634}
4635
4636bool IRTranslatorImpl::translate(const Instruction &Inst) {
4637 CurBuilder->setDebugLoc(Inst.getDebugLoc());
4638 CurBuilder->setPCSections(Inst.getMetadata(KindID: LLVMContext::MD_pcsections));
4639 CurBuilder->setMMRAMetadata(Inst.getMetadata(KindID: LLVMContext::MD_mmra));
4640
4641 if (TLI->fallBackToDAGISel(Inst))
4642 return false;
4643
4644 switch (Inst.getOpcode()) {
4645#define HANDLE_INST(NUM, OPCODE, CLASS) \
4646 case Instruction::OPCODE: \
4647 return translate##OPCODE(Inst, *CurBuilder.get());
4648#include "llvm/IR/Instruction.def"
4649 default:
4650 return false;
4651 }
4652}
4653
4654bool IRTranslatorImpl::translate(const Constant &C, Register Reg) {
4655 // We only emit constants into the entry block from here. To prevent jumpy
4656 // debug behaviour remove debug line.
4657 if (auto CurrInstDL = CurBuilder->getDL())
4658 EntryBuilder->setDebugLoc(DebugLoc());
4659
4660 if (auto CI = dyn_cast<ConstantInt>(Val: &C)) {
4661 // buildConstant expects a to-be-splatted scalar ConstantInt.
4662 if (isa<VectorType>(Val: CI->getType()))
4663 CI = ConstantInt::get(Context&: CI->getContext(), V: CI->getValue());
4664 EntryBuilder->buildConstant(Res: Reg, Val: *CI);
4665 } else if (auto CB = dyn_cast<ConstantByte>(Val: &C)) {
4666 // Byte constants share G_CONSTANT with integers; the destination Reg's
4667 // LLT (an integer LLT, see getLLTForType) determines vector splatting.
4668 EntryBuilder->buildConstant(Res: Reg, Val: CB->getValue());
4669 } else if (auto CF = dyn_cast<ConstantFP>(Val: &C)) {
4670 // buildFConstant expects a to-be-splatted scalar ConstantFP.
4671 if (isa<VectorType>(Val: CF->getType()))
4672 CF = ConstantFP::get(Context&: CF->getContext(), V: CF->getValue());
4673 EntryBuilder->buildFConstant(Res: Reg, Val: *CF);
4674 } else if (isa<UndefValue>(Val: C))
4675 EntryBuilder->buildUndef(Res: Reg);
4676 else if (isa<ConstantPointerNull>(Val: C))
4677 EntryBuilder->buildConstant(Res: Reg, Val: 0);
4678 else if (auto GV = dyn_cast<GlobalValue>(Val: &C))
4679 EntryBuilder->buildGlobalValue(Res: Reg, GV);
4680 else if (auto CPA = dyn_cast<ConstantPtrAuth>(Val: &C)) {
4681 Register Addr = getOrCreateVReg(Val: *CPA->getPointer());
4682 Register AddrDisc = getOrCreateVReg(Val: *CPA->getAddrDiscriminator());
4683 EntryBuilder->buildConstantPtrAuth(Res: Reg, CPA, Addr, AddrDisc);
4684 } else if (auto CAZ = dyn_cast<ConstantAggregateZero>(Val: &C)) {
4685 Constant &Elt = *CAZ->getElementValue(Idx: 0u);
4686 if (isa<ScalableVectorType>(Val: CAZ->getType())) {
4687 EntryBuilder->buildSplatVector(Res: Reg, Val: getOrCreateVReg(Val: Elt));
4688 return true;
4689 }
4690 // Return the scalar if it is a <1 x Ty> vector.
4691 unsigned NumElts = CAZ->getElementCount().getFixedValue();
4692 if (NumElts == 1)
4693 return translateCopy(U: C, V: Elt, MIRBuilder&: *EntryBuilder);
4694 // All elements are zero so we can just use the first one.
4695 EntryBuilder->buildSplatBuildVector(Res: Reg, Src: getOrCreateVReg(Val: Elt));
4696 } else if (auto CV = dyn_cast<ConstantDataVector>(Val: &C)) {
4697 // Return the scalar if it is a <1 x Ty> vector.
4698 if (CV->getNumElements() == 1)
4699 return translateCopy(U: C, V: *CV->getElementAsConstant(i: 0), MIRBuilder&: *EntryBuilder);
4700 SmallVector<Register, 4> Ops;
4701 for (unsigned i = 0; i < CV->getNumElements(); ++i) {
4702 Constant &Elt = *CV->getElementAsConstant(i);
4703 Ops.push_back(Elt: getOrCreateVReg(Val: Elt));
4704 }
4705 EntryBuilder->buildBuildVector(Res: Reg, Ops);
4706 } else if (auto CE = dyn_cast<ConstantExpr>(Val: &C)) {
4707 switch(CE->getOpcode()) {
4708#define HANDLE_INST(NUM, OPCODE, CLASS) \
4709 case Instruction::OPCODE: \
4710 return translate##OPCODE(*CE, *EntryBuilder.get());
4711#include "llvm/IR/Instruction.def"
4712 default:
4713 return false;
4714 }
4715 } else if (auto CV = dyn_cast<ConstantVector>(Val: &C)) {
4716 if (CV->getNumOperands() == 1)
4717 return translateCopy(U: C, V: *CV->getOperand(i_nocapture: 0), MIRBuilder&: *EntryBuilder);
4718 SmallVector<Register, 4> Ops;
4719 for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
4720 Ops.push_back(Elt: getOrCreateVReg(Val: *CV->getOperand(i_nocapture: i)));
4721 }
4722 EntryBuilder->buildBuildVector(Res: Reg, Ops);
4723 } else if (auto *BA = dyn_cast<BlockAddress>(Val: &C)) {
4724 EntryBuilder->buildBlockAddress(Res: Reg, BA);
4725 } else
4726 return false;
4727
4728 return true;
4729}
4730
4731bool IRTranslatorImpl::mayTranslateUserTypes(const User &U) const {
4732 const TargetMachine &TM = TLI->getTargetMachine();
4733 if (LLT::getUseExtended())
4734 return true;
4735
4736 // BF16 cannot currently be represented by default LLT. To avoid miscompiles
4737 // we prevent any instructions using them by default in all targets that do
4738 // not explicitly enable it via LLT::setUseExtended(true).
4739 // SPIRV target is exception.
4740 return TM.getTargetTriple().isSPIRV() ||
4741 (!U.getType()->getScalarType()->isBFloatTy() &&
4742 !any_of(Range: U.operands(), P: [](Value *V) {
4743 return V->getType()->getScalarType()->isBFloatTy();
4744 }));
4745}
4746
4747bool IRTranslatorImpl::finalizeBasicBlock(const BasicBlock &BB,
4748 MachineBasicBlock &MBB) {
4749 for (auto &BTB : SL->BitTestCases) {
4750 // Emit header first, if it wasn't already emitted.
4751 if (!BTB.Emitted)
4752 emitBitTestHeader(B&: BTB, SwitchBB: BTB.Parent);
4753
4754 BranchProbability UnhandledProb = BTB.Prob;
4755 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
4756 UnhandledProb -= BTB.Cases[j].ExtraProb;
4757 // Set the current basic block to the mbb we wish to insert the code into
4758 MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
4759 // If all cases cover a contiguous range, it is not necessary to jump to
4760 // the default block after the last bit test fails. This is because the
4761 // range check during bit test header creation has guaranteed that every
4762 // case here doesn't go outside the range. In this case, there is no need
4763 // to perform the last bit test, as it will always be true. Instead, make
4764 // the second-to-last bit-test fall through to the target of the last bit
4765 // test, and delete the last bit test.
4766
4767 MachineBasicBlock *NextMBB;
4768 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
4769 // Second-to-last bit-test with contiguous range: fall through to the
4770 // target of the final bit test.
4771 NextMBB = BTB.Cases[j + 1].TargetBB;
4772 } else if (j + 1 == ej) {
4773 // For the last bit test, fall through to Default.
4774 NextMBB = BTB.Default;
4775 } else {
4776 // Otherwise, fall through to the next bit test.
4777 NextMBB = BTB.Cases[j + 1].ThisBB;
4778 }
4779
4780 emitBitTestCase(BB&: BTB, NextMBB, BranchProbToNext: UnhandledProb, Reg: BTB.Reg, B&: BTB.Cases[j], SwitchBB: MBB);
4781
4782 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
4783 // We need to record the replacement phi edge here that normally
4784 // happens in emitBitTestCase before we delete the case, otherwise the
4785 // phi edge will be lost.
4786 addMachineCFGPred(Edge: {BTB.Parent->getBasicBlock(),
4787 BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
4788 NewPred: MBB);
4789 // Since we're not going to use the final bit test, remove it.
4790 BTB.Cases.pop_back();
4791 break;
4792 }
4793 }
4794 // This is "default" BB. We have two jumps to it. From "header" BB and from
4795 // last "case" BB, unless the latter was skipped.
4796 CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
4797 BTB.Default->getBasicBlock()};
4798 addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Parent);
4799 if (!BTB.ContiguousRange) {
4800 addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Cases.back().ThisBB);
4801 }
4802 }
4803 SL->BitTestCases.clear();
4804
4805 for (auto &JTCase : SL->JTCases) {
4806 // Emit header first, if it wasn't already emitted.
4807 if (!JTCase.first.Emitted)
4808 emitJumpTableHeader(JT&: JTCase.second, JTH&: JTCase.first, HeaderBB: JTCase.first.HeaderBB);
4809
4810 emitJumpTable(JT&: JTCase.second, MBB: JTCase.second.MBB);
4811 }
4812 SL->JTCases.clear();
4813
4814 for (auto &SwCase : SL->SwitchCases)
4815 emitSwitchCase(CB&: SwCase, SwitchBB: &CurBuilder->getMBB(), MIB&: *CurBuilder);
4816 SL->SwitchCases.clear();
4817
4818 // Check if we need to generate stack-protector guard checks.
4819 if (SPInfo->shouldEmitSDCheck(BB)) {
4820 bool FunctionBasedInstrumentation =
4821 TLI->getSSPStackGuardCheck(M: *MF->getFunction().getParent(), Libcalls: *Libcalls);
4822 SPDescriptor.initialize(BB: &BB, MBB: &MBB, FunctionBasedInstrumentation);
4823 }
4824 // Handle stack protector.
4825 if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
4826 LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
4827 return false;
4828 } else if (SPDescriptor.shouldEmitStackProtector()) {
4829 MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
4830 MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
4831
4832 // Find the split point to split the parent mbb. At the same time copy all
4833 // physical registers used in the tail of parent mbb into virtual registers
4834 // before the split point and back into physical registers after the split
4835 // point. This prevents us needing to deal with Live-ins and many other
4836 // register allocation issues caused by us splitting the parent mbb. The
4837 // register allocator will clean up said virtual copies later on.
4838 MachineBasicBlock::iterator SplitPoint = findSplitPointForStackProtector(
4839 BB: ParentMBB, TII: *MF->getSubtarget().getInstrInfo());
4840
4841 // Splice the terminator of ParentMBB into SuccessMBB.
4842 SuccessMBB->splice(Where: SuccessMBB->end(), Other: ParentMBB, From: SplitPoint,
4843 To: ParentMBB->end());
4844
4845 // Add compare/jump on neq/jump to the parent BB.
4846 if (!emitSPDescriptorParent(SPD&: SPDescriptor, ParentBB: ParentMBB))
4847 return false;
4848
4849 // CodeGen Failure MBB if we have not codegened it yet.
4850 MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
4851 if (FailureMBB->empty()) {
4852 if (!emitSPDescriptorFailure(SPD&: SPDescriptor, FailureBB: FailureMBB))
4853 return false;
4854 }
4855
4856 // Clear the Per-BB State.
4857 SPDescriptor.resetPerBBState();
4858 }
4859 return true;
4860}
4861
4862bool IRTranslatorImpl::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
4863 MachineBasicBlock *ParentBB) {
4864 CurBuilder->setInsertPt(MBB&: *ParentBB, II: ParentBB->end());
4865 // First create the loads to the guard/stack slot for the comparison.
4866 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
4867 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
4868 LLT PtrMemTy = getLLTForMVT(Ty: TLI->getPointerMemTy(DL: *DL));
4869
4870 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
4871 int FI = MFI.getStackProtectorIndex();
4872
4873 Register Guard;
4874 Register StackSlotPtr = CurBuilder->buildFrameIndex(Res: PtrTy, Idx: FI).getReg(Idx: 0);
4875 const Module &M = *ParentBB->getParent()->getFunction().getParent();
4876 Align Align = DL->getPrefTypeAlign(Ty: PointerType::getUnqual(C&: M.getContext()));
4877
4878 // Generate code to load the content of the guard slot.
4879 Register GuardVal =
4880 CurBuilder
4881 ->buildLoad(Res: PtrMemTy, Addr: StackSlotPtr,
4882 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
4883 MMOFlags: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile)
4884 .getReg(Idx: 0);
4885
4886 // Retrieve guard check function, nullptr if instrumentation is inlined.
4887 if (const Function *GuardCheckFn = TLI->getSSPStackGuardCheck(M, Libcalls: *Libcalls)) {
4888 // This path is currently untestable on GlobalISel, since the only platform
4889 // that needs this seems to be Windows, and we fall back on that currently.
4890 // The code still lives here in case that changes.
4891 // Silence warning about unused variable until the code below that uses
4892 // 'GuardCheckFn' is enabled.
4893 (void)GuardCheckFn;
4894 return false;
4895#if 0
4896 // The target provides a guard check function to validate the guard value.
4897 // Generate a call to that function with the content of the guard slot as
4898 // argument.
4899 FunctionType *FnTy = GuardCheckFn->getFunctionType();
4900 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
4901 ISD::ArgFlagsTy Flags;
4902 if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
4903 Flags.setInReg();
4904 CallLowering::ArgInfo GuardArgInfo(
4905 {GuardVal, FnTy->getParamType(0), {Flags}});
4906
4907 CallLowering::CallLoweringInfo Info;
4908 Info.OrigArgs.push_back(GuardArgInfo);
4909 Info.CallConv = GuardCheckFn->getCallingConv();
4910 Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
4911 Info.OrigRet = {Register(), FnTy->getReturnType()};
4912 if (!CLI->lowerCall(MIRBuilder, Info)) {
4913 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
4914 return false;
4915 }
4916 return true;
4917#endif
4918 }
4919
4920 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
4921 // Otherwise, emit a volatile load to retrieve the stack guard value.
4922 if (TLI->useLoadStackGuardNode(M: *ParentBB->getBasicBlock()->getModule())) {
4923 Guard = MRI->createGenericVirtualRegister(Ty: PtrMemTy);
4924 getStackGuard(DstReg: Guard, MIRBuilder&: *CurBuilder);
4925 } else {
4926 // TODO: test using android subtarget when we support @llvm.thread.pointer.
4927 const Value *IRGuard = TLI->getSDagStackGuard(M, Libcalls: *Libcalls);
4928 Register GuardPtr = getOrCreateVReg(Val: *IRGuard);
4929
4930 Guard = CurBuilder
4931 ->buildLoad(Res: PtrMemTy, Addr: GuardPtr,
4932 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
4933 MMOFlags: MachineMemOperand::MOLoad |
4934 MachineMemOperand::MOVolatile)
4935 .getReg(Idx: 0);
4936 }
4937
4938 // Perform the comparison.
4939 auto Cmp =
4940 CurBuilder->buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::integer(SizeInBits: 1), Op0: Guard, Op1: GuardVal);
4941 // If the guard/stackslot do not equal, branch to failure MBB.
4942 CurBuilder->buildBrCond(Tst: Cmp, Dest&: *SPD.getFailureMBB());
4943 // Otherwise branch to success MBB.
4944 CurBuilder->buildBr(Dest&: *SPD.getSuccessMBB());
4945 return true;
4946}
4947
4948bool IRTranslatorImpl::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
4949 MachineBasicBlock *FailureBB) {
4950 const RTLIB::LibcallImpl LibcallImpl =
4951 Libcalls->getLibcallImpl(Call: RTLIB::STACKPROTECTOR_CHECK_FAIL);
4952 if (LibcallImpl == RTLIB::Unsupported)
4953 return false;
4954
4955 CurBuilder->setInsertPt(MBB&: *FailureBB, II: FailureBB->end());
4956
4957 CallLowering::CallLoweringInfo Info;
4958 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: LibcallImpl);
4959
4960 StringRef LibcallName =
4961 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LibcallImpl);
4962 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
4963 Info.OrigRet = {Register(), Type::getVoidTy(C&: MF->getFunction().getContext()),
4964 0};
4965 if (!CLI->lowerCall(MIRBuilder&: *CurBuilder, Info)) {
4966 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
4967 return false;
4968 }
4969
4970 // Emit a trap instruction if we are required to do so.
4971 const TargetOptions &TargetOpts = TLI->getTargetMachine().Options;
4972 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
4973 CurBuilder->buildInstr(Opcode: TargetOpcode::G_TRAP);
4974
4975 return true;
4976}
4977
4978void IRTranslatorImpl::finalizeFunction() {
4979 // Release the memory used by the different maps we
4980 // needed during the translation.
4981 PendingPHIs.clear();
4982 VMap.reset();
4983 FrameIndices.clear();
4984 MachinePreds.clear();
4985 // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
4986 // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
4987 // destroying it twice (in ~IRTranslator() and ~LLVMContext())
4988 EntryBuilder.reset();
4989 CurBuilder.reset();
4990 FuncInfo.clear();
4991 SPDescriptor.resetPerFunctionState();
4992}
4993
4994/// Returns true if a BasicBlock \p BB within a variadic function contains a
4995/// variadic musttail call.
4996static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
4997 if (!IsVarArg)
4998 return false;
4999
5000 // Walk the block backwards, because tail calls usually only appear at the end
5001 // of a block.
5002 return llvm::any_of(Range: llvm::reverse(C: BB), P: [](const Instruction &I) {
5003 const auto *CI = dyn_cast<CallInst>(Val: &I);
5004 return CI && CI->isMustTailCall();
5005 });
5006}
5007
5008bool IRTranslatorImpl::runOnMachineFunction(
5009 MachineFunction &CurMF, function_ref<GISelCSEInfo *()> GetCSEInfo,
5010 bool ShouldSkipOpts, function_ref<AAResults *()> GetAAResults,
5011 function_ref<BranchProbabilityInfo *()> GetBPI,
5012 function_ref<AssumptionCache *()> GetAC, TargetLibraryInfo *LibraryInfo,
5013 const LibcallLoweringInfo *LibcallInfo, SSPLayoutInfo *StackProtectorInfo) {
5014 MF = &CurMF;
5015 const Function &F = MF->getFunction();
5016 ORE = std::make_unique<OptimizationRemarkEmitter>(args: &F);
5017 CLI = MF->getSubtarget().getCallLowering();
5018 SPInfo = StackProtectorInfo;
5019
5020 if (CLI->fallBackToDAGISel(MF: *MF)) {
5021 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
5022 F.getSubprogram(), &F.getEntryBlock());
5023 R << "unable to lower function: "
5024 << ore::NV("Prototype", F.getFunctionType());
5025
5026 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5027 return false;
5028 }
5029
5030 // Set the CSEConfig and run the analysis.
5031 GISelCSEInfo *CSEInfo = nullptr;
5032
5033 bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
5034 ? EnableCSEInIRTranslator
5035 : true;
5036
5037 const TargetSubtargetInfo &Subtarget = MF->getSubtarget();
5038 TLI = Subtarget.getTargetLowering();
5039
5040 if (EnableCSE) {
5041 EntryBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
5042 CSEInfo = GetCSEInfo();
5043 EntryBuilder->setCSEInfo(CSEInfo);
5044 CurBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
5045 CurBuilder->setCSEInfo(CSEInfo);
5046 } else {
5047 EntryBuilder = std::make_unique<MachineIRBuilder>();
5048 CurBuilder = std::make_unique<MachineIRBuilder>();
5049 }
5050 CLI = Subtarget.getCallLowering();
5051 CurBuilder->setMF(*MF);
5052 EntryBuilder->setMF(*MF);
5053 MRI = &MF->getRegInfo();
5054 DL = &F.getDataLayout();
5055 const TargetMachine &TM = MF->getTarget();
5056 EnableOpts = OptLevel != CodeGenOptLevel::None && !ShouldSkipOpts;
5057 FuncInfo.MF = MF;
5058 // Prefer the "exception-model" module flag, else the TargetOptions default.
5059 FuncInfo.ExceptionModel = F.getParent()->getExceptionModel();
5060 if (FuncInfo.ExceptionModel == ExceptionHandling::Default)
5061 FuncInfo.ExceptionModel = TM.getExceptionModel();
5062 if (EnableOpts) {
5063 AA = GetAAResults();
5064 FuncInfo.BPI = GetBPI();
5065 AC = GetAC();
5066 } else {
5067 AA = nullptr;
5068 FuncInfo.BPI = nullptr;
5069 AC = nullptr;
5070 }
5071 LibInfo = LibraryInfo;
5072 Libcalls = LibcallInfo;
5073
5074 FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(MF&: *MF);
5075
5076 SL = std::make_unique<GISelSwitchLowering>(args: this, args&: FuncInfo);
5077 SL->init(tli: *TLI, tm: TM, dl: *DL);
5078
5079 assert(PendingPHIs.empty() && "stale PHIs");
5080
5081 // Targets which want to use big endian can enable it using
5082 // enableBigEndian()
5083 if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
5084 // Currently we don't properly handle big endian code.
5085 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
5086 F.getSubprogram(), &F.getEntryBlock());
5087 R << "unable to translate in big endian mode";
5088 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5089 return false;
5090 }
5091
5092 // Release the per-function state when we return, whether we succeeded or not.
5093 llvm::scope_exit FinalizeOnReturn([this]() { finalizeFunction(); });
5094
5095 // Setup a separate basic-block for the arguments and constants
5096 MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
5097 MF->push_back(MBB: EntryBB);
5098 EntryBuilder->setMBB(*EntryBB);
5099
5100 DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHIIt()->getDebugLoc();
5101 SwiftError.setFunction(CurMF);
5102 SwiftError.createEntriesInEntryBlock(DbgLoc);
5103
5104 bool IsVarArg = F.isVarArg();
5105 bool HasMustTailInVarArgFn = false;
5106 // Use arguments and instructions to estimate the number of mapped values and
5107 // virtual registers.
5108 unsigned NumValues = F.arg_size();
5109
5110 // Create all blocks, in IR order, to preserve the layout.
5111 FuncInfo.MBBMap.resize(N: F.getMaxBlockNumber());
5112 for (const BasicBlock &BB: F) {
5113 NumValues += BB.size();
5114 auto *&MBB = FuncInfo.MBBMap[BB.getNumber()];
5115
5116 MBB = MF->CreateMachineBasicBlock(BB: &BB);
5117 MF->push_back(MBB);
5118
5119 // Only mark the block if the BlockAddress actually has users. The
5120 // hasAddressTaken flag may be stale if the BlockAddress was optimized away
5121 // but the constant still exists in the uniquing table.
5122 if (BB.hasAddressTaken()) {
5123 if (BlockAddress *BA = BlockAddress::lookup(BB: &BB))
5124 if (!BA->hasZeroLiveUses())
5125 MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
5126 }
5127
5128 if (!HasMustTailInVarArgFn)
5129 HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
5130 }
5131
5132 VMap.reserveVRegs(NumValues);
5133 MRI->reserveVirtRegs(NumVirtRegs: NumValues);
5134
5135 MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
5136
5137 // Make our arguments/constants entry block fallthrough to the IR entry block.
5138 EntryBB->addSuccessor(Succ: &getMBB(BB: F.front()));
5139
5140 // Lower the actual args into this basic block.
5141 SmallVector<ArrayRef<Register>, 8> VRegArgs;
5142 for (const Argument &Arg: F.args()) {
5143 if (DL->getTypeStoreSize(Ty: Arg.getType()).isZero())
5144 continue; // Don't handle zero sized types.
5145 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: Arg);
5146 VRegArgs.push_back(Elt: VRegs);
5147
5148 if (CLI->supportSwiftError() && Arg.hasSwiftErrorAttr()) {
5149 assert(VRegs.size() == 1 && "Too many vregs for Swift error");
5150 SwiftError.setCurrentVReg(MBB: EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
5151 }
5152 }
5153
5154 if (!CLI->lowerFormalArguments(MIRBuilder&: *EntryBuilder, F, VRegs: VRegArgs, FLI&: FuncInfo)) {
5155 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
5156 F.getSubprogram(), &F.getEntryBlock());
5157 R << "unable to lower arguments: "
5158 << ore::NV("Prototype", F.getFunctionType());
5159 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5160 return false;
5161 }
5162
5163 // Need to visit defs before uses when translating instructions.
5164 GISelObserverWrapper WrapperObserver;
5165 if (EnableCSE && CSEInfo)
5166 WrapperObserver.addObserver(O: CSEInfo);
5167 {
5168 ReversePostOrderTraversal<const Function *> RPOT(&F);
5169#ifndef NDEBUG
5170 DILocationVerifier Verifier;
5171 WrapperObserver.addObserver(&Verifier);
5172#endif // ifndef NDEBUG
5173 RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);
5174 for (const BasicBlock *BB : RPOT) {
5175 MachineBasicBlock &MBB = getMBB(BB: *BB);
5176 // Set the insertion point of all the following translations to
5177 // the end of this basic block.
5178 CurBuilder->setMBB(MBB);
5179 HasTailCall = false;
5180 for (const Instruction &Inst : *BB) {
5181 // If we translated a tail call in the last step, then we know
5182 // everything after the call is either a return, or something that is
5183 // handled by the call itself. (E.g. a lifetime marker or assume
5184 // intrinsic.) In this case, we should stop translating the block and
5185 // move on.
5186 if (HasTailCall)
5187 break;
5188#ifndef NDEBUG
5189 Verifier.setCurrentInst(&Inst);
5190#endif // ifndef NDEBUG
5191
5192 // Translate any debug-info attached to the instruction.
5193 translateDbgInfo(Inst, MIRBuilder&: *CurBuilder);
5194
5195 if (translate(Inst))
5196 continue;
5197
5198 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
5199 Inst.getDebugLoc(), BB);
5200 R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
5201
5202 if (ORE->allowExtraAnalysis(PassName: "gisel-irtranslator")) {
5203 std::string InstStrStorage;
5204 raw_string_ostream InstStr(InstStrStorage);
5205 InstStr << Inst;
5206
5207 R << ": '" << InstStrStorage << "'";
5208 }
5209
5210 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5211 return false;
5212 }
5213
5214 if (!finalizeBasicBlock(BB: *BB, MBB)) {
5215 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
5216 BB->getTerminator()->getDebugLoc(), BB);
5217 R << "unable to translate basic block";
5218 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5219 return false;
5220 }
5221 }
5222#ifndef NDEBUG
5223 WrapperObserver.removeObserver(&Verifier);
5224#endif
5225 }
5226
5227 finishPendingPhis();
5228
5229 SwiftError.propagateVRegs();
5230
5231 // Merge the argument lowering and constants block with its single
5232 // successor, the LLVM-IR entry block. We want the basic block to
5233 // be maximal.
5234 assert(EntryBB->succ_size() == 1 &&
5235 "Custom BB used for lowering should have only one successor");
5236 // Get the successor of the current entry block.
5237 MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
5238 assert(NewEntryBB.pred_size() == 1 &&
5239 "LLVM-IR entry block has a predecessor!?");
5240 // Move all the instruction from the current entry block to the
5241 // new entry block.
5242 NewEntryBB.splice(Where: NewEntryBB.begin(), Other: EntryBB, From: EntryBB->begin(),
5243 To: EntryBB->end());
5244
5245 // Update the live-in information for the new entry block.
5246 for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
5247 NewEntryBB.addLiveIn(RegMaskPair: LiveIn);
5248 NewEntryBB.sortUniqueLiveIns();
5249
5250 // Get rid of the now empty basic block.
5251 EntryBB->removeSuccessor(Succ: &NewEntryBB);
5252 MF->remove(MBBI: EntryBB);
5253 MF->deleteMachineBasicBlock(MBB: EntryBB);
5254
5255 assert(&MF->front() == &NewEntryBB &&
5256 "New entry wasn't next in the list of basic block!");
5257
5258 // Initialize stack protector information.
5259 SPInfo->copyToMachineFrameInfo(MFI&: MF->getFrameInfo());
5260
5261 return false;
5262}
5263
5264bool IRTranslatorLegacy::runOnMachineFunction(MachineFunction &MF) {
5265 const TargetSubtargetInfo &Subtarget = MF.getSubtarget();
5266 Function &F = MF.getFunction();
5267
5268 bool ShouldSkipOpts = skipFunction(F: MF.getFunction());
5269 return Impl->runOnMachineFunction(
5270 CurMF&: MF,
5271 GetCSEInfo: [&]() {
5272 TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
5273 GISelCSEAnalysisWrapper &Wrapper =
5274 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
5275 return &Wrapper.get(CSEOpt: TPC.getCSEConfig());
5276 },
5277 ShouldSkipOpts,
5278 GetAAResults: [&]() { return &getAnalysis<AAResultsWrapperPass>().getAAResults(); },
5279 GetBPI: [&]() {
5280 return &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
5281 },
5282 GetAC: [&]() {
5283 return &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
5284 F&: MF.getFunction());
5285 },
5286 LibraryInfo: &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F: MF.getFunction()),
5287 LibcallInfo: &getAnalysis<LibcallLoweringInfoWrapper>().getLibcallLowering(
5288 M: *F.getParent(), Subtarget),
5289 StackProtectorInfo: &getAnalysis<StackProtector>().getLayoutInfo());
5290}
5291
5292IRTranslatorPass::IRTranslatorPass(CodeGenOptLevel OptLevel)
5293 : Impl(std::make_unique<IRTranslatorImpl>(args&: OptLevel)) {}
5294
5295IRTranslatorPass::~IRTranslatorPass() = default;
5296IRTranslatorPass::IRTranslatorPass(IRTranslatorPass &&) = default;
5297
5298PreservedAnalyses IRTranslatorPass::run(MachineFunction &MF,
5299 MachineFunctionAnalysisManager &MFAM) {
5300 const TargetSubtargetInfo &Subtarget = MF.getSubtarget();
5301 Function &F = MF.getFunction();
5302
5303 bool ShouldSkipOpts = MF.getFunction().hasOptNone();
5304 auto &FAM = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
5305 .getManager();
5306 auto &MAMProxy =
5307 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF);
5308 const ModuleLibcallLoweringInfo *MLLI =
5309 MAMProxy.getCachedResult<LibcallLoweringModuleAnalysis>(IR&: *F.getParent());
5310 if (!MLLI)
5311 reportFatalUsageError(
5312 reason: "LibcallLoweringModuleAnalysis must be available for IRTranslator");
5313 Impl->runOnMachineFunction(
5314 CurMF&: MF, GetCSEInfo: [&]() { return MFAM.getResult<GISelCSEAnalysis>(IR&: MF).get(); },
5315 ShouldSkipOpts, GetAAResults: [&]() { return &FAM.getResult<AAManager>(IR&: F); },
5316 GetBPI: [&]() { return &FAM.getResult<BranchProbabilityAnalysis>(IR&: F); },
5317 GetAC: [&]() { return &FAM.getResult<AssumptionAnalysis>(IR&: F); },
5318 LibraryInfo: &FAM.getResult<TargetLibraryAnalysis>(IR&: F),
5319 LibcallInfo: &getLibcallLowering(ModuleInfo: *MLLI, Subtarget),
5320 StackProtectorInfo: &FAM.getResult<SSPLayoutAnalysis>(IR&: F));
5321
5322 return getMachineFunctionPassPreservedAnalyses();
5323}
5324