1//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the IRTranslator class.
10//===----------------------------------------------------------------------===//
11
12#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
13#include "llvm/ADT/PostOrderIterator.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/Analysis/AliasAnalysis.h"
18#include "llvm/Analysis/AssumptionCache.h"
19#include "llvm/Analysis/BranchProbabilityInfo.h"
20#include "llvm/Analysis/Loads.h"
21#include "llvm/Analysis/OptimizationRemarkEmitter.h"
22#include "llvm/Analysis/ValueTracking.h"
23#include "llvm/Analysis/VectorUtils.h"
24#include "llvm/CodeGen/Analysis.h"
25#include "llvm/CodeGen/CodeGenCommonISel.h"
26#include "llvm/CodeGen/FunctionLoweringInfo.h"
27#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
28#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
29#include "llvm/CodeGen/GlobalISel/CallLowering.h"
30#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
31#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
32#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
33#include "llvm/CodeGen/LowLevelTypeUtils.h"
34#include "llvm/CodeGen/MachineBasicBlock.h"
35#include "llvm/CodeGen/MachineFrameInfo.h"
36#include "llvm/CodeGen/MachineFunction.h"
37#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
38#include "llvm/CodeGen/MachineInstrBuilder.h"
39#include "llvm/CodeGen/MachineMemOperand.h"
40#include "llvm/CodeGen/MachineModuleInfo.h"
41#include "llvm/CodeGen/MachineOperand.h"
42#include "llvm/CodeGen/MachinePassManager.h"
43#include "llvm/CodeGen/MachineRegisterInfo.h"
44#include "llvm/CodeGen/StackProtector.h"
45#include "llvm/CodeGen/SwiftErrorValueTracking.h"
46#include "llvm/CodeGen/SwitchLoweringUtils.h"
47#include "llvm/CodeGen/TargetFrameLowering.h"
48#include "llvm/CodeGen/TargetInstrInfo.h"
49#include "llvm/CodeGen/TargetLowering.h"
50#include "llvm/CodeGen/TargetOpcodes.h"
51#include "llvm/CodeGen/TargetPassConfig.h"
52#include "llvm/CodeGen/TargetRegisterInfo.h"
53#include "llvm/CodeGen/TargetSubtargetInfo.h"
54#include "llvm/CodeGenTypes/LowLevelType.h"
55#include "llvm/IR/Analysis.h"
56#include "llvm/IR/BasicBlock.h"
57#include "llvm/IR/CFG.h"
58#include "llvm/IR/Constant.h"
59#include "llvm/IR/Constants.h"
60#include "llvm/IR/DataLayout.h"
61#include "llvm/IR/DerivedTypes.h"
62#include "llvm/IR/DiagnosticInfo.h"
63#include "llvm/IR/Function.h"
64#include "llvm/IR/GetElementPtrTypeIterator.h"
65#include "llvm/IR/InlineAsm.h"
66#include "llvm/IR/InstrTypes.h"
67#include "llvm/IR/Instructions.h"
68#include "llvm/IR/IntrinsicInst.h"
69#include "llvm/IR/Intrinsics.h"
70#include "llvm/IR/IntrinsicsAMDGPU.h"
71#include "llvm/IR/LLVMContext.h"
72#include "llvm/IR/Metadata.h"
73#include "llvm/IR/Module.h"
74#include "llvm/IR/PatternMatch.h"
75#include "llvm/IR/Statepoint.h"
76#include "llvm/IR/Type.h"
77#include "llvm/IR/User.h"
78#include "llvm/IR/Value.h"
79#include "llvm/InitializePasses.h"
80#include "llvm/MC/MCContext.h"
81#include "llvm/Pass.h"
82#include "llvm/Support/Casting.h"
83#include "llvm/Support/CodeGen.h"
84#include "llvm/Support/Debug.h"
85#include "llvm/Support/ErrorHandling.h"
86#include "llvm/Support/MathExtras.h"
87#include "llvm/Support/raw_ostream.h"
88#include "llvm/Target/TargetMachine.h"
89#include "llvm/Transforms/Utils/Local.h"
90#include "llvm/Transforms/Utils/MemoryOpRemark.h"
91#include <algorithm>
92#include <cassert>
93#include <cstdint>
94#include <iterator>
95#include <optional>
96#include <string>
97#include <utility>
98#include <vector>
99
100#define DEBUG_TYPE "ir-translator"
101
102using namespace llvm;
103
104static cl::opt<bool>
105 EnableCSEInIRTranslator("enable-cse-in-ir-translator",
106 cl::desc("Should enable CSE in ir-translator"),
107 cl::init(Val: false));
108
109namespace llvm {
110
111class IRTranslatorImpl {
112 /// Interface used to lower the everything related to calls.
113 const CallLowering *CLI = nullptr;
114
115 SSPLayoutInfo *SPInfo = nullptr;
116
117 /// This class contains the mapping between the Values to vreg related data.
118 class ValueToVRegInfo {
119 public:
120 ValueToVRegInfo() = default;
121
122 using VRegListT = SmallVector<Register, 1>;
123 using OffsetListT = SmallVector<uint64_t, 1>;
124
125 using const_vreg_iterator =
126 DenseMap<const Value *, VRegListT *>::const_iterator;
127 using const_offset_iterator =
128 DenseMap<const Value *, OffsetListT *>::const_iterator;
129
130 inline const_vreg_iterator vregs_end() const { return ValToVRegs.end(); }
131
132 VRegListT *getVRegs(const Value &V) {
133 auto [It, Inserted] = ValToVRegs.try_emplace(Key: &V);
134 if (!Inserted)
135 return It->second;
136
137 // We placement new using our fast allocator since we never try to free
138 // the vectors until translation is finished.
139 It->second = new (VRegAlloc.Allocate()) VRegListT();
140 return It->second;
141 }
142
143 OffsetListT *getOffsets(const Value &V) {
144 assert(V.getType()->isAggregateType() &&
145 "Offsets are for aggregate values");
146 auto [It, Inserted] = TypeToOffsets.try_emplace(Key: V.getType());
147 if (!Inserted)
148 return It->second;
149
150 It->second = new (OffsetAlloc.Allocate()) OffsetListT();
151 return It->second;
152 }
153
154 const_vreg_iterator findVRegs(const Value &V) const {
155 return ValToVRegs.find(Val: &V);
156 }
157
158 bool contains(const Value &V) const { return ValToVRegs.contains(Val: &V); }
159
160 void reserveVRegs(unsigned NumValues) { ValToVRegs.reserve(NumEntries: NumValues); }
161
162 void reset() {
163 ValToVRegs.clear();
164 TypeToOffsets.clear();
165 VRegAlloc.DestroyAll();
166 OffsetAlloc.DestroyAll();
167 }
168
169 private:
170 SpecificBumpPtrAllocator<VRegListT> VRegAlloc;
171 SpecificBumpPtrAllocator<OffsetListT> OffsetAlloc;
172
173 // We store pointers to vectors here since references may be invalidated
174 // while we hold them if we stored the vectors directly.
175 DenseMap<const Value *, VRegListT *> ValToVRegs;
176 DenseMap<const Type *, OffsetListT *> TypeToOffsets;
177 };
178
179 /// Mapping of the values of the current LLVM IR function to the related
180 /// virtual registers and offsets.
181 ValueToVRegInfo VMap;
182
183 // One BasicBlock can be translated to multiple MachineBasicBlocks. For such
184 // BasicBlocks translated to multiple MachineBasicBlocks, MachinePreds retains
185 // a mapping between the edges arriving at the BasicBlock to the corresponding
186 // created MachineBasicBlocks. Some BasicBlocks that get translated to a
187 // single MachineBasicBlock may also end up in this Map.
188 using CFGEdge = std::pair<const BasicBlock *, const BasicBlock *>;
189 DenseMap<CFGEdge, SmallVector<MachineBasicBlock *, 1>> MachinePreds;
190
191 // List of stubbed PHI instructions, for values and basic blocks to be filled
192 // in once all MachineBasicBlocks have been created.
193 SmallVector<std::pair<const PHINode *, SmallVector<MachineInstr *, 1>>, 4>
194 PendingPHIs;
195
196 // Constant DbgValues that cannot be materialized as DBG_VALUEs
197 // because they are not an immediate or symbol reference.
198 SmallVector<std::pair<MachineInstr *, const Constant *>, 4>
199 PendingConstDbgValues;
200
201 /// Record of what frame index has been allocated to specified allocas for
202 /// this function.
203 DenseMap<const AllocaInst *, int> FrameIndices;
204
205 SwiftErrorValueTracking SwiftError;
206
207 /// \name Methods for translating form LLVM IR to MachineInstr.
208 /// \see ::translate for general information on the translate methods.
209 /// @{
210
211 /// Translate \p Inst into its corresponding MachineInstr instruction(s).
212 /// Insert the newly translated instruction(s) right where the CurBuilder
213 /// is set.
214 ///
215 /// The general algorithm is:
216 /// 1. Look for a virtual register for each operand or
217 /// create one.
218 /// 2 Update the VMap accordingly.
219 /// 2.alt. For constant arguments, if they are compile time constants,
220 /// produce an immediate in the right operand and do not touch
221 /// ValToReg. Actually we will go with a virtual register for each
222 /// constants because it may be expensive to actually materialize the
223 /// constant. Moreover, if the constant spans on several instructions,
224 /// CSE may not catch them.
225 /// => Update ValToVReg and remember that we saw a constant in Constants.
226 /// We will materialize all the constants in finalize.
227 /// Note: we would need to do something so that we can recognize such operand
228 /// as constants.
229 /// 3. Create the generic instruction.
230 ///
231 /// \return true if the translation succeeded.
232 bool translate(const Instruction &Inst);
233
234 /// Materialize \p C into virtual-register \p Reg. The generic instructions
235 /// performing this materialization will be inserted into the entry block of
236 /// the function.
237 ///
238 /// \return true if the materialization succeeded.
239 bool translate(const Constant &C, Register Reg);
240
241 /// Examine any debug-info attached to the instruction (in the form of
242 /// DbgRecords) and translate it.
243 void translateDbgInfo(const Instruction &Inst, MachineIRBuilder &MIRBuilder);
244
245 /// Translate a debug-info record of a dbg.value into a DBG_* instruction.
246 /// Pass in all the contents of the record, rather than relying on how it's
247 /// stored.
248 void translateDbgValueRecord(Value *V, bool HasArgList,
249 const DILocalVariable *Variable,
250 const DIExpression *Expression,
251 const DebugLoc &DL,
252 MachineIRBuilder &MIRBuilder);
253
254 /// Translate a debug-info record of a dbg.declare into an indirect DBG_*
255 /// instruction. Pass in all the contents of the record, rather than relying
256 /// on how it's stored.
257 void translateDbgDeclareRecord(Value *Address, bool HasArgList,
258 const DILocalVariable *Variable,
259 const DIExpression *Expression,
260 const DebugLoc &DL,
261 MachineIRBuilder &MIRBuilder);
262
263 // Translate U as a copy of V.
264 bool translateCopy(const User &U, const Value &V,
265 MachineIRBuilder &MIRBuilder);
266 bool translateCopy(const User &U, Register Src, MachineIRBuilder &MIRBuilder);
267
268 /// Translate an LLVM bitcast into generic IR. Either a COPY or a G_BITCAST is
269 /// emitted.
270 bool translateBitCast(const User &U, MachineIRBuilder &MIRBuilder);
271
272 /// Translate an LLVM load instruction into generic IR.
273 bool translateLoad(const User &U, MachineIRBuilder &MIRBuilder);
274
275 /// Translate an LLVM store instruction into generic IR.
276 bool translateStore(const User &U, MachineIRBuilder &MIRBuilder);
277
278 /// Translate an LLVM string intrinsic (memcpy, memset, ...).
279 bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder,
280 unsigned Opcode);
281
282 /// Translate an LLVM trap intrinsic (trap, debugtrap, ubsantrap).
283 bool translateTrap(const CallInst &U, MachineIRBuilder &MIRBuilder,
284 unsigned Opcode);
285
286 // Translate @llvm.vector.interleave2 and
287 // @llvm.vector.deinterleave2 intrinsics for fixed-width vector
288 // types into vector shuffles.
289 bool translateVectorInterleave2Intrinsic(const CallInst &CI,
290 MachineIRBuilder &MIRBuilder);
291 bool translateVectorDeinterleave2Intrinsic(const CallInst &CI,
292 MachineIRBuilder &MIRBuilder);
293
294 void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder);
295
296 bool translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
297 MachineIRBuilder &MIRBuilder);
298 bool translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
299 MachineIRBuilder &MIRBuilder);
300
301 /// Helper function for translateSimpleIntrinsic.
302 /// \return The generic opcode for \p IntrinsicID if \p IntrinsicID is a
303 /// simple intrinsic (ceil, fabs, etc.). Otherwise, returns
304 /// Intrinsic::not_intrinsic.
305 unsigned getSimpleIntrinsicOpcode(Intrinsic::ID ID);
306
307 /// Translates the intrinsics defined in getSimpleIntrinsicOpcode.
308 /// \return true if the translation succeeded.
309 bool translateSimpleIntrinsic(const CallInst &CI, Intrinsic::ID ID,
310 MachineIRBuilder &MIRBuilder);
311
312 bool translateConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI,
313 MachineIRBuilder &MIRBuilder);
314
315 bool translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
316 MachineIRBuilder &MIRBuilder);
317
318 /// Returns the single livein physical register Arg was lowered to, if
319 /// possible.
320 std::optional<MCRegister> getArgPhysReg(Argument &Arg);
321
322 /// If debug-info targets an Argument and its expression is an EntryValue,
323 /// lower it as either an entry in the MF debug table (dbg.declare), or a
324 /// DBG_VALUE targeting the corresponding livein register for that Argument
325 /// (dbg.value).
326 bool translateIfEntryValueArgument(bool isDeclare, Value *Arg,
327 const DILocalVariable *Var,
328 const DIExpression *Expr,
329 const DebugLoc &DL,
330 MachineIRBuilder &MIRBuilder);
331
332 bool translateInlineAsm(const CallBase &CB, MachineIRBuilder &MIRBuilder);
333
334 /// Common code for translating normal calls or invokes.
335 bool translateCallBase(const CallBase &CB, MachineIRBuilder &MIRBuilder);
336
337 /// Translate call instruction.
338 /// \pre \p U is a call instruction.
339 bool translateCall(const User &U, MachineIRBuilder &MIRBuilder);
340
341 bool translateIntrinsic(
342 const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
343 ArrayRef<TargetLowering::IntrinsicInfo> TgtMemIntrinsicInfos = {});
344
345 /// Report an intrinsic the subtarget does not support and define its results
346 /// with G_IMPLICIT_DEF. Prevents creating a malformed MIR.
347 bool handleUnsupportedIntrinsic(const CallBase &CB, Intrinsic::ID ID,
348 MachineIRBuilder &MIRBuilder);
349
350 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
351 /// many places it could ultimately go. In the IR, we have a single unwind
352 /// destination, but in the machine CFG, we enumerate all the possible blocks.
353 /// This function skips over imaginary basic blocks that hold catchswitch
354 /// instructions, and finds all the "real" machine
355 /// basic block destinations. As those destinations may not be successors of
356 /// EHPadBB, here we also calculate the edge probability to those
357 /// destinations. The passed-in Prob is the edge probability to EHPadBB.
358 bool findUnwindDestinations(
359 const BasicBlock *EHPadBB, BranchProbability Prob,
360 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
361 &UnwindDests);
362
363 bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder);
364
365 bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder);
366
367 bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder);
368
369 /// Translate one of LLVM's cast instructions into MachineInstrs, with the
370 /// given generic Opcode.
371 bool translateCast(unsigned Opcode, const User &U,
372 MachineIRBuilder &MIRBuilder);
373
374 /// Translate a phi instruction.
375 bool translatePHI(const User &U, MachineIRBuilder &MIRBuilder);
376
377 /// Translate a comparison (icmp or fcmp) instruction or constant.
378 bool translateCompare(const User &U, MachineIRBuilder &MIRBuilder);
379
380 /// Translate an integer compare instruction (or constant).
381 bool translateICmp(const User &U, MachineIRBuilder &MIRBuilder) {
382 return translateCompare(U, MIRBuilder);
383 }
384
385 /// Translate a floating-point compare instruction (or constant).
386 bool translateFCmp(const User &U, MachineIRBuilder &MIRBuilder) {
387 return translateCompare(U, MIRBuilder);
388 }
389
390 /// Add remaining operands onto phis we've translated. Executed after all
391 /// MachineBasicBlocks for the function have been created.
392 void finishPendingPhis();
393
394 /// Describe any constants in PendingConstDbgValues that are
395 /// materialized as vregs.
396 void finishPendingConstDbgValues();
397
398 /// Translate \p Inst into a unary operation \p Opcode.
399 /// \pre \p U is a unary operation.
400 bool translateUnaryOp(unsigned Opcode, const User &U,
401 MachineIRBuilder &MIRBuilder);
402
403 /// Translate \p Inst into a binary operation \p Opcode.
404 /// \pre \p U is a binary operation.
405 bool translateBinaryOp(unsigned Opcode, const User &U,
406 MachineIRBuilder &MIRBuilder);
407
408 /// If the set of cases should be emitted as a series of branches, return
409 /// true. If we should emit this as a bunch of and/or'd together conditions,
410 /// return false.
411 bool shouldEmitAsBranches(const std::vector<SwitchCG::CaseBlock> &Cases);
412 /// Helper method for findMergedConditions.
413 /// This function emits a branch and is used at the leaves of an OR or an
414 /// AND operator tree.
415 void emitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB,
416 MachineBasicBlock *FBB,
417 MachineBasicBlock *CurBB,
418 MachineBasicBlock *SwitchBB,
419 BranchProbability TProb,
420 BranchProbability FProb, bool InvertCond);
421 /// Used during condbr translation to find trees of conditions that can be
422 /// optimized.
423 void findMergedConditions(const Value *Cond, MachineBasicBlock *TBB,
424 MachineBasicBlock *FBB, MachineBasicBlock *CurBB,
425 MachineBasicBlock *SwitchBB,
426 Instruction::BinaryOps Opc, BranchProbability TProb,
427 BranchProbability FProb, bool InvertCond);
428
429 /// Translate branch (br) instruction.
430 /// \pre \p U is a branch instruction.
431 bool translateUncondBr(const User &U, MachineIRBuilder &MIRBuilder);
432 bool translateCondBr(const User &U, MachineIRBuilder &MIRBuilder);
433
434 // Begin switch lowering functions.
435 bool emitJumpTableHeader(SwitchCG::JumpTable &JT,
436 SwitchCG::JumpTableHeader &JTH,
437 MachineBasicBlock *HeaderBB);
438 void emitJumpTable(SwitchCG::JumpTable &JT, MachineBasicBlock *MBB);
439
440 void emitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB,
441 MachineIRBuilder &MIB);
442
443 /// Generate for the BitTest header block, which precedes each sequence of
444 /// BitTestCases.
445 void emitBitTestHeader(SwitchCG::BitTestBlock &BTB,
446 MachineBasicBlock *SwitchMBB);
447 /// Generate code to produces one "bit test" for a given BitTestCase \p B.
448 void emitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB,
449 BranchProbability BranchProbToNext, Register Reg,
450 SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB);
451
452 void splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
453 const SwitchCG::SwitchWorkListItem &W, Value *Cond,
454 MachineBasicBlock *SwitchMBB, MachineIRBuilder &MIB);
455
456 bool lowerJumpTableWorkItem(
457 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
458 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
459 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
460 BranchProbability UnhandledProbs, SwitchCG::CaseClusterIt I,
461 MachineBasicBlock *Fallthrough, bool FallthroughUnreachable);
462
463 bool lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I, Value *Cond,
464 MachineBasicBlock *Fallthrough,
465 bool FallthroughUnreachable,
466 BranchProbability UnhandledProbs,
467 MachineBasicBlock *CurMBB,
468 MachineIRBuilder &MIB,
469 MachineBasicBlock *SwitchMBB);
470
471 bool lowerBitTestWorkItem(
472 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
473 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
474 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
475 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
476 SwitchCG::CaseClusterIt I, MachineBasicBlock *Fallthrough,
477 bool FallthroughUnreachable);
478
479 bool lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W, Value *Cond,
480 MachineBasicBlock *SwitchMBB,
481 MachineBasicBlock *DefaultMBB,
482 MachineIRBuilder &MIB);
483
484 bool translateSwitch(const User &U, MachineIRBuilder &MIRBuilder);
485 // End switch lowering section.
486
487 bool translateIndirectBr(const User &U, MachineIRBuilder &MIRBuilder);
488
489 bool translateExtractValue(const User &U, MachineIRBuilder &MIRBuilder);
490
491 bool translateInsertValue(const User &U, MachineIRBuilder &MIRBuilder);
492
493 bool translateSelect(const User &U, MachineIRBuilder &MIRBuilder);
494
495 bool translateGetElementPtr(const User &U, MachineIRBuilder &MIRBuilder);
496
497 bool translateAlloca(const User &U, MachineIRBuilder &MIRBuilder);
498
499 /// Translate return (ret) instruction.
500 /// The target needs to implement CallLowering::lowerReturn for
501 /// this to succeed.
502 /// \pre \p U is a return instruction.
503 bool translateRet(const User &U, MachineIRBuilder &MIRBuilder);
504
505 bool translateFNeg(const User &U, MachineIRBuilder &MIRBuilder);
506
507 bool translateAdd(const User &U, MachineIRBuilder &MIRBuilder) {
508 return translateBinaryOp(Opcode: TargetOpcode::G_ADD, U, MIRBuilder);
509 }
510 bool translateSub(const User &U, MachineIRBuilder &MIRBuilder) {
511 return translateBinaryOp(Opcode: TargetOpcode::G_SUB, U, MIRBuilder);
512 }
513 bool translateAnd(const User &U, MachineIRBuilder &MIRBuilder) {
514 return translateBinaryOp(Opcode: TargetOpcode::G_AND, U, MIRBuilder);
515 }
516 bool translateMul(const User &U, MachineIRBuilder &MIRBuilder) {
517 return translateBinaryOp(Opcode: TargetOpcode::G_MUL, U, MIRBuilder);
518 }
519 bool translateOr(const User &U, MachineIRBuilder &MIRBuilder) {
520 return translateBinaryOp(Opcode: TargetOpcode::G_OR, U, MIRBuilder);
521 }
522 bool translateXor(const User &U, MachineIRBuilder &MIRBuilder) {
523 return translateBinaryOp(Opcode: TargetOpcode::G_XOR, U, MIRBuilder);
524 }
525
526 bool translateUDiv(const User &U, MachineIRBuilder &MIRBuilder) {
527 return translateBinaryOp(Opcode: TargetOpcode::G_UDIV, U, MIRBuilder);
528 }
529 bool translateSDiv(const User &U, MachineIRBuilder &MIRBuilder) {
530 return translateBinaryOp(Opcode: TargetOpcode::G_SDIV, U, MIRBuilder);
531 }
532 bool translateURem(const User &U, MachineIRBuilder &MIRBuilder) {
533 return translateBinaryOp(Opcode: TargetOpcode::G_UREM, U, MIRBuilder);
534 }
535 bool translateSRem(const User &U, MachineIRBuilder &MIRBuilder) {
536 return translateBinaryOp(Opcode: TargetOpcode::G_SREM, U, MIRBuilder);
537 }
538 bool translateIntToPtr(const User &U, MachineIRBuilder &MIRBuilder) {
539 return translateCast(Opcode: TargetOpcode::G_INTTOPTR, U, MIRBuilder);
540 }
541 bool translatePtrToInt(const User &U, MachineIRBuilder &MIRBuilder) {
542 return translateCast(Opcode: TargetOpcode::G_PTRTOINT, U, MIRBuilder);
543 }
544 bool translatePtrToAddr(const User &U, MachineIRBuilder &MIRBuilder) {
545 // FIXME: this is not correct for pointers with addr width != pointer width
546 return translatePtrToInt(U, MIRBuilder);
547 }
548 bool translateTrunc(const User &U, MachineIRBuilder &MIRBuilder) {
549 return translateCast(Opcode: TargetOpcode::G_TRUNC, U, MIRBuilder);
550 }
551 bool translateFPTrunc(const User &U, MachineIRBuilder &MIRBuilder) {
552 return translateCast(Opcode: TargetOpcode::G_FPTRUNC, U, MIRBuilder);
553 }
554 bool translateFPExt(const User &U, MachineIRBuilder &MIRBuilder) {
555 return translateCast(Opcode: TargetOpcode::G_FPEXT, U, MIRBuilder);
556 }
557 bool translateFPToUI(const User &U, MachineIRBuilder &MIRBuilder) {
558 return translateCast(Opcode: TargetOpcode::G_FPTOUI, U, MIRBuilder);
559 }
560 bool translateFPToSI(const User &U, MachineIRBuilder &MIRBuilder) {
561 return translateCast(Opcode: TargetOpcode::G_FPTOSI, U, MIRBuilder);
562 }
563 bool translateUIToFP(const User &U, MachineIRBuilder &MIRBuilder) {
564 return translateCast(Opcode: TargetOpcode::G_UITOFP, U, MIRBuilder);
565 }
566 bool translateSIToFP(const User &U, MachineIRBuilder &MIRBuilder) {
567 return translateCast(Opcode: TargetOpcode::G_SITOFP, U, MIRBuilder);
568 }
569 bool translateUnreachable(const User &U, MachineIRBuilder &MIRBuilder);
570
571 bool translateSExt(const User &U, MachineIRBuilder &MIRBuilder) {
572 return translateCast(Opcode: TargetOpcode::G_SEXT, U, MIRBuilder);
573 }
574
575 bool translateZExt(const User &U, MachineIRBuilder &MIRBuilder) {
576 return translateCast(Opcode: TargetOpcode::G_ZEXT, U, MIRBuilder);
577 }
578
579 bool translateShl(const User &U, MachineIRBuilder &MIRBuilder) {
580 return translateBinaryOp(Opcode: TargetOpcode::G_SHL, U, MIRBuilder);
581 }
582 bool translateLShr(const User &U, MachineIRBuilder &MIRBuilder) {
583 return translateBinaryOp(Opcode: TargetOpcode::G_LSHR, U, MIRBuilder);
584 }
585 bool translateAShr(const User &U, MachineIRBuilder &MIRBuilder) {
586 return translateBinaryOp(Opcode: TargetOpcode::G_ASHR, U, MIRBuilder);
587 }
588
589 bool translateFAdd(const User &U, MachineIRBuilder &MIRBuilder) {
590 return translateBinaryOp(Opcode: TargetOpcode::G_FADD, U, MIRBuilder);
591 }
592 bool translateFSub(const User &U, MachineIRBuilder &MIRBuilder) {
593 return translateBinaryOp(Opcode: TargetOpcode::G_FSUB, U, MIRBuilder);
594 }
595 bool translateFMul(const User &U, MachineIRBuilder &MIRBuilder) {
596 return translateBinaryOp(Opcode: TargetOpcode::G_FMUL, U, MIRBuilder);
597 }
598 bool translateFDiv(const User &U, MachineIRBuilder &MIRBuilder) {
599 return translateBinaryOp(Opcode: TargetOpcode::G_FDIV, U, MIRBuilder);
600 }
601 bool translateFRem(const User &U, MachineIRBuilder &MIRBuilder) {
602 return translateBinaryOp(Opcode: TargetOpcode::G_FREM, U, MIRBuilder);
603 }
604
605 bool translateVAArg(const User &U, MachineIRBuilder &MIRBuilder);
606
607 bool translateInsertElement(const User &U, MachineIRBuilder &MIRBuilder);
608 bool translateInsertVector(const User &U, MachineIRBuilder &MIRBuilder);
609
610 bool translateExtractElement(const User &U, MachineIRBuilder &MIRBuilder);
611 bool translateExtractVector(const User &U, MachineIRBuilder &MIRBuilder);
612
613 bool translateShuffleVector(const User &U, MachineIRBuilder &MIRBuilder);
614
615 bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder);
616 bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder);
617 bool translateFence(const User &U, MachineIRBuilder &MIRBuilder);
618 bool translateFreeze(const User &U, MachineIRBuilder &MIRBuilder);
619
620 bool translateBitExtract(const User &U, MachineIRBuilder &MIRBuilder);
621 bool translateBitInsert(const User &U, MachineIRBuilder &MIRBuilder);
622
623 // Stubs to keep the compiler happy while we implement the rest of the
624 // translation.
625 bool translateResume(const User &U, MachineIRBuilder &MIRBuilder) {
626 return false;
627 }
628 bool translateCleanupRet(const User &U, MachineIRBuilder &MIRBuilder) {
629 return false;
630 }
631 bool translateCatchRet(const User &U, MachineIRBuilder &MIRBuilder) {
632 return false;
633 }
634 bool translateCatchSwitch(const User &U, MachineIRBuilder &MIRBuilder) {
635 return false;
636 }
637 bool translateAddrSpaceCast(const User &U, MachineIRBuilder &MIRBuilder) {
638 return translateCast(Opcode: TargetOpcode::G_ADDRSPACE_CAST, U, MIRBuilder);
639 }
640 bool translateCleanupPad(const User &U, MachineIRBuilder &MIRBuilder) {
641 return false;
642 }
643 bool translateCatchPad(const User &U, MachineIRBuilder &MIRBuilder) {
644 return false;
645 }
646 bool translateUserOp1(const User &U, MachineIRBuilder &MIRBuilder) {
647 return false;
648 }
649 bool translateUserOp2(const User &U, MachineIRBuilder &MIRBuilder) {
650 return false;
651 }
652
653 bool translateConvergenceControlIntrinsic(const CallInst &CI,
654 Intrinsic::ID ID,
655 MachineIRBuilder &MIRBuilder);
656
657 /// @}
658
659 // Builder for machine instruction a la IRBuilder.
660 // I.e., compared to regular MIBuilder, this one also inserts the instruction
661 // in the current block, it can creates block, etc., basically a kind of
662 // IRBuilder, but for Machine IR.
663 // CSEMIRBuilder CurBuilder;
664 std::unique_ptr<MachineIRBuilder> CurBuilder;
665
666 // Builder set to the entry block (just after ABI lowering instructions). Used
667 // as a convenient location for Constants.
668 // CSEMIRBuilder EntryBuilder;
669 std::unique_ptr<MachineIRBuilder> EntryBuilder;
670
671 // The MachineFunction currently being translated.
672 MachineFunction *MF = nullptr;
673
674 /// MachineRegisterInfo used to create virtual registers.
675 MachineRegisterInfo *MRI = nullptr;
676
677 const DataLayout *DL = nullptr;
678
679 CodeGenOptLevel OptLevel;
680
681 /// Current optimization remark emitter. Used to report failures.
682 std::unique_ptr<OptimizationRemarkEmitter> ORE;
683
684 AAResults *AA = nullptr;
685 AssumptionCache *AC = nullptr;
686 const TargetLibraryInfo *LibInfo = nullptr;
687 const LibcallLoweringInfo *Libcalls = nullptr;
688 const TargetLowering *TLI = nullptr;
689 FunctionLoweringInfo FuncInfo;
690
691 // True when either the Target Machine specifies no optimizations or the
692 // function has the optnone attribute.
693 bool EnableOpts = false;
694
695 /// True when the block contains a tail call. This allows the IRTranslator to
696 /// stop translating such blocks early.
697 bool HasTailCall = false;
698
699 StackProtectorDescriptor SPDescriptor;
700
701 bool mayTranslateUserTypes(const User &U) const;
702
703 /// Switch analysis and optimization.
704 class GISelSwitchLowering : public SwitchCG::SwitchLowering {
705 public:
706 GISelSwitchLowering(IRTranslatorImpl *irt, FunctionLoweringInfo &funcinfo)
707 : SwitchLowering(funcinfo), IRT(irt) {
708 assert(irt && "irt is null!");
709 }
710
711 void addSuccessorWithProb(
712 MachineBasicBlock *Src, MachineBasicBlock *Dst,
713 BranchProbability Prob = BranchProbability::getUnknown()) override {
714 IRT->addSuccessorWithProb(Src, Dst, Prob);
715 }
716
717 ~GISelSwitchLowering() override = default;
718
719 private:
720 IRTranslatorImpl *IRT;
721 };
722
723 std::unique_ptr<GISelSwitchLowering> SL;
724
725 // * Insert all the code needed to materialize the constants
726 // at the proper place. E.g., Entry block or dominator block
727 // of each constant depending on how fancy we want to be.
728 // * Clear the different maps.
729 void finalizeFunction();
730
731 // Processing steps done per block. E.g. emitting jump tables, stack
732 // protectors etc. Returns true if no errors, false if there was a problem
733 // that caused an abort.
734 bool finalizeBasicBlock(const BasicBlock &BB, MachineBasicBlock &MBB);
735
736 /// Codegen a new tail for a stack protector check ParentMBB which has had its
737 /// tail spliced into a stack protector check success bb.
738 ///
739 /// For a high level explanation of how this fits into the stack protector
740 /// generation see the comment on the declaration of class
741 /// StackProtectorDescriptor.
742 ///
743 /// \return true if there were no problems.
744 bool emitSPDescriptorParent(StackProtectorDescriptor &SPD,
745 MachineBasicBlock *ParentBB);
746
747 /// Codegen the failure basic block for a stack protector check.
748 ///
749 /// A failure stack protector machine basic block consists simply of a call to
750 /// __stack_chk_fail().
751 ///
752 /// For a high level explanation of how this fits into the stack protector
753 /// generation see the comment on the declaration of class
754 /// StackProtectorDescriptor.
755 ///
756 /// \return true if there were no problems.
757 bool emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
758 MachineBasicBlock *FailureBB);
759
760 /// Get the VRegs that represent \p Val.
761 /// Non-aggregate types have just one corresponding VReg and the list can be
762 /// used as a single "unsigned". Aggregates get flattened. If such VRegs do
763 /// not exist, they are created.
764 ArrayRef<Register> getOrCreateVRegs(const Value &Val);
765
766 Register getOrCreateVReg(const Value &Val) {
767 auto Regs = getOrCreateVRegs(Val);
768 if (Regs.empty())
769 return 0;
770 assert(Regs.size() == 1 &&
771 "attempt to get single VReg for aggregate or void");
772 return Regs[0];
773 }
774
775 Register getOrCreateConvergenceTokenVReg(const Value &Token) {
776 assert(Token.getType()->isTokenTy());
777 auto &Regs = *VMap.getVRegs(V: Token);
778 if (!Regs.empty()) {
779 assert(Regs.size() == 1 &&
780 "Expected a single register for convergence tokens.");
781 return Regs[0];
782 }
783
784 auto Reg = MRI->createGenericVirtualRegister(Ty: LLT::token());
785 Regs.push_back(Elt: Reg);
786 return Reg;
787 }
788
789 /// Allocate empty vregs for \p Val. For aggregate values, also populate
790 /// their offsets.
791 ValueToVRegInfo::VRegListT &allocateVRegs(const Value &Val);
792
793 /// Get the frame index that represents \p Val.
794 /// If such VReg does not exist, it is created.
795 int getOrCreateFrameIndex(const AllocaInst &AI);
796
797 /// Get the alignment of the given memory operation instruction. This will
798 /// either be the explicitly specified value or the ABI-required alignment for
799 /// the type being accessed (according to the Module's DataLayout).
800 Align getMemOpAlign(const Instruction &I);
801
802 /// Get the MachineBasicBlock that represents \p BB. Specifically, the block
803 /// returned will be the head of the translated block (suitable for branch
804 /// destinations).
805 MachineBasicBlock &getMBB(const BasicBlock &BB);
806
807 /// Record \p NewPred as a Machine predecessor to `Edge.second`, corresponding
808 /// to `Edge.first` at the IR level. This is used when IRTranslation creates
809 /// multiple MachineBasicBlocks for a given IR block and the CFG is no longer
810 /// represented simply by the IR-level CFG.
811 void addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred);
812
813 /// Returns the Machine IR predecessors for the given IR CFG edge. Usually
814 /// this is just the single MachineBasicBlock corresponding to the predecessor
815 /// in the IR. More complex lowering can result in multiple MachineBasicBlocks
816 /// preceding the original though (e.g. switch instructions).
817 SmallVector<MachineBasicBlock *, 1> getMachinePredBBs(CFGEdge Edge) {
818 auto RemappedEdge = MachinePreds.find(Val: Edge);
819 if (RemappedEdge != MachinePreds.end())
820 return RemappedEdge->second;
821 return SmallVector<MachineBasicBlock *, 4>(1, &getMBB(BB: *Edge.first));
822 }
823
824 /// Return branch probability calculated by BranchProbabilityInfo for IR
825 /// blocks.
826 BranchProbability getEdgeProbability(const MachineBasicBlock *Src,
827 const MachineBasicBlock *Dst) const;
828
829 void addSuccessorWithProb(
830 MachineBasicBlock *Src, MachineBasicBlock *Dst,
831 BranchProbability Prob = BranchProbability::getUnknown());
832
833public:
834 IRTranslatorImpl(CodeGenOptLevel OptLevel = CodeGenOptLevel::None)
835 : OptLevel(OptLevel) {}
836
837 // Algo:
838 // CallLowering = MF.subtarget.getCallLowering()
839 // F = MF.getParent()
840 // MIRBuilder.reset(MF)
841 // getMBB(F.getEntryBB())
842 // CallLowering->translateArguments(MIRBuilder, F, ValToVReg)
843 // for each bb in F
844 // getMBB(bb)
845 // for each inst in bb
846 // if (!translate(MIRBuilder, inst, ValToVReg, ConstantToSequence))
847 // reportFatalUsageError("Don't know how to translate input");
848 // finalize()
849 bool runOnMachineFunction(MachineFunction &MF,
850 function_ref<GISelCSEInfo *()> GetCSEInfo,
851 bool ShouldSkipOpts,
852 function_ref<AAResults *()> GetAAResults,
853 function_ref<BranchProbabilityInfo *()> GetBPI,
854 function_ref<AssumptionCache *()> GetAC,
855 TargetLibraryInfo *LibraryInfo,
856 const LibcallLoweringInfo *LibcallInfo,
857 SSPLayoutInfo *StackProtectorInfo);
858};
859
860} // namespace llvm
861
862char IRTranslatorLegacy::ID = 0;
863
864INITIALIZE_PASS_BEGIN(IRTranslatorLegacy, DEBUG_TYPE,
865 "IRTranslator LLVM IR -> MI", false, false)
866INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
867INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
868INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
869INITIALIZE_PASS_DEPENDENCY(StackProtector)
870INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
871INITIALIZE_PASS_END(IRTranslatorLegacy, DEBUG_TYPE,
872 "IRTranslator LLVM IR -> MI", false, false)
873
874static void reportTranslationError(MachineFunction &MF,
875 OptimizationRemarkEmitter &ORE,
876 OptimizationRemarkMissed &R) {
877 MF.getProperties().setFailedISel();
878 bool IsGlobalISelAbortEnabled =
879 MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
880
881 // Print the function name explicitly if we don't have a debug location (which
882 // makes the diagnostic less useful) or if we're going to emit a raw error.
883 if (!R.getLocation().isValid() || IsGlobalISelAbortEnabled)
884 R << (" (in function: " + MF.getName() + ")").str();
885
886 if (IsGlobalISelAbortEnabled)
887 report_fatal_error(reason: Twine(R.getMsg()));
888 else
889 ORE.emit(OptDiag&: R);
890}
891
892IRTranslatorLegacy::IRTranslatorLegacy(CodeGenOptLevel OptLevel)
893 : MachineFunctionPass(ID), OptLevel(OptLevel),
894 Impl(std::make_unique<IRTranslatorImpl>(args&: OptLevel)) {}
895
896IRTranslatorLegacy::~IRTranslatorLegacy() = default;
897
898#ifndef NDEBUG
899namespace {
900/// Verify that every instruction created has the same DILocation as the
901/// instruction being translated.
902class DILocationVerifier : public GISelChangeObserver {
903 const Instruction *CurrInst = nullptr;
904
905public:
906 DILocationVerifier() = default;
907 ~DILocationVerifier() override = default;
908
909 const Instruction *getCurrentInst() const { return CurrInst; }
910 void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
911
912 void erasingInstr(MachineInstr &MI) override {}
913 void changingInstr(MachineInstr &MI) override {}
914 void changedInstr(MachineInstr &MI) override {}
915
916 void createdInstr(MachineInstr &MI) override {
917 assert(getCurrentInst() && "Inserted instruction without a current MI");
918
919 // Only print the check message if we're actually checking it.
920#ifndef NDEBUG
921 LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
922 << " was copied to " << MI);
923#endif
924 // We allow insts in the entry block to have no debug loc because
925 // they could have originated from constants, and we don't want a jumpy
926 // debug experience.
927 assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
928 (MI.getParent()->isEntryBlock() && !MI.getDebugLoc()) ||
929 (MI.isDebugInstr())) &&
930 "Line info was not transferred to all instructions");
931 }
932};
933} // namespace
934#endif // ifndef NDEBUG
935
936void IRTranslatorLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
937 AU.addRequired<StackProtector>();
938 AU.addRequired<TargetPassConfig>();
939 AU.addRequired<GISelCSEAnalysisWrapperPass>();
940 if (OptLevel != CodeGenOptLevel::None) {
941 AU.addRequired<AssumptionCacheTracker>();
942 AU.addRequired<BranchProbabilityInfoWrapperPass>();
943 AU.addRequired<AAResultsWrapperPass>();
944 }
945 AU.addRequired<TargetLibraryInfoWrapperPass>();
946 AU.addPreserved<TargetLibraryInfoWrapperPass>();
947 AU.addRequired<LibcallLoweringInfoWrapper>();
948
949 getSelectionDAGFallbackAnalysisUsage(AU);
950 MachineFunctionPass::getAnalysisUsage(AU);
951}
952
953IRTranslatorImpl::ValueToVRegInfo::VRegListT &
954IRTranslatorImpl::allocateVRegs(const Value &Val) {
955 auto VRegsIt = VMap.findVRegs(V: Val);
956 if (VRegsIt != VMap.vregs_end())
957 return *VRegsIt->second;
958 auto *Regs = VMap.getVRegs(V: Val);
959 if (!Val.getType()->isAggregateType()) {
960 Regs->push_back(Elt: 0);
961 return *Regs;
962 }
963
964 auto *Offsets = VMap.getOffsets(V: Val);
965 SmallVector<LLT, 4> SplitTys;
966 computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueLLTs&: SplitTys,
967 FixedOffsets: Offsets->empty() ? Offsets : nullptr);
968 for (unsigned i = 0; i < SplitTys.size(); ++i)
969 Regs->push_back(Elt: 0);
970 return *Regs;
971}
972
973ArrayRef<Register> IRTranslatorImpl::getOrCreateVRegs(const Value &Val) {
974 auto VRegsIt = VMap.findVRegs(V: Val);
975 if (VRegsIt != VMap.vregs_end())
976 return *VRegsIt->second;
977
978 if (Val.getType()->isVoidTy())
979 return *VMap.getVRegs(V: Val);
980
981 // Create entry for this type.
982 auto *VRegs = VMap.getVRegs(V: Val);
983
984 if (!Val.getType()->isTokenTy())
985 assert(Val.getType()->isSized() &&
986 "Don't know how to create an empty vreg");
987
988 // Fast-path values that lower to a single vreg.
989 if (!Val.getType()->isAggregateType()) {
990 LLT Ty = getLLTForType(Ty&: *Val.getType(), DL: *DL);
991 VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty));
992 if (isa<Constant>(Val)) {
993 bool Success = translate(C: cast<Constant>(Val), Reg: VRegs->front());
994 if (!Success) {
995 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
996 MF->getFunction().getSubprogram(),
997 &MF->getFunction().getEntryBlock());
998 R << "unable to translate constant: " << ore::NV("Type", Val.getType());
999 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
1000 }
1001 }
1002 return *VRegs;
1003 }
1004
1005 SmallVector<LLT, 4> SplitTys;
1006 auto *Offsets = VMap.getOffsets(V: Val);
1007 computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueLLTs&: SplitTys,
1008 FixedOffsets: Offsets->empty() ? Offsets : nullptr);
1009
1010 if (!isa<Constant>(Val)) {
1011 for (auto Ty : SplitTys)
1012 VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty));
1013 return *VRegs;
1014 }
1015
1016 // UndefValue, ConstantAggregateZero
1017 auto &C = cast<Constant>(Val);
1018 unsigned Idx = 0;
1019 while (auto Elt = C.getAggregateElement(Elt: Idx++)) {
1020 auto EltRegs = getOrCreateVRegs(Val: *Elt);
1021 llvm::append_range(C&: *VRegs, R&: EltRegs);
1022 }
1023
1024 return *VRegs;
1025}
1026
1027int IRTranslatorImpl::getOrCreateFrameIndex(const AllocaInst &AI) {
1028 auto [MapEntry, Inserted] = FrameIndices.try_emplace(Key: &AI);
1029 if (!Inserted)
1030 return MapEntry->second;
1031
1032 TypeSize TySize = AI.getAllocationSize(DL: *DL).value_or(u: TypeSize::getZero());
1033 uint64_t Size = TySize.getKnownMinValue();
1034
1035 // Always allocate at least one byte.
1036 Size = std::max<uint64_t>(a: Size, b: 1u);
1037
1038 int &FI = MapEntry->second;
1039 FI = MF->getFrameInfo().CreateStackObject(Size, Alignment: AI.getAlign(), isSpillSlot: false, Alloca: &AI);
1040
1041 // Scalable vectors and structures that contain scalable vectors may
1042 // need a special StackID to distinguish them from other (fixed size)
1043 // stack objects.
1044 if (TySize.isScalable()) {
1045 auto StackID =
1046 MF->getSubtarget().getFrameLowering()->getStackIDForScalableVectors();
1047 MF->getFrameInfo().setStackID(ObjectIdx: FI, ID: StackID);
1048 }
1049
1050 return FI;
1051}
1052
1053Align IRTranslatorImpl::getMemOpAlign(const Instruction &I) {
1054 if (const StoreInst *SI = dyn_cast<StoreInst>(Val: &I))
1055 return SI->getAlign();
1056 if (const LoadInst *LI = dyn_cast<LoadInst>(Val: &I))
1057 return LI->getAlign();
1058 if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(Val: &I))
1059 return AI->getAlign();
1060 if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(Val: &I))
1061 return AI->getAlign();
1062
1063 OptimizationRemarkMissed R("gisel-ir-translator", "", &I);
1064 R << "unable to translate memop: " << ore::NV("Opcode", &I);
1065 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
1066 return Align(1);
1067}
1068
1069MachineBasicBlock &IRTranslatorImpl::getMBB(const BasicBlock &BB) {
1070 MachineBasicBlock *MBB = FuncInfo.getMBB(BB: &BB);
1071 assert(MBB && "BasicBlock was not encountered before");
1072 return *MBB;
1073}
1074
1075void IRTranslatorImpl::addMachineCFGPred(CFGEdge Edge,
1076 MachineBasicBlock *NewPred) {
1077 assert(NewPred && "new predecessor must be a real MachineBasicBlock");
1078 MachinePreds[Edge].push_back(Elt: NewPred);
1079}
1080
1081bool IRTranslatorImpl::translateBinaryOp(unsigned Opcode, const User &U,
1082 MachineIRBuilder &MIRBuilder) {
1083 if (!mayTranslateUserTypes(U))
1084 return false;
1085
1086 // Get or create a virtual register for each value.
1087 // Unless the value is a Constant => loadimm cst?
1088 // or inline constant each time?
1089 // Creation of a virtual register needs to have a size.
1090 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
1091 Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
1092 Register Res = getOrCreateVReg(Val: U);
1093 uint32_t Flags = 0;
1094 if (isa<Instruction>(Val: U)) {
1095 const Instruction &I = cast<Instruction>(Val: U);
1096 Flags = MachineInstr::copyFlagsFromInstruction(I);
1097 }
1098
1099 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0, Op1}, Flags);
1100 return true;
1101}
1102
1103bool IRTranslatorImpl::translateUnaryOp(unsigned Opcode, const User &U,
1104 MachineIRBuilder &MIRBuilder) {
1105 if (!mayTranslateUserTypes(U))
1106 return false;
1107
1108 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
1109 Register Res = getOrCreateVReg(Val: U);
1110 uint32_t Flags = 0;
1111 if (isa<Instruction>(Val: U)) {
1112 const Instruction &I = cast<Instruction>(Val: U);
1113 Flags = MachineInstr::copyFlagsFromInstruction(I);
1114 }
1115 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0}, Flags);
1116 return true;
1117}
1118
1119bool IRTranslatorImpl::translateFNeg(const User &U,
1120 MachineIRBuilder &MIRBuilder) {
1121 return translateUnaryOp(Opcode: TargetOpcode::G_FNEG, U, MIRBuilder);
1122}
1123
1124bool IRTranslatorImpl::translateCompare(const User &U,
1125 MachineIRBuilder &MIRBuilder) {
1126 if (!mayTranslateUserTypes(U))
1127 return false;
1128
1129 auto *CI = cast<CmpInst>(Val: &U);
1130 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
1131 Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
1132 Register Res = getOrCreateVReg(Val: U);
1133 CmpInst::Predicate Pred = CI->getPredicate();
1134 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: *CI);
1135 if (CmpInst::isIntPredicate(P: Pred))
1136 MIRBuilder.buildICmp(Pred, Res, Op0, Op1, Flags);
1137 else if (Pred == CmpInst::FCMP_FALSE)
1138 MIRBuilder.buildCopy(
1139 Res, Op: getOrCreateVReg(Val: *Constant::getNullValue(Ty: U.getType())));
1140 else if (Pred == CmpInst::FCMP_TRUE)
1141 MIRBuilder.buildCopy(
1142 Res, Op: getOrCreateVReg(Val: *Constant::getAllOnesValue(Ty: U.getType())));
1143 else
1144 MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
1145
1146 return true;
1147}
1148
1149bool IRTranslatorImpl::translateRet(const User &U,
1150 MachineIRBuilder &MIRBuilder) {
1151 const ReturnInst &RI = cast<ReturnInst>(Val: U);
1152 const Value *Ret = RI.getReturnValue();
1153 if (Ret && DL->getTypeStoreSize(Ty: Ret->getType()).isZero())
1154 Ret = nullptr;
1155
1156 ArrayRef<Register> VRegs;
1157 if (Ret)
1158 VRegs = getOrCreateVRegs(Val: *Ret);
1159
1160 Register SwiftErrorVReg = 0;
1161 if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
1162 SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
1163 &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
1164 }
1165
1166 // The target may mess up with the insertion point, but
1167 // this is not important as a return is the last instruction
1168 // of the block anyway.
1169 return CLI->lowerReturn(MIRBuilder, Val: Ret, VRegs, FLI&: FuncInfo, SwiftErrorVReg);
1170}
1171
1172void IRTranslatorImpl::emitBranchForMergedCondition(
1173 const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
1174 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
1175 BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
1176 // If the leaf of the tree is a comparison, merge the condition into
1177 // the caseblock.
1178 if (const CmpInst *BOp = dyn_cast<CmpInst>(Val: Cond)) {
1179 CmpInst::Predicate Condition;
1180 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Val: Cond)) {
1181 Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
1182 } else {
1183 const FCmpInst *FC = cast<FCmpInst>(Val: Cond);
1184 Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
1185 }
1186
1187 SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(i_nocapture: 0),
1188 BOp->getOperand(i_nocapture: 1), nullptr, TBB, FBB, CurBB,
1189 CurBuilder->getDebugLoc(), TProb, FProb);
1190 SL->SwitchCases.push_back(x: CB);
1191 return;
1192 }
1193
1194 // Create a CaseBlock record representing this branch.
1195 CmpInst::Predicate Pred = InvertCond ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ;
1196 SwitchCG::CaseBlock CB(
1197 Pred, false, Cond, ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
1198 nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
1199 SL->SwitchCases.push_back(x: CB);
1200}
1201
1202static bool isValInBlock(const Value *V, const BasicBlock *BB) {
1203 if (const Instruction *I = dyn_cast<Instruction>(Val: V))
1204 return I->getParent() == BB;
1205 return true;
1206}
1207
1208void IRTranslatorImpl::findMergedConditions(
1209 const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
1210 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
1211 Instruction::BinaryOps Opc, BranchProbability TProb,
1212 BranchProbability FProb, bool InvertCond) {
1213 using namespace PatternMatch;
1214 assert((Opc == Instruction::And || Opc == Instruction::Or) &&
1215 "Expected Opc to be AND/OR");
1216 // Skip over not part of the tree and remember to invert op and operands at
1217 // next level.
1218 Value *NotCond;
1219 if (match(V: Cond, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: NotCond)))) &&
1220 isValInBlock(V: NotCond, BB: CurBB->getBasicBlock())) {
1221 findMergedConditions(Cond: NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
1222 InvertCond: !InvertCond);
1223 return;
1224 }
1225
1226 const Instruction *BOp = dyn_cast<Instruction>(Val: Cond);
1227 const Value *BOpOp0, *BOpOp1;
1228 // Compute the effective opcode for Cond, taking into account whether it needs
1229 // to be inverted, e.g.
1230 // and (not (or A, B)), C
1231 // gets lowered as
1232 // and (and (not A, not B), C)
1233 Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
1234 if (BOp) {
1235 BOpc = match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
1236 ? Instruction::And
1237 : (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
1238 ? Instruction::Or
1239 : (Instruction::BinaryOps)0);
1240 if (InvertCond) {
1241 if (BOpc == Instruction::And)
1242 BOpc = Instruction::Or;
1243 else if (BOpc == Instruction::Or)
1244 BOpc = Instruction::And;
1245 }
1246 }
1247
1248 // If this node is not part of the or/and tree, emit it as a branch.
1249 // Note that all nodes in the tree should have same opcode.
1250 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
1251 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
1252 !isValInBlock(V: BOpOp0, BB: CurBB->getBasicBlock()) ||
1253 !isValInBlock(V: BOpOp1, BB: CurBB->getBasicBlock())) {
1254 emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
1255 InvertCond);
1256 return;
1257 }
1258
1259 // Create TmpBB after CurBB.
1260 MachineFunction::iterator BBI(CurBB);
1261 MachineBasicBlock *TmpBB =
1262 MF->CreateMachineBasicBlock(BB: CurBB->getBasicBlock());
1263 CurBB->getParent()->insert(MBBI: ++BBI, MBB: TmpBB);
1264
1265 if (Opc == Instruction::Or) {
1266 // Codegen X | Y as:
1267 // BB1:
1268 // jmp_if_X TBB
1269 // jmp TmpBB
1270 // TmpBB:
1271 // jmp_if_Y TBB
1272 // jmp FBB
1273 //
1274
1275 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1276 // The requirement is that
1277 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1278 // = TrueProb for original BB.
1279 // Assuming the original probabilities are A and B, one choice is to set
1280 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1281 // A/(1+B) and 2B/(1+B). This choice assumes that
1282 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1283 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1284 // TmpBB, but the math is more complicated.
1285
1286 auto NewTrueProb = TProb / 2;
1287 auto NewFalseProb = TProb / 2 + FProb;
1288 // Emit the LHS condition.
1289 findMergedConditions(Cond: BOpOp0, TBB, FBB: TmpBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
1290 FProb: NewFalseProb, InvertCond);
1291
1292 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1293 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1294 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
1295 // Emit the RHS condition into TmpBB.
1296 findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
1297 FProb: Probs[1], InvertCond);
1298 } else {
1299 assert(Opc == Instruction::And && "Unknown merge op!");
1300 // Codegen X & Y as:
1301 // BB1:
1302 // jmp_if_X TmpBB
1303 // jmp FBB
1304 // TmpBB:
1305 // jmp_if_Y TBB
1306 // jmp FBB
1307 //
1308 // This requires creation of TmpBB after CurBB.
1309
1310 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1311 // The requirement is that
1312 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1313 // = FalseProb for original BB.
1314 // Assuming the original probabilities are A and B, one choice is to set
1315 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1316 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1317 // TrueProb for BB1 * FalseProb for TmpBB.
1318
1319 auto NewTrueProb = TProb + FProb / 2;
1320 auto NewFalseProb = FProb / 2;
1321 // Emit the LHS condition.
1322 findMergedConditions(Cond: BOpOp0, TBB: TmpBB, FBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
1323 FProb: NewFalseProb, InvertCond);
1324
1325 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1326 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1327 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
1328 // Emit the RHS condition into TmpBB.
1329 findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
1330 FProb: Probs[1], InvertCond);
1331 }
1332}
1333
1334bool IRTranslatorImpl::shouldEmitAsBranches(
1335 const std::vector<SwitchCG::CaseBlock> &Cases) {
1336 // For multiple cases, it's better to emit as branches.
1337 if (Cases.size() != 2)
1338 return true;
1339
1340 // If this is two comparisons of the same values or'd or and'd together, they
1341 // will get folded into a single comparison, so don't emit two blocks.
1342 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1343 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1344 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1345 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1346 return false;
1347 }
1348
1349 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1350 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1351 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1352 Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
1353 isa<Constant>(Val: Cases[0].CmpRHS) &&
1354 cast<Constant>(Val: Cases[0].CmpRHS)->isNullValue()) {
1355 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
1356 Cases[0].TrueBB == Cases[1].ThisBB)
1357 return false;
1358 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
1359 Cases[0].FalseBB == Cases[1].ThisBB)
1360 return false;
1361 }
1362
1363 return true;
1364}
1365
1366bool IRTranslatorImpl::translateUncondBr(const User &U,
1367 MachineIRBuilder &MIRBuilder) {
1368 const UncondBrInst &BrInst = cast<UncondBrInst>(Val: U);
1369 auto &CurMBB = MIRBuilder.getMBB();
1370 auto *Succ0MBB = &getMBB(BB: *BrInst.getSuccessor(i: 0));
1371
1372 // If the unconditional target is the layout successor, fallthrough.
1373 if (OptLevel == CodeGenOptLevel::None || !CurMBB.isLayoutSuccessor(MBB: Succ0MBB))
1374 MIRBuilder.buildBr(Dest&: *Succ0MBB);
1375
1376 // Link successors.
1377 for (const BasicBlock *Succ : successors(I: &BrInst))
1378 CurMBB.addSuccessor(Succ: &getMBB(BB: *Succ));
1379 return true;
1380}
1381
1382bool IRTranslatorImpl::translateCondBr(const User &U,
1383 MachineIRBuilder &MIRBuilder) {
1384 const CondBrInst &BrInst = cast<CondBrInst>(Val: U);
1385 auto &CurMBB = MIRBuilder.getMBB();
1386 auto *Succ0MBB = &getMBB(BB: *BrInst.getSuccessor(i: 0));
1387
1388 // If this condition is one of the special cases we handle, do special stuff
1389 // now.
1390 const Value *CondVal = BrInst.getCondition();
1391 MachineBasicBlock *Succ1MBB = &getMBB(BB: *BrInst.getSuccessor(i: 1));
1392
1393 // If this is a series of conditions that are or'd or and'd together, emit
1394 // this as a sequence of branches instead of setcc's with and/or operations.
1395 // As long as jumps are not expensive (exceptions for multi-use logic ops,
1396 // unpredictable branches, and vector extracts because those jumps are likely
1397 // expensive for any target), this should improve performance.
1398 // For example, instead of something like:
1399 // cmp A, B
1400 // C = seteq
1401 // cmp D, E
1402 // F = setle
1403 // or C, F
1404 // jnz foo
1405 // Emit:
1406 // cmp A, B
1407 // je foo
1408 // cmp D, E
1409 // jle foo
1410 using namespace PatternMatch;
1411 const Instruction *CondI = dyn_cast<Instruction>(Val: CondVal);
1412 if (!TLI->isJumpExpensive() && CondI && CondI->hasOneUse() &&
1413 !BrInst.hasMetadata(KindID: LLVMContext::MD_unpredictable)) {
1414 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
1415 Value *Vec;
1416 const Value *BOp0, *BOp1;
1417 if (match(V: CondI, P: m_LogicalAnd(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
1418 Opcode = Instruction::And;
1419 else if (match(V: CondI, P: m_LogicalOr(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
1420 Opcode = Instruction::Or;
1421
1422 if (Opcode && !(match(V: BOp0, P: m_ExtractElt(Val: m_Value(V&: Vec), Idx: m_Value())) &&
1423 match(V: BOp1, P: m_ExtractElt(Val: m_Specific(V: Vec), Idx: m_Value())))) {
1424 findMergedConditions(Cond: CondI, TBB: Succ0MBB, FBB: Succ1MBB, CurBB: &CurMBB, SwitchBB: &CurMBB, Opc: Opcode,
1425 TProb: getEdgeProbability(Src: &CurMBB, Dst: Succ0MBB),
1426 FProb: getEdgeProbability(Src: &CurMBB, Dst: Succ1MBB),
1427 /*InvertCond=*/false);
1428 assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
1429
1430 // Allow some cases to be rejected.
1431 if (shouldEmitAsBranches(Cases: SL->SwitchCases)) {
1432 // Emit the branch for this block.
1433 emitSwitchCase(CB&: SL->SwitchCases[0], SwitchBB: &CurMBB, MIB&: *CurBuilder);
1434 SL->SwitchCases.erase(position: SL->SwitchCases.begin());
1435 return true;
1436 }
1437
1438 // Okay, we decided not to do this, remove any inserted MBB's and clear
1439 // SwitchCases.
1440 for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
1441 MF->erase(MBBI: SL->SwitchCases[I].ThisBB);
1442
1443 SL->SwitchCases.clear();
1444 }
1445 }
1446
1447 // Create a CaseBlock record representing this branch.
1448 SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
1449 ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
1450 nullptr, Succ0MBB, Succ1MBB, &CurMBB,
1451 CurBuilder->getDebugLoc());
1452
1453 // Use emitSwitchCase to actually insert the fast branch sequence for this
1454 // cond branch.
1455 emitSwitchCase(CB, SwitchBB: &CurMBB, MIB&: *CurBuilder);
1456 return true;
1457}
1458
1459void IRTranslatorImpl::addSuccessorWithProb(MachineBasicBlock *Src,
1460 MachineBasicBlock *Dst,
1461 BranchProbability Prob) {
1462 if (!FuncInfo.BPI) {
1463 Src->addSuccessorWithoutProb(Succ: Dst);
1464 return;
1465 }
1466 if (Prob.isUnknown())
1467 Prob = getEdgeProbability(Src, Dst);
1468 Src->addSuccessor(Succ: Dst, Prob);
1469}
1470
1471BranchProbability
1472IRTranslatorImpl::getEdgeProbability(const MachineBasicBlock *Src,
1473 const MachineBasicBlock *Dst) const {
1474 const BasicBlock *SrcBB = Src->getBasicBlock();
1475 const BasicBlock *DstBB = Dst->getBasicBlock();
1476 if (!FuncInfo.BPI) {
1477 // If BPI is not available, set the default probability as 1 / N, where N is
1478 // the number of successors.
1479 auto SuccSize = std::max<uint32_t>(a: succ_size(BB: SrcBB), b: 1);
1480 return BranchProbability(1, SuccSize);
1481 }
1482 return FuncInfo.BPI->getEdgeProbability(Src: SrcBB, Dst: DstBB);
1483}
1484
1485bool IRTranslatorImpl::translateSwitch(const User &U, MachineIRBuilder &MIB) {
1486 using namespace SwitchCG;
1487 // Extract cases from the switch.
1488 const SwitchInst &SI = cast<SwitchInst>(Val: U);
1489 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1490 CaseClusterVector Clusters;
1491 Clusters.reserve(n: SI.getNumCases());
1492 for (const auto &I : SI.cases()) {
1493 MachineBasicBlock *Succ = &getMBB(BB: *I.getCaseSuccessor());
1494 assert(Succ && "Could not find successor mbb in mapping");
1495 const ConstantInt *CaseVal = I.getCaseValue();
1496 BranchProbability Prob =
1497 BPI ? BPI->getEdgeProbability(Src: SI.getParent(), IndexInSuccessors: I.getSuccessorIndex())
1498 : BranchProbability(1, SI.getNumCases() + 1);
1499 Clusters.push_back(x: CaseCluster::range(Low: CaseVal, High: CaseVal, MBB: Succ, Prob));
1500 }
1501
1502 MachineBasicBlock *DefaultMBB = &getMBB(BB: *SI.getDefaultDest());
1503
1504 // Cluster adjacent cases with the same destination. We do this at all
1505 // optimization levels because it's cheap to do and will make codegen faster
1506 // if there are many clusters.
1507 sortAndRangeify(Clusters);
1508
1509 MachineBasicBlock *SwitchMBB = &getMBB(BB: *SI.getParent());
1510
1511 // If there is only the default destination, jump there directly.
1512 if (Clusters.empty()) {
1513 SwitchMBB->addSuccessor(Succ: DefaultMBB);
1514 if (DefaultMBB != SwitchMBB->getNextNode())
1515 MIB.buildBr(Dest&: *DefaultMBB);
1516 return true;
1517 }
1518
1519 SL->findJumpTables(Clusters, SI: &SI, SL: std::nullopt, DefaultMBB, PSI: nullptr, BFI: nullptr);
1520 SL->findBitTestClusters(Clusters, SI: &SI);
1521
1522 LLVM_DEBUG({
1523 dbgs() << "Case clusters: ";
1524 for (const CaseCluster &C : Clusters) {
1525 if (C.Kind == CC_JumpTable)
1526 dbgs() << "JT:";
1527 if (C.Kind == CC_BitTests)
1528 dbgs() << "BT:";
1529
1530 C.Low->getValue().print(dbgs(), true);
1531 if (C.Low != C.High) {
1532 dbgs() << '-';
1533 C.High->getValue().print(dbgs(), true);
1534 }
1535 dbgs() << ' ';
1536 }
1537 dbgs() << '\n';
1538 });
1539
1540 assert(!Clusters.empty());
1541 SwitchWorkList WorkList;
1542 CaseClusterIt First = Clusters.begin();
1543 CaseClusterIt Last = Clusters.end() - 1;
1544 auto DefaultProb = getEdgeProbability(Src: SwitchMBB, Dst: DefaultMBB);
1545 WorkList.push_back(Elt: {.MBB: SwitchMBB, .FirstCluster: First, .LastCluster: Last, .GE: nullptr, .LT: nullptr, .DefaultProb: DefaultProb});
1546
1547 while (!WorkList.empty()) {
1548 SwitchWorkListItem W = WorkList.pop_back_val();
1549
1550 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
1551 // For optimized builds, lower large range as a balanced binary tree.
1552 if (NumClusters > 3 &&
1553 MF->getTarget().getOptLevel() != CodeGenOptLevel::None &&
1554 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
1555 splitWorkItem(WorkList, W, Cond: SI.getCondition(), SwitchMBB, MIB);
1556 continue;
1557 }
1558
1559 if (!lowerSwitchWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
1560 return false;
1561 }
1562 return true;
1563}
1564
1565void IRTranslatorImpl::splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
1566 const SwitchCG::SwitchWorkListItem &W,
1567 Value *Cond, MachineBasicBlock *SwitchMBB,
1568 MachineIRBuilder &MIB) {
1569 using namespace SwitchCG;
1570 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
1571 "Clusters not sorted?");
1572 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
1573
1574 auto [LastLeft, FirstRight, LeftProb, RightProb] =
1575 SL->computeSplitWorkItemInfo(W);
1576
1577 // Use the first element on the right as pivot since we will make less-than
1578 // comparisons against it.
1579 CaseClusterIt PivotCluster = FirstRight;
1580 assert(PivotCluster > W.FirstCluster);
1581 assert(PivotCluster <= W.LastCluster);
1582
1583 CaseClusterIt FirstLeft = W.FirstCluster;
1584 CaseClusterIt LastRight = W.LastCluster;
1585
1586 const ConstantInt *Pivot = PivotCluster->Low;
1587
1588 // New blocks will be inserted immediately after the current one.
1589 MachineFunction::iterator BBI(W.MBB);
1590 ++BBI;
1591
1592 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
1593 // we can branch to its destination directly if it's squeezed exactly in
1594 // between the known lower bound and Pivot - 1.
1595 MachineBasicBlock *LeftMBB;
1596 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
1597 FirstLeft->Low == W.GE &&
1598 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
1599 LeftMBB = FirstLeft->MBB;
1600 } else {
1601 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
1602 FuncInfo.MF->insert(MBBI: BBI, MBB: LeftMBB);
1603 WorkList.push_back(
1604 Elt: {.MBB: LeftMBB, .FirstCluster: FirstLeft, .LastCluster: LastLeft, .GE: W.GE, .LT: Pivot, .DefaultProb: W.DefaultProb / 2});
1605 }
1606
1607 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
1608 // single cluster, RHS.Low == Pivot, and we can branch to its destination
1609 // directly if RHS.High equals the current upper bound.
1610 MachineBasicBlock *RightMBB;
1611 if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT &&
1612 (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
1613 RightMBB = FirstRight->MBB;
1614 } else {
1615 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
1616 FuncInfo.MF->insert(MBBI: BBI, MBB: RightMBB);
1617 WorkList.push_back(
1618 Elt: {.MBB: RightMBB, .FirstCluster: FirstRight, .LastCluster: LastRight, .GE: Pivot, .LT: W.LT, .DefaultProb: W.DefaultProb / 2});
1619 }
1620
1621 // Create the CaseBlock record that will be used to lower the branch.
1622 CaseBlock CB(ICmpInst::Predicate::ICMP_SLT, false, Cond, Pivot, nullptr,
1623 LeftMBB, RightMBB, W.MBB, MIB.getDebugLoc(), LeftProb,
1624 RightProb);
1625
1626 if (W.MBB == SwitchMBB)
1627 emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
1628 else
1629 SL->SwitchCases.push_back(x: CB);
1630}
1631
1632void IRTranslatorImpl::emitJumpTable(SwitchCG::JumpTable &JT,
1633 MachineBasicBlock *MBB) {
1634 // Emit the code for the jump table
1635 assert(JT.Reg && "Should lower JT Header first!");
1636 MachineIRBuilder MIB(*MBB->getParent());
1637 MIB.setMBB(*MBB);
1638 MIB.setDebugLoc(CurBuilder->getDebugLoc());
1639
1640 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
1641 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1642
1643 auto Table = MIB.buildJumpTable(PtrTy, JTI: JT.JTI);
1644 MIB.buildBrJT(TablePtr: Table.getReg(Idx: 0), JTI: JT.JTI, IndexReg: JT.Reg);
1645}
1646
1647bool IRTranslatorImpl::emitJumpTableHeader(SwitchCG::JumpTable &JT,
1648 SwitchCG::JumpTableHeader &JTH,
1649 MachineBasicBlock *HeaderBB) {
1650 MachineIRBuilder MIB(*HeaderBB->getParent());
1651 MIB.setMBB(*HeaderBB);
1652 MIB.setDebugLoc(CurBuilder->getDebugLoc());
1653
1654 const Value &SValue = *JTH.SValue;
1655 // Subtract the lowest switch case value from the value being switched on.
1656 const LLT SwitchTy = getLLTForType(Ty&: *SValue.getType(), DL: *DL);
1657 Register SwitchOpReg = getOrCreateVReg(Val: SValue);
1658 auto FirstCst = MIB.buildConstant(Res: SwitchTy, Val: JTH.First);
1659 auto Sub = MIB.buildSub(Dst: {SwitchTy}, Src0: SwitchOpReg, Src1: FirstCst);
1660
1661 // This value may be smaller or larger than the target's pointer type, and
1662 // therefore require extension or truncating.
1663 auto *PtrIRTy = PointerType::getUnqual(C&: SValue.getContext());
1664 const LLT PtrScalarTy = LLT::integer(SizeInBits: DL->getTypeSizeInBits(Ty: PtrIRTy));
1665 auto Index = MIB.buildZExtOrTrunc(Res: PtrScalarTy, Op: Sub);
1666
1667 JT.Reg = Index.getReg(Idx: 0);
1668
1669 if (JTH.FallthroughUnreachable) {
1670 if (JT.MBB != HeaderBB->getNextNode())
1671 MIB.buildBr(Dest&: *JT.MBB);
1672 return true;
1673 }
1674
1675 // Emit the range check for the jump table, and branch to the default block
1676 // for the switch statement if the value being switched on exceeds the
1677 // largest case in the switch.
1678 auto Cst = getOrCreateVReg(
1679 Val: *ConstantInt::get(Ty: SValue.getType(), V: JTH.Last - JTH.First));
1680 auto Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_UGT, Res: LLT::integer(SizeInBits: 1), Op0: Sub, Op1: Cst);
1681
1682 auto BrCond = MIB.buildBrCond(Tst: Cmp.getReg(Idx: 0), Dest&: *JT.Default);
1683
1684 // Avoid emitting unnecessary branches to the next block.
1685 if (JT.MBB != HeaderBB->getNextNode())
1686 BrCond = MIB.buildBr(Dest&: *JT.MBB);
1687 return true;
1688}
1689
1690void IRTranslatorImpl::emitSwitchCase(SwitchCG::CaseBlock &CB,
1691 MachineBasicBlock *SwitchBB,
1692 MachineIRBuilder &MIB) {
1693 Register CondLHS = getOrCreateVReg(Val: *CB.CmpLHS);
1694 Register Cond;
1695 DebugLoc OldDbgLoc = MIB.getDebugLoc();
1696 MIB.setDebugLoc(CB.DbgLoc);
1697 MIB.setMBB(*CB.ThisBB);
1698
1699 if (CB.PredInfo.NoCmp) {
1700 // Branch or fall through to TrueBB.
1701 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
1702 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
1703 NewPred: CB.ThisBB);
1704 CB.ThisBB->normalizeSuccProbs();
1705 if (CB.TrueBB != CB.ThisBB->getNextNode())
1706 MIB.buildBr(Dest&: *CB.TrueBB);
1707 MIB.setDebugLoc(OldDbgLoc);
1708 return;
1709 }
1710
1711 const LLT i1Ty = LLT::integer(SizeInBits: 1);
1712 // Build the compare.
1713 if (!CB.CmpMHS) {
1714 const auto *CI = dyn_cast<ConstantInt>(Val: CB.CmpRHS);
1715 // For conditional branch lowering, we might try to do something silly like
1716 // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
1717 // just re-use the existing condition vreg.
1718 if (MRI->getType(Reg: CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
1719 CB.PredInfo.Pred == CmpInst::ICMP_EQ) {
1720 Cond = CondLHS;
1721 } else {
1722 Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
1723 if (CmpInst::isFPPredicate(P: CB.PredInfo.Pred))
1724 Cond =
1725 MIB.buildFCmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
1726 else
1727 Cond =
1728 MIB.buildICmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
1729 }
1730 } else {
1731 assert(CB.PredInfo.Pred == CmpInst::ICMP_SLE &&
1732 "Can only handle SLE ranges");
1733
1734 const APInt& Low = cast<ConstantInt>(Val: CB.CmpLHS)->getValue();
1735 const APInt& High = cast<ConstantInt>(Val: CB.CmpRHS)->getValue();
1736
1737 Register CmpOpReg = getOrCreateVReg(Val: *CB.CmpMHS);
1738 if (cast<ConstantInt>(Val: CB.CmpLHS)->isMinValue(IsSigned: true)) {
1739 Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
1740 Cond =
1741 MIB.buildICmp(Pred: CmpInst::ICMP_SLE, Res: i1Ty, Op0: CmpOpReg, Op1: CondRHS).getReg(Idx: 0);
1742 } else {
1743 const LLT CmpTy = MRI->getType(Reg: CmpOpReg);
1744 auto Sub = MIB.buildSub(Dst: {CmpTy}, Src0: CmpOpReg, Src1: CondLHS);
1745 auto Diff = MIB.buildConstant(Res: CmpTy, Val: High - Low);
1746 Cond = MIB.buildICmp(Pred: CmpInst::ICMP_ULE, Res: i1Ty, Op0: Sub, Op1: Diff).getReg(Idx: 0);
1747 }
1748 }
1749
1750 // Update successor info
1751 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
1752
1753 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
1754 NewPred: CB.ThisBB);
1755
1756 // TrueBB and FalseBB are always different unless the incoming IR is
1757 // degenerate. This only happens when running llc on weird IR.
1758 if (CB.TrueBB != CB.FalseBB)
1759 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.FalseBB, Prob: CB.FalseProb);
1760 CB.ThisBB->normalizeSuccProbs();
1761
1762 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
1763 NewPred: CB.ThisBB);
1764
1765 MIB.buildBrCond(Tst: Cond, Dest&: *CB.TrueBB);
1766 MIB.buildBr(Dest&: *CB.FalseBB);
1767 MIB.setDebugLoc(OldDbgLoc);
1768}
1769
1770bool IRTranslatorImpl::lowerJumpTableWorkItem(
1771 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
1772 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1773 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
1774 BranchProbability UnhandledProbs, SwitchCG::CaseClusterIt I,
1775 MachineBasicBlock *Fallthrough, bool FallthroughUnreachable) {
1776 using namespace SwitchCG;
1777 MachineFunction *CurMF = SwitchMBB->getParent();
1778 // FIXME: Optimize away range check based on pivot comparisons.
1779 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
1780 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
1781 BranchProbability DefaultProb = W.DefaultProb;
1782
1783 // The jump block hasn't been inserted yet; insert it here.
1784 MachineBasicBlock *JumpMBB = JT->MBB;
1785 CurMF->insert(MBBI: BBI, MBB: JumpMBB);
1786
1787 // Since the jump table block is separate from the switch block, we need
1788 // to keep track of it as a machine predecessor to the default block,
1789 // otherwise we lose the phi edges.
1790 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1791 NewPred: CurMBB);
1792 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1793 NewPred: JumpMBB);
1794
1795 auto JumpProb = I->Prob;
1796 auto FallthroughProb = UnhandledProbs;
1797
1798 // If the default statement is a target of the jump table, we evenly
1799 // distribute the default probability to successors of CurMBB. Also
1800 // update the probability on the edge from JumpMBB to Fallthrough.
1801 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
1802 SE = JumpMBB->succ_end();
1803 SI != SE; ++SI) {
1804 if (*SI == DefaultMBB) {
1805 JumpProb += DefaultProb / 2;
1806 FallthroughProb -= DefaultProb / 2;
1807 JumpMBB->setSuccProbability(I: SI, Prob: DefaultProb / 2);
1808 JumpMBB->normalizeSuccProbs();
1809 } else {
1810 // Also record edges from the jump table block to it's successors.
1811 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
1812 NewPred: JumpMBB);
1813 }
1814 }
1815
1816 if (FallthroughUnreachable)
1817 JTH->FallthroughUnreachable = true;
1818
1819 if (!JTH->FallthroughUnreachable)
1820 addSuccessorWithProb(Src: CurMBB, Dst: Fallthrough, Prob: FallthroughProb);
1821 addSuccessorWithProb(Src: CurMBB, Dst: JumpMBB, Prob: JumpProb);
1822 CurMBB->normalizeSuccProbs();
1823
1824 // The jump table header will be inserted in our current block, do the
1825 // range check, and fall through to our fallthrough block.
1826 JTH->HeaderBB = CurMBB;
1827 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
1828
1829 // If we're in the right place, emit the jump table header right now.
1830 if (CurMBB == SwitchMBB) {
1831 if (!emitJumpTableHeader(JT&: *JT, JTH&: *JTH, HeaderBB: CurMBB))
1832 return false;
1833 JTH->Emitted = true;
1834 }
1835 return true;
1836}
1837bool IRTranslatorImpl::lowerSwitchRangeWorkItem(
1838 SwitchCG::CaseClusterIt I, Value *Cond, MachineBasicBlock *Fallthrough,
1839 bool FallthroughUnreachable, BranchProbability UnhandledProbs,
1840 MachineBasicBlock *CurMBB, MachineIRBuilder &MIB,
1841 MachineBasicBlock *SwitchMBB) {
1842 using namespace SwitchCG;
1843 const Value *RHS, *LHS, *MHS;
1844 CmpInst::Predicate Pred;
1845 if (I->Low == I->High) {
1846 // Check Cond == I->Low.
1847 Pred = CmpInst::ICMP_EQ;
1848 LHS = Cond;
1849 RHS = I->Low;
1850 MHS = nullptr;
1851 } else {
1852 // Check I->Low <= Cond <= I->High.
1853 Pred = CmpInst::ICMP_SLE;
1854 LHS = I->Low;
1855 MHS = Cond;
1856 RHS = I->High;
1857 }
1858
1859 // If Fallthrough is unreachable, fold away the comparison.
1860 // The false probability is the sum of all unhandled cases.
1861 CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
1862 CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
1863
1864 emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
1865 return true;
1866}
1867
1868void IRTranslatorImpl::emitBitTestHeader(SwitchCG::BitTestBlock &B,
1869 MachineBasicBlock *SwitchBB) {
1870 MachineIRBuilder &MIB = *CurBuilder;
1871 MIB.setMBB(*SwitchBB);
1872
1873 // Subtract the minimum value.
1874 Register SwitchOpReg = getOrCreateVReg(Val: *B.SValue);
1875
1876 LLT SwitchOpTy = MRI->getType(Reg: SwitchOpReg);
1877 Register MinValReg = MIB.buildConstant(Res: SwitchOpTy, Val: B.First).getReg(Idx: 0);
1878 auto RangeSub = MIB.buildSub(Dst: SwitchOpTy, Src0: SwitchOpReg, Src1: MinValReg);
1879
1880 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
1881 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1882
1883 LLT MaskTy = SwitchOpTy;
1884 if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
1885 !llvm::has_single_bit<uint32_t>(Value: MaskTy.getSizeInBits()))
1886 MaskTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
1887 else {
1888 // Ensure that the type will fit the mask value.
1889 for (const SwitchCG::BitTestCase &Case : B.Cases) {
1890 if (!isUIntN(N: SwitchOpTy.getSizeInBits(), x: Case.Mask)) {
1891 // Switch table case range are encoded into series of masks.
1892 // Just use pointer type, it's guaranteed to fit.
1893 MaskTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
1894 break;
1895 }
1896 }
1897 }
1898 Register SubReg = RangeSub.getReg(Idx: 0);
1899 if (SwitchOpTy != MaskTy)
1900 SubReg = MIB.buildZExtOrTrunc(Res: MaskTy, Op: SubReg).getReg(Idx: 0);
1901
1902 B.RegVT = getMVTForLLT(Ty: MaskTy);
1903 B.Reg = SubReg;
1904
1905 MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1906
1907 if (!B.FallthroughUnreachable)
1908 addSuccessorWithProb(Src: SwitchBB, Dst: B.Default, Prob: B.DefaultProb);
1909 addSuccessorWithProb(Src: SwitchBB, Dst: MBB, Prob: B.Prob);
1910
1911 SwitchBB->normalizeSuccProbs();
1912
1913 if (!B.FallthroughUnreachable) {
1914 // Conditional branch to the default block.
1915 auto RangeCst = MIB.buildConstant(Res: SwitchOpTy, Val: B.Range);
1916 auto RangeCmp = MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: LLT::integer(SizeInBits: 1),
1917 Op0: RangeSub, Op1: RangeCst);
1918 MIB.buildBrCond(Tst: RangeCmp, Dest&: *B.Default);
1919 }
1920
1921 // Avoid emitting unnecessary branches to the next block.
1922 if (MBB != SwitchBB->getNextNode())
1923 MIB.buildBr(Dest&: *MBB);
1924}
1925
1926void IRTranslatorImpl::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1927 MachineBasicBlock *NextMBB,
1928 BranchProbability BranchProbToNext,
1929 Register Reg, SwitchCG::BitTestCase &B,
1930 MachineBasicBlock *SwitchBB) {
1931 MachineIRBuilder &MIB = *CurBuilder;
1932 MIB.setMBB(*SwitchBB);
1933
1934 LLT SwitchTy = getLLTForMVT(Ty: BB.RegVT);
1935 Register Cmp;
1936 unsigned PopCount = llvm::popcount(Value: B.Mask);
1937 if (PopCount == 1) {
1938 // Testing for a single bit; just compare the shift count with what it
1939 // would need to be to shift a 1 bit in that position.
1940 auto MaskTrailingZeros =
1941 MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_zero(Val: B.Mask));
1942 Cmp = MIB.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: LLT::integer(SizeInBits: 1), Op0: Reg,
1943 Op1: MaskTrailingZeros)
1944 .getReg(Idx: 0);
1945 } else if (PopCount == BB.Range) {
1946 // There is only one zero bit in the range, test for it directly.
1947 auto MaskTrailingOnes =
1948 MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_one(Value: B.Mask));
1949 Cmp =
1950 MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::integer(SizeInBits: 1), Op0: Reg, Op1: MaskTrailingOnes)
1951 .getReg(Idx: 0);
1952 } else {
1953 // Make desired shift.
1954 auto CstOne = MIB.buildConstant(Res: SwitchTy, Val: 1);
1955 auto SwitchVal = MIB.buildShl(Dst: SwitchTy, Src0: CstOne, Src1: Reg);
1956
1957 // Emit bit tests and jumps.
1958 auto CstMask = MIB.buildConstant(Res: SwitchTy, Val: B.Mask);
1959 auto AndOp = MIB.buildAnd(Dst: SwitchTy, Src0: SwitchVal, Src1: CstMask);
1960 auto CstZero = MIB.buildConstant(Res: SwitchTy, Val: 0);
1961 Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::integer(SizeInBits: 1), Op0: AndOp, Op1: CstZero)
1962 .getReg(Idx: 0);
1963 }
1964
1965 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1966 addSuccessorWithProb(Src: SwitchBB, Dst: B.TargetBB, Prob: B.ExtraProb);
1967 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1968 addSuccessorWithProb(Src: SwitchBB, Dst: NextMBB, Prob: BranchProbToNext);
1969 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1970 // one as they are relative probabilities (and thus work more like weights),
1971 // and hence we need to normalize them to let the sum of them become one.
1972 SwitchBB->normalizeSuccProbs();
1973
1974 // Record the fact that the IR edge from the header to the bit test target
1975 // will go through our new block. Neeeded for PHIs to have nodes added.
1976 addMachineCFGPred(Edge: {BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1977 NewPred: SwitchBB);
1978
1979 MIB.buildBrCond(Tst: Cmp, Dest&: *B.TargetBB);
1980
1981 // Avoid emitting unnecessary branches to the next block.
1982 if (NextMBB != SwitchBB->getNextNode())
1983 MIB.buildBr(Dest&: *NextMBB);
1984}
1985
1986bool IRTranslatorImpl::lowerBitTestWorkItem(
1987 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
1988 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1989 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
1990 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1991 SwitchCG::CaseClusterIt I, MachineBasicBlock *Fallthrough,
1992 bool FallthroughUnreachable) {
1993 using namespace SwitchCG;
1994 MachineFunction *CurMF = SwitchMBB->getParent();
1995 // FIXME: Optimize away range check based on pivot comparisons.
1996 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
1997 // The bit test blocks haven't been inserted yet; insert them here.
1998 for (BitTestCase &BTC : BTB->Cases)
1999 CurMF->insert(MBBI: BBI, MBB: BTC.ThisBB);
2000
2001 // Fill in fields of the BitTestBlock.
2002 BTB->Parent = CurMBB;
2003 BTB->Default = Fallthrough;
2004
2005 BTB->DefaultProb = UnhandledProbs;
2006 // If the cases in bit test don't form a contiguous range, we evenly
2007 // distribute the probability on the edge to Fallthrough to two
2008 // successors of CurMBB.
2009 if (!BTB->ContiguousRange) {
2010 BTB->Prob += DefaultProb / 2;
2011 BTB->DefaultProb -= DefaultProb / 2;
2012 }
2013
2014 if (FallthroughUnreachable)
2015 BTB->FallthroughUnreachable = true;
2016
2017 // If we're in the right place, emit the bit test header right now.
2018 if (CurMBB == SwitchMBB) {
2019 emitBitTestHeader(B&: *BTB, SwitchBB: SwitchMBB);
2020 BTB->Emitted = true;
2021 }
2022 return true;
2023}
2024
2025bool IRTranslatorImpl::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
2026 Value *Cond,
2027 MachineBasicBlock *SwitchMBB,
2028 MachineBasicBlock *DefaultMBB,
2029 MachineIRBuilder &MIB) {
2030 using namespace SwitchCG;
2031 MachineFunction *CurMF = FuncInfo.MF;
2032 MachineBasicBlock *NextMBB = nullptr;
2033 MachineFunction::iterator BBI(W.MBB);
2034 if (++BBI != FuncInfo.MF->end())
2035 NextMBB = &*BBI;
2036
2037 if (EnableOpts) {
2038 // Here, we order cases by probability so the most likely case will be
2039 // checked first. However, two clusters can have the same probability in
2040 // which case their relative ordering is non-deterministic. So we use Low
2041 // as a tie-breaker as clusters are guaranteed to never overlap.
2042 llvm::sort(Start: W.FirstCluster, End: W.LastCluster + 1,
2043 Comp: [](const CaseCluster &a, const CaseCluster &b) {
2044 return a.Prob != b.Prob
2045 ? a.Prob > b.Prob
2046 : a.Low->getValue().slt(RHS: b.Low->getValue());
2047 });
2048
2049 // Rearrange the case blocks so that the last one falls through if possible
2050 // without changing the order of probabilities.
2051 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
2052 --I;
2053 if (I->Prob > W.LastCluster->Prob)
2054 break;
2055 if (I->Kind == CC_Range && I->MBB == NextMBB) {
2056 std::swap(a&: *I, b&: *W.LastCluster);
2057 break;
2058 }
2059 }
2060 }
2061
2062 // Compute total probability.
2063 BranchProbability DefaultProb = W.DefaultProb;
2064 BranchProbability UnhandledProbs = DefaultProb;
2065 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
2066 UnhandledProbs += I->Prob;
2067
2068 MachineBasicBlock *CurMBB = W.MBB;
2069 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
2070 bool FallthroughUnreachable = false;
2071 MachineBasicBlock *Fallthrough;
2072 if (I == W.LastCluster) {
2073 // For the last cluster, fall through to the default destination.
2074 Fallthrough = DefaultMBB;
2075 FallthroughUnreachable =
2076 isa<UnreachableInst>(Val: DefaultMBB->getBasicBlock()->getFirstNonPHIIt());
2077 } else {
2078 Fallthrough = CurMF->CreateMachineBasicBlock(BB: CurMBB->getBasicBlock());
2079 CurMF->insert(MBBI: BBI, MBB: Fallthrough);
2080 }
2081 UnhandledProbs -= I->Prob;
2082
2083 switch (I->Kind) {
2084 case CC_BitTests: {
2085 if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
2086 DefaultProb, UnhandledProbs, I, Fallthrough,
2087 FallthroughUnreachable)) {
2088 LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
2089 return false;
2090 }
2091 break;
2092 }
2093
2094 case CC_JumpTable: {
2095 if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
2096 UnhandledProbs, I, Fallthrough,
2097 FallthroughUnreachable)) {
2098 LLVM_DEBUG(dbgs() << "Failed to lower jump table");
2099 return false;
2100 }
2101 break;
2102 }
2103 case CC_Range: {
2104 if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
2105 FallthroughUnreachable, UnhandledProbs,
2106 CurMBB, MIB, SwitchMBB)) {
2107 LLVM_DEBUG(dbgs() << "Failed to lower switch range");
2108 return false;
2109 }
2110 break;
2111 }
2112 }
2113 CurMBB = Fallthrough;
2114 }
2115
2116 return true;
2117}
2118
2119bool IRTranslatorImpl::translateIndirectBr(const User &U,
2120 MachineIRBuilder &MIRBuilder) {
2121 const IndirectBrInst &BrInst = cast<IndirectBrInst>(Val: U);
2122
2123 const Register Tgt = getOrCreateVReg(Val: *BrInst.getAddress());
2124 MIRBuilder.buildBrIndirect(Tgt);
2125
2126 // Link successors.
2127 SmallPtrSet<const BasicBlock *, 32> AddedSuccessors;
2128 MachineBasicBlock &CurBB = MIRBuilder.getMBB();
2129 for (const BasicBlock *Succ : successors(I: &BrInst)) {
2130 // It's legal for indirectbr instructions to have duplicate blocks in the
2131 // destination list. We don't allow this in MIR. Skip anything that's
2132 // already a successor.
2133 if (!AddedSuccessors.insert(Ptr: Succ).second)
2134 continue;
2135 CurBB.addSuccessor(Succ: &getMBB(BB: *Succ));
2136 }
2137
2138 return true;
2139}
2140
2141static bool isSwiftError(const Value *V) {
2142 if (auto Arg = dyn_cast<Argument>(Val: V))
2143 return Arg->hasSwiftErrorAttr();
2144 if (auto AI = dyn_cast<AllocaInst>(Val: V))
2145 return AI->isSwiftError();
2146 return false;
2147}
2148
2149bool IRTranslatorImpl::translateLoad(const User &U,
2150 MachineIRBuilder &MIRBuilder) {
2151 const LoadInst &LI = cast<LoadInst>(Val: U);
2152 TypeSize StoreSize = DL->getTypeStoreSize(Ty: LI.getType());
2153 if (StoreSize.isZero())
2154 return true;
2155
2156 ArrayRef<Register> Regs = getOrCreateVRegs(Val: LI);
2157 Register Base = getOrCreateVReg(Val: *LI.getPointerOperand());
2158 AAMDNodes AAInfo = LI.getAAMetadata();
2159
2160 const Value *Ptr = LI.getPointerOperand();
2161
2162 if (CLI->supportSwiftError() && isSwiftError(V: Ptr)) {
2163 assert(Regs.size() == 1 && "swifterror should be single pointer");
2164 Register VReg =
2165 SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);
2166 MIRBuilder.buildCopy(Res: Regs[0], Op: VReg);
2167 return true;
2168 }
2169
2170 MachineMemOperand::Flags Flags =
2171 TLI->getLoadMemOperandFlags(LI, DL: *DL, AC, LibInfo, OptLevel);
2172 if (AA && !(Flags & MachineMemOperand::MOInvariant)) {
2173 if (AA->pointsToConstantMemory(
2174 Loc: MemoryLocation(Ptr, LocationSize::precise(Value: StoreSize), AAInfo))) {
2175 Flags |= MachineMemOperand::MOInvariant;
2176 }
2177 }
2178
2179 // Fast-path the common single-register load.
2180 if (Regs.size() == 1) {
2181 auto *MMO = MF->getMachineMemOperand(
2182 PtrInfo: MachinePointerInfo(LI.getPointerOperand()), F: Flags,
2183 MemTy: MRI->getType(Reg: Regs[0]), BaseAlignment: getMemOpAlign(I: LI),
2184 Metadata: MMOMetadata(AAInfo, LI.getMetadata(KindID: LLVMContext::MD_range)),
2185 SSID: LI.getSyncScopeID(), Ordering: LI.getOrdering());
2186 MIRBuilder.buildLoad(Res: Regs[0], Addr: Base, MMO&: *MMO);
2187 return true;
2188 }
2189
2190 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: LI);
2191 Type *OffsetIRTy = DL->getIndexType(PtrTy: Ptr->getType());
2192 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2193 for (unsigned i = 0; i < Regs.size(); ++i) {
2194 Register Addr;
2195 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i]);
2196
2197 MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i]);
2198 Align BaseAlign = getMemOpAlign(I: LI);
2199 auto *MMO =
2200 MF->getMachineMemOperand(PtrInfo: Ptr, F: Flags, MemTy: MRI->getType(Reg: Regs[i]),
2201 BaseAlignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]), Metadata: AAInfo,
2202 SSID: LI.getSyncScopeID(), Ordering: LI.getOrdering());
2203 MIRBuilder.buildLoad(Res: Regs[i], Addr, MMO&: *MMO);
2204 }
2205
2206 return true;
2207}
2208
2209bool IRTranslatorImpl::translateStore(const User &U,
2210 MachineIRBuilder &MIRBuilder) {
2211 const StoreInst &SI = cast<StoreInst>(Val: U);
2212 if (DL->getTypeStoreSize(Ty: SI.getValueOperand()->getType()).isZero())
2213 return true;
2214
2215 ArrayRef<Register> Vals = getOrCreateVRegs(Val: *SI.getValueOperand());
2216 Register Base = getOrCreateVReg(Val: *SI.getPointerOperand());
2217
2218 if (CLI->supportSwiftError() && isSwiftError(V: SI.getPointerOperand())) {
2219 assert(Vals.size() == 1 && "swifterror should be single pointer");
2220
2221 Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
2222 SI.getPointerOperand());
2223 MIRBuilder.buildCopy(Res: VReg, Op: Vals[0]);
2224 return true;
2225 }
2226
2227 MachineMemOperand::Flags Flags = TLI->getStoreMemOperandFlags(SI, DL: *DL);
2228 // Fast-path the common single-register store.
2229 if (Vals.size() == 1) {
2230 auto *MMO = MF->getMachineMemOperand(
2231 PtrInfo: MachinePointerInfo(SI.getPointerOperand()), F: Flags,
2232 MemTy: MRI->getType(Reg: Vals[0]), BaseAlignment: getMemOpAlign(I: SI), Metadata: SI.getAAMetadata(),
2233 SSID: SI.getSyncScopeID(), Ordering: SI.getOrdering());
2234 MIRBuilder.buildStore(Val: Vals[0], Addr: Base, MMO&: *MMO);
2235 return true;
2236 }
2237
2238 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *SI.getValueOperand());
2239 Type *OffsetIRTy = DL->getIndexType(PtrTy: SI.getPointerOperandType());
2240 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2241 for (unsigned i = 0; i < Vals.size(); ++i) {
2242 Register Addr;
2243 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i]);
2244
2245 MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i]);
2246 Align BaseAlign = getMemOpAlign(I: SI);
2247 auto *MMO = MF->getMachineMemOperand(PtrInfo: Ptr, F: Flags, MemTy: MRI->getType(Reg: Vals[i]),
2248 BaseAlignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]),
2249 Metadata: SI.getAAMetadata(),
2250 SSID: SI.getSyncScopeID(), Ordering: SI.getOrdering());
2251 MIRBuilder.buildStore(Val: Vals[i], Addr, MMO&: *MMO);
2252 }
2253 return true;
2254}
2255
2256static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
2257 const Value *Src = U.getOperand(i: 0);
2258 Type *Int32Ty = Type::getInt32Ty(C&: U.getContext());
2259
2260 // getIndexedOffsetInType is designed for GEPs, so the first index is the
2261 // usual array element rather than looking into the actual aggregate.
2262 SmallVector<Value *, 1> Indices;
2263 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: 0));
2264
2265 if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: &U)) {
2266 for (auto Idx : EVI->indices())
2267 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
2268 } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(Val: &U)) {
2269 for (auto Idx : IVI->indices())
2270 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
2271 } else {
2272 llvm::append_range(C&: Indices, R: drop_begin(RangeOrContainer: U.operands()));
2273 }
2274
2275 return static_cast<uint64_t>(
2276 DL.getIndexedOffsetInType(ElemTy: Src->getType(), Indices));
2277}
2278
2279bool IRTranslatorImpl::translateExtractValue(const User &U,
2280 MachineIRBuilder &MIRBuilder) {
2281 const Value *Src = U.getOperand(i: 0);
2282 uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
2283 ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
2284 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *Src);
2285 unsigned Idx = llvm::lower_bound(Range&: Offsets, Value&: Offset) - Offsets.begin();
2286 auto &DstRegs = allocateVRegs(Val: U);
2287
2288 for (unsigned i = 0; i < DstRegs.size(); ++i)
2289 DstRegs[i] = SrcRegs[Idx++];
2290
2291 return true;
2292}
2293
2294bool IRTranslatorImpl::translateInsertValue(const User &U,
2295 MachineIRBuilder &MIRBuilder) {
2296 const Value *Src = U.getOperand(i: 0);
2297 uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
2298 auto &DstRegs = allocateVRegs(Val: U);
2299 ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(V: U);
2300 ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
2301 ArrayRef<Register> InsertedRegs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
2302 auto *InsertedIt = InsertedRegs.begin();
2303
2304 for (unsigned i = 0; i < DstRegs.size(); ++i) {
2305 if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
2306 DstRegs[i] = *InsertedIt++;
2307 else
2308 DstRegs[i] = SrcRegs[i];
2309 }
2310
2311 return true;
2312}
2313
2314bool IRTranslatorImpl::translateSelect(const User &U,
2315 MachineIRBuilder &MIRBuilder) {
2316 Register Tst = getOrCreateVReg(Val: *U.getOperand(i: 0));
2317 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: U);
2318 ArrayRef<Register> Op0Regs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
2319 ArrayRef<Register> Op1Regs = getOrCreateVRegs(Val: *U.getOperand(i: 2));
2320
2321 uint32_t Flags = 0;
2322 if (const SelectInst *SI = dyn_cast<SelectInst>(Val: &U))
2323 Flags = MachineInstr::copyFlagsFromInstruction(I: *SI);
2324
2325 for (unsigned i = 0; i < ResRegs.size(); ++i) {
2326 MIRBuilder.buildSelect(Res: ResRegs[i], Tst, Op0: Op0Regs[i], Op1: Op1Regs[i], Flags);
2327 }
2328
2329 return true;
2330}
2331
2332bool IRTranslatorImpl::translateCopy(const User &U, const Value &V,
2333 MachineIRBuilder &MIRBuilder) {
2334 return translateCopy(U, Src: getOrCreateVReg(Val: V), MIRBuilder);
2335}
2336
2337bool IRTranslatorImpl::translateCopy(const User &U, Register Src,
2338 MachineIRBuilder &MIRBuilder) {
2339 auto &Regs = *VMap.getVRegs(V: U);
2340 if (Regs.empty()) {
2341 Regs.push_back(Elt: Src);
2342 } else {
2343 // If we already assigned a vreg for this instruction, we can't change that.
2344 // Emit a copy to satisfy the users we already emitted.
2345 MIRBuilder.buildCopy(Res: Regs[0], Op: Src);
2346 }
2347 return true;
2348}
2349
2350bool IRTranslatorImpl::translateBitCast(const User &U,
2351 MachineIRBuilder &MIRBuilder) {
2352 Type *SrcTy = U.getOperand(i: 0)->getType();
2353 Type *DstTy = U.getType();
2354
2355 // If we're bitcasting to the source type, we can reuse the source vreg.
2356 if (getLLTForType(Ty&: *SrcTy, DL: *DL) == getLLTForType(Ty&: *DstTy, DL: *DL)) {
2357 // If the source is a ConstantInt then it was probably created by
2358 // ConstantHoisting and we should leave it alone.
2359 if (isa<ConstantInt>(Val: U.getOperand(i: 0)))
2360 return translateCast(Opcode: TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
2361 MIRBuilder);
2362 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
2363 }
2364
2365 // The IR only allows pointer/non-pointer bitcasts with byte types, but
2366 // G_BITCAST can't convert between pointers and other types. Go through an
2367 // integer with the pointer's shape instead: `bitcast <2 x b32> to ptr`
2368 // becomes a G_BITCAST to i64 and a G_INTTOPTR.
2369 if (SrcTy->isPtrOrPtrVectorTy() != DstTy->isPtrOrPtrVectorTy()) {
2370 assert((SrcTy->isByteOrByteVectorTy() || DstTy->isByteOrByteVectorTy()) &&
2371 "only byte types can be bitcast to or from pointers");
2372 Type *PtrIRTy = SrcTy->isPtrOrPtrVectorTy() ? SrcTy : DstTy;
2373 LLT IntTy = getLLTForType(Ty&: *DL->getIntPtrType(PtrIRTy), DL: *DL);
2374 Register Src = getOrCreateVReg(Val: *U.getOperand(i: 0));
2375 Register Dst = getOrCreateVReg(Val: U);
2376 if (MRI->getType(Reg: Src) != IntTy && MRI->getType(Reg: Dst) != IntTy)
2377 Src = MIRBuilder.buildCast(Dst: IntTy, Src).getReg(Idx: 0);
2378 MIRBuilder.buildCast(Dst, Src);
2379 return true;
2380 }
2381
2382 return translateCast(Opcode: TargetOpcode::G_BITCAST, U, MIRBuilder);
2383}
2384
2385bool IRTranslatorImpl::translateCast(unsigned Opcode, const User &U,
2386 MachineIRBuilder &MIRBuilder) {
2387 if (!mayTranslateUserTypes(U))
2388 return false;
2389
2390 uint32_t Flags = 0;
2391 if (const Instruction *I = dyn_cast<Instruction>(Val: &U))
2392 Flags = MachineInstr::copyFlagsFromInstruction(I: *I);
2393
2394 Register Op = getOrCreateVReg(Val: *U.getOperand(i: 0));
2395 Register Res = getOrCreateVReg(Val: U);
2396 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op}, Flags);
2397 return true;
2398}
2399
2400bool IRTranslatorImpl::translateGetElementPtr(const User &U,
2401 MachineIRBuilder &MIRBuilder) {
2402 Value &Op0 = *U.getOperand(i: 0);
2403 Register BaseReg = getOrCreateVReg(Val: Op0);
2404 Type *PtrIRTy = Op0.getType();
2405 LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
2406 Type *OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
2407 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2408
2409 uint32_t PtrAddFlags = 0;
2410 // Each PtrAdd generated to implement the GEP inherits its nuw, nusw, inbounds
2411 // flags.
2412 if (const Instruction *I = dyn_cast<Instruction>(Val: &U))
2413 PtrAddFlags = MachineInstr::copyFlagsFromInstruction(I: *I);
2414
2415 auto PtrAddFlagsWithConst = [&](int64_t Offset) {
2416 // For nusw/inbounds GEP with an offset that is nonnegative when interpreted
2417 // as signed, assume there is no unsigned overflow.
2418 if (Offset >= 0 && (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap))
2419 return PtrAddFlags | MachineInstr::MIFlag::NoUWrap;
2420 return PtrAddFlags;
2421 };
2422
2423 // Normalize Vector GEP - all scalar operands should be converted to the
2424 // splat vector.
2425 unsigned VectorWidth = 0;
2426
2427 // True if we should use a splat vector; using VectorWidth alone is not
2428 // sufficient.
2429 bool WantSplatVector = false;
2430 if (auto *VT = dyn_cast<VectorType>(Val: U.getType())) {
2431 VectorWidth = cast<FixedVectorType>(Val: VT)->getNumElements();
2432 // We don't produce 1 x N vectors; those are treated as scalars.
2433 WantSplatVector = VectorWidth > 1;
2434 }
2435
2436 if (cast<GEPOperator>(Val: U).hasAllZeroIndices())
2437 return translateCopy(U, Src: BaseReg, MIRBuilder);
2438
2439 // We might need to splat the base pointer into a vector if the offsets
2440 // are vectors.
2441 if (WantSplatVector && !PtrTy.isVector()) {
2442 BaseReg = MIRBuilder
2443 .buildSplatBuildVector(Res: LLT::fixed_vector(NumElements: VectorWidth, ScalarTy: PtrTy),
2444 Src: BaseReg)
2445 .getReg(Idx: 0);
2446 PtrIRTy = FixedVectorType::get(ElementType: PtrIRTy, NumElts: VectorWidth);
2447 PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
2448 OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
2449 OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2450 }
2451
2452 int64_t Offset = 0;
2453 for (gep_type_iterator GTI = gep_type_begin(GEP: &U), E = gep_type_end(GEP: &U);
2454 GTI != E; ++GTI) {
2455 const Value *Idx = GTI.getOperand();
2456 if (StructType *StTy = GTI.getStructTypeOrNull()) {
2457 unsigned Field = cast<Constant>(Val: Idx)->getUniqueInteger().getZExtValue();
2458 Offset += DL->getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
2459 continue;
2460 } else {
2461 uint64_t ElementSize = GTI.getSequentialElementStride(DL: *DL);
2462
2463 // If this is a scalar constant or a splat vector of constants,
2464 // handle it quickly.
2465 if (const auto *CI = dyn_cast<ConstantInt>(Val: Idx)) {
2466 if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {
2467 Offset += ElementSize * *Val;
2468 continue;
2469 }
2470 }
2471
2472 if (Offset != 0) {
2473 auto OffsetMIB = MIRBuilder.buildConstant(Res: {OffsetTy}, Val: Offset);
2474 BaseReg = MIRBuilder
2475 .buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0),
2476 Flags: PtrAddFlagsWithConst(Offset))
2477 .getReg(Idx: 0);
2478 Offset = 0;
2479 }
2480
2481 Register IdxReg = getOrCreateVReg(Val: *Idx);
2482 LLT IdxTy = MRI->getType(Reg: IdxReg);
2483 if (IdxTy != OffsetTy) {
2484 if (!IdxTy.isVector() && WantSplatVector) {
2485 IdxReg = MIRBuilder
2486 .buildSplatBuildVector(Res: OffsetTy.changeElementType(NewEltTy: IdxTy),
2487 Src: IdxReg)
2488 .getReg(Idx: 0);
2489 }
2490
2491 IdxReg = MIRBuilder.buildSExtOrTrunc(Res: OffsetTy, Op: IdxReg).getReg(Idx: 0);
2492 }
2493
2494 // N = N + Idx * ElementSize;
2495 // Avoid doing it for ElementSize of 1.
2496 Register GepOffsetReg;
2497 if (ElementSize != 1) {
2498 auto ElementSizeMIB = MIRBuilder.buildConstant(
2499 Res: getLLTForType(Ty&: *OffsetIRTy, DL: *DL), Val: ElementSize);
2500
2501 // The multiplication is NUW if the GEP is NUW and NSW if the GEP is
2502 // NUSW.
2503 uint32_t ScaleFlags = PtrAddFlags & MachineInstr::MIFlag::NoUWrap;
2504 if (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap)
2505 ScaleFlags |= MachineInstr::MIFlag::NoSWrap;
2506
2507 GepOffsetReg =
2508 MIRBuilder.buildMul(Dst: OffsetTy, Src0: IdxReg, Src1: ElementSizeMIB, Flags: ScaleFlags)
2509 .getReg(Idx: 0);
2510 } else {
2511 GepOffsetReg = IdxReg;
2512 }
2513
2514 BaseReg =
2515 MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: GepOffsetReg, Flags: PtrAddFlags)
2516 .getReg(Idx: 0);
2517 }
2518 }
2519
2520 if (Offset != 0) {
2521 auto OffsetMIB =
2522 MIRBuilder.buildConstant(Res: OffsetTy, Val: Offset);
2523
2524 MIRBuilder.buildPtrAdd(Res: getOrCreateVReg(Val: U), Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0),
2525 Flags: PtrAddFlagsWithConst(Offset));
2526 return true;
2527 }
2528
2529 return translateCopy(U, Src: BaseReg, MIRBuilder);
2530}
2531
2532bool IRTranslatorImpl::translateMemFunc(const CallInst &CI,
2533 MachineIRBuilder &MIRBuilder,
2534 unsigned Opcode) {
2535 const Value *SrcPtr = CI.getArgOperand(i: 1);
2536 // If the source is undef, then just emit a nop.
2537 if (isa<UndefValue>(Val: SrcPtr))
2538 return true;
2539
2540 SmallVector<Register, 3> SrcRegs;
2541
2542 unsigned MinPtrSize = UINT_MAX;
2543 for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(x: AI) != AE; ++AI) {
2544 Register SrcReg = getOrCreateVReg(Val: **AI);
2545 LLT SrcTy = MRI->getType(Reg: SrcReg);
2546 if (SrcTy.isPointer())
2547 MinPtrSize = std::min<unsigned>(a: SrcTy.getSizeInBits(), b: MinPtrSize);
2548 SrcRegs.push_back(Elt: SrcReg);
2549 }
2550
2551 LLT SizeTy = LLT::integer(SizeInBits: MinPtrSize);
2552
2553 // The size operand should be the minimum of the pointer sizes.
2554 Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
2555 if (MRI->getType(Reg: SizeOpReg) != SizeTy)
2556 SizeOpReg = MIRBuilder.buildZExtOrTrunc(Res: SizeTy, Op: SizeOpReg).getReg(Idx: 0);
2557
2558 auto ICall = MIRBuilder.buildInstr(Opcode);
2559 for (Register SrcReg : SrcRegs)
2560 ICall.addUse(RegNo: SrcReg);
2561
2562 Align DstAlign;
2563 Align SrcAlign;
2564 unsigned IsVol =
2565 cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1))->getZExtValue();
2566
2567 ConstantInt *CopySize = nullptr;
2568
2569 if (auto *MCI = dyn_cast<MemCpyInst>(Val: &CI)) {
2570 DstAlign = MCI->getDestAlign().valueOrOne();
2571 SrcAlign = MCI->getSourceAlign().valueOrOne();
2572 CopySize = dyn_cast<ConstantInt>(Val: MCI->getArgOperand(i: 2));
2573 } else if (auto *MMI = dyn_cast<MemMoveInst>(Val: &CI)) {
2574 DstAlign = MMI->getDestAlign().valueOrOne();
2575 SrcAlign = MMI->getSourceAlign().valueOrOne();
2576 CopySize = dyn_cast<ConstantInt>(Val: MMI->getArgOperand(i: 2));
2577 } else {
2578 auto *MSI = cast<MemSetInst>(Val: &CI);
2579 DstAlign = MSI->getDestAlign().valueOrOne();
2580 }
2581
2582 if (Opcode != TargetOpcode::G_MEMCPY_INLINE &&
2583 Opcode != TargetOpcode::G_MEMSET_INLINE) {
2584 // We need to propagate the tail call flag from the IR inst as an argument.
2585 // Otherwise, we have to pessimize and assume later that we cannot tail call
2586 // any memory intrinsics.
2587 ICall.addImm(Val: CI.isTailCall() ? 1 : 0);
2588 }
2589
2590 // Create mem operands to store the alignment and volatile info.
2591 MachineMemOperand::Flags LoadFlags = MachineMemOperand::MOLoad;
2592 MachineMemOperand::Flags StoreFlags = MachineMemOperand::MOStore;
2593 if (IsVol) {
2594 LoadFlags |= MachineMemOperand::MOVolatile;
2595 StoreFlags |= MachineMemOperand::MOVolatile;
2596 }
2597
2598 AAMDNodes AAInfo = CI.getAAMetadata();
2599 if (AA && CopySize &&
2600 AA->pointsToConstantMemory(Loc: MemoryLocation(
2601 SrcPtr, LocationSize::precise(Value: CopySize->getZExtValue()), AAInfo))) {
2602 LoadFlags |= MachineMemOperand::MOInvariant;
2603
2604 // FIXME: pointsToConstantMemory probably does not imply dereferenceable,
2605 // but the previous usage implied it did. Probably should check
2606 // isDereferenceableAndAlignedPointer.
2607 LoadFlags |= MachineMemOperand::MODereferenceable;
2608 }
2609
2610 ICall.addMemOperand(
2611 MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(CI.getArgOperand(i: 0)),
2612 F: StoreFlags, Size: 1, BaseAlignment: DstAlign, Metadata: AAInfo));
2613 if (Opcode != TargetOpcode::G_MEMSET &&
2614 Opcode != TargetOpcode::G_MEMSET_INLINE)
2615 ICall.addMemOperand(MMO: MF->getMachineMemOperand(
2616 PtrInfo: MachinePointerInfo(SrcPtr), F: LoadFlags, Size: 1, BaseAlignment: SrcAlign, Metadata: AAInfo));
2617
2618 return true;
2619}
2620
2621bool IRTranslatorImpl::translateTrap(const CallInst &CI,
2622 MachineIRBuilder &MIRBuilder,
2623 unsigned Opcode) {
2624 StringRef TrapFuncName =
2625 CI.getAttributes().getFnAttr(Kind: "trap-func-name").getValueAsString();
2626 if (TrapFuncName.empty()) {
2627 MachineInstrBuilder MIB;
2628 if (Opcode == TargetOpcode::G_UBSANTRAP) {
2629 uint64_t Code = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 0))->getZExtValue();
2630 MIB = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: ArrayRef<llvm::SrcOp>{Code});
2631 } else {
2632 MIB = MIRBuilder.buildInstr(Opcode);
2633 }
2634 if (CI.cannotMerge())
2635 MIB.setMIFlag(MachineInstr::MIFlag::NoMerge);
2636 return true;
2637 }
2638
2639 CallLowering::CallLoweringInfo Info;
2640 if (Opcode == TargetOpcode::G_UBSANTRAP)
2641 Info.OrigArgs.push_back(Elt: {getOrCreateVRegs(Val: *CI.getArgOperand(i: 0)),
2642 CI.getArgOperand(i: 0)->getType(), 0});
2643
2644 Info.Callee = MachineOperand::CreateES(SymName: TrapFuncName.data());
2645 Info.CB = &CI;
2646 Info.OrigRet = {Register(), Type::getVoidTy(C&: CI.getContext()), 0};
2647 Info.NoMerge = CI.cannotMerge();
2648 return CLI->lowerCall(MIRBuilder, Info);
2649}
2650
2651bool IRTranslatorImpl::translateVectorInterleave2Intrinsic(
2652 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
2653 assert(CI.getIntrinsicID() == Intrinsic::vector_interleave2 &&
2654 "This function can only be called on the interleave2 intrinsic!");
2655 // Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG).
2656 Register Op0 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
2657 Register Op1 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1));
2658 Register Res = getOrCreateVReg(Val: CI);
2659
2660 LLT OpTy = MRI->getType(Reg: Op0);
2661 MIRBuilder.buildShuffleVector(Res, Src1: Op0, Src2: Op1,
2662 Mask: createInterleaveMask(VF: OpTy.getNumElements(), NumVecs: 2));
2663
2664 return true;
2665}
2666
2667bool IRTranslatorImpl::translateVectorDeinterleave2Intrinsic(
2668 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
2669 assert(CI.getIntrinsicID() == Intrinsic::vector_deinterleave2 &&
2670 "This function can only be called on the deinterleave2 intrinsic!");
2671 // Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to
2672 // SelectionDAG).
2673 Register Op = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
2674 auto Undef = MIRBuilder.buildUndef(Res: MRI->getType(Reg: Op));
2675 ArrayRef<Register> Res = getOrCreateVRegs(Val: CI);
2676
2677 LLT ResTy = MRI->getType(Reg: Res[0]);
2678 if (ResTy.isScalar()) {
2679 MIRBuilder.buildExtractVectorElementConstant(Res: Res[0], Val: Op, Idx: 0);
2680 MIRBuilder.buildExtractVectorElementConstant(Res: Res[1], Val: Op, Idx: 1);
2681
2682 return true;
2683 }
2684
2685 assert(ResTy.isVector() && "Expected vector result type");
2686 MIRBuilder.buildShuffleVector(Res: Res[0], Src1: Op, Src2: Undef,
2687 Mask: createStrideMask(Start: 0, Stride: 2, VF: ResTy.getNumElements()));
2688 MIRBuilder.buildShuffleVector(Res: Res[1], Src1: Op, Src2: Undef,
2689 Mask: createStrideMask(Start: 1, Stride: 2, VF: ResTy.getNumElements()));
2690
2691 return true;
2692}
2693
2694void IRTranslatorImpl::getStackGuard(Register DstReg,
2695 MachineIRBuilder &MIRBuilder) {
2696 Value *Global =
2697 TLI->getSDagStackGuard(M: *MF->getFunction().getParent(), Libcalls: *Libcalls);
2698 if (!Global) {
2699 LLVMContext &Ctx = MIRBuilder.getContext();
2700 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
2701 MIRBuilder.buildUndef(Res: DstReg);
2702 return;
2703 }
2704
2705 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
2706 MRI->setRegClass(Reg: DstReg,
2707 RC: TII.getRegClass(MCID: TII.get(Opcode: TargetOpcode::LOAD_STACK_GUARD), OpNum: 0));
2708 auto MIB =
2709 MIRBuilder.buildInstr(Opc: TargetOpcode::LOAD_STACK_GUARD, DstOps: {DstReg}, SrcOps: {});
2710
2711 unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
2712 LLT PtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL->getPointerSizeInBits(AS: AddrSpace));
2713
2714 MachinePointerInfo MPInfo(Global);
2715 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2716 MachineMemOperand::MODereferenceable;
2717 MachineMemOperand *MemRef = MF->getMachineMemOperand(
2718 PtrInfo: MPInfo, F: Flags, MemTy: PtrTy, BaseAlignment: DL->getPointerABIAlignment(AS: AddrSpace));
2719 MIB.setMemRefs({MemRef});
2720}
2721
2722bool IRTranslatorImpl::translateOverflowIntrinsic(
2723 const CallInst &CI, unsigned Op, MachineIRBuilder &MIRBuilder) {
2724 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: CI);
2725 MIRBuilder.buildInstr(
2726 Opc: Op, DstOps: {ResRegs[0], ResRegs[1]},
2727 SrcOps: {getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)), getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1))});
2728
2729 return true;
2730}
2731
2732bool IRTranslatorImpl::translateFixedPointIntrinsic(
2733 unsigned Op, const CallInst &CI, MachineIRBuilder &MIRBuilder) {
2734 Register Dst = getOrCreateVReg(Val: CI);
2735 Register Src0 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
2736 Register Src1 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1));
2737 uint64_t Scale = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
2738 MIRBuilder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: { Src0, Src1, Scale });
2739 return true;
2740}
2741
2742unsigned IRTranslatorImpl::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
2743 switch (ID) {
2744 default:
2745 break;
2746 case Intrinsic::acos:
2747 return TargetOpcode::G_FACOS;
2748 case Intrinsic::asin:
2749 return TargetOpcode::G_FASIN;
2750 case Intrinsic::atan:
2751 return TargetOpcode::G_FATAN;
2752 case Intrinsic::atan2:
2753 return TargetOpcode::G_FATAN2;
2754 case Intrinsic::bswap:
2755 return TargetOpcode::G_BSWAP;
2756 case Intrinsic::bitreverse:
2757 return TargetOpcode::G_BITREVERSE;
2758 case Intrinsic::clmul:
2759 return TargetOpcode::G_CLMUL;
2760 case Intrinsic::fshl:
2761 return TargetOpcode::G_FSHL;
2762 case Intrinsic::fshr:
2763 return TargetOpcode::G_FSHR;
2764 case Intrinsic::ceil:
2765 return TargetOpcode::G_FCEIL;
2766 case Intrinsic::cos:
2767 return TargetOpcode::G_FCOS;
2768 case Intrinsic::cosh:
2769 return TargetOpcode::G_FCOSH;
2770 case Intrinsic::ctpop:
2771 return TargetOpcode::G_CTPOP;
2772 case Intrinsic::smulh:
2773 return TargetOpcode::G_SMULH;
2774 case Intrinsic::umulh:
2775 return TargetOpcode::G_UMULH;
2776 case Intrinsic::exp:
2777 return TargetOpcode::G_FEXP;
2778 case Intrinsic::exp2:
2779 return TargetOpcode::G_FEXP2;
2780 case Intrinsic::exp10:
2781 return TargetOpcode::G_FEXP10;
2782 case Intrinsic::fabs:
2783 return TargetOpcode::G_FABS;
2784 case Intrinsic::copysign:
2785 return TargetOpcode::G_FCOPYSIGN;
2786 case Intrinsic::minnum:
2787 return TargetOpcode::G_FMINNUM;
2788 case Intrinsic::maxnum:
2789 return TargetOpcode::G_FMAXNUM;
2790 case Intrinsic::minimum:
2791 return TargetOpcode::G_FMINIMUM;
2792 case Intrinsic::maximum:
2793 return TargetOpcode::G_FMAXIMUM;
2794 case Intrinsic::minimumnum:
2795 return TargetOpcode::G_FMINIMUMNUM;
2796 case Intrinsic::maximumnum:
2797 return TargetOpcode::G_FMAXIMUMNUM;
2798 case Intrinsic::canonicalize:
2799 return TargetOpcode::G_FCANONICALIZE;
2800 case Intrinsic::floor:
2801 return TargetOpcode::G_FFLOOR;
2802 case Intrinsic::fma:
2803 return TargetOpcode::G_FMA;
2804 case Intrinsic::log:
2805 return TargetOpcode::G_FLOG;
2806 case Intrinsic::log2:
2807 return TargetOpcode::G_FLOG2;
2808 case Intrinsic::log10:
2809 return TargetOpcode::G_FLOG10;
2810 case Intrinsic::ldexp:
2811 return TargetOpcode::G_FLDEXP;
2812 case Intrinsic::nearbyint:
2813 return TargetOpcode::G_FNEARBYINT;
2814 case Intrinsic::pow:
2815 return TargetOpcode::G_FPOW;
2816 case Intrinsic::powi:
2817 return TargetOpcode::G_FPOWI;
2818 case Intrinsic::rint:
2819 return TargetOpcode::G_FRINT;
2820 case Intrinsic::round:
2821 return TargetOpcode::G_INTRINSIC_ROUND;
2822 case Intrinsic::roundeven:
2823 return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
2824 case Intrinsic::sin:
2825 return TargetOpcode::G_FSIN;
2826 case Intrinsic::sinh:
2827 return TargetOpcode::G_FSINH;
2828 case Intrinsic::sqrt:
2829 return TargetOpcode::G_FSQRT;
2830 case Intrinsic::tan:
2831 return TargetOpcode::G_FTAN;
2832 case Intrinsic::tanh:
2833 return TargetOpcode::G_FTANH;
2834 case Intrinsic::trunc:
2835 return TargetOpcode::G_INTRINSIC_TRUNC;
2836 case Intrinsic::readcyclecounter:
2837 return TargetOpcode::G_READCYCLECOUNTER;
2838 case Intrinsic::readsteadycounter:
2839 return TargetOpcode::G_READSTEADYCOUNTER;
2840 case Intrinsic::ptrmask:
2841 return TargetOpcode::G_PTRMASK;
2842 case Intrinsic::lrint:
2843 return TargetOpcode::G_INTRINSIC_LRINT;
2844 case Intrinsic::llrint:
2845 return TargetOpcode::G_INTRINSIC_LLRINT;
2846 // FADD/FMUL require checking the FMF, so are handled elsewhere.
2847 case Intrinsic::vector_reduce_fmin:
2848 return TargetOpcode::G_VECREDUCE_FMIN;
2849 case Intrinsic::vector_reduce_fmax:
2850 return TargetOpcode::G_VECREDUCE_FMAX;
2851 case Intrinsic::vector_reduce_fminimum:
2852 return TargetOpcode::G_VECREDUCE_FMINIMUM;
2853 case Intrinsic::vector_reduce_fmaximum:
2854 return TargetOpcode::G_VECREDUCE_FMAXIMUM;
2855 case Intrinsic::vector_reduce_fminimumnum:
2856 return TargetOpcode::G_VECREDUCE_FMINIMUMNUM;
2857 case Intrinsic::vector_reduce_fmaximumnum:
2858 return TargetOpcode::G_VECREDUCE_FMAXIMUMNUM;
2859 case Intrinsic::vector_reduce_add:
2860 return TargetOpcode::G_VECREDUCE_ADD;
2861 case Intrinsic::vector_reduce_mul:
2862 return TargetOpcode::G_VECREDUCE_MUL;
2863 case Intrinsic::vector_reduce_and:
2864 return TargetOpcode::G_VECREDUCE_AND;
2865 case Intrinsic::vector_reduce_or:
2866 return TargetOpcode::G_VECREDUCE_OR;
2867 case Intrinsic::vector_reduce_xor:
2868 return TargetOpcode::G_VECREDUCE_XOR;
2869 case Intrinsic::vector_reduce_smax:
2870 return TargetOpcode::G_VECREDUCE_SMAX;
2871 case Intrinsic::vector_reduce_smin:
2872 return TargetOpcode::G_VECREDUCE_SMIN;
2873 case Intrinsic::vector_reduce_umax:
2874 return TargetOpcode::G_VECREDUCE_UMAX;
2875 case Intrinsic::vector_reduce_umin:
2876 return TargetOpcode::G_VECREDUCE_UMIN;
2877 case Intrinsic::experimental_vector_compress:
2878 return TargetOpcode::G_VECTOR_COMPRESS;
2879 case Intrinsic::lround:
2880 return TargetOpcode::G_LROUND;
2881 case Intrinsic::llround:
2882 return TargetOpcode::G_LLROUND;
2883 case Intrinsic::get_fpenv:
2884 return TargetOpcode::G_GET_FPENV;
2885 case Intrinsic::get_fpmode:
2886 return TargetOpcode::G_GET_FPMODE;
2887 }
2888 return Intrinsic::not_intrinsic;
2889}
2890
2891bool IRTranslatorImpl::translateSimpleIntrinsic(const CallInst &CI,
2892 Intrinsic::ID ID,
2893 MachineIRBuilder &MIRBuilder) {
2894
2895 unsigned Op = getSimpleIntrinsicOpcode(ID);
2896
2897 // Is this a simple intrinsic?
2898 if (Op == Intrinsic::not_intrinsic)
2899 return false;
2900
2901 // Yes. Let's translate it.
2902 SmallVector<llvm::SrcOp, 4> VRegs;
2903 for (const auto &Arg : CI.args())
2904 VRegs.push_back(Elt: getOrCreateVReg(Val: *Arg));
2905
2906 MIRBuilder.buildInstr(Opc: Op, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: VRegs,
2907 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2908 return true;
2909}
2910
2911// TODO: Include ConstainedOps.def when all strict instructions are defined.
2912static unsigned getConstrainedOpcode(Intrinsic::ID ID) {
2913 switch (ID) {
2914 case Intrinsic::experimental_constrained_fadd:
2915 return TargetOpcode::G_STRICT_FADD;
2916 case Intrinsic::experimental_constrained_fsub:
2917 return TargetOpcode::G_STRICT_FSUB;
2918 case Intrinsic::experimental_constrained_fmul:
2919 return TargetOpcode::G_STRICT_FMUL;
2920 case Intrinsic::experimental_constrained_fdiv:
2921 return TargetOpcode::G_STRICT_FDIV;
2922 case Intrinsic::experimental_constrained_frem:
2923 return TargetOpcode::G_STRICT_FREM;
2924 case Intrinsic::experimental_constrained_fma:
2925 return TargetOpcode::G_STRICT_FMA;
2926 case Intrinsic::experimental_constrained_sqrt:
2927 return TargetOpcode::G_STRICT_FSQRT;
2928 case Intrinsic::experimental_constrained_ldexp:
2929 return TargetOpcode::G_STRICT_FLDEXP;
2930 case Intrinsic::experimental_constrained_fcmp:
2931 return TargetOpcode::G_STRICT_FCMP;
2932 case Intrinsic::experimental_constrained_fcmps:
2933 return TargetOpcode::G_STRICT_FCMPS;
2934 default:
2935 return 0;
2936 }
2937}
2938
2939bool IRTranslatorImpl::translateConstrainedFPIntrinsic(
2940 const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
2941 fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
2942
2943 unsigned Opcode = getConstrainedOpcode(ID: FPI.getIntrinsicID());
2944 if (!Opcode)
2945 return false;
2946
2947 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: FPI);
2948 if (EB == fp::ExceptionBehavior::ebIgnore)
2949 Flags |= MachineInstr::NoFPExcept;
2950
2951 if (Opcode == TargetOpcode::G_STRICT_FCMP ||
2952 Opcode == TargetOpcode::G_STRICT_FCMPS) {
2953 auto *FPCmp = cast<ConstrainedFPCmpIntrinsic>(Val: &FPI);
2954 Register Operand0 = getOrCreateVReg(Val: *FPCmp->getArgOperand(i: 0));
2955 Register Operand1 = getOrCreateVReg(Val: *FPCmp->getArgOperand(i: 1));
2956 Register Result = getOrCreateVReg(Val: FPI);
2957 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Result}, SrcOps: {}, Flags)
2958 .addPredicate(Pred: FPCmp->getPredicate())
2959 .addUse(RegNo: Operand0)
2960 .addUse(RegNo: Operand1);
2961 return true;
2962 }
2963
2964 SmallVector<llvm::SrcOp, 4> VRegs;
2965 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
2966 VRegs.push_back(Elt: getOrCreateVReg(Val: *FPI.getArgOperand(i: I)));
2967
2968 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: FPI)}, SrcOps: VRegs, Flags);
2969 return true;
2970}
2971
2972std::optional<MCRegister> IRTranslatorImpl::getArgPhysReg(Argument &Arg) {
2973 auto VRegs = getOrCreateVRegs(Val: Arg);
2974 if (VRegs.size() != 1)
2975 return std::nullopt;
2976
2977 // Arguments are lowered as a copy of a livein physical register.
2978 auto *VRegDef = MF->getRegInfo().getVRegDef(Reg: VRegs[0]);
2979 if (!VRegDef || !VRegDef->isCopy())
2980 return std::nullopt;
2981 return VRegDef->getOperand(i: 1).getReg().asMCReg();
2982}
2983
2984bool IRTranslatorImpl::translateIfEntryValueArgument(
2985 bool isDeclare, Value *Val, const DILocalVariable *Var,
2986 const DIExpression *Expr, const DebugLoc &DL,
2987 MachineIRBuilder &MIRBuilder) {
2988 auto *Arg = dyn_cast<Argument>(Val);
2989 if (!Arg)
2990 return false;
2991
2992 if (!Expr->isEntryValue())
2993 return false;
2994
2995 std::optional<MCRegister> PhysReg = getArgPhysReg(Arg&: *Arg);
2996 if (!PhysReg) {
2997 LLVM_DEBUG(dbgs() << "Dropping dbg." << (isDeclare ? "declare" : "value")
2998 << ": expression is entry_value but "
2999 << "couldn't find a physical register\n");
3000 LLVM_DEBUG(dbgs() << *Var << "\n");
3001 return true;
3002 }
3003
3004 if (isDeclare) {
3005 // Append an op deref to account for the fact that this is a dbg_declare.
3006 Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
3007 MF->setVariableDbgInfo(Var, Expr, Reg: *PhysReg, Loc: DL);
3008 } else {
3009 MIRBuilder.buildDirectDbgValue(Reg: *PhysReg, Variable: Var, Expr);
3010 }
3011
3012 return true;
3013}
3014
3015static unsigned getConvOpcode(Intrinsic::ID ID) {
3016 switch (ID) {
3017 default:
3018 llvm_unreachable("Unexpected intrinsic");
3019 case Intrinsic::experimental_convergence_anchor:
3020 return TargetOpcode::CONVERGENCECTRL_ANCHOR;
3021 case Intrinsic::experimental_convergence_entry:
3022 return TargetOpcode::CONVERGENCECTRL_ENTRY;
3023 case Intrinsic::experimental_convergence_loop:
3024 return TargetOpcode::CONVERGENCECTRL_LOOP;
3025 }
3026}
3027
3028bool IRTranslatorImpl::translateConvergenceControlIntrinsic(
3029 const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
3030 MachineInstrBuilder MIB = MIRBuilder.buildInstr(Opcode: getConvOpcode(ID));
3031 Register OutputReg = getOrCreateConvergenceTokenVReg(Token: CI);
3032 MIB.addDef(RegNo: OutputReg);
3033
3034 if (ID == Intrinsic::experimental_convergence_loop) {
3035 auto Bundle = CI.getOperandBundle(ID: LLVMContext::OB_convergencectrl);
3036 assert(Bundle && "Expected a convergence control token.");
3037 Register InputReg =
3038 getOrCreateConvergenceTokenVReg(Token: *Bundle->Inputs[0].get());
3039 MIB.addUse(RegNo: InputReg);
3040 }
3041
3042 return true;
3043}
3044
3045bool IRTranslatorImpl::translateKnownIntrinsic(const CallInst &CI,
3046 Intrinsic::ID ID,
3047 MachineIRBuilder &MIRBuilder) {
3048 if (auto *MI = dyn_cast<AnyMemIntrinsic>(Val: &CI)) {
3049 if (ORE->enabled()) {
3050 if (MemoryOpRemark::canHandle(I: MI, TLI: *LibInfo)) {
3051 MemoryOpRemark R(*ORE, "gisel-ir-translator-memsize", *DL, *LibInfo);
3052 R.visit(I: MI);
3053 }
3054 }
3055 }
3056
3057 // If this is a simple intrinsic (that is, we just need to add a def of
3058 // a vreg, and uses for each arg operand, then translate it.
3059 if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
3060 return true;
3061
3062 switch (ID) {
3063 default:
3064 break;
3065 case Intrinsic::lifetime_start:
3066 case Intrinsic::lifetime_end: {
3067 // No stack colouring in O0, discard region information.
3068 if (MF->getTarget().getOptLevel() == CodeGenOptLevel::None ||
3069 MF->getFunction().hasOptNone())
3070 return true;
3071
3072 unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
3073 : TargetOpcode::LIFETIME_END;
3074
3075 const AllocaInst *AI = dyn_cast<AllocaInst>(Val: CI.getArgOperand(i: 0));
3076 if (!AI || !AI->isStaticAlloca())
3077 return true;
3078
3079 MIRBuilder.buildInstr(Opcode: Op).addFrameIndex(Idx: getOrCreateFrameIndex(AI: *AI));
3080 return true;
3081 }
3082 case Intrinsic::fake_use: {
3083 SmallVector<llvm::SrcOp, 4> VRegs;
3084 for (const auto &Arg : CI.args())
3085 llvm::append_range(C&: VRegs, R: getOrCreateVRegs(Val: *Arg));
3086 MIRBuilder.buildInstr(Opc: TargetOpcode::FAKE_USE, DstOps: {}, SrcOps: VRegs);
3087 MF->setHasFakeUses(true);
3088 return true;
3089 }
3090 case Intrinsic::vaend:
3091 // No target I know of cares about va_end. Certainly no in-tree target
3092 // does. Simplest intrinsic ever!
3093 return true;
3094 case Intrinsic::vastart: {
3095 Value *Ptr = CI.getArgOperand(i: 0);
3096 unsigned ListSize = TLI->getVaListSizeInBits(DL: *DL) / 8;
3097 Align Alignment = getKnownAlignment(V: Ptr, DL: *DL);
3098
3099 MIRBuilder.buildInstr(Opc: TargetOpcode::G_VASTART, DstOps: {}, SrcOps: {getOrCreateVReg(Val: *Ptr)})
3100 .addMemOperand(MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Ptr),
3101 F: MachineMemOperand::MOStore,
3102 Size: ListSize, BaseAlignment: Alignment));
3103 return true;
3104 }
3105 case Intrinsic::uadd_with_overflow:
3106 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UADDO, MIRBuilder);
3107 case Intrinsic::sadd_with_overflow:
3108 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SADDO, MIRBuilder);
3109 case Intrinsic::usub_with_overflow:
3110 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_USUBO, MIRBuilder);
3111 case Intrinsic::ssub_with_overflow:
3112 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SSUBO, MIRBuilder);
3113 case Intrinsic::umul_with_overflow:
3114 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UMULO, MIRBuilder);
3115 case Intrinsic::smul_with_overflow:
3116 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SMULO, MIRBuilder);
3117 case Intrinsic::uadd_sat:
3118 return translateBinaryOp(Opcode: TargetOpcode::G_UADDSAT, U: CI, MIRBuilder);
3119 case Intrinsic::sadd_sat:
3120 return translateBinaryOp(Opcode: TargetOpcode::G_SADDSAT, U: CI, MIRBuilder);
3121 case Intrinsic::usub_sat:
3122 return translateBinaryOp(Opcode: TargetOpcode::G_USUBSAT, U: CI, MIRBuilder);
3123 case Intrinsic::ssub_sat:
3124 return translateBinaryOp(Opcode: TargetOpcode::G_SSUBSAT, U: CI, MIRBuilder);
3125 case Intrinsic::ushl_sat:
3126 return translateBinaryOp(Opcode: TargetOpcode::G_USHLSAT, U: CI, MIRBuilder);
3127 case Intrinsic::sshl_sat:
3128 return translateBinaryOp(Opcode: TargetOpcode::G_SSHLSAT, U: CI, MIRBuilder);
3129 case Intrinsic::umin:
3130 return translateBinaryOp(Opcode: TargetOpcode::G_UMIN, U: CI, MIRBuilder);
3131 case Intrinsic::umax:
3132 return translateBinaryOp(Opcode: TargetOpcode::G_UMAX, U: CI, MIRBuilder);
3133 case Intrinsic::smin:
3134 return translateBinaryOp(Opcode: TargetOpcode::G_SMIN, U: CI, MIRBuilder);
3135 case Intrinsic::smax:
3136 return translateBinaryOp(Opcode: TargetOpcode::G_SMAX, U: CI, MIRBuilder);
3137 case Intrinsic::abs:
3138 // TODO: Preserve "int min is poison" arg in GMIR?
3139 return translateUnaryOp(Opcode: TargetOpcode::G_ABS, U: CI, MIRBuilder);
3140 case Intrinsic::smul_fix:
3141 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIX, CI, MIRBuilder);
3142 case Intrinsic::umul_fix:
3143 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIX, CI, MIRBuilder);
3144 case Intrinsic::smul_fix_sat:
3145 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
3146 case Intrinsic::umul_fix_sat:
3147 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
3148 case Intrinsic::sdiv_fix:
3149 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
3150 case Intrinsic::udiv_fix:
3151 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
3152 case Intrinsic::sdiv_fix_sat:
3153 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
3154 case Intrinsic::udiv_fix_sat:
3155 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
3156 case Intrinsic::fmuladd: {
3157 Register Dst = getOrCreateVReg(Val: CI);
3158 Register Op0 = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
3159 Register Op1 = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
3160 Register Op2 = getOrCreateVReg(Val: *CI.getArgOperand(i: 2));
3161 if (TLI->isFMAFasterThanFMulAndFAdd(MF: *MF,
3162 TLI->getValueType(DL: *DL, Ty: CI.getType()))) {
3163 // TODO: Revisit this to see if we should move this part of the
3164 // lowering to the combiner.
3165 MIRBuilder.buildFMA(Dst, Src0: Op0, Src1: Op1, Src2: Op2,
3166 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3167 } else {
3168 LLT Ty = getLLTForType(Ty&: *CI.getType(), DL: *DL);
3169 auto FMul = MIRBuilder.buildFMul(
3170 Dst: Ty, Src0: Op0, Src1: Op1, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3171 MIRBuilder.buildFAdd(Dst, Src0: FMul, Src1: Op2,
3172 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3173 }
3174 return true;
3175 }
3176 case Intrinsic::frexp: {
3177 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
3178 MIRBuilder.buildFFrexp(Fract: VRegs[0], Exp: VRegs[1],
3179 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
3180 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3181 return true;
3182 }
3183 case Intrinsic::modf: {
3184 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
3185 MIRBuilder.buildModf(Fract: VRegs[0], Int: VRegs[1],
3186 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
3187 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3188 return true;
3189 }
3190 case Intrinsic::sincos: {
3191 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
3192 MIRBuilder.buildFSincos(Sin: VRegs[0], Cos: VRegs[1],
3193 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
3194 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3195 return true;
3196 }
3197 case Intrinsic::fptosi_sat:
3198 MIRBuilder.buildFPTOSI_SAT(Dst: getOrCreateVReg(Val: CI),
3199 Src0: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
3200 return true;
3201 case Intrinsic::fptoui_sat:
3202 MIRBuilder.buildFPTOUI_SAT(Dst: getOrCreateVReg(Val: CI),
3203 Src0: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
3204 return true;
3205 case Intrinsic::memcpy_inline:
3206 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY_INLINE);
3207 case Intrinsic::memcpy:
3208 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY);
3209 case Intrinsic::memmove:
3210 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMMOVE);
3211 case Intrinsic::memset:
3212 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMSET);
3213 case Intrinsic::memset_inline:
3214 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMSET_INLINE);
3215 case Intrinsic::eh_typeid_for: {
3216 GlobalValue *GV = ExtractTypeInfo(V: CI.getArgOperand(i: 0));
3217 Register Reg = getOrCreateVReg(Val: CI);
3218 unsigned TypeID = MF->getTypeIDFor(TI: GV);
3219 MIRBuilder.buildConstant(Res: Reg, Val: TypeID);
3220 return true;
3221 }
3222 case Intrinsic::objectsize:
3223 llvm_unreachable("llvm.objectsize.* should have been lowered already");
3224
3225 case Intrinsic::is_constant:
3226 llvm_unreachable("llvm.is.constant.* should have been lowered already");
3227
3228 case Intrinsic::stackguard:
3229 getStackGuard(DstReg: getOrCreateVReg(Val: CI), MIRBuilder);
3230 return true;
3231 case Intrinsic::stackprotector: {
3232 LLT PtrTy = getLLTForType(Ty&: *CI.getArgOperand(i: 0)->getType(), DL: *DL);
3233 Register GuardVal;
3234 if (TLI->useLoadStackGuardNode(M: *CI.getModule())) {
3235 GuardVal = MRI->createGenericVirtualRegister(Ty: PtrTy);
3236 getStackGuard(DstReg: GuardVal, MIRBuilder);
3237 } else
3238 GuardVal = getOrCreateVReg(Val: *CI.getArgOperand(i: 0)); // The guard's value.
3239
3240 AllocaInst *Slot = cast<AllocaInst>(Val: CI.getArgOperand(i: 1));
3241 int FI = getOrCreateFrameIndex(AI: *Slot);
3242 MF->getFrameInfo().setStackProtectorIndex(FI);
3243
3244 MIRBuilder.buildStore(
3245 Val: GuardVal, Addr: getOrCreateVReg(Val: *Slot),
3246 MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI),
3247 F: MachineMemOperand::MOStore |
3248 MachineMemOperand::MOVolatile,
3249 MemTy: PtrTy, BaseAlignment: Align(8)));
3250 return true;
3251 }
3252 case Intrinsic::stacksave: {
3253 MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKSAVE, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {});
3254 return true;
3255 }
3256 case Intrinsic::stackrestore: {
3257 MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKRESTORE, DstOps: {},
3258 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
3259 return true;
3260 }
3261 case Intrinsic::cttz:
3262 case Intrinsic::ctlz: {
3263 ConstantInt *Cst = cast<ConstantInt>(Val: CI.getArgOperand(i: 1));
3264 bool isTrailing = ID == Intrinsic::cttz;
3265 unsigned Opcode = isTrailing ? Cst->isZero()
3266 ? TargetOpcode::G_CTTZ
3267 : TargetOpcode::G_CTTZ_ZERO_POISON
3268 : Cst->isZero() ? TargetOpcode::G_CTLZ
3269 : TargetOpcode::G_CTLZ_ZERO_POISON;
3270 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: CI)},
3271 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
3272 return true;
3273 }
3274 case Intrinsic::invariant_start: {
3275 MIRBuilder.buildUndef(Res: getOrCreateVReg(Val: CI));
3276 return true;
3277 }
3278 case Intrinsic::invariant_end:
3279 return true;
3280 case Intrinsic::expect:
3281 case Intrinsic::expect_with_probability:
3282 case Intrinsic::annotation:
3283 case Intrinsic::ptr_annotation:
3284 case Intrinsic::launder_invariant_group:
3285 case Intrinsic::threadlocal_address: {
3286 // Drop the intrinsic, but forward the value.
3287 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: CI),
3288 Op: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
3289 return true;
3290 }
3291 case Intrinsic::assume:
3292 case Intrinsic::experimental_noalias_scope_decl:
3293 case Intrinsic::var_annotation:
3294 case Intrinsic::sideeffect:
3295 // Discard annotate attributes, assumptions, and artificial side-effects.
3296 return true;
3297 case Intrinsic::read_volatile_register:
3298 case Intrinsic::read_register: {
3299 Value *Arg = CI.getArgOperand(i: 0);
3300 MIRBuilder
3301 .buildInstr(Opc: TargetOpcode::G_READ_REGISTER, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {})
3302 .addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()));
3303 return true;
3304 }
3305 case Intrinsic::write_register: {
3306 Value *Arg = CI.getArgOperand(i: 0);
3307 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_WRITE_REGISTER)
3308 .addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()))
3309 .addUse(RegNo: getOrCreateVReg(Val: *CI.getArgOperand(i: 1)));
3310 return true;
3311 }
3312 case Intrinsic::localescape: {
3313 MachineBasicBlock &EntryMBB = MF->front();
3314 StringRef EscapedName = GlobalValue::dropLLVMManglingEscape(Name: MF->getName());
3315
3316 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
3317 // is the same on all targets.
3318 for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
3319 Value *Arg = CI.getArgOperand(i: Idx)->stripPointerCasts();
3320 if (isa<ConstantPointerNull>(Val: Arg))
3321 continue; // Skip null pointers. They represent a hole in index space.
3322
3323 int FI = getOrCreateFrameIndex(AI: *cast<AllocaInst>(Val: Arg));
3324 MCSymbol *FrameAllocSym =
3325 MF->getContext().getOrCreateFrameAllocSymbol(FuncName: EscapedName, Idx);
3326
3327 // This should be inserted at the start of the entry block.
3328 auto LocalEscape =
3329 MIRBuilder.buildInstrNoInsert(Opcode: TargetOpcode::LOCAL_ESCAPE)
3330 .addSym(Sym: FrameAllocSym)
3331 .addFrameIndex(Idx: FI);
3332
3333 EntryMBB.insert(I: EntryMBB.begin(), MI: LocalEscape);
3334 }
3335
3336 return true;
3337 }
3338 case Intrinsic::vector_reduce_fadd:
3339 case Intrinsic::vector_reduce_fmul: {
3340 // Need to check for the reassoc flag to decide whether we want a
3341 // sequential reduction opcode or not.
3342 Register Dst = getOrCreateVReg(Val: CI);
3343 Register ScalarSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
3344 Register VecSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
3345 unsigned Opc = 0;
3346 if (!CI.hasAllowReassoc()) {
3347 // The sequential ordering case.
3348 Opc = ID == Intrinsic::vector_reduce_fadd
3349 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
3350 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
3351 if (!MRI->getType(Reg: VecSrc).isVector())
3352 Opc = ID == Intrinsic::vector_reduce_fadd ? TargetOpcode::G_FADD
3353 : TargetOpcode::G_FMUL;
3354 MIRBuilder.buildInstr(Opc, DstOps: {Dst}, SrcOps: {ScalarSrc, VecSrc},
3355 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3356 return true;
3357 }
3358 // We split the operation into a separate G_FADD/G_FMUL + the reduce,
3359 // since the associativity doesn't matter.
3360 unsigned ScalarOpc;
3361 if (ID == Intrinsic::vector_reduce_fadd) {
3362 Opc = TargetOpcode::G_VECREDUCE_FADD;
3363 ScalarOpc = TargetOpcode::G_FADD;
3364 } else {
3365 Opc = TargetOpcode::G_VECREDUCE_FMUL;
3366 ScalarOpc = TargetOpcode::G_FMUL;
3367 }
3368 LLT DstTy = MRI->getType(Reg: Dst);
3369 auto Rdx = MIRBuilder.buildInstr(
3370 Opc, DstOps: {DstTy}, SrcOps: {VecSrc}, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3371 MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {Dst}, SrcOps: {ScalarSrc, Rdx},
3372 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3373
3374 return true;
3375 }
3376 case Intrinsic::trap:
3377 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_TRAP);
3378 case Intrinsic::debugtrap:
3379 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_DEBUGTRAP);
3380 case Intrinsic::ubsantrap:
3381 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_UBSANTRAP);
3382 case Intrinsic::allow_runtime_check:
3383 case Intrinsic::allow_ubsan_check:
3384 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: CI),
3385 Op: getOrCreateVReg(Val: *ConstantInt::getTrue(Ty: CI.getType())));
3386 return true;
3387 case Intrinsic::amdgcn_cs_chain:
3388 case Intrinsic::amdgcn_call_whole_wave:
3389 return translateCallBase(CB: CI, MIRBuilder);
3390 case Intrinsic::fptrunc_round: {
3391 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: CI);
3392
3393 // Convert the metadata argument to a constant integer
3394 Metadata *MD = cast<MetadataAsValue>(Val: CI.getArgOperand(i: 1))->getMetadata();
3395 std::optional<RoundingMode> RoundMode =
3396 convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
3397
3398 // Add the Rounding mode as an integer
3399 MIRBuilder
3400 .buildInstr(Opc: TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
3401 DstOps: {getOrCreateVReg(Val: CI)},
3402 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))}, Flags)
3403 .addImm(Val: (int)*RoundMode);
3404
3405 return true;
3406 }
3407 case Intrinsic::is_fpclass: {
3408 Value *FpValue = CI.getOperand(i_nocapture: 0);
3409 ConstantInt *TestMaskValue = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1));
3410
3411 MIRBuilder
3412 .buildInstr(Opc: TargetOpcode::G_IS_FPCLASS, DstOps: {getOrCreateVReg(Val: CI)},
3413 SrcOps: {getOrCreateVReg(Val: *FpValue)})
3414 .addImm(Val: TestMaskValue->getZExtValue());
3415
3416 return true;
3417 }
3418 case Intrinsic::set_fpenv: {
3419 Value *FPEnv = CI.getOperand(i_nocapture: 0);
3420 MIRBuilder.buildSetFPEnv(Src: getOrCreateVReg(Val: *FPEnv));
3421 return true;
3422 }
3423 case Intrinsic::reset_fpenv:
3424 MIRBuilder.buildResetFPEnv();
3425 return true;
3426 case Intrinsic::set_fpmode: {
3427 Value *FPState = CI.getOperand(i_nocapture: 0);
3428 MIRBuilder.buildSetFPMode(Src: getOrCreateVReg(Val: *FPState));
3429 return true;
3430 }
3431 case Intrinsic::reset_fpmode:
3432 MIRBuilder.buildResetFPMode();
3433 return true;
3434 case Intrinsic::get_rounding:
3435 MIRBuilder.buildGetRounding(Dst: getOrCreateVReg(Val: CI));
3436 return true;
3437 case Intrinsic::set_rounding:
3438 MIRBuilder.buildSetRounding(Src: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)));
3439 return true;
3440 case Intrinsic::vscale: {
3441 MIRBuilder.buildVScale(Res: getOrCreateVReg(Val: CI), MinElts: 1);
3442 return true;
3443 }
3444 case Intrinsic::scmp:
3445 MIRBuilder.buildSCmp(Res: getOrCreateVReg(Val: CI),
3446 Op0: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)),
3447 Op1: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1)));
3448 return true;
3449 case Intrinsic::ucmp:
3450 MIRBuilder.buildUCmp(Res: getOrCreateVReg(Val: CI),
3451 Op0: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)),
3452 Op1: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1)));
3453 return true;
3454 case Intrinsic::vector_extract:
3455 return translateExtractVector(U: CI, MIRBuilder);
3456 case Intrinsic::vector_insert:
3457 return translateInsertVector(U: CI, MIRBuilder);
3458 case Intrinsic::stepvector: {
3459 MIRBuilder.buildStepVector(Res: getOrCreateVReg(Val: CI), Step: 1);
3460 return true;
3461 }
3462 case Intrinsic::prefetch: {
3463 Value *Addr = CI.getOperand(i_nocapture: 0);
3464 unsigned RW = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1))->getZExtValue();
3465 unsigned Locality = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
3466 unsigned CacheType = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 3))->getZExtValue();
3467
3468 auto Flags = RW ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
3469 auto &MMO = *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Addr), F: Flags,
3470 MemTy: LLT(), BaseAlignment: Align());
3471
3472 MIRBuilder.buildPrefetch(Addr: getOrCreateVReg(Val: *Addr), RW, Locality, CacheType,
3473 MMO);
3474
3475 return true;
3476 }
3477
3478 case Intrinsic::speculative_load: {
3479 // Only the pointer operand is needed at codegen; the remaining arguments
3480 // carry IR-level semantics only.
3481 const Value *Ptr = CI.getArgOperand(i: 0);
3482 Register Dst = getOrCreateVReg(Val: CI);
3483 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad;
3484 Flags |= TLI->getTargetMMOFlags(I: CI);
3485 if (CI.hasMetadata(KindID: LLVMContext::MD_nontemporal))
3486 Flags |= MachineMemOperand::MONonTemporal;
3487 if (CI.hasMetadata(KindID: LLVMContext::MD_invariant_load))
3488 Flags |= MachineMemOperand::MOInvariant;
3489 auto *MMO = MF->getMachineMemOperand(
3490 PtrInfo: MachinePointerInfo(Ptr), F: Flags, MemTy: MRI->getType(Reg: Dst),
3491 BaseAlignment: CI.getParamAlign(ArgNo: 0).valueOrOne(), Metadata: MMOMetadata(CI.getAAMetadata()));
3492 MIRBuilder.buildLoad(Res: Dst, Addr: getOrCreateVReg(Val: *Ptr), MMO&: *MMO);
3493 return true;
3494 }
3495
3496 case Intrinsic::vector_interleave2:
3497 case Intrinsic::vector_deinterleave2: {
3498 // Both intrinsics have at least one operand.
3499 Value *Op0 = CI.getOperand(i_nocapture: 0);
3500 LLT ResTy = getLLTForType(Ty&: *Op0->getType(), DL: MIRBuilder.getDataLayout());
3501 if (!ResTy.isFixedVector())
3502 return false;
3503
3504 if (CI.getIntrinsicID() == Intrinsic::vector_interleave2)
3505 return translateVectorInterleave2Intrinsic(CI, MIRBuilder);
3506
3507 return translateVectorDeinterleave2Intrinsic(CI, MIRBuilder);
3508 }
3509
3510#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
3511 case Intrinsic::INTRINSIC:
3512#include "llvm/IR/ConstrainedOps.def"
3513 return translateConstrainedFPIntrinsic(FPI: cast<ConstrainedFPIntrinsic>(Val: CI),
3514 MIRBuilder);
3515 case Intrinsic::experimental_convergence_anchor:
3516 case Intrinsic::experimental_convergence_entry:
3517 case Intrinsic::experimental_convergence_loop:
3518 return translateConvergenceControlIntrinsic(CI, ID, MIRBuilder);
3519 case Intrinsic::reloc_none: {
3520 Metadata *MD = cast<MetadataAsValue>(Val: CI.getArgOperand(i: 0))->getMetadata();
3521 StringRef SymbolName = cast<MDString>(Val: MD)->getString();
3522 MIRBuilder.buildInstr(Opcode: TargetOpcode::RELOC_NONE)
3523 .addExternalSymbol(FnName: SymbolName.data());
3524 return true;
3525 }
3526 }
3527 return false;
3528}
3529
3530bool IRTranslatorImpl::translateInlineAsm(const CallBase &CB,
3531 MachineIRBuilder &MIRBuilder) {
3532 if (!mayTranslateUserTypes(U: CB))
3533 return false;
3534
3535 const InlineAsmLowering *ALI = MF->getSubtarget().getInlineAsmLowering();
3536
3537 if (!ALI) {
3538 LLVM_DEBUG(
3539 dbgs() << "Inline asm lowering is not supported for this target yet\n");
3540 return false;
3541 }
3542
3543 return ALI->lowerInlineAsm(
3544 MIRBuilder, CB, GetOrCreateVRegs: [&](const Value &Val) { return getOrCreateVRegs(Val); });
3545}
3546
3547bool IRTranslatorImpl::translateCallBase(const CallBase &CB,
3548 MachineIRBuilder &MIRBuilder) {
3549 ArrayRef<Register> Res = getOrCreateVRegs(Val: CB);
3550
3551 SmallVector<ArrayRef<Register>, 8> Args;
3552 Register SwiftInVReg = 0;
3553 Register SwiftErrorVReg = 0;
3554 for (const auto &Arg : CB.args()) {
3555 if (CLI->supportSwiftError() && isSwiftError(V: Arg)) {
3556 assert(SwiftInVReg == 0 && "Expected only one swift error argument");
3557 LLT Ty = getLLTForType(Ty&: *Arg->getType(), DL: *DL);
3558 SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
3559 MIRBuilder.buildCopy(Res: SwiftInVReg, Op: SwiftError.getOrCreateVRegUseAt(
3560 &CB, &MIRBuilder.getMBB(), Arg));
3561 Args.emplace_back(Args: ArrayRef(SwiftInVReg));
3562 SwiftErrorVReg =
3563 SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
3564 continue;
3565 }
3566 Args.push_back(Elt: getOrCreateVRegs(Val: *Arg));
3567 }
3568
3569 if (auto *CI = dyn_cast<CallInst>(Val: &CB)) {
3570 if (ORE->enabled()) {
3571 if (MemoryOpRemark::canHandle(I: CI, TLI: *LibInfo)) {
3572 MemoryOpRemark R(*ORE, "gisel-ir-translator-memsize", *DL, *LibInfo);
3573 R.visit(I: CI);
3574 }
3575 }
3576 }
3577
3578 std::optional<CallLowering::PtrAuthInfo> PAI;
3579 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_ptrauth)) {
3580 // Functions should never be ptrauth-called directly.
3581 assert(!CB.getCalledFunction() && "invalid direct ptrauth call");
3582
3583 const Value *Key = Bundle->Inputs[0];
3584 const Value *Discriminator = Bundle->Inputs[1];
3585
3586 // Look through ptrauth constants to try to eliminate the matching bundle
3587 // and turn this into a direct call with no ptrauth.
3588 // CallLowering will use the raw pointer if it doesn't find the PAI.
3589 const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(Val: CB.getCalledOperand());
3590 if (!CalleeCPA || !isa<Function>(Val: CalleeCPA->getPointer()) ||
3591 !CalleeCPA->isKnownCompatibleWith(Key, Discriminator, DL: *DL)) {
3592 // If we can't make it direct, package the bundle into PAI.
3593 Register DiscReg = getOrCreateVReg(Val: *Discriminator);
3594 PAI = CallLowering::PtrAuthInfo{.Key: cast<ConstantInt>(Val: Key)->getZExtValue(),
3595 .Discriminator: DiscReg};
3596 }
3597 }
3598
3599 Register ConvergenceCtrlToken = 0;
3600 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
3601 const auto &Token = *Bundle->Inputs[0].get();
3602 ConvergenceCtrlToken = getOrCreateConvergenceTokenVReg(Token);
3603 }
3604
3605 // We don't set HasCalls on MFI here yet because call lowering may decide to
3606 // optimize into tail calls. Instead, we defer that to selection where a final
3607 // scan is done to check if any instructions are calls.
3608 bool Success = CLI->lowerCall(
3609 MIRBuilder, Call: CB, ResRegs: Res, ArgRegs: Args, SwiftErrorVReg, PAI, ConvergenceCtrlToken,
3610 GetCalleeReg: [&]() { return getOrCreateVReg(Val: *CB.getCalledOperand()); });
3611
3612 // Check if we just inserted a tail call.
3613 if (Success) {
3614 assert(!HasTailCall && "Can't tail call return twice from block?");
3615 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
3616 HasTailCall = TII->isTailCall(Inst: *std::prev(x: MIRBuilder.getInsertPt()));
3617 }
3618
3619 return Success;
3620}
3621
3622bool IRTranslatorImpl::translateCall(const User &U,
3623 MachineIRBuilder &MIRBuilder) {
3624 if (!mayTranslateUserTypes(U))
3625 return false;
3626
3627 const CallInst &CI = cast<CallInst>(Val: U);
3628 const Function *F = CI.getCalledFunction();
3629
3630 // FIXME: support Windows dllimport function calls and calls through
3631 // weak symbols.
3632 if (F && (F->hasDLLImportStorageClass() ||
3633 (MF->getTarget().getTargetTriple().isOSWindows() &&
3634 F->hasExternalWeakLinkage())))
3635 return false;
3636
3637 // FIXME: support control flow guard targets.
3638 if (CI.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
3639 return false;
3640
3641 // FIXME: support statepoints and related.
3642 if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(Val: U))
3643 return false;
3644
3645 if (CI.isInlineAsm())
3646 return translateInlineAsm(CB: CI, MIRBuilder);
3647
3648 Intrinsic::ID ID = F ? F->getIntrinsicID() : Intrinsic::not_intrinsic;
3649 if (!F || ID == Intrinsic::not_intrinsic) {
3650 if (translateCallBase(CB: CI, MIRBuilder)) {
3651 diagnoseDontCall(CI);
3652 return true;
3653 }
3654 return false;
3655 }
3656
3657 assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
3658
3659 if (!MF->getSubtarget().isIntrinsicSupported(IntrinsicID: ID))
3660 return handleUnsupportedIntrinsic(CB: CI, ID, MIRBuilder);
3661
3662 if (translateKnownIntrinsic(CI, ID, MIRBuilder))
3663 return true;
3664
3665 SmallVector<TargetLowering::IntrinsicInfo> Infos;
3666 TLI->getTgtMemIntrinsic(Infos, I: CI, MF&: *MF, Intrinsic: ID);
3667
3668 return translateIntrinsic(CB: CI, ID, MIRBuilder, TgtMemIntrinsicInfos: Infos);
3669}
3670
3671bool IRTranslatorImpl::handleUnsupportedIntrinsic(
3672 const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
3673 const Function &F = MF->getFunction();
3674 F.getContext().diagnose(
3675 DI: DiagnosticInfoUnsupportedTargetIntrinsic(F, ID, CB.getDebugLoc()));
3676
3677 if (!CB.getType()->isVoidTy()) {
3678 for (Register Reg : getOrCreateVRegs(Val: CB))
3679 MIRBuilder.buildUndef(Res: Reg);
3680 }
3681
3682 return true;
3683}
3684
3685/// Translate a call or callbr to an intrinsic.
3686bool IRTranslatorImpl::translateIntrinsic(
3687 const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
3688 ArrayRef<TargetLowering::IntrinsicInfo> TgtMemIntrinsicInfos) {
3689 if (!MF->getSubtarget().isIntrinsicSupported(IntrinsicID: ID))
3690 return handleUnsupportedIntrinsic(CB, ID, MIRBuilder);
3691
3692 ArrayRef<Register> ResultRegs;
3693 if (!CB.getType()->isVoidTy())
3694 ResultRegs = getOrCreateVRegs(Val: CB);
3695
3696 // Ignore the callsite attributes. Backend code is most likely not expecting
3697 // an intrinsic to sometimes have side effects and sometimes not.
3698 MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, Res: ResultRegs);
3699 if (isa<FPMathOperator>(Val: CB))
3700 MIB->copyIRFlags(I: CB);
3701
3702 for (const auto &Arg : enumerate(First: CB.args())) {
3703 // If this is required to be an immediate, don't materialize it in a
3704 // register.
3705 if (CB.paramHasAttr(ArgNo: Arg.index(), Kind: Attribute::ImmArg)) {
3706 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Arg.value())) {
3707 // imm arguments are more convenient than cimm (and realistically
3708 // probably sufficient), so use them.
3709 assert(CI->getBitWidth() <= 64 &&
3710 "large intrinsic immediates not handled");
3711 MIB.addImm(Val: CI->getSExtValue());
3712 } else {
3713 MIB.addFPImm(Val: cast<ConstantFP>(Val: Arg.value()));
3714 }
3715 } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Val: Arg.value())) {
3716 auto *MD = MDVal->getMetadata();
3717 auto *MDN = dyn_cast<MDNode>(Val: MD);
3718 if (!MDN) {
3719 if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(Val: MD))
3720 MDN = MDNode::get(Context&: MF->getFunction().getContext(), MDs: ConstMD);
3721 else // This was probably an MDString.
3722 return false;
3723 }
3724 MIB.addMetadata(MD: MDN);
3725 } else {
3726 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: *Arg.value());
3727 if (VRegs.size() > 1)
3728 return false;
3729 MIB.addUse(RegNo: VRegs[0]);
3730 }
3731 }
3732
3733 // Add MachineMemOperands for each memory access described by the target.
3734 for (const auto &Info : TgtMemIntrinsicInfos) {
3735 Align Alignment = Info.align.value_or(
3736 u: DL->getABITypeAlign(Ty: Info.memVT.getTypeForEVT(Context&: CB.getContext())));
3737 LLT MemTy = Info.memVT.isSimple()
3738 ? getLLTForMVT(Ty: Info.memVT.getSimpleVT())
3739 : LLT::scalar(SizeInBits: Info.memVT.getStoreSizeInBits());
3740
3741 // TODO: We currently just fallback to address space 0 if
3742 // getTgtMemIntrinsic didn't yield anything useful.
3743 MachinePointerInfo MPI;
3744 if (Info.ptrVal) {
3745 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
3746 } else if (Info.fallbackAddressSpace) {
3747 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
3748 }
3749 MIB.addMemOperand(MMO: MF->getMachineMemOperand(
3750 PtrInfo: MPI, F: Info.flags, MemTy, BaseAlignment: Alignment, Metadata: CB.getAAMetadata(), SSID: Info.ssid,
3751 Ordering: Info.order, FailureOrdering: Info.failureOrder));
3752 }
3753
3754 if (CB.isConvergent()) {
3755 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
3756 auto *Token = Bundle->Inputs[0].get();
3757 Register TokenReg = getOrCreateVReg(Val: *Token);
3758 MIB.addUse(RegNo: TokenReg, Flags: RegState::Implicit);
3759 }
3760 }
3761
3762 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol))
3763 MIB->setDeactivationSymbol(MF&: *MF, DS: Bundle->Inputs[0].get());
3764
3765 return true;
3766}
3767
3768bool IRTranslatorImpl::findUnwindDestinations(
3769 const BasicBlock *EHPadBB, BranchProbability Prob,
3770 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
3771 &UnwindDests) {
3772 EHPersonality Personality = classifyEHPersonality(
3773 Pers: EHPadBB->getParent()->getFunction().getPersonalityFn());
3774 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
3775 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
3776 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
3777 bool IsWasmD = Personality == EHPersonality::Wasm_D;
3778 bool IsSEH = isAsynchronousEHPersonality(Pers: Personality);
3779
3780 if (IsWasmCXX || IsWasmD) {
3781 // Ignore this for now.
3782 return false;
3783 }
3784
3785 while (EHPadBB) {
3786 BasicBlock::const_iterator Pad = EHPadBB->getFirstNonPHIIt();
3787 BasicBlock *NewEHPadBB = nullptr;
3788 if (isa<LandingPadInst>(Val: Pad)) {
3789 // Stop on landingpads. They are not funclets.
3790 UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
3791 break;
3792 }
3793 if (isa<CleanupPadInst>(Val: Pad)) {
3794 // Stop on cleanup pads. Cleanups are always funclet entries for all known
3795 // personalities.
3796 UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
3797 UnwindDests.back().first->setIsEHScopeEntry();
3798 UnwindDests.back().first->setIsEHFuncletEntry();
3799 break;
3800 }
3801 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: Pad)) {
3802 // Add the catchpad handlers to the possible destinations.
3803 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
3804 UnwindDests.emplace_back(Args: &getMBB(BB: *CatchPadBB), Args&: Prob);
3805 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
3806 if (IsMSVCCXX || IsCoreCLR)
3807 UnwindDests.back().first->setIsEHFuncletEntry();
3808 if (!IsSEH)
3809 UnwindDests.back().first->setIsEHScopeEntry();
3810 }
3811 NewEHPadBB = CatchSwitch->getUnwindDest();
3812 } else {
3813 continue;
3814 }
3815
3816 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3817 if (BPI && NewEHPadBB)
3818 Prob *= BPI->getEdgeProbability(Src: EHPadBB, Dst: NewEHPadBB);
3819 EHPadBB = NewEHPadBB;
3820 }
3821 return true;
3822}
3823
3824bool IRTranslatorImpl::translateInvoke(const User &U,
3825 MachineIRBuilder &MIRBuilder) {
3826 const InvokeInst &I = cast<InvokeInst>(Val: U);
3827 MCContext &Context = MF->getContext();
3828
3829 const BasicBlock *ReturnBB = I.getSuccessor(i: 0);
3830 const BasicBlock *EHPadBB = I.getSuccessor(i: 1);
3831
3832 const Function *Fn = I.getCalledFunction();
3833
3834 // FIXME: support invoking patchpoint and statepoint intrinsics.
3835 if (Fn && Fn->isIntrinsic())
3836 return false;
3837
3838 // FIXME: support whatever these are.
3839 if (I.hasDeoptState())
3840 return false;
3841
3842 // FIXME: support control flow guard targets.
3843 if (I.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
3844 return false;
3845
3846 // FIXME: support Windows exception handling.
3847 if (!isa<LandingPadInst>(Val: EHPadBB->getFirstNonPHIIt()))
3848 return false;
3849
3850 // FIXME: support Windows dllimport function calls and calls through
3851 // weak symbols.
3852 if (Fn && (Fn->hasDLLImportStorageClass() ||
3853 (MF->getTarget().getTargetTriple().isOSWindows() &&
3854 Fn->hasExternalWeakLinkage())))
3855 return false;
3856
3857 bool LowerInlineAsm = I.isInlineAsm();
3858 bool NeedEHLabel = true;
3859
3860 // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
3861 // the region covered by the try.
3862 MCSymbol *BeginSymbol = nullptr;
3863 if (NeedEHLabel) {
3864 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_INVOKE_REGION_START);
3865 BeginSymbol = Context.createTempSymbol();
3866 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: BeginSymbol);
3867 }
3868
3869 if (LowerInlineAsm) {
3870 if (!translateInlineAsm(CB: I, MIRBuilder))
3871 return false;
3872 } else if (!translateCallBase(CB: I, MIRBuilder))
3873 return false;
3874
3875 MCSymbol *EndSymbol = nullptr;
3876 if (NeedEHLabel) {
3877 EndSymbol = Context.createTempSymbol();
3878 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: EndSymbol);
3879 }
3880
3881 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
3882 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3883 MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
3884 BranchProbability EHPadBBProb =
3885 BPI ? BPI->getEdgeProbability(Src: InvokeMBB->getBasicBlock(), Dst: EHPadBB)
3886 : BranchProbability::getZero();
3887
3888 if (!findUnwindDestinations(EHPadBB, Prob: EHPadBBProb, UnwindDests))
3889 return false;
3890
3891 MachineBasicBlock &EHPadMBB = getMBB(BB: *EHPadBB),
3892 &ReturnMBB = getMBB(BB: *ReturnBB);
3893 // Update successor info.
3894 addSuccessorWithProb(Src: InvokeMBB, Dst: &ReturnMBB);
3895 for (auto &UnwindDest : UnwindDests) {
3896 UnwindDest.first->setIsEHPad();
3897 addSuccessorWithProb(Src: InvokeMBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
3898 }
3899 InvokeMBB->normalizeSuccProbs();
3900
3901 if (NeedEHLabel) {
3902 assert(BeginSymbol && "Expected a begin symbol!");
3903 assert(EndSymbol && "Expected an end symbol!");
3904 MF->addInvoke(LandingPad: &EHPadMBB, BeginLabel: BeginSymbol, EndLabel: EndSymbol);
3905 }
3906
3907 MIRBuilder.buildBr(Dest&: ReturnMBB);
3908 return true;
3909}
3910
3911/// The intrinsics currently supported by callbr are implicit control flow
3912/// intrinsics such as amdgcn.kill.
3913bool IRTranslatorImpl::translateCallBr(const User &U,
3914 MachineIRBuilder &MIRBuilder) {
3915 if (!mayTranslateUserTypes(U))
3916 return false; // see translateCall
3917
3918 const CallBrInst &I = cast<CallBrInst>(Val: U);
3919 MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB();
3920
3921 Intrinsic::ID IID = I.getIntrinsicID();
3922 if (I.isInlineAsm()) {
3923 // FIXME: inline asm is not yet supported for callbr in GlobalISel. As soon
3924 // as we add support, we need to handle the indirect asm targets, see
3925 // SelectionDAGBuilder::visitCallBr().
3926 return false;
3927 }
3928 if (!translateIntrinsic(CB: I, ID: IID, MIRBuilder))
3929 return false;
3930
3931 // Retrieve successors.
3932 SmallPtrSet<BasicBlock *, 8> Dests = {I.getDefaultDest()};
3933 MachineBasicBlock *Return = &getMBB(BB: *I.getDefaultDest());
3934
3935 // Update successor info.
3936 addSuccessorWithProb(Src: CallBrMBB, Dst: Return, Prob: BranchProbability::getOne());
3937
3938 // Add indirect targets as successors. For intrinsic callbr, these represent
3939 // implicit control flow (e.g., the "kill" path for amdgcn.kill). We mark them
3940 // with setIsInlineAsmBrIndirectTarget so the machine verifier accepts them as
3941 // valid successors, even though they're not from inline asm.
3942 for (BasicBlock *Dest : I.getIndirectDests()) {
3943 MachineBasicBlock &Target = getMBB(BB: *Dest);
3944 Target.setIsInlineAsmBrIndirectTarget();
3945 Target.setLabelMustBeEmitted();
3946 // Don't add duplicate machine successors.
3947 if (Dests.insert(Ptr: Dest).second)
3948 addSuccessorWithProb(Src: CallBrMBB, Dst: &Target, Prob: BranchProbability::getZero());
3949 }
3950
3951 CallBrMBB->normalizeSuccProbs();
3952
3953 // Drop into default successor.
3954 MIRBuilder.buildBr(Dest&: *Return);
3955
3956 return true;
3957}
3958
3959bool IRTranslatorImpl::translateLandingPad(const User &U,
3960 MachineIRBuilder &MIRBuilder) {
3961 const LandingPadInst &LP = cast<LandingPadInst>(Val: U);
3962
3963 MachineBasicBlock &MBB = MIRBuilder.getMBB();
3964
3965 MBB.setIsEHPad();
3966
3967 // If there aren't registers to copy the values into (e.g., during SjLj
3968 // exceptions), then don't bother.
3969 const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
3970 if (TLI->getExceptionPointerRegister(EH: FuncInfo.ExceptionModel,
3971 PersonalityFn) == 0 &&
3972 TLI->getExceptionSelectorRegister(EH: FuncInfo.ExceptionModel,
3973 PersonalityFn) == 0)
3974 return true;
3975
3976 // If landingpad's return type is token type, we don't create DAG nodes
3977 // for its exception pointer and selector value. The extraction of exception
3978 // pointer or selector value from token type landingpads is not currently
3979 // supported.
3980 if (LP.getType()->isTokenTy())
3981 return true;
3982
3983 if (!isExceptionPointerAndSelectorType(Ty: LP.getType()))
3984 return false;
3985
3986 // Add a label to mark the beginning of the landing pad. Deletion of the
3987 // landing pad can thus be detected via the MachineModuleInfo.
3988 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL)
3989 .addSym(Sym: MF->addLandingPad(LandingPad: &MBB));
3990
3991 // If the unwinder does not preserve all registers, ensure that the
3992 // function marks the clobbered registers as used.
3993 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
3994 if (auto *RegMask = TRI.getCustomEHPadPreservedMask(MF: *MF))
3995 MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask);
3996
3997 LLT Ty = getLLTForType(Ty&: *LP.getType(), DL: *DL);
3998 Register Undef = MRI->createGenericVirtualRegister(Ty);
3999 MIRBuilder.buildUndef(Res: Undef);
4000
4001 SmallVector<LLT, 2> Tys;
4002 for (Type *Ty : cast<StructType>(Val: LP.getType())->elements())
4003 Tys.push_back(Elt: getLLTForType(Ty&: *Ty, DL: *DL));
4004 assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
4005
4006 // Mark exception register as live in.
4007 Register ExceptionReg =
4008 TLI->getExceptionPointerRegister(EH: FuncInfo.ExceptionModel, PersonalityFn);
4009 if (!ExceptionReg)
4010 return false;
4011
4012 MBB.addLiveIn(PhysReg: ExceptionReg);
4013 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: LP);
4014 MIRBuilder.buildCopy(Res: ResRegs[0], Op: ExceptionReg);
4015
4016 Register SelectorReg =
4017 TLI->getExceptionSelectorRegister(EH: FuncInfo.ExceptionModel, PersonalityFn);
4018 if (!SelectorReg)
4019 return false;
4020
4021 MBB.addLiveIn(PhysReg: SelectorReg);
4022 Register PtrVReg = MRI->createGenericVirtualRegister(Ty: Tys[0]);
4023 MIRBuilder.buildCopy(Res: PtrVReg, Op: SelectorReg);
4024 MIRBuilder.buildCast(Dst: ResRegs[1], Src: PtrVReg);
4025
4026 return true;
4027}
4028
4029bool IRTranslatorImpl::translateAlloca(const User &U,
4030 MachineIRBuilder &MIRBuilder) {
4031 auto &AI = cast<AllocaInst>(Val: U);
4032
4033 if (AI.isSwiftError())
4034 return true;
4035
4036 if (AI.isStaticAlloca()) {
4037 Register Res = getOrCreateVReg(Val: AI);
4038 int FI = getOrCreateFrameIndex(AI);
4039 auto MIB = MIRBuilder.buildFrameIndex(Res, Idx: FI);
4040 // The frame address may be materialized before the alloca's source line is
4041 // reached. Clear its debug location to avoid reporting that line early.
4042 if (AI.getDebugLoc())
4043 MIB->setDebugLoc(DebugLoc::getDropped());
4044 return true;
4045 }
4046
4047 // FIXME: support stack probing for Windows.
4048 if (MF->getTarget().getTargetTriple().isOSWindows())
4049 return false;
4050
4051 // Now we're in the harder dynamic case.
4052 Register NumElts = getOrCreateVReg(Val: *AI.getArraySize());
4053 Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
4054 LLT IntPtrTy = getLLTForType(Ty&: *IntPtrIRTy, DL: *DL);
4055 if (MRI->getType(Reg: NumElts) != IntPtrTy) {
4056 Register ExtElts = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
4057 MIRBuilder.buildZExtOrTrunc(Res: ExtElts, Op: NumElts);
4058 NumElts = ExtElts;
4059 }
4060
4061 TypeSize TySize = AI.getAllocationBaseSize(DL: *DL);
4062
4063 Register AllocSize = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
4064 Register TySizeReg;
4065 if (TySize.isScalable()) {
4066 // For scalable types, use vscale * min_value
4067 TySizeReg = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
4068 MIRBuilder.buildVScale(Res: TySizeReg, MinElts: TySize.getKnownMinValue());
4069 } else {
4070 // For fixed types, use a constant
4071 TySizeReg =
4072 getOrCreateVReg(Val: *ConstantInt::get(Ty: IntPtrIRTy, V: TySize.getFixedValue()));
4073 }
4074 MIRBuilder.buildMul(Dst: AllocSize, Src0: NumElts, Src1: TySizeReg);
4075
4076 // Round the size of the allocation up to the stack alignment size
4077 // by add SA-1 to the size. This doesn't overflow because we're computing
4078 // an address inside an alloca.
4079 Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();
4080 auto SAMinusOne = MIRBuilder.buildConstant(Res: IntPtrTy, Val: StackAlign.value() - 1);
4081 auto AllocAdd = MIRBuilder.buildAdd(Dst: IntPtrTy, Src0: AllocSize, Src1: SAMinusOne,
4082 Flags: MachineInstr::NoUWrap);
4083 auto AlignCst =
4084 MIRBuilder.buildConstant(Res: IntPtrTy, Val: ~(uint64_t)(StackAlign.value() - 1));
4085 auto AlignedAlloc = MIRBuilder.buildAnd(Dst: IntPtrTy, Src0: AllocAdd, Src1: AlignCst);
4086
4087 Align Alignment = AI.getAlign();
4088 if (Alignment <= StackAlign)
4089 Alignment = Align(1);
4090 MIRBuilder.buildDynStackAlloc(Res: getOrCreateVReg(Val: AI), Size: AlignedAlloc, Alignment);
4091
4092 MF->getFrameInfo().CreateVariableSizedObject(Alignment, Alloca: &AI);
4093 assert(MF->getFrameInfo().hasVarSizedObjects());
4094 return true;
4095}
4096
4097bool IRTranslatorImpl::translateVAArg(const User &U,
4098 MachineIRBuilder &MIRBuilder) {
4099 // FIXME: We may need more info about the type. Because of how LLT works,
4100 // we're completely discarding the i64/double distinction here (amongst
4101 // others). Fortunately the ABIs I know of where that matters don't use va_arg
4102 // anyway but that's not guaranteed.
4103 MIRBuilder.buildInstr(Opc: TargetOpcode::G_VAARG, DstOps: {getOrCreateVReg(Val: U)},
4104 SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
4105 DL->getABITypeAlign(Ty: U.getType()).value()});
4106 return true;
4107}
4108
4109bool IRTranslatorImpl::translateUnreachable(const User &U,
4110 MachineIRBuilder &MIRBuilder) {
4111 auto &UI = cast<UnreachableInst>(Val: U);
4112 if (!UI.shouldLowerToTrap(TrapUnreachable: MF->getTarget().Options.TrapUnreachable,
4113 NoTrapAfterNoreturn: MF->getTarget().Options.NoTrapAfterNoreturn))
4114 return true;
4115
4116 MIRBuilder.buildTrap();
4117 return true;
4118}
4119
4120bool IRTranslatorImpl::translateInsertElement(const User &U,
4121 MachineIRBuilder &MIRBuilder) {
4122 // If it is a <1 x Ty> vector, use the scalar as it is
4123 // not a legal vector type in LLT.
4124 if (auto *FVT = dyn_cast<FixedVectorType>(Val: U.getType());
4125 FVT && FVT->getNumElements() == 1)
4126 return translateCopy(U, V: *U.getOperand(i: 1), MIRBuilder);
4127
4128 Register Res = getOrCreateVReg(Val: U);
4129 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
4130 Register Elt = getOrCreateVReg(Val: *U.getOperand(i: 1));
4131 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4132 Register Idx;
4133 if (auto *CI = dyn_cast<ConstantInt>(Val: U.getOperand(i: 2))) {
4134 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4135 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4136 auto *NewIdxCI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4137 Idx = getOrCreateVReg(Val: *NewIdxCI);
4138 }
4139 }
4140 if (!Idx)
4141 Idx = getOrCreateVReg(Val: *U.getOperand(i: 2));
4142 if (MRI->getType(Reg: Idx).getSizeInBits() != PreferredVecIdxWidth) {
4143 const LLT VecIdxTy =
4144 MRI->getType(Reg: Idx).changeElementSize(NewEltSize: PreferredVecIdxWidth);
4145 Idx = MIRBuilder.buildZExtOrTrunc(Res: VecIdxTy, Op: Idx).getReg(Idx: 0);
4146 }
4147 MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
4148 return true;
4149}
4150
4151bool IRTranslatorImpl::translateInsertVector(const User &U,
4152 MachineIRBuilder &MIRBuilder) {
4153 Register Dst = getOrCreateVReg(Val: U);
4154 Register Vec = getOrCreateVReg(Val: *U.getOperand(i: 0));
4155 Register Elt = getOrCreateVReg(Val: *U.getOperand(i: 1));
4156
4157 ConstantInt *CI = cast<ConstantInt>(Val: U.getOperand(i: 2));
4158 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4159
4160 // Resize Index to preferred index width.
4161 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4162 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4163 CI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4164 }
4165
4166 // If it is a <1 x Ty> vector, we have to use other means.
4167 if (auto *ResultType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 1)->getType());
4168 ResultType && ResultType->getNumElements() == 1) {
4169 if (auto *InputType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType());
4170 InputType && InputType->getNumElements() == 1) {
4171 // We are inserting an illegal fixed vector into an illegal
4172 // fixed vector, use the scalar as it is not a legal vector type
4173 // in LLT.
4174 return translateCopy(U, Src: Vec, MIRBuilder);
4175 }
4176 if (isa<FixedVectorType>(Val: U.getOperand(i: 0)->getType())) {
4177 // We are inserting an illegal fixed vector into a legal fixed
4178 // vector, use the scalar as it is not a legal vector type in
4179 // LLT.
4180 Register Idx = getOrCreateVReg(Val: *CI);
4181 MIRBuilder.buildInsertVectorElement(Res: Dst, Val: Vec, Elt, Idx);
4182 return true;
4183 }
4184 if (isa<ScalableVectorType>(Val: U.getOperand(i: 0)->getType())) {
4185 // We are inserting an illegal fixed vector into a scalable
4186 // vector, use a scalar element insert.
4187 LLT VecIdxTy = LLT::integer(SizeInBits: PreferredVecIdxWidth);
4188 Register Idx = getOrCreateVReg(Val: *CI);
4189 auto ScaledIndex = MIRBuilder.buildMul(
4190 Dst: VecIdxTy, Src0: MIRBuilder.buildVScale(Res: VecIdxTy, MinElts: 1), Src1: Idx);
4191 MIRBuilder.buildInsertVectorElement(Res: Dst, Val: Vec, Elt, Idx: ScaledIndex);
4192 return true;
4193 }
4194 }
4195
4196 MIRBuilder.buildInsertSubvector(Res: Dst, Src0: Vec, Src1: Elt, Index: CI->getZExtValue());
4197 return true;
4198}
4199
4200bool IRTranslatorImpl::translateExtractElement(const User &U,
4201 MachineIRBuilder &MIRBuilder) {
4202 // If it is a <1 x Ty> vector, use the scalar as it is
4203 // not a legal vector type in LLT.
4204 if (const FixedVectorType *FVT =
4205 dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType()))
4206 if (FVT->getNumElements() == 1)
4207 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
4208
4209 Register Res = getOrCreateVReg(Val: U);
4210 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
4211 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4212 Register Idx;
4213 if (auto *CI = dyn_cast<ConstantInt>(Val: U.getOperand(i: 1))) {
4214 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4215 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4216 auto *NewIdxCI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4217 Idx = getOrCreateVReg(Val: *NewIdxCI);
4218 }
4219 }
4220 if (!Idx)
4221 Idx = getOrCreateVReg(Val: *U.getOperand(i: 1));
4222 if (MRI->getType(Reg: Idx).getSizeInBits() != PreferredVecIdxWidth) {
4223 const LLT VecIdxTy =
4224 MRI->getType(Reg: Idx).changeElementSize(NewEltSize: PreferredVecIdxWidth);
4225 Idx = MIRBuilder.buildZExtOrTrunc(Res: VecIdxTy, Op: Idx).getReg(Idx: 0);
4226 }
4227 MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
4228 return true;
4229}
4230
4231bool IRTranslatorImpl::translateExtractVector(const User &U,
4232 MachineIRBuilder &MIRBuilder) {
4233 Register Res = getOrCreateVReg(Val: U);
4234 Register Vec = getOrCreateVReg(Val: *U.getOperand(i: 0));
4235 ConstantInt *CI = cast<ConstantInt>(Val: U.getOperand(i: 1));
4236 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4237
4238 // Resize Index to preferred index width.
4239 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4240 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4241 CI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4242 }
4243
4244 // If it is a <1 x Ty> vector, we have to use other means.
4245 if (auto *ResultType = dyn_cast<FixedVectorType>(Val: U.getType());
4246 ResultType && ResultType->getNumElements() == 1) {
4247 if (auto *InputType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType());
4248 InputType && InputType->getNumElements() == 1) {
4249 // We are extracting an illegal fixed vector from an illegal fixed vector,
4250 // use the scalar as it is not a legal vector type in LLT.
4251 return translateCopy(U, Src: Vec, MIRBuilder);
4252 }
4253 if (isa<FixedVectorType>(Val: U.getOperand(i: 0)->getType())) {
4254 // We are extracting an illegal fixed vector from a legal fixed
4255 // vector, use the scalar as it is not a legal vector type in
4256 // LLT.
4257 Register Idx = getOrCreateVReg(Val: *CI);
4258 MIRBuilder.buildExtractVectorElement(Res, Val: Vec, Idx);
4259 return true;
4260 }
4261 if (isa<ScalableVectorType>(Val: U.getOperand(i: 0)->getType())) {
4262 // We are extracting an illegal fixed vector from a scalable
4263 // vector, use a scalar element extract.
4264 LLT VecIdxTy = LLT::integer(SizeInBits: PreferredVecIdxWidth);
4265 Register Idx = getOrCreateVReg(Val: *CI);
4266 auto ScaledIndex = MIRBuilder.buildMul(
4267 Dst: VecIdxTy, Src0: MIRBuilder.buildVScale(Res: VecIdxTy, MinElts: 1), Src1: Idx);
4268 MIRBuilder.buildExtractVectorElement(Res, Val: Vec, Idx: ScaledIndex);
4269 return true;
4270 }
4271 }
4272
4273 MIRBuilder.buildExtractSubvector(Res, Src: Vec, Index: CI->getZExtValue());
4274 return true;
4275}
4276
4277bool IRTranslatorImpl::translateShuffleVector(const User &U,
4278 MachineIRBuilder &MIRBuilder) {
4279 // A ShuffleVector that operates on scalable vectors is a splat vector where
4280 // the value of the splat vector is the 0th element of the first operand,
4281 // since the index mask operand is the zeroinitializer (undef and
4282 // poison are treated as zeroinitializer here).
4283 if (U.getOperand(i: 0)->getType()->isScalableTy()) {
4284 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
4285 auto SplatVal = MIRBuilder.buildExtractVectorElementConstant(
4286 Res: MRI->getType(Reg: Val).getElementType(), Val, Idx: 0);
4287 MIRBuilder.buildSplatVector(Res: getOrCreateVReg(Val: U), Val: SplatVal);
4288 return true;
4289 }
4290
4291 ArrayRef<int> Mask;
4292 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: &U))
4293 Mask = SVI->getShuffleMask();
4294 else
4295 Mask = cast<ConstantExpr>(Val: U).getShuffleMask();
4296
4297 // As GISel does not represent <1 x > vectors as a separate type from scalars,
4298 // we transform shuffle_vector with a scalar output to an
4299 // ExtractVectorElement. If the input type is also scalar it becomes a Copy.
4300 unsigned DstElts = cast<FixedVectorType>(Val: U.getType())->getNumElements();
4301 unsigned SrcElts =
4302 cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType())->getNumElements();
4303 if (DstElts == 1) {
4304 unsigned M = Mask[0];
4305 if (SrcElts == 1) {
4306 if (M == 0 || M == 1)
4307 return translateCopy(U, V: *U.getOperand(i: M), MIRBuilder);
4308 MIRBuilder.buildUndef(Res: getOrCreateVReg(Val: U));
4309 } else {
4310 Register Dst = getOrCreateVReg(Val: U);
4311 if (M < SrcElts) {
4312 MIRBuilder.buildExtractVectorElementConstant(
4313 Res: Dst, Val: getOrCreateVReg(Val: *U.getOperand(i: 0)), Idx: M);
4314 } else if (M < SrcElts * 2) {
4315 MIRBuilder.buildExtractVectorElementConstant(
4316 Res: Dst, Val: getOrCreateVReg(Val: *U.getOperand(i: 1)), Idx: M - SrcElts);
4317 } else {
4318 MIRBuilder.buildUndef(Res: Dst);
4319 }
4320 }
4321 return true;
4322 }
4323
4324 // A single element src is transformed to a build_vector.
4325 if (SrcElts == 1) {
4326 SmallVector<Register> Ops;
4327 Register Undef;
4328 for (int M : Mask) {
4329 LLT SrcTy = getLLTForType(Ty&: *U.getOperand(i: 0)->getType(), DL: *DL);
4330 if (M == 0 || M == 1) {
4331 Ops.push_back(Elt: getOrCreateVReg(Val: *U.getOperand(i: M)));
4332 } else {
4333 if (!Undef.isValid()) {
4334 Undef = MRI->createGenericVirtualRegister(Ty: SrcTy);
4335 MIRBuilder.buildUndef(Res: Undef);
4336 }
4337 Ops.push_back(Elt: Undef);
4338 }
4339 }
4340 MIRBuilder.buildBuildVector(Res: getOrCreateVReg(Val: U), Ops);
4341 return true;
4342 }
4343
4344 ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
4345 MIRBuilder
4346 .buildInstr(Opc: TargetOpcode::G_SHUFFLE_VECTOR, DstOps: {getOrCreateVReg(Val: U)},
4347 SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
4348 getOrCreateVReg(Val: *U.getOperand(i: 1))})
4349 .addShuffleMask(Val: MaskAlloc);
4350 return true;
4351}
4352
4353bool IRTranslatorImpl::translateBitInsert(const User &U,
4354 MachineIRBuilder &MIRBuilder) {
4355 Register Res = getOrCreateVReg(Val: U);
4356 Register Base = getOrCreateVReg(Val: *U.getOperand(i: 0));
4357 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 1));
4358 Register Offset = getOrCreateVReg(Val: *U.getOperand(i: 2));
4359 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4360 LLT BaseTy = MRI.getType(Reg: Base);
4361 LLT ValTy = MRI.getType(Reg: Val);
4362
4363 assert(BaseTy.getSizeInBits() >= ValTy.getSizeInBits() &&
4364 "bitinsert val wider than base should be rejected by verifier");
4365
4366 // If Val is a floating-point type, bitcast it to an integer of the same
4367 // size so buildZExtOrTrunc can safely extend or truncate it.
4368 if (ValTy.isFloat()) {
4369 ValTy = LLT::scalar(SizeInBits: ValTy.getSizeInBits());
4370 Val = MIRBuilder.buildBitcast(Dst: ValTy, Src: Val).getReg(Idx: 0);
4371 } else if (ValTy.isPointer()) {
4372 ValTy = LLT::scalar(SizeInBits: ValTy.getSizeInBits());
4373 Val = MIRBuilder.buildPtrToInt(Dst: ValTy, Src: Val).getReg(Idx: 0);
4374 }
4375
4376 // Convert Offset to the target's preferred shift amount type.
4377 LLT ShiftAmtTy = TLI->getPreferredShiftAmountTy(ShiftValueTy: BaseTy);
4378 Register LegalOffset =
4379 MIRBuilder.buildZExtOrTrunc(Res: ShiftAmtTy, Op: Offset).getReg(Idx: 0);
4380
4381 // Truncate or extend Val to BaseTy so only the inserted bit range remains.
4382 Register ExtVal = MIRBuilder.buildZExtOrTrunc(Res: BaseTy, Op: Val).getReg(Idx: 0);
4383
4384 unsigned BaseBitWidth = BaseTy.getSizeInBits();
4385 unsigned ValBitWidth = ValTy.getSizeInBits();
4386 APInt InsertMask = APInt::getLowBitsSet(numBits: BaseBitWidth, loBitsSet: ValBitWidth);
4387 Register MaskConst = MIRBuilder.buildConstant(Res: BaseTy, Val: InsertMask).getReg(Idx: 0);
4388 Register ShiftedMask =
4389 MIRBuilder.buildShl(Dst: BaseTy, Src0: MaskConst, Src1: LegalOffset).getReg(Idx: 0);
4390 Register ClearMask = MIRBuilder.buildNot(Dst: BaseTy, Src0: ShiftedMask).getReg(Idx: 0);
4391 Register ClearedBase = MIRBuilder.buildAnd(Dst: BaseTy, Src0: Base, Src1: ClearMask).getReg(Idx: 0);
4392 Register ShiftedVal =
4393 MIRBuilder.buildShl(Dst: BaseTy, Src0: ExtVal, Src1: LegalOffset).getReg(Idx: 0);
4394 MIRBuilder.buildOr(Dst: Res, Src0: ClearedBase, Src1: ShiftedVal);
4395 return true;
4396}
4397
4398bool IRTranslatorImpl::translateBitExtract(const User &U,
4399 MachineIRBuilder &MIRBuilder) {
4400 Register Res = getOrCreateVReg(Val: U);
4401 Register Src = getOrCreateVReg(Val: *U.getOperand(i: 0));
4402 Register Offset = getOrCreateVReg(Val: *U.getOperand(i: 1));
4403 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4404 LLT SrcTy = MRI.getType(Reg: Src);
4405 LLT ResTy = MRI.getType(Reg: Res);
4406
4407 assert(ResTy.getSizeInBits() <= SrcTy.getSizeInBits() &&
4408 "bitextract result wider than source should be rejected by verifier");
4409
4410 // Convert Offset to the target's preferred shift amount type.
4411 LLT ShiftAmtTy = TLI->getPreferredShiftAmountTy(ShiftValueTy: SrcTy);
4412 Register LegalOffset =
4413 MIRBuilder.buildZExtOrTrunc(Res: ShiftAmtTy, Op: Offset).getReg(Idx: 0);
4414
4415 // Shift right by Offset to bring the target field down to bit 0.
4416 Register Shifted = MIRBuilder.buildLShr(Dst: SrcTy, Src0: Src, Src1: LegalOffset).getReg(Idx: 0);
4417
4418 if (ResTy.isFloat()) {
4419 // Drop into the integer domain to safely handle the size conversion
4420 LLT IntResTy = LLT::scalar(SizeInBits: ResTy.getSizeInBits());
4421 Register IntRes = MRI.createGenericVirtualRegister(Ty: IntResTy);
4422
4423 if (SrcTy == IntResTy)
4424 MIRBuilder.buildCopy(Res: IntRes, Op: Shifted);
4425 else
4426 MIRBuilder.buildTrunc(Res: IntRes, Op: Shifted);
4427
4428 // Bitcast the raw integer bits back into the requested floating-point
4429 // register
4430 MIRBuilder.buildBitcast(Dst: Res, Src: IntRes);
4431 } else if (ResTy.isPointer()) {
4432 // Drop into the integer domain to safely handle the size conversion
4433 LLT IntResTy = LLT::scalar(SizeInBits: ResTy.getSizeInBits());
4434 Register IntRes = MRI.createGenericVirtualRegister(Ty: IntResTy);
4435
4436 if (SrcTy == IntResTy)
4437 MIRBuilder.buildCopy(Res: IntRes, Op: Shifted);
4438 else
4439 MIRBuilder.buildTrunc(Res: IntRes, Op: Shifted);
4440
4441 MIRBuilder.buildIntToPtr(Dst: Res, Src: IntRes);
4442 } else {
4443 // Normal integer path
4444 if (SrcTy == ResTy)
4445 MIRBuilder.buildCopy(Res, Op: Shifted);
4446 else
4447 MIRBuilder.buildTrunc(Res, Op: Shifted);
4448 }
4449
4450 return true;
4451}
4452
4453bool IRTranslatorImpl::translatePHI(const User &U,
4454 MachineIRBuilder &MIRBuilder) {
4455 const PHINode &PI = cast<PHINode>(Val: U);
4456
4457 SmallVector<MachineInstr *, 4> Insts;
4458 for (auto Reg : getOrCreateVRegs(Val: PI)) {
4459 auto MIB = MIRBuilder.buildInstr(Opc: TargetOpcode::G_PHI, DstOps: {Reg}, SrcOps: {});
4460 Insts.push_back(Elt: MIB.getInstr());
4461 }
4462
4463 PendingPHIs.emplace_back(Args: &PI, Args: std::move(Insts));
4464 return true;
4465}
4466
4467bool IRTranslatorImpl::translateAtomicCmpXchg(const User &U,
4468 MachineIRBuilder &MIRBuilder) {
4469 const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(Val: U);
4470
4471 auto Flags = TLI->getAtomicMemOperandFlags(AI: I, DL: *DL);
4472
4473 auto Res = getOrCreateVRegs(Val: I);
4474 Register OldValRes = Res[0];
4475 Register SuccessRes = Res[1];
4476 Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
4477 Register Cmp = getOrCreateVReg(Val: *I.getCompareOperand());
4478 Register NewVal = getOrCreateVReg(Val: *I.getNewValOperand());
4479
4480 MIRBuilder.buildAtomicCmpXchgWithSuccess(
4481 OldValRes, SuccessRes, Addr, CmpVal: Cmp, NewVal,
4482 MMO&: *MF->getMachineMemOperand(
4483 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, MemTy: MRI->getType(Reg: Cmp),
4484 BaseAlignment: getMemOpAlign(I), Metadata: I.getAAMetadata(), SSID: I.getSyncScopeID(),
4485 Ordering: I.getSuccessOrdering(), FailureOrdering: I.getFailureOrdering()));
4486 return true;
4487}
4488
4489bool IRTranslatorImpl::translateAtomicRMW(const User &U,
4490 MachineIRBuilder &MIRBuilder) {
4491 if (!mayTranslateUserTypes(U))
4492 return false;
4493
4494 const AtomicRMWInst &I = cast<AtomicRMWInst>(Val: U);
4495 auto Flags = TLI->getAtomicMemOperandFlags(AI: I, DL: *DL);
4496
4497 Register Res = getOrCreateVReg(Val: I);
4498 Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
4499 Register Val = getOrCreateVReg(Val: *I.getValOperand());
4500
4501 unsigned Opcode = 0;
4502 switch (I.getOperation()) {
4503 default:
4504 return false;
4505 case AtomicRMWInst::Xchg:
4506 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
4507 break;
4508 case AtomicRMWInst::Add:
4509 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
4510 break;
4511 case AtomicRMWInst::Sub:
4512 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
4513 break;
4514 case AtomicRMWInst::And:
4515 Opcode = TargetOpcode::G_ATOMICRMW_AND;
4516 break;
4517 case AtomicRMWInst::Nand:
4518 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
4519 break;
4520 case AtomicRMWInst::Or:
4521 Opcode = TargetOpcode::G_ATOMICRMW_OR;
4522 break;
4523 case AtomicRMWInst::Xor:
4524 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
4525 break;
4526 case AtomicRMWInst::Max:
4527 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
4528 break;
4529 case AtomicRMWInst::Min:
4530 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
4531 break;
4532 case AtomicRMWInst::UMax:
4533 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
4534 break;
4535 case AtomicRMWInst::UMin:
4536 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
4537 break;
4538 case AtomicRMWInst::FAdd:
4539 Opcode = TargetOpcode::G_ATOMICRMW_FADD;
4540 break;
4541 case AtomicRMWInst::FSub:
4542 Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
4543 break;
4544 case AtomicRMWInst::FMax:
4545 Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
4546 break;
4547 case AtomicRMWInst::FMin:
4548 Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
4549 break;
4550 case AtomicRMWInst::FMaximum:
4551 Opcode = TargetOpcode::G_ATOMICRMW_FMAXIMUM;
4552 break;
4553 case AtomicRMWInst::FMinimum:
4554 Opcode = TargetOpcode::G_ATOMICRMW_FMINIMUM;
4555 break;
4556 case AtomicRMWInst::FMaximumNum:
4557 Opcode = TargetOpcode::G_ATOMICRMW_FMAXIMUMNUM;
4558 break;
4559 case AtomicRMWInst::FMinimumNum:
4560 Opcode = TargetOpcode::G_ATOMICRMW_FMINIMUMNUM;
4561 break;
4562 case AtomicRMWInst::UIncWrap:
4563 Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
4564 break;
4565 case AtomicRMWInst::UDecWrap:
4566 Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
4567 break;
4568 case AtomicRMWInst::USubCond:
4569 Opcode = TargetOpcode::G_ATOMICRMW_USUB_COND;
4570 break;
4571 case AtomicRMWInst::USubSat:
4572 Opcode = TargetOpcode::G_ATOMICRMW_USUB_SAT;
4573 break;
4574 }
4575
4576 MIRBuilder.buildAtomicRMW(
4577 Opcode, OldValRes: Res, Addr, Val,
4578 MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(I.getPointerOperand()),
4579 F: Flags, MemTy: MRI->getType(Reg: Val), BaseAlignment: getMemOpAlign(I),
4580 Metadata: I.getAAMetadata(), SSID: I.getSyncScopeID(),
4581 Ordering: I.getOrdering()));
4582 return true;
4583}
4584
4585bool IRTranslatorImpl::translateFence(const User &U,
4586 MachineIRBuilder &MIRBuilder) {
4587 const FenceInst &Fence = cast<FenceInst>(Val: U);
4588 MIRBuilder.buildFence(Ordering: static_cast<unsigned>(Fence.getOrdering()),
4589 Scope: Fence.getSyncScopeID());
4590 return true;
4591}
4592
4593bool IRTranslatorImpl::translateFreeze(const User &U,
4594 MachineIRBuilder &MIRBuilder) {
4595 const ArrayRef<Register> DstRegs = getOrCreateVRegs(Val: U);
4596 const ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *U.getOperand(i: 0));
4597
4598 assert(DstRegs.size() == SrcRegs.size() &&
4599 "Freeze with different source and destination type?");
4600
4601 for (unsigned I = 0; I < DstRegs.size(); ++I) {
4602 MIRBuilder.buildFreeze(Dst: DstRegs[I], Src: SrcRegs[I]);
4603 }
4604
4605 return true;
4606}
4607
4608void IRTranslatorImpl::finishPendingConstDbgValues() {
4609 for (auto [MI, C] : PendingConstDbgValues) {
4610 auto VRegsIt = VMap.findVRegs(V: *C);
4611 if (VRegsIt == VMap.vregs_end() || VRegsIt->second->size() != 1)
4612 continue;
4613 // Constants are materialized in the entry block, so the vreg dominates
4614 // every debug value; where it is not live, the location is dropped later.
4615 MachineIRBuilder Builder(*MI);
4616 Builder.buildDirectDbgValue(Reg: VRegsIt->second->front(),
4617 Variable: MI->getDebugVariable(),
4618 Expr: MI->getDebugExpression());
4619 MI->eraseFromParent();
4620 }
4621 PendingConstDbgValues.clear();
4622}
4623
4624void IRTranslatorImpl::finishPendingPhis() {
4625#ifndef NDEBUG
4626 DILocationVerifier Verifier;
4627 GISelObserverWrapper WrapperObserver(&Verifier);
4628 RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);
4629#endif // ifndef NDEBUG
4630 for (auto &Phi : PendingPHIs) {
4631 const PHINode *PI = Phi.first;
4632 if (PI->getType()->isEmptyTy())
4633 continue;
4634 ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
4635 MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
4636 EntryBuilder->setDebugLoc(PI->getDebugLoc());
4637#ifndef NDEBUG
4638 Verifier.setCurrentInst(PI);
4639#endif // ifndef NDEBUG
4640
4641 SmallPtrSet<const MachineBasicBlock *, 16> SeenPreds;
4642 for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
4643 auto IRPred = PI->getIncomingBlock(i);
4644 ArrayRef<Register> ValRegs = getOrCreateVRegs(Val: *PI->getIncomingValue(i));
4645 for (auto *Pred : getMachinePredBBs(Edge: {IRPred, PI->getParent()})) {
4646 if (SeenPreds.count(Ptr: Pred) || !PhiMBB->isPredecessor(MBB: Pred))
4647 continue;
4648 SeenPreds.insert(Ptr: Pred);
4649 for (unsigned j = 0; j < ValRegs.size(); ++j) {
4650 MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
4651 MIB.addUse(RegNo: ValRegs[j]);
4652 MIB.addMBB(MBB: Pred);
4653 }
4654 }
4655 }
4656 }
4657}
4658
4659void IRTranslatorImpl::translateDbgValueRecord(Value *V, bool HasArgList,
4660 const DILocalVariable *Variable,
4661 const DIExpression *Expression,
4662 const DebugLoc &DL,
4663 MachineIRBuilder &MIRBuilder) {
4664 assert(Variable->isValidLocationForIntrinsic(DL) &&
4665 "Expected inlined-at fields to agree");
4666 // Act as if we're handling a debug intrinsic.
4667 MIRBuilder.setDebugLoc(DL);
4668
4669 if (!V || HasArgList) {
4670 // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
4671 // terminate any prior location.
4672 MIRBuilder.buildIndirectDbgValue(Reg: 0, Variable, Expr: Expression);
4673 return;
4674 }
4675
4676 if (const auto *CI = dyn_cast<Constant>(Val: V)) {
4677 MachineInstr *MI = MIRBuilder.buildConstDbgValue(C: *CI, Variable, Expr: Expression);
4678 if (MI->getDebugOperand(Index: 0).isReg() && !isa<UndefValue>(Val: CI))
4679 PendingConstDbgValues.emplace_back(Args&: MI, Args&: CI);
4680 return;
4681 }
4682
4683 if (auto *AI = dyn_cast<AllocaInst>(Val: V);
4684 AI && AI->isStaticAlloca() && Expression->startsWithDeref()) {
4685 // If the value is an alloca and the expression starts with a
4686 // dereference, track a stack slot instead of a register, as registers
4687 // may be clobbered.
4688 auto ExprOperands = Expression->getElements();
4689 auto *ExprDerefRemoved =
4690 DIExpression::get(Context&: AI->getContext(), Elements: ExprOperands.drop_front());
4691 MIRBuilder.buildFIDbgValue(FI: getOrCreateFrameIndex(AI: *AI), Variable,
4692 Expr: ExprDerefRemoved);
4693 return;
4694 }
4695 if (translateIfEntryValueArgument(isDeclare: false, Val: V, Var: Variable, Expr: Expression, DL,
4696 MIRBuilder))
4697 return;
4698 for (Register Reg : getOrCreateVRegs(Val: *V)) {
4699 // FIXME: This does not handle register-indirect values at offset 0. The
4700 // direct/indirect thing shouldn't really be handled by something as
4701 // implicit as reg+noreg vs reg+imm in the first place, but it seems
4702 // pretty baked in right now.
4703 MIRBuilder.buildDirectDbgValue(Reg, Variable, Expr: Expression);
4704 }
4705}
4706
4707void IRTranslatorImpl::translateDbgDeclareRecord(
4708 Value *Address, bool HasArgList, const DILocalVariable *Variable,
4709 const DIExpression *Expression, const DebugLoc &DL,
4710 MachineIRBuilder &MIRBuilder) {
4711 if (!Address || isa<UndefValue>(Val: Address)) {
4712 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *Variable << "\n");
4713 return;
4714 }
4715
4716 assert(Variable->isValidLocationForIntrinsic(DL) &&
4717 "Expected inlined-at fields to agree");
4718 auto AI = dyn_cast<AllocaInst>(Val: Address);
4719 if (AI && AI->isStaticAlloca()) {
4720 // Static allocas are tracked at the MF level, no need for DBG_VALUE
4721 // instructions (in fact, they get ignored if they *do* exist).
4722 MF->setVariableDbgInfo(Var: Variable, Expr: Expression,
4723 Slot: getOrCreateFrameIndex(AI: *AI), Loc: DL);
4724 return;
4725 }
4726
4727 if (translateIfEntryValueArgument(isDeclare: true, Val: Address, Var: Variable,
4728 Expr: Expression, DL,
4729 MIRBuilder))
4730 return;
4731
4732 // A dbg.declare describes the address of a source variable, so lower it
4733 // into an indirect DBG_VALUE.
4734 MIRBuilder.setDebugLoc(DL);
4735 MIRBuilder.buildIndirectDbgValue(Reg: getOrCreateVReg(Val: *Address), Variable,
4736 Expr: Expression);
4737}
4738
4739void IRTranslatorImpl::translateDbgInfo(const Instruction &Inst,
4740 MachineIRBuilder &MIRBuilder) {
4741 for (DbgRecord &DR : Inst.getDbgRecordRange()) {
4742 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
4743 MIRBuilder.setDebugLoc(DLR->getDebugLoc());
4744 assert(DLR->getLabel() && "Missing label");
4745 assert(DLR->getLabel()->isValidLocationForIntrinsic(
4746 MIRBuilder.getDebugLoc()) &&
4747 "Expected inlined-at fields to agree");
4748 MIRBuilder.buildDbgLabel(Label: DLR->getLabel());
4749 continue;
4750 }
4751 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
4752 const DILocalVariable *Variable = DVR.getVariable();
4753 const DIExpression *Expression = DVR.getExpression();
4754 Value *V = DVR.getVariableLocationOp(OpIdx: 0);
4755 if (DVR.isDbgDeclare())
4756 translateDbgDeclareRecord(Address: V, HasArgList: DVR.hasArgList(), Variable, Expression,
4757 DL: DVR.getDebugLoc(), MIRBuilder);
4758 else
4759 translateDbgValueRecord(V, HasArgList: DVR.hasArgList(), Variable, Expression,
4760 DL: DVR.getDebugLoc(), MIRBuilder);
4761 }
4762}
4763
4764bool IRTranslatorImpl::translate(const Instruction &Inst) {
4765 CurBuilder->setDebugLoc(Inst.getDebugLoc());
4766 CurBuilder->setPCSections(Inst.getMetadata(KindID: LLVMContext::MD_pcsections));
4767 CurBuilder->setMMRAMetadata(Inst.getMetadata(KindID: LLVMContext::MD_mmra));
4768
4769 if (TLI->fallBackToDAGISel(Inst))
4770 return false;
4771
4772 switch (Inst.getOpcode()) {
4773#define HANDLE_INST(NUM, OPCODE, CLASS) \
4774 case Instruction::OPCODE: \
4775 return translate##OPCODE(Inst, *CurBuilder.get());
4776#include "llvm/IR/Instruction.def"
4777 default:
4778 return false;
4779 }
4780}
4781
4782bool IRTranslatorImpl::translate(const Constant &C, Register Reg) {
4783 // We only emit constants into the entry block from here. To prevent jumpy
4784 // debug behaviour remove debug line.
4785 if (auto CurrInstDL = CurBuilder->getDL())
4786 EntryBuilder->setDebugLoc(DebugLoc());
4787
4788 if (auto CI = dyn_cast<ConstantInt>(Val: &C)) {
4789 // buildConstant expects a to-be-splatted scalar ConstantInt.
4790 if (isa<VectorType>(Val: CI->getType()))
4791 CI = ConstantInt::get(Context&: CI->getContext(), V: CI->getValue());
4792 EntryBuilder->buildConstant(Res: Reg, Val: *CI);
4793 } else if (auto CB = dyn_cast<ConstantByte>(Val: &C)) {
4794 // Byte constants share G_CONSTANT with integers; the destination Reg's
4795 // LLT (an integer LLT, see getLLTForType) determines vector splatting.
4796 EntryBuilder->buildConstant(Res: Reg, Val: CB->getValue());
4797 } else if (auto CF = dyn_cast<ConstantFP>(Val: &C)) {
4798 // buildFConstant expects a to-be-splatted scalar ConstantFP.
4799 if (isa<VectorType>(Val: CF->getType()))
4800 CF = ConstantFP::get(Context&: CF->getContext(), V: CF->getValue());
4801 EntryBuilder->buildFConstant(Res: Reg, Val: *CF);
4802 } else if (isa<UndefValue>(Val: C))
4803 EntryBuilder->buildUndef(Res: Reg);
4804 else if (isa<ConstantPointerNull>(Val: C))
4805 EntryBuilder->buildConstant(Res: Reg, Val: 0);
4806 else if (auto GV = dyn_cast<GlobalValue>(Val: &C))
4807 EntryBuilder->buildGlobalValue(Res: Reg, GV);
4808 else if (auto CPA = dyn_cast<ConstantPtrAuth>(Val: &C)) {
4809 Register Addr = getOrCreateVReg(Val: *CPA->getPointer());
4810 Register AddrDisc = getOrCreateVReg(Val: *CPA->getAddrDiscriminator());
4811 EntryBuilder->buildConstantPtrAuth(Res: Reg, CPA, Addr, AddrDisc);
4812 } else if (auto CAZ = dyn_cast<ConstantAggregateZero>(Val: &C)) {
4813 Constant &Elt = *CAZ->getElementValue(Idx: 0u);
4814 if (isa<ScalableVectorType>(Val: CAZ->getType())) {
4815 EntryBuilder->buildSplatVector(Res: Reg, Val: getOrCreateVReg(Val: Elt));
4816 return true;
4817 }
4818 // Return the scalar if it is a <1 x Ty> vector.
4819 unsigned NumElts = CAZ->getElementCount().getFixedValue();
4820 if (NumElts == 1)
4821 return translateCopy(U: C, V: Elt, MIRBuilder&: *EntryBuilder);
4822 // All elements are zero so we can just use the first one.
4823 EntryBuilder->buildSplatBuildVector(Res: Reg, Src: getOrCreateVReg(Val: Elt));
4824 } else if (auto CV = dyn_cast<ConstantDataVector>(Val: &C)) {
4825 // Return the scalar if it is a <1 x Ty> vector.
4826 if (CV->getNumElements() == 1)
4827 return translateCopy(U: C, V: *CV->getElementAsConstant(i: 0), MIRBuilder&: *EntryBuilder);
4828 SmallVector<Register, 4> Ops;
4829 for (unsigned i = 0; i < CV->getNumElements(); ++i) {
4830 Constant &Elt = *CV->getElementAsConstant(i);
4831 Ops.push_back(Elt: getOrCreateVReg(Val: Elt));
4832 }
4833 EntryBuilder->buildBuildVector(Res: Reg, Ops);
4834 } else if (auto CE = dyn_cast<ConstantExpr>(Val: &C)) {
4835 switch(CE->getOpcode()) {
4836#define HANDLE_INST(NUM, OPCODE, CLASS) \
4837 case Instruction::OPCODE: \
4838 return translate##OPCODE(*CE, *EntryBuilder.get());
4839#include "llvm/IR/Instruction.def"
4840 default:
4841 return false;
4842 }
4843 } else if (auto CV = dyn_cast<ConstantVector>(Val: &C)) {
4844 if (CV->getNumOperands() == 1)
4845 return translateCopy(U: C, V: *CV->getOperand(i_nocapture: 0), MIRBuilder&: *EntryBuilder);
4846 SmallVector<Register, 4> Ops;
4847 for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
4848 Ops.push_back(Elt: getOrCreateVReg(Val: *CV->getOperand(i_nocapture: i)));
4849 }
4850 EntryBuilder->buildBuildVector(Res: Reg, Ops);
4851 } else if (auto *BA = dyn_cast<BlockAddress>(Val: &C)) {
4852 EntryBuilder->buildBlockAddress(Res: Reg, BA);
4853 } else
4854 return false;
4855
4856 return true;
4857}
4858
4859bool IRTranslatorImpl::mayTranslateUserTypes(const User &U) const {
4860 const TargetMachine &TM = TLI->getTargetMachine();
4861 if (LLT::getUseExtended())
4862 return true;
4863
4864 // BF16 cannot currently be represented by default LLT. To avoid miscompiles
4865 // we prevent any instructions using them by default in all targets that do
4866 // not explicitly enable it via LLT::setUseExtended(true).
4867 // SPIRV target is exception.
4868 return TM.getTargetTriple().isSPIRV() ||
4869 (!U.getType()->getScalarType()->isBFloatTy() &&
4870 !any_of(Range: U.operands(), P: [](Value *V) {
4871 return V->getType()->getScalarType()->isBFloatTy();
4872 }));
4873}
4874
4875bool IRTranslatorImpl::finalizeBasicBlock(const BasicBlock &BB,
4876 MachineBasicBlock &MBB) {
4877 for (auto &BTB : SL->BitTestCases) {
4878 // Emit header first, if it wasn't already emitted.
4879 if (!BTB.Emitted)
4880 emitBitTestHeader(B&: BTB, SwitchBB: BTB.Parent);
4881
4882 BranchProbability UnhandledProb = BTB.Prob;
4883 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
4884 UnhandledProb -= BTB.Cases[j].ExtraProb;
4885 // Set the current basic block to the mbb we wish to insert the code into
4886 MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
4887 // If all cases cover a contiguous range, it is not necessary to jump to
4888 // the default block after the last bit test fails. This is because the
4889 // range check during bit test header creation has guaranteed that every
4890 // case here doesn't go outside the range. In this case, there is no need
4891 // to perform the last bit test, as it will always be true. Instead, make
4892 // the second-to-last bit-test fall through to the target of the last bit
4893 // test, and delete the last bit test.
4894
4895 MachineBasicBlock *NextMBB;
4896 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
4897 // Second-to-last bit-test with contiguous range: fall through to the
4898 // target of the final bit test.
4899 NextMBB = BTB.Cases[j + 1].TargetBB;
4900 } else if (j + 1 == ej) {
4901 // For the last bit test, fall through to Default.
4902 NextMBB = BTB.Default;
4903 } else {
4904 // Otherwise, fall through to the next bit test.
4905 NextMBB = BTB.Cases[j + 1].ThisBB;
4906 }
4907
4908 emitBitTestCase(BB&: BTB, NextMBB, BranchProbToNext: UnhandledProb, Reg: BTB.Reg, B&: BTB.Cases[j], SwitchBB: MBB);
4909
4910 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
4911 // We need to record the replacement phi edge here that normally
4912 // happens in emitBitTestCase before we delete the case, otherwise the
4913 // phi edge will be lost.
4914 addMachineCFGPred(Edge: {BTB.Parent->getBasicBlock(),
4915 BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
4916 NewPred: MBB);
4917 // Since we're not going to use the final bit test, remove it.
4918 BTB.Cases.pop_back();
4919 break;
4920 }
4921 }
4922 // This is "default" BB. We have two jumps to it. From "header" BB and from
4923 // last "case" BB, unless the latter was skipped.
4924 CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
4925 BTB.Default->getBasicBlock()};
4926 addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Parent);
4927 if (!BTB.ContiguousRange) {
4928 addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Cases.back().ThisBB);
4929 }
4930 }
4931 SL->BitTestCases.clear();
4932
4933 for (auto &JTCase : SL->JTCases) {
4934 // Emit header first, if it wasn't already emitted.
4935 if (!JTCase.first.Emitted)
4936 emitJumpTableHeader(JT&: JTCase.second, JTH&: JTCase.first, HeaderBB: JTCase.first.HeaderBB);
4937
4938 emitJumpTable(JT&: JTCase.second, MBB: JTCase.second.MBB);
4939 }
4940 SL->JTCases.clear();
4941
4942 for (auto &SwCase : SL->SwitchCases)
4943 emitSwitchCase(CB&: SwCase, SwitchBB: &CurBuilder->getMBB(), MIB&: *CurBuilder);
4944 SL->SwitchCases.clear();
4945
4946 // Check if we need to generate stack-protector guard checks.
4947 if (SPInfo->shouldEmitSDCheck(BB)) {
4948 bool FunctionBasedInstrumentation =
4949 TLI->getSSPStackGuardCheck(M: *MF->getFunction().getParent(), Libcalls: *Libcalls);
4950 SPDescriptor.initialize(BB: &BB, MBB: &MBB, FunctionBasedInstrumentation);
4951 }
4952 // Handle stack protector.
4953 if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
4954 LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
4955 return false;
4956 } else if (SPDescriptor.shouldEmitStackProtector()) {
4957 MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
4958 MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
4959
4960 // Find the split point to split the parent mbb. At the same time copy all
4961 // physical registers used in the tail of parent mbb into virtual registers
4962 // before the split point and back into physical registers after the split
4963 // point. This prevents us needing to deal with Live-ins and many other
4964 // register allocation issues caused by us splitting the parent mbb. The
4965 // register allocator will clean up said virtual copies later on.
4966 MachineBasicBlock::iterator SplitPoint = findSplitPointForStackProtector(
4967 BB: ParentMBB, TII: *MF->getSubtarget().getInstrInfo());
4968
4969 // Splice the terminator of ParentMBB into SuccessMBB.
4970 SuccessMBB->splice(Where: SuccessMBB->end(), Other: ParentMBB, From: SplitPoint,
4971 To: ParentMBB->end());
4972
4973 // Add compare/jump on neq/jump to the parent BB.
4974 if (!emitSPDescriptorParent(SPD&: SPDescriptor, ParentBB: ParentMBB))
4975 return false;
4976
4977 // CodeGen Failure MBB if we have not codegened it yet.
4978 MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
4979 if (FailureMBB->empty()) {
4980 if (!emitSPDescriptorFailure(SPD&: SPDescriptor, FailureBB: FailureMBB))
4981 return false;
4982 }
4983
4984 // Clear the Per-BB State.
4985 SPDescriptor.resetPerBBState();
4986 }
4987 return true;
4988}
4989
4990bool IRTranslatorImpl::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
4991 MachineBasicBlock *ParentBB) {
4992 CurBuilder->setInsertPt(MBB&: *ParentBB, II: ParentBB->end());
4993 // First create the loads to the guard/stack slot for the comparison.
4994 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
4995 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
4996 LLT PtrMemTy = getLLTForMVT(Ty: TLI->getPointerMemTy(DL: *DL));
4997
4998 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
4999 int FI = MFI.getStackProtectorIndex();
5000
5001 Register Guard;
5002 Register StackSlotPtr = CurBuilder->buildFrameIndex(Res: PtrTy, Idx: FI).getReg(Idx: 0);
5003 const Module &M = *ParentBB->getParent()->getFunction().getParent();
5004 Align Align = DL->getPrefTypeAlign(Ty: PointerType::getUnqual(C&: M.getContext()));
5005
5006 // Generate code to load the content of the guard slot.
5007 Register GuardVal =
5008 CurBuilder
5009 ->buildLoad(Res: PtrMemTy, Addr: StackSlotPtr,
5010 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
5011 MMOFlags: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile)
5012 .getReg(Idx: 0);
5013
5014 // Retrieve guard check function, nullptr if instrumentation is inlined.
5015 if (const Function *GuardCheckFn = TLI->getSSPStackGuardCheck(M, Libcalls: *Libcalls)) {
5016 // This path is currently untestable on GlobalISel, since the only platform
5017 // that needs this seems to be Windows, and we fall back on that currently.
5018 // The code still lives here in case that changes.
5019 // Silence warning about unused variable until the code below that uses
5020 // 'GuardCheckFn' is enabled.
5021 (void)GuardCheckFn;
5022 return false;
5023#if 0
5024 // The target provides a guard check function to validate the guard value.
5025 // Generate a call to that function with the content of the guard slot as
5026 // argument.
5027 FunctionType *FnTy = GuardCheckFn->getFunctionType();
5028 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
5029 ISD::ArgFlagsTy Flags;
5030 if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
5031 Flags.setInReg();
5032 CallLowering::ArgInfo GuardArgInfo(
5033 {GuardVal, FnTy->getParamType(0), {Flags}});
5034
5035 CallLowering::CallLoweringInfo Info;
5036 Info.OrigArgs.push_back(GuardArgInfo);
5037 Info.CallConv = GuardCheckFn->getCallingConv();
5038 Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
5039 Info.OrigRet = {Register(), FnTy->getReturnType()};
5040 if (!CLI->lowerCall(MIRBuilder, Info)) {
5041 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
5042 return false;
5043 }
5044 return true;
5045#endif
5046 }
5047
5048 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
5049 // Otherwise, emit a volatile load to retrieve the stack guard value.
5050 if (TLI->useLoadStackGuardNode(M: *ParentBB->getBasicBlock()->getModule())) {
5051 Guard = MRI->createGenericVirtualRegister(Ty: PtrMemTy);
5052 getStackGuard(DstReg: Guard, MIRBuilder&: *CurBuilder);
5053 } else {
5054 // TODO: test using android subtarget when we support @llvm.thread.pointer.
5055 const Value *IRGuard = TLI->getSDagStackGuard(M, Libcalls: *Libcalls);
5056 if (IRGuard) {
5057 Register GuardPtr = getOrCreateVReg(Val: *IRGuard);
5058
5059 Guard = CurBuilder
5060 ->buildLoad(Res: PtrMemTy, Addr: GuardPtr,
5061 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
5062 MMOFlags: MachineMemOperand::MOLoad |
5063 MachineMemOperand::MOVolatile)
5064 .getReg(Idx: 0);
5065 } else {
5066 LLVMContext &Ctx = CurBuilder->getContext();
5067 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
5068 Guard = CurBuilder->buildUndef(Res: PtrMemTy).getReg(Idx: 0);
5069 }
5070 }
5071
5072 // Perform the comparison.
5073 auto Cmp =
5074 CurBuilder->buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::integer(SizeInBits: 1), Op0: Guard, Op1: GuardVal);
5075 // If the guard/stackslot do not equal, branch to failure MBB.
5076 CurBuilder->buildBrCond(Tst: Cmp, Dest&: *SPD.getFailureMBB());
5077 // Otherwise branch to success MBB.
5078 CurBuilder->buildBr(Dest&: *SPD.getSuccessMBB());
5079 return true;
5080}
5081
5082bool IRTranslatorImpl::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
5083 MachineBasicBlock *FailureBB) {
5084 const RTLIB::LibcallImpl LibcallImpl =
5085 Libcalls->getLibcallImpl(Call: RTLIB::STACKPROTECTOR_CHECK_FAIL);
5086 if (LibcallImpl == RTLIB::Unsupported)
5087 return false;
5088
5089 CurBuilder->setInsertPt(MBB&: *FailureBB, II: FailureBB->end());
5090
5091 CallLowering::CallLoweringInfo Info;
5092 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: LibcallImpl);
5093
5094 StringRef LibcallName =
5095 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LibcallImpl);
5096 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
5097 Info.OrigRet = {Register(), Type::getVoidTy(C&: MF->getFunction().getContext()),
5098 0};
5099 if (!CLI->lowerCall(MIRBuilder&: *CurBuilder, Info)) {
5100 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
5101 return false;
5102 }
5103
5104 // Emit a trap instruction if we are required to do so.
5105 const TargetOptions &TargetOpts = TLI->getTargetMachine().Options;
5106 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
5107 CurBuilder->buildInstr(Opcode: TargetOpcode::G_TRAP);
5108
5109 return true;
5110}
5111
5112void IRTranslatorImpl::finalizeFunction() {
5113 // Release the memory used by the different maps we
5114 // needed during the translation.
5115 PendingPHIs.clear();
5116 PendingConstDbgValues.clear();
5117 VMap.reset();
5118 FrameIndices.clear();
5119 MachinePreds.clear();
5120 // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
5121 // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
5122 // destroying it twice (in ~IRTranslator() and ~LLVMContext())
5123 EntryBuilder.reset();
5124 CurBuilder.reset();
5125 FuncInfo.clear();
5126 SPDescriptor.resetPerFunctionState();
5127}
5128
5129/// Returns true if a BasicBlock \p BB within a variadic function contains a
5130/// variadic musttail call.
5131static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
5132 if (!IsVarArg)
5133 return false;
5134
5135 // Walk the block backwards, because tail calls usually only appear at the end
5136 // of a block.
5137 return llvm::any_of(Range: llvm::reverse(C: BB), P: [](const Instruction &I) {
5138 const auto *CI = dyn_cast<CallInst>(Val: &I);
5139 return CI && CI->isMustTailCall();
5140 });
5141}
5142
5143bool IRTranslatorImpl::runOnMachineFunction(
5144 MachineFunction &CurMF, function_ref<GISelCSEInfo *()> GetCSEInfo,
5145 bool ShouldSkipOpts, function_ref<AAResults *()> GetAAResults,
5146 function_ref<BranchProbabilityInfo *()> GetBPI,
5147 function_ref<AssumptionCache *()> GetAC, TargetLibraryInfo *LibraryInfo,
5148 const LibcallLoweringInfo *LibcallInfo, SSPLayoutInfo *StackProtectorInfo) {
5149 MF = &CurMF;
5150 const Function &F = MF->getFunction();
5151 ORE = std::make_unique<OptimizationRemarkEmitter>(args: &F);
5152 CLI = MF->getSubtarget().getCallLowering();
5153 SPInfo = StackProtectorInfo;
5154
5155 if (CLI->fallBackToDAGISel(MF: *MF)) {
5156 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
5157 F.getSubprogram(), &F.getEntryBlock());
5158 R << "unable to lower function: "
5159 << ore::NV("Prototype", F.getFunctionType());
5160
5161 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5162 return false;
5163 }
5164
5165 // Set the CSEConfig and run the analysis.
5166 GISelCSEInfo *CSEInfo = nullptr;
5167
5168 bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
5169 ? EnableCSEInIRTranslator
5170 : true;
5171
5172 const TargetSubtargetInfo &Subtarget = MF->getSubtarget();
5173 TLI = Subtarget.getTargetLowering();
5174
5175 if (EnableCSE) {
5176 EntryBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
5177 CSEInfo = GetCSEInfo();
5178 EntryBuilder->setCSEInfo(CSEInfo);
5179 CurBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
5180 CurBuilder->setCSEInfo(CSEInfo);
5181 } else {
5182 EntryBuilder = std::make_unique<MachineIRBuilder>();
5183 CurBuilder = std::make_unique<MachineIRBuilder>();
5184 }
5185 CLI = Subtarget.getCallLowering();
5186 CurBuilder->setMF(*MF);
5187 EntryBuilder->setMF(*MF);
5188 MRI = &MF->getRegInfo();
5189 DL = &F.getDataLayout();
5190 const TargetMachine &TM = MF->getTarget();
5191 EnableOpts = OptLevel != CodeGenOptLevel::None && !ShouldSkipOpts;
5192 FuncInfo.MF = MF;
5193 // Prefer the "exception-model" module flag, else the TargetOptions default.
5194 FuncInfo.ExceptionModel = F.getParent()->getExceptionModel();
5195 if (FuncInfo.ExceptionModel == ExceptionHandling::Default)
5196 FuncInfo.ExceptionModel = TM.getExceptionModel();
5197 if (EnableOpts) {
5198 AA = GetAAResults();
5199 FuncInfo.BPI = GetBPI();
5200 AC = GetAC();
5201 } else {
5202 AA = nullptr;
5203 FuncInfo.BPI = nullptr;
5204 AC = nullptr;
5205 }
5206 LibInfo = LibraryInfo;
5207 Libcalls = LibcallInfo;
5208
5209 FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(MF&: *MF);
5210
5211 SL = std::make_unique<GISelSwitchLowering>(args: this, args&: FuncInfo);
5212 SL->init(tli: *TLI, tm: TM, dl: *DL);
5213
5214 assert(PendingPHIs.empty() && "stale PHIs");
5215
5216 // Targets which want to use big endian can enable it using
5217 // enableBigEndian()
5218 if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
5219 // Currently we don't properly handle big endian code.
5220 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
5221 F.getSubprogram(), &F.getEntryBlock());
5222 R << "unable to translate in big endian mode";
5223 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5224 return false;
5225 }
5226
5227 // Release the per-function state when we return, whether we succeeded or not.
5228 llvm::scope_exit FinalizeOnReturn([this]() { finalizeFunction(); });
5229
5230 // Setup a separate basic-block for the arguments and constants
5231 MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
5232 MF->push_back(MBB: EntryBB);
5233 EntryBuilder->setMBB(*EntryBB);
5234
5235 DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHIIt()->getDebugLoc();
5236 SwiftError.setFunction(CurMF);
5237 SwiftError.createEntriesInEntryBlock(DbgLoc);
5238
5239 bool IsVarArg = F.isVarArg();
5240 bool HasMustTailInVarArgFn = false;
5241 // Use arguments and instructions to estimate the number of mapped values and
5242 // virtual registers.
5243 unsigned NumValues = F.arg_size();
5244
5245 // Create all blocks, in IR order, to preserve the layout.
5246 FuncInfo.MBBMap.resize(N: F.getMaxBlockNumber());
5247 for (const BasicBlock &BB: F) {
5248 NumValues += BB.size();
5249 auto *&MBB = FuncInfo.MBBMap[BB.getNumber()];
5250
5251 MBB = MF->CreateMachineBasicBlock(BB: &BB);
5252 MF->push_back(MBB);
5253
5254 // Only mark the block if the BlockAddress actually has users. The
5255 // hasAddressTaken flag may be stale if the BlockAddress was optimized away
5256 // but the constant still exists in the uniquing table.
5257 if (BB.hasAddressTaken()) {
5258 if (BlockAddress *BA = BlockAddress::lookup(BB: &BB))
5259 if (!BA->hasZeroLiveUses())
5260 MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
5261 }
5262
5263 if (!HasMustTailInVarArgFn)
5264 HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
5265 }
5266
5267 VMap.reserveVRegs(NumValues);
5268 MRI->reserveVirtRegs(NumVirtRegs: NumValues);
5269
5270 MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
5271
5272 // Make our arguments/constants entry block fallthrough to the IR entry block.
5273 EntryBB->addSuccessor(Succ: &getMBB(BB: F.front()));
5274
5275 // Lower the actual args into this basic block.
5276 SmallVector<ArrayRef<Register>, 8> VRegArgs;
5277 for (const Argument &Arg: F.args()) {
5278 if (DL->getTypeStoreSize(Ty: Arg.getType()).isZero())
5279 continue; // Don't handle zero sized types.
5280 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: Arg);
5281 VRegArgs.push_back(Elt: VRegs);
5282
5283 if (CLI->supportSwiftError() && Arg.hasSwiftErrorAttr()) {
5284 assert(VRegs.size() == 1 && "Too many vregs for Swift error");
5285 SwiftError.setCurrentVReg(MBB: EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
5286 }
5287 }
5288
5289 if (!CLI->lowerFormalArguments(MIRBuilder&: *EntryBuilder, F, VRegs: VRegArgs, FLI&: FuncInfo)) {
5290 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
5291 F.getSubprogram(), &F.getEntryBlock());
5292 R << "unable to lower arguments: "
5293 << ore::NV("Prototype", F.getFunctionType());
5294 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5295 return false;
5296 }
5297
5298 // Need to visit defs before uses when translating instructions.
5299 GISelObserverWrapper WrapperObserver;
5300 if (EnableCSE && CSEInfo)
5301 WrapperObserver.addObserver(O: CSEInfo);
5302 {
5303 ReversePostOrderTraversal<const Function *> RPOT(&F);
5304#ifndef NDEBUG
5305 DILocationVerifier Verifier;
5306 WrapperObserver.addObserver(&Verifier);
5307#endif // ifndef NDEBUG
5308 RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);
5309 for (const BasicBlock *BB : RPOT) {
5310 MachineBasicBlock &MBB = getMBB(BB: *BB);
5311 // Set the insertion point of all the following translations to
5312 // the end of this basic block.
5313 CurBuilder->setMBB(MBB);
5314 HasTailCall = false;
5315 for (const Instruction &Inst : *BB) {
5316 // If we translated a tail call in the last step, then we know
5317 // everything after the call is either a return, or something that is
5318 // handled by the call itself. (E.g. a lifetime marker or assume
5319 // intrinsic.) In this case, we should stop translating the block and
5320 // move on.
5321 if (HasTailCall)
5322 break;
5323#ifndef NDEBUG
5324 Verifier.setCurrentInst(&Inst);
5325#endif // ifndef NDEBUG
5326
5327 // Translate any debug-info attached to the instruction.
5328 translateDbgInfo(Inst, MIRBuilder&: *CurBuilder);
5329
5330 if (translate(Inst))
5331 continue;
5332
5333 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
5334 Inst.getDebugLoc(), BB);
5335 R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
5336
5337 if (ORE->allowExtraAnalysis(PassName: "gisel-ir-translator")) {
5338 std::string InstStrStorage;
5339 raw_string_ostream InstStr(InstStrStorage);
5340 InstStr << Inst;
5341
5342 R << ": '" << InstStrStorage << "'";
5343 }
5344
5345 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5346 return false;
5347 }
5348
5349 if (!finalizeBasicBlock(BB: *BB, MBB)) {
5350 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
5351 BB->getTerminator()->getDebugLoc(), BB);
5352 R << "unable to translate basic block";
5353 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5354 return false;
5355 }
5356 }
5357#ifndef NDEBUG
5358 WrapperObserver.removeObserver(&Verifier);
5359#endif
5360 }
5361
5362 finishPendingPhis();
5363 finishPendingConstDbgValues();
5364
5365 SwiftError.propagateVRegs();
5366
5367 // Merge the argument lowering and constants block with its single
5368 // successor, the LLVM-IR entry block. We want the basic block to
5369 // be maximal.
5370 assert(EntryBB->succ_size() == 1 &&
5371 "Custom BB used for lowering should have only one successor");
5372 // Get the successor of the current entry block.
5373 MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
5374 assert(NewEntryBB.pred_size() == 1 &&
5375 "LLVM-IR entry block has a predecessor!?");
5376 // Move all the instruction from the current entry block to the
5377 // new entry block.
5378 NewEntryBB.splice(Where: NewEntryBB.begin(), Other: EntryBB, From: EntryBB->begin(),
5379 To: EntryBB->end());
5380
5381 // Update the live-in information for the new entry block.
5382 for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
5383 NewEntryBB.addLiveIn(RegMaskPair: LiveIn);
5384 NewEntryBB.sortUniqueLiveIns();
5385
5386 // Get rid of the now empty basic block.
5387 EntryBB->removeSuccessor(Succ: &NewEntryBB);
5388 MF->remove(MBBI: EntryBB);
5389 MF->deleteMachineBasicBlock(MBB: EntryBB);
5390
5391 assert(&MF->front() == &NewEntryBB &&
5392 "New entry wasn't next in the list of basic block!");
5393
5394 // Initialize stack protector information.
5395 SPInfo->copyToMachineFrameInfo(MFI&: MF->getFrameInfo());
5396
5397 return false;
5398}
5399
5400bool IRTranslatorLegacy::runOnMachineFunction(MachineFunction &MF) {
5401 const TargetSubtargetInfo &Subtarget = MF.getSubtarget();
5402 Function &F = MF.getFunction();
5403
5404 bool ShouldSkipOpts = skipFunction(F: MF.getFunction());
5405 return Impl->runOnMachineFunction(
5406 CurMF&: MF,
5407 GetCSEInfo: [&]() {
5408 TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
5409 GISelCSEAnalysisWrapper &Wrapper =
5410 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
5411 return &Wrapper.get(CSEOpt: TPC.getCSEConfig());
5412 },
5413 ShouldSkipOpts,
5414 GetAAResults: [&]() { return &getAnalysis<AAResultsWrapperPass>().getAAResults(); },
5415 GetBPI: [&]() {
5416 return &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
5417 },
5418 GetAC: [&]() {
5419 return &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
5420 F&: MF.getFunction());
5421 },
5422 LibraryInfo: &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F: MF.getFunction()),
5423 LibcallInfo: &getAnalysis<LibcallLoweringInfoWrapper>().getLibcallLowering(
5424 M: *F.getParent(), Subtarget),
5425 StackProtectorInfo: &getAnalysis<StackProtector>().getLayoutInfo());
5426}
5427
5428IRTranslatorPass::IRTranslatorPass(CodeGenOptLevel OptLevel)
5429 : Impl(std::make_unique<IRTranslatorImpl>(args&: OptLevel)) {}
5430
5431IRTranslatorPass::~IRTranslatorPass() = default;
5432IRTranslatorPass::IRTranslatorPass(IRTranslatorPass &&) = default;
5433
5434PreservedAnalyses IRTranslatorPass::run(MachineFunction &MF,
5435 MachineFunctionAnalysisManager &MFAM) {
5436 const TargetSubtargetInfo &Subtarget = MF.getSubtarget();
5437 Function &F = MF.getFunction();
5438
5439 bool ShouldSkipOpts = MF.getFunction().hasOptNone() ||
5440 shouldSkipOptimizationForOptBisect(IR: MF.getFunction());
5441 auto &FAM = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
5442 .getManager();
5443 auto &MAMProxy =
5444 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF);
5445 const ModuleLibcallLoweringInfo *MLLI =
5446 MAMProxy.getCachedResult<LibcallLoweringModuleAnalysis>(IR&: *F.getParent());
5447 if (!MLLI)
5448 reportFatalUsageError(
5449 reason: "LibcallLoweringModuleAnalysis must be available for IRTranslator");
5450 Impl->runOnMachineFunction(
5451 CurMF&: MF, GetCSEInfo: [&]() { return MFAM.getResult<GISelCSEAnalysis>(IR&: MF).get(); },
5452 ShouldSkipOpts, GetAAResults: [&]() { return &FAM.getResult<AAManager>(IR&: F); },
5453 GetBPI: [&]() { return &FAM.getResult<BranchProbabilityAnalysis>(IR&: F); },
5454 GetAC: [&]() { return &FAM.getResult<AssumptionAnalysis>(IR&: F); },
5455 LibraryInfo: &FAM.getResult<TargetLibraryAnalysis>(IR&: F),
5456 LibcallInfo: &getLibcallLowering(ModuleInfo: *MLLI, Subtarget),
5457 StackProtectorInfo: &FAM.getResult<SSPLayoutAnalysis>(IR&: F));
5458
5459 return getMachineFunctionPassPreservedAnalyses();
5460}
5461