1//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the IRTranslator class.
10//===----------------------------------------------------------------------===//
11
12#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
13#include "llvm/ADT/PostOrderIterator.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/Analysis/AliasAnalysis.h"
18#include "llvm/Analysis/AssumptionCache.h"
19#include "llvm/Analysis/BranchProbabilityInfo.h"
20#include "llvm/Analysis/Loads.h"
21#include "llvm/Analysis/OptimizationRemarkEmitter.h"
22#include "llvm/Analysis/ValueTracking.h"
23#include "llvm/Analysis/VectorUtils.h"
24#include "llvm/CodeGen/Analysis.h"
25#include "llvm/CodeGen/CodeGenCommonISel.h"
26#include "llvm/CodeGen/FunctionLoweringInfo.h"
27#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
28#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
29#include "llvm/CodeGen/GlobalISel/CallLowering.h"
30#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
31#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
32#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
33#include "llvm/CodeGen/LowLevelTypeUtils.h"
34#include "llvm/CodeGen/MachineBasicBlock.h"
35#include "llvm/CodeGen/MachineFrameInfo.h"
36#include "llvm/CodeGen/MachineFunction.h"
37#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
38#include "llvm/CodeGen/MachineInstrBuilder.h"
39#include "llvm/CodeGen/MachineMemOperand.h"
40#include "llvm/CodeGen/MachineModuleInfo.h"
41#include "llvm/CodeGen/MachineOperand.h"
42#include "llvm/CodeGen/MachinePassManager.h"
43#include "llvm/CodeGen/MachineRegisterInfo.h"
44#include "llvm/CodeGen/StackProtector.h"
45#include "llvm/CodeGen/SwiftErrorValueTracking.h"
46#include "llvm/CodeGen/SwitchLoweringUtils.h"
47#include "llvm/CodeGen/TargetFrameLowering.h"
48#include "llvm/CodeGen/TargetInstrInfo.h"
49#include "llvm/CodeGen/TargetLowering.h"
50#include "llvm/CodeGen/TargetOpcodes.h"
51#include "llvm/CodeGen/TargetPassConfig.h"
52#include "llvm/CodeGen/TargetRegisterInfo.h"
53#include "llvm/CodeGen/TargetSubtargetInfo.h"
54#include "llvm/CodeGenTypes/LowLevelType.h"
55#include "llvm/IR/Analysis.h"
56#include "llvm/IR/BasicBlock.h"
57#include "llvm/IR/CFG.h"
58#include "llvm/IR/Constant.h"
59#include "llvm/IR/Constants.h"
60#include "llvm/IR/DataLayout.h"
61#include "llvm/IR/DerivedTypes.h"
62#include "llvm/IR/DiagnosticInfo.h"
63#include "llvm/IR/Function.h"
64#include "llvm/IR/GetElementPtrTypeIterator.h"
65#include "llvm/IR/InlineAsm.h"
66#include "llvm/IR/InstrTypes.h"
67#include "llvm/IR/Instructions.h"
68#include "llvm/IR/IntrinsicInst.h"
69#include "llvm/IR/Intrinsics.h"
70#include "llvm/IR/IntrinsicsAMDGPU.h"
71#include "llvm/IR/LLVMContext.h"
72#include "llvm/IR/Metadata.h"
73#include "llvm/IR/Module.h"
74#include "llvm/IR/PatternMatch.h"
75#include "llvm/IR/Statepoint.h"
76#include "llvm/IR/Type.h"
77#include "llvm/IR/User.h"
78#include "llvm/IR/Value.h"
79#include "llvm/InitializePasses.h"
80#include "llvm/MC/MCContext.h"
81#include "llvm/Pass.h"
82#include "llvm/Support/Casting.h"
83#include "llvm/Support/CodeGen.h"
84#include "llvm/Support/Debug.h"
85#include "llvm/Support/ErrorHandling.h"
86#include "llvm/Support/MathExtras.h"
87#include "llvm/Support/raw_ostream.h"
88#include "llvm/Target/TargetMachine.h"
89#include "llvm/Transforms/Utils/Local.h"
90#include "llvm/Transforms/Utils/MemoryOpRemark.h"
91#include <algorithm>
92#include <cassert>
93#include <cstdint>
94#include <iterator>
95#include <optional>
96#include <string>
97#include <utility>
98#include <vector>
99
100#define DEBUG_TYPE "ir-translator"
101
102using namespace llvm;
103
104static cl::opt<bool>
105 EnableCSEInIRTranslator("enable-cse-in-ir-translator",
106 cl::desc("Should enable CSE in ir-translator"),
107 cl::init(Val: false));
108
109namespace llvm {
110
111class IRTranslatorImpl {
112 /// Interface used to lower the everything related to calls.
113 const CallLowering *CLI = nullptr;
114
115 SSPLayoutInfo *SPInfo = nullptr;
116
117 /// This class contains the mapping between the Values to vreg related data.
118 class ValueToVRegInfo {
119 public:
120 ValueToVRegInfo() = default;
121
122 using VRegListT = SmallVector<Register, 1>;
123 using OffsetListT = SmallVector<uint64_t, 1>;
124
125 using const_vreg_iterator =
126 DenseMap<const Value *, VRegListT *>::const_iterator;
127 using const_offset_iterator =
128 DenseMap<const Value *, OffsetListT *>::const_iterator;
129
130 inline const_vreg_iterator vregs_end() const { return ValToVRegs.end(); }
131
132 VRegListT *getVRegs(const Value &V) {
133 auto [It, Inserted] = ValToVRegs.try_emplace(Key: &V);
134 if (!Inserted)
135 return It->second;
136
137 // We placement new using our fast allocator since we never try to free
138 // the vectors until translation is finished.
139 It->second = new (VRegAlloc.Allocate()) VRegListT();
140 return It->second;
141 }
142
143 OffsetListT *getOffsets(const Value &V) {
144 assert(V.getType()->isAggregateType() &&
145 "Offsets are for aggregate values");
146 auto [It, Inserted] = TypeToOffsets.try_emplace(Key: V.getType());
147 if (!Inserted)
148 return It->second;
149
150 It->second = new (OffsetAlloc.Allocate()) OffsetListT();
151 return It->second;
152 }
153
154 const_vreg_iterator findVRegs(const Value &V) const {
155 return ValToVRegs.find(Val: &V);
156 }
157
158 bool contains(const Value &V) const { return ValToVRegs.contains(Val: &V); }
159
160 void reserveVRegs(unsigned NumValues) { ValToVRegs.reserve(NumEntries: NumValues); }
161
162 void reset() {
163 ValToVRegs.clear();
164 TypeToOffsets.clear();
165 VRegAlloc.DestroyAll();
166 OffsetAlloc.DestroyAll();
167 }
168
169 private:
170 SpecificBumpPtrAllocator<VRegListT> VRegAlloc;
171 SpecificBumpPtrAllocator<OffsetListT> OffsetAlloc;
172
173 // We store pointers to vectors here since references may be invalidated
174 // while we hold them if we stored the vectors directly.
175 DenseMap<const Value *, VRegListT *> ValToVRegs;
176 DenseMap<const Type *, OffsetListT *> TypeToOffsets;
177 };
178
179 /// Mapping of the values of the current LLVM IR function to the related
180 /// virtual registers and offsets.
181 ValueToVRegInfo VMap;
182
183 // One BasicBlock can be translated to multiple MachineBasicBlocks. For such
184 // BasicBlocks translated to multiple MachineBasicBlocks, MachinePreds retains
185 // a mapping between the edges arriving at the BasicBlock to the corresponding
186 // created MachineBasicBlocks. Some BasicBlocks that get translated to a
187 // single MachineBasicBlock may also end up in this Map.
188 using CFGEdge = std::pair<const BasicBlock *, const BasicBlock *>;
189 DenseMap<CFGEdge, SmallVector<MachineBasicBlock *, 1>> MachinePreds;
190
191 // List of stubbed PHI instructions, for values and basic blocks to be filled
192 // in once all MachineBasicBlocks have been created.
193 SmallVector<std::pair<const PHINode *, SmallVector<MachineInstr *, 1>>, 4>
194 PendingPHIs;
195
196 /// Record of what frame index has been allocated to specified allocas for
197 /// this function.
198 DenseMap<const AllocaInst *, int> FrameIndices;
199
200 SwiftErrorValueTracking SwiftError;
201
202 /// \name Methods for translating form LLVM IR to MachineInstr.
203 /// \see ::translate for general information on the translate methods.
204 /// @{
205
206 /// Translate \p Inst into its corresponding MachineInstr instruction(s).
207 /// Insert the newly translated instruction(s) right where the CurBuilder
208 /// is set.
209 ///
210 /// The general algorithm is:
211 /// 1. Look for a virtual register for each operand or
212 /// create one.
213 /// 2 Update the VMap accordingly.
214 /// 2.alt. For constant arguments, if they are compile time constants,
215 /// produce an immediate in the right operand and do not touch
216 /// ValToReg. Actually we will go with a virtual register for each
217 /// constants because it may be expensive to actually materialize the
218 /// constant. Moreover, if the constant spans on several instructions,
219 /// CSE may not catch them.
220 /// => Update ValToVReg and remember that we saw a constant in Constants.
221 /// We will materialize all the constants in finalize.
222 /// Note: we would need to do something so that we can recognize such operand
223 /// as constants.
224 /// 3. Create the generic instruction.
225 ///
226 /// \return true if the translation succeeded.
227 bool translate(const Instruction &Inst);
228
229 /// Materialize \p C into virtual-register \p Reg. The generic instructions
230 /// performing this materialization will be inserted into the entry block of
231 /// the function.
232 ///
233 /// \return true if the materialization succeeded.
234 bool translate(const Constant &C, Register Reg);
235
236 /// Examine any debug-info attached to the instruction (in the form of
237 /// DbgRecords) and translate it.
238 void translateDbgInfo(const Instruction &Inst, MachineIRBuilder &MIRBuilder);
239
240 /// Translate a debug-info record of a dbg.value into a DBG_* instruction.
241 /// Pass in all the contents of the record, rather than relying on how it's
242 /// stored.
243 void translateDbgValueRecord(Value *V, bool HasArgList,
244 const DILocalVariable *Variable,
245 const DIExpression *Expression,
246 const DebugLoc &DL,
247 MachineIRBuilder &MIRBuilder);
248
249 /// Translate a debug-info record of a dbg.declare into an indirect DBG_*
250 /// instruction. Pass in all the contents of the record, rather than relying
251 /// on how it's stored.
252 void translateDbgDeclareRecord(Value *Address, bool HasArgList,
253 const DILocalVariable *Variable,
254 const DIExpression *Expression,
255 const DebugLoc &DL,
256 MachineIRBuilder &MIRBuilder);
257
258 // Translate U as a copy of V.
259 bool translateCopy(const User &U, const Value &V,
260 MachineIRBuilder &MIRBuilder);
261 bool translateCopy(const User &U, Register Src, MachineIRBuilder &MIRBuilder);
262
263 /// Translate an LLVM bitcast into generic IR. Either a COPY or a G_BITCAST is
264 /// emitted.
265 bool translateBitCast(const User &U, MachineIRBuilder &MIRBuilder);
266
267 /// Translate an LLVM load instruction into generic IR.
268 bool translateLoad(const User &U, MachineIRBuilder &MIRBuilder);
269
270 /// Translate an LLVM store instruction into generic IR.
271 bool translateStore(const User &U, MachineIRBuilder &MIRBuilder);
272
273 /// Translate an LLVM string intrinsic (memcpy, memset, ...).
274 bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder,
275 unsigned Opcode);
276
277 /// Translate an LLVM trap intrinsic (trap, debugtrap, ubsantrap).
278 bool translateTrap(const CallInst &U, MachineIRBuilder &MIRBuilder,
279 unsigned Opcode);
280
281 // Translate @llvm.vector.interleave2 and
282 // @llvm.vector.deinterleave2 intrinsics for fixed-width vector
283 // types into vector shuffles.
284 bool translateVectorInterleave2Intrinsic(const CallInst &CI,
285 MachineIRBuilder &MIRBuilder);
286 bool translateVectorDeinterleave2Intrinsic(const CallInst &CI,
287 MachineIRBuilder &MIRBuilder);
288
289 void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder);
290
291 bool translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
292 MachineIRBuilder &MIRBuilder);
293 bool translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
294 MachineIRBuilder &MIRBuilder);
295
296 /// Helper function for translateSimpleIntrinsic.
297 /// \return The generic opcode for \p IntrinsicID if \p IntrinsicID is a
298 /// simple intrinsic (ceil, fabs, etc.). Otherwise, returns
299 /// Intrinsic::not_intrinsic.
300 unsigned getSimpleIntrinsicOpcode(Intrinsic::ID ID);
301
302 /// Translates the intrinsics defined in getSimpleIntrinsicOpcode.
303 /// \return true if the translation succeeded.
304 bool translateSimpleIntrinsic(const CallInst &CI, Intrinsic::ID ID,
305 MachineIRBuilder &MIRBuilder);
306
307 bool translateConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI,
308 MachineIRBuilder &MIRBuilder);
309
310 bool translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
311 MachineIRBuilder &MIRBuilder);
312
313 /// Returns the single livein physical register Arg was lowered to, if
314 /// possible.
315 std::optional<MCRegister> getArgPhysReg(Argument &Arg);
316
317 /// If debug-info targets an Argument and its expression is an EntryValue,
318 /// lower it as either an entry in the MF debug table (dbg.declare), or a
319 /// DBG_VALUE targeting the corresponding livein register for that Argument
320 /// (dbg.value).
321 bool translateIfEntryValueArgument(bool isDeclare, Value *Arg,
322 const DILocalVariable *Var,
323 const DIExpression *Expr,
324 const DebugLoc &DL,
325 MachineIRBuilder &MIRBuilder);
326
327 bool translateInlineAsm(const CallBase &CB, MachineIRBuilder &MIRBuilder);
328
329 /// Common code for translating normal calls or invokes.
330 bool translateCallBase(const CallBase &CB, MachineIRBuilder &MIRBuilder);
331
332 /// Translate call instruction.
333 /// \pre \p U is a call instruction.
334 bool translateCall(const User &U, MachineIRBuilder &MIRBuilder);
335
336 bool translateIntrinsic(
337 const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
338 ArrayRef<TargetLowering::IntrinsicInfo> TgtMemIntrinsicInfos = {});
339
340 /// Report an intrinsic the subtarget does not support and define its results
341 /// with G_IMPLICIT_DEF. Prevents creating a malformed MIR.
342 bool handleUnsupportedIntrinsic(const CallBase &CB, Intrinsic::ID ID,
343 MachineIRBuilder &MIRBuilder);
344
345 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
346 /// many places it could ultimately go. In the IR, we have a single unwind
347 /// destination, but in the machine CFG, we enumerate all the possible blocks.
348 /// This function skips over imaginary basic blocks that hold catchswitch
349 /// instructions, and finds all the "real" machine
350 /// basic block destinations. As those destinations may not be successors of
351 /// EHPadBB, here we also calculate the edge probability to those
352 /// destinations. The passed-in Prob is the edge probability to EHPadBB.
353 bool findUnwindDestinations(
354 const BasicBlock *EHPadBB, BranchProbability Prob,
355 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
356 &UnwindDests);
357
358 bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder);
359
360 bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder);
361
362 bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder);
363
364 /// Translate one of LLVM's cast instructions into MachineInstrs, with the
365 /// given generic Opcode.
366 bool translateCast(unsigned Opcode, const User &U,
367 MachineIRBuilder &MIRBuilder);
368
369 /// Translate a phi instruction.
370 bool translatePHI(const User &U, MachineIRBuilder &MIRBuilder);
371
372 /// Translate a comparison (icmp or fcmp) instruction or constant.
373 bool translateCompare(const User &U, MachineIRBuilder &MIRBuilder);
374
375 /// Translate an integer compare instruction (or constant).
376 bool translateICmp(const User &U, MachineIRBuilder &MIRBuilder) {
377 return translateCompare(U, MIRBuilder);
378 }
379
380 /// Translate a floating-point compare instruction (or constant).
381 bool translateFCmp(const User &U, MachineIRBuilder &MIRBuilder) {
382 return translateCompare(U, MIRBuilder);
383 }
384
385 /// Add remaining operands onto phis we've translated. Executed after all
386 /// MachineBasicBlocks for the function have been created.
387 void finishPendingPhis();
388
389 /// Translate \p Inst into a unary operation \p Opcode.
390 /// \pre \p U is a unary operation.
391 bool translateUnaryOp(unsigned Opcode, const User &U,
392 MachineIRBuilder &MIRBuilder);
393
394 /// Translate \p Inst into a binary operation \p Opcode.
395 /// \pre \p U is a binary operation.
396 bool translateBinaryOp(unsigned Opcode, const User &U,
397 MachineIRBuilder &MIRBuilder);
398
399 /// If the set of cases should be emitted as a series of branches, return
400 /// true. If we should emit this as a bunch of and/or'd together conditions,
401 /// return false.
402 bool shouldEmitAsBranches(const std::vector<SwitchCG::CaseBlock> &Cases);
403 /// Helper method for findMergedConditions.
404 /// This function emits a branch and is used at the leaves of an OR or an
405 /// AND operator tree.
406 void emitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB,
407 MachineBasicBlock *FBB,
408 MachineBasicBlock *CurBB,
409 MachineBasicBlock *SwitchBB,
410 BranchProbability TProb,
411 BranchProbability FProb, bool InvertCond);
412 /// Used during condbr translation to find trees of conditions that can be
413 /// optimized.
414 void findMergedConditions(const Value *Cond, MachineBasicBlock *TBB,
415 MachineBasicBlock *FBB, MachineBasicBlock *CurBB,
416 MachineBasicBlock *SwitchBB,
417 Instruction::BinaryOps Opc, BranchProbability TProb,
418 BranchProbability FProb, bool InvertCond);
419
420 /// Translate branch (br) instruction.
421 /// \pre \p U is a branch instruction.
422 bool translateUncondBr(const User &U, MachineIRBuilder &MIRBuilder);
423 bool translateCondBr(const User &U, MachineIRBuilder &MIRBuilder);
424
425 // Begin switch lowering functions.
426 bool emitJumpTableHeader(SwitchCG::JumpTable &JT,
427 SwitchCG::JumpTableHeader &JTH,
428 MachineBasicBlock *HeaderBB);
429 void emitJumpTable(SwitchCG::JumpTable &JT, MachineBasicBlock *MBB);
430
431 void emitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB,
432 MachineIRBuilder &MIB);
433
434 /// Generate for the BitTest header block, which precedes each sequence of
435 /// BitTestCases.
436 void emitBitTestHeader(SwitchCG::BitTestBlock &BTB,
437 MachineBasicBlock *SwitchMBB);
438 /// Generate code to produces one "bit test" for a given BitTestCase \p B.
439 void emitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB,
440 BranchProbability BranchProbToNext, Register Reg,
441 SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB);
442
443 void splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
444 const SwitchCG::SwitchWorkListItem &W, Value *Cond,
445 MachineBasicBlock *SwitchMBB, MachineIRBuilder &MIB);
446
447 bool lowerJumpTableWorkItem(
448 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
449 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
450 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
451 BranchProbability UnhandledProbs, SwitchCG::CaseClusterIt I,
452 MachineBasicBlock *Fallthrough, bool FallthroughUnreachable);
453
454 bool lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I, Value *Cond,
455 MachineBasicBlock *Fallthrough,
456 bool FallthroughUnreachable,
457 BranchProbability UnhandledProbs,
458 MachineBasicBlock *CurMBB,
459 MachineIRBuilder &MIB,
460 MachineBasicBlock *SwitchMBB);
461
462 bool lowerBitTestWorkItem(
463 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
464 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
465 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
466 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
467 SwitchCG::CaseClusterIt I, MachineBasicBlock *Fallthrough,
468 bool FallthroughUnreachable);
469
470 bool lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W, Value *Cond,
471 MachineBasicBlock *SwitchMBB,
472 MachineBasicBlock *DefaultMBB,
473 MachineIRBuilder &MIB);
474
475 bool translateSwitch(const User &U, MachineIRBuilder &MIRBuilder);
476 // End switch lowering section.
477
478 bool translateIndirectBr(const User &U, MachineIRBuilder &MIRBuilder);
479
480 bool translateExtractValue(const User &U, MachineIRBuilder &MIRBuilder);
481
482 bool translateInsertValue(const User &U, MachineIRBuilder &MIRBuilder);
483
484 bool translateSelect(const User &U, MachineIRBuilder &MIRBuilder);
485
486 bool translateGetElementPtr(const User &U, MachineIRBuilder &MIRBuilder);
487
488 bool translateAlloca(const User &U, MachineIRBuilder &MIRBuilder);
489
490 /// Translate return (ret) instruction.
491 /// The target needs to implement CallLowering::lowerReturn for
492 /// this to succeed.
493 /// \pre \p U is a return instruction.
494 bool translateRet(const User &U, MachineIRBuilder &MIRBuilder);
495
496 bool translateFNeg(const User &U, MachineIRBuilder &MIRBuilder);
497
498 bool translateAdd(const User &U, MachineIRBuilder &MIRBuilder) {
499 return translateBinaryOp(Opcode: TargetOpcode::G_ADD, U, MIRBuilder);
500 }
501 bool translateSub(const User &U, MachineIRBuilder &MIRBuilder) {
502 return translateBinaryOp(Opcode: TargetOpcode::G_SUB, U, MIRBuilder);
503 }
504 bool translateAnd(const User &U, MachineIRBuilder &MIRBuilder) {
505 return translateBinaryOp(Opcode: TargetOpcode::G_AND, U, MIRBuilder);
506 }
507 bool translateMul(const User &U, MachineIRBuilder &MIRBuilder) {
508 return translateBinaryOp(Opcode: TargetOpcode::G_MUL, U, MIRBuilder);
509 }
510 bool translateOr(const User &U, MachineIRBuilder &MIRBuilder) {
511 return translateBinaryOp(Opcode: TargetOpcode::G_OR, U, MIRBuilder);
512 }
513 bool translateXor(const User &U, MachineIRBuilder &MIRBuilder) {
514 return translateBinaryOp(Opcode: TargetOpcode::G_XOR, U, MIRBuilder);
515 }
516
517 bool translateUDiv(const User &U, MachineIRBuilder &MIRBuilder) {
518 return translateBinaryOp(Opcode: TargetOpcode::G_UDIV, U, MIRBuilder);
519 }
520 bool translateSDiv(const User &U, MachineIRBuilder &MIRBuilder) {
521 return translateBinaryOp(Opcode: TargetOpcode::G_SDIV, U, MIRBuilder);
522 }
523 bool translateURem(const User &U, MachineIRBuilder &MIRBuilder) {
524 return translateBinaryOp(Opcode: TargetOpcode::G_UREM, U, MIRBuilder);
525 }
526 bool translateSRem(const User &U, MachineIRBuilder &MIRBuilder) {
527 return translateBinaryOp(Opcode: TargetOpcode::G_SREM, U, MIRBuilder);
528 }
529 bool translateIntToPtr(const User &U, MachineIRBuilder &MIRBuilder) {
530 return translateCast(Opcode: TargetOpcode::G_INTTOPTR, U, MIRBuilder);
531 }
532 bool translatePtrToInt(const User &U, MachineIRBuilder &MIRBuilder) {
533 return translateCast(Opcode: TargetOpcode::G_PTRTOINT, U, MIRBuilder);
534 }
535 bool translatePtrToAddr(const User &U, MachineIRBuilder &MIRBuilder) {
536 // FIXME: this is not correct for pointers with addr width != pointer width
537 return translatePtrToInt(U, MIRBuilder);
538 }
539 bool translateTrunc(const User &U, MachineIRBuilder &MIRBuilder) {
540 return translateCast(Opcode: TargetOpcode::G_TRUNC, U, MIRBuilder);
541 }
542 bool translateFPTrunc(const User &U, MachineIRBuilder &MIRBuilder) {
543 return translateCast(Opcode: TargetOpcode::G_FPTRUNC, U, MIRBuilder);
544 }
545 bool translateFPExt(const User &U, MachineIRBuilder &MIRBuilder) {
546 return translateCast(Opcode: TargetOpcode::G_FPEXT, U, MIRBuilder);
547 }
548 bool translateFPToUI(const User &U, MachineIRBuilder &MIRBuilder) {
549 return translateCast(Opcode: TargetOpcode::G_FPTOUI, U, MIRBuilder);
550 }
551 bool translateFPToSI(const User &U, MachineIRBuilder &MIRBuilder) {
552 return translateCast(Opcode: TargetOpcode::G_FPTOSI, U, MIRBuilder);
553 }
554 bool translateUIToFP(const User &U, MachineIRBuilder &MIRBuilder) {
555 return translateCast(Opcode: TargetOpcode::G_UITOFP, U, MIRBuilder);
556 }
557 bool translateSIToFP(const User &U, MachineIRBuilder &MIRBuilder) {
558 return translateCast(Opcode: TargetOpcode::G_SITOFP, U, MIRBuilder);
559 }
560 bool translateUnreachable(const User &U, MachineIRBuilder &MIRBuilder);
561
562 bool translateSExt(const User &U, MachineIRBuilder &MIRBuilder) {
563 return translateCast(Opcode: TargetOpcode::G_SEXT, U, MIRBuilder);
564 }
565
566 bool translateZExt(const User &U, MachineIRBuilder &MIRBuilder) {
567 return translateCast(Opcode: TargetOpcode::G_ZEXT, U, MIRBuilder);
568 }
569
570 bool translateShl(const User &U, MachineIRBuilder &MIRBuilder) {
571 return translateBinaryOp(Opcode: TargetOpcode::G_SHL, U, MIRBuilder);
572 }
573 bool translateLShr(const User &U, MachineIRBuilder &MIRBuilder) {
574 return translateBinaryOp(Opcode: TargetOpcode::G_LSHR, U, MIRBuilder);
575 }
576 bool translateAShr(const User &U, MachineIRBuilder &MIRBuilder) {
577 return translateBinaryOp(Opcode: TargetOpcode::G_ASHR, U, MIRBuilder);
578 }
579
580 bool translateFAdd(const User &U, MachineIRBuilder &MIRBuilder) {
581 return translateBinaryOp(Opcode: TargetOpcode::G_FADD, U, MIRBuilder);
582 }
583 bool translateFSub(const User &U, MachineIRBuilder &MIRBuilder) {
584 return translateBinaryOp(Opcode: TargetOpcode::G_FSUB, U, MIRBuilder);
585 }
586 bool translateFMul(const User &U, MachineIRBuilder &MIRBuilder) {
587 return translateBinaryOp(Opcode: TargetOpcode::G_FMUL, U, MIRBuilder);
588 }
589 bool translateFDiv(const User &U, MachineIRBuilder &MIRBuilder) {
590 return translateBinaryOp(Opcode: TargetOpcode::G_FDIV, U, MIRBuilder);
591 }
592 bool translateFRem(const User &U, MachineIRBuilder &MIRBuilder) {
593 return translateBinaryOp(Opcode: TargetOpcode::G_FREM, U, MIRBuilder);
594 }
595
596 bool translateVAArg(const User &U, MachineIRBuilder &MIRBuilder);
597
598 bool translateInsertElement(const User &U, MachineIRBuilder &MIRBuilder);
599 bool translateInsertVector(const User &U, MachineIRBuilder &MIRBuilder);
600
601 bool translateExtractElement(const User &U, MachineIRBuilder &MIRBuilder);
602 bool translateExtractVector(const User &U, MachineIRBuilder &MIRBuilder);
603
604 bool translateShuffleVector(const User &U, MachineIRBuilder &MIRBuilder);
605
606 bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder);
607 bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder);
608 bool translateFence(const User &U, MachineIRBuilder &MIRBuilder);
609 bool translateFreeze(const User &U, MachineIRBuilder &MIRBuilder);
610
611 bool translateBitExtract(const User &U, MachineIRBuilder &MIRBuilder);
612 bool translateBitInsert(const User &U, MachineIRBuilder &MIRBuilder);
613
614 // Stubs to keep the compiler happy while we implement the rest of the
615 // translation.
616 bool translateResume(const User &U, MachineIRBuilder &MIRBuilder) {
617 return false;
618 }
619 bool translateCleanupRet(const User &U, MachineIRBuilder &MIRBuilder) {
620 return false;
621 }
622 bool translateCatchRet(const User &U, MachineIRBuilder &MIRBuilder) {
623 return false;
624 }
625 bool translateCatchSwitch(const User &U, MachineIRBuilder &MIRBuilder) {
626 return false;
627 }
628 bool translateAddrSpaceCast(const User &U, MachineIRBuilder &MIRBuilder) {
629 return translateCast(Opcode: TargetOpcode::G_ADDRSPACE_CAST, U, MIRBuilder);
630 }
631 bool translateCleanupPad(const User &U, MachineIRBuilder &MIRBuilder) {
632 return false;
633 }
634 bool translateCatchPad(const User &U, MachineIRBuilder &MIRBuilder) {
635 return false;
636 }
637 bool translateUserOp1(const User &U, MachineIRBuilder &MIRBuilder) {
638 return false;
639 }
640 bool translateUserOp2(const User &U, MachineIRBuilder &MIRBuilder) {
641 return false;
642 }
643
644 bool translateConvergenceControlIntrinsic(const CallInst &CI,
645 Intrinsic::ID ID,
646 MachineIRBuilder &MIRBuilder);
647
648 /// @}
649
650 // Builder for machine instruction a la IRBuilder.
651 // I.e., compared to regular MIBuilder, this one also inserts the instruction
652 // in the current block, it can creates block, etc., basically a kind of
653 // IRBuilder, but for Machine IR.
654 // CSEMIRBuilder CurBuilder;
655 std::unique_ptr<MachineIRBuilder> CurBuilder;
656
657 // Builder set to the entry block (just after ABI lowering instructions). Used
658 // as a convenient location for Constants.
659 // CSEMIRBuilder EntryBuilder;
660 std::unique_ptr<MachineIRBuilder> EntryBuilder;
661
662 // The MachineFunction currently being translated.
663 MachineFunction *MF = nullptr;
664
665 /// MachineRegisterInfo used to create virtual registers.
666 MachineRegisterInfo *MRI = nullptr;
667
668 const DataLayout *DL = nullptr;
669
670 CodeGenOptLevel OptLevel;
671
672 /// Current optimization remark emitter. Used to report failures.
673 std::unique_ptr<OptimizationRemarkEmitter> ORE;
674
675 AAResults *AA = nullptr;
676 AssumptionCache *AC = nullptr;
677 const TargetLibraryInfo *LibInfo = nullptr;
678 const LibcallLoweringInfo *Libcalls = nullptr;
679 const TargetLowering *TLI = nullptr;
680 FunctionLoweringInfo FuncInfo;
681
682 // True when either the Target Machine specifies no optimizations or the
683 // function has the optnone attribute.
684 bool EnableOpts = false;
685
686 /// True when the block contains a tail call. This allows the IRTranslator to
687 /// stop translating such blocks early.
688 bool HasTailCall = false;
689
690 StackProtectorDescriptor SPDescriptor;
691
692 bool mayTranslateUserTypes(const User &U) const;
693
694 /// Switch analysis and optimization.
695 class GISelSwitchLowering : public SwitchCG::SwitchLowering {
696 public:
697 GISelSwitchLowering(IRTranslatorImpl *irt, FunctionLoweringInfo &funcinfo)
698 : SwitchLowering(funcinfo), IRT(irt) {
699 assert(irt && "irt is null!");
700 }
701
702 void addSuccessorWithProb(
703 MachineBasicBlock *Src, MachineBasicBlock *Dst,
704 BranchProbability Prob = BranchProbability::getUnknown()) override {
705 IRT->addSuccessorWithProb(Src, Dst, Prob);
706 }
707
708 ~GISelSwitchLowering() override = default;
709
710 private:
711 IRTranslatorImpl *IRT;
712 };
713
714 std::unique_ptr<GISelSwitchLowering> SL;
715
716 // * Insert all the code needed to materialize the constants
717 // at the proper place. E.g., Entry block or dominator block
718 // of each constant depending on how fancy we want to be.
719 // * Clear the different maps.
720 void finalizeFunction();
721
722 // Processing steps done per block. E.g. emitting jump tables, stack
723 // protectors etc. Returns true if no errors, false if there was a problem
724 // that caused an abort.
725 bool finalizeBasicBlock(const BasicBlock &BB, MachineBasicBlock &MBB);
726
727 /// Codegen a new tail for a stack protector check ParentMBB which has had its
728 /// tail spliced into a stack protector check success bb.
729 ///
730 /// For a high level explanation of how this fits into the stack protector
731 /// generation see the comment on the declaration of class
732 /// StackProtectorDescriptor.
733 ///
734 /// \return true if there were no problems.
735 bool emitSPDescriptorParent(StackProtectorDescriptor &SPD,
736 MachineBasicBlock *ParentBB);
737
738 /// Codegen the failure basic block for a stack protector check.
739 ///
740 /// A failure stack protector machine basic block consists simply of a call to
741 /// __stack_chk_fail().
742 ///
743 /// For a high level explanation of how this fits into the stack protector
744 /// generation see the comment on the declaration of class
745 /// StackProtectorDescriptor.
746 ///
747 /// \return true if there were no problems.
748 bool emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
749 MachineBasicBlock *FailureBB);
750
751 /// Get the VRegs that represent \p Val.
752 /// Non-aggregate types have just one corresponding VReg and the list can be
753 /// used as a single "unsigned". Aggregates get flattened. If such VRegs do
754 /// not exist, they are created.
755 ArrayRef<Register> getOrCreateVRegs(const Value &Val);
756
757 Register getOrCreateVReg(const Value &Val) {
758 auto Regs = getOrCreateVRegs(Val);
759 if (Regs.empty())
760 return 0;
761 assert(Regs.size() == 1 &&
762 "attempt to get single VReg for aggregate or void");
763 return Regs[0];
764 }
765
766 Register getOrCreateConvergenceTokenVReg(const Value &Token) {
767 assert(Token.getType()->isTokenTy());
768 auto &Regs = *VMap.getVRegs(V: Token);
769 if (!Regs.empty()) {
770 assert(Regs.size() == 1 &&
771 "Expected a single register for convergence tokens.");
772 return Regs[0];
773 }
774
775 auto Reg = MRI->createGenericVirtualRegister(Ty: LLT::token());
776 Regs.push_back(Elt: Reg);
777 return Reg;
778 }
779
780 /// Allocate empty vregs for \p Val. For aggregate values, also populate
781 /// their offsets.
782 ValueToVRegInfo::VRegListT &allocateVRegs(const Value &Val);
783
784 /// Get the frame index that represents \p Val.
785 /// If such VReg does not exist, it is created.
786 int getOrCreateFrameIndex(const AllocaInst &AI);
787
788 /// Get the alignment of the given memory operation instruction. This will
789 /// either be the explicitly specified value or the ABI-required alignment for
790 /// the type being accessed (according to the Module's DataLayout).
791 Align getMemOpAlign(const Instruction &I);
792
793 /// Get the MachineBasicBlock that represents \p BB. Specifically, the block
794 /// returned will be the head of the translated block (suitable for branch
795 /// destinations).
796 MachineBasicBlock &getMBB(const BasicBlock &BB);
797
798 /// Record \p NewPred as a Machine predecessor to `Edge.second`, corresponding
799 /// to `Edge.first` at the IR level. This is used when IRTranslation creates
800 /// multiple MachineBasicBlocks for a given IR block and the CFG is no longer
801 /// represented simply by the IR-level CFG.
802 void addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred);
803
804 /// Returns the Machine IR predecessors for the given IR CFG edge. Usually
805 /// this is just the single MachineBasicBlock corresponding to the predecessor
806 /// in the IR. More complex lowering can result in multiple MachineBasicBlocks
807 /// preceding the original though (e.g. switch instructions).
808 SmallVector<MachineBasicBlock *, 1> getMachinePredBBs(CFGEdge Edge) {
809 auto RemappedEdge = MachinePreds.find(Val: Edge);
810 if (RemappedEdge != MachinePreds.end())
811 return RemappedEdge->second;
812 return SmallVector<MachineBasicBlock *, 4>(1, &getMBB(BB: *Edge.first));
813 }
814
815 /// Return branch probability calculated by BranchProbabilityInfo for IR
816 /// blocks.
817 BranchProbability getEdgeProbability(const MachineBasicBlock *Src,
818 const MachineBasicBlock *Dst) const;
819
820 void addSuccessorWithProb(
821 MachineBasicBlock *Src, MachineBasicBlock *Dst,
822 BranchProbability Prob = BranchProbability::getUnknown());
823
824public:
825 IRTranslatorImpl(CodeGenOptLevel OptLevel = CodeGenOptLevel::None)
826 : OptLevel(OptLevel) {}
827
828 // Algo:
829 // CallLowering = MF.subtarget.getCallLowering()
830 // F = MF.getParent()
831 // MIRBuilder.reset(MF)
832 // getMBB(F.getEntryBB())
833 // CallLowering->translateArguments(MIRBuilder, F, ValToVReg)
834 // for each bb in F
835 // getMBB(bb)
836 // for each inst in bb
837 // if (!translate(MIRBuilder, inst, ValToVReg, ConstantToSequence))
838 // reportFatalUsageError("Don't know how to translate input");
839 // finalize()
840 bool runOnMachineFunction(MachineFunction &MF,
841 function_ref<GISelCSEInfo *()> GetCSEInfo,
842 bool ShouldSkipOpts,
843 function_ref<AAResults *()> GetAAResults,
844 function_ref<BranchProbabilityInfo *()> GetBPI,
845 function_ref<AssumptionCache *()> GetAC,
846 TargetLibraryInfo *LibraryInfo,
847 const LibcallLoweringInfo *LibcallInfo,
848 SSPLayoutInfo *StackProtectorInfo);
849};
850
851} // namespace llvm
852
853char IRTranslatorLegacy::ID = 0;
854
855INITIALIZE_PASS_BEGIN(IRTranslatorLegacy, DEBUG_TYPE,
856 "IRTranslator LLVM IR -> MI", false, false)
857INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
858INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
859INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
860INITIALIZE_PASS_DEPENDENCY(StackProtector)
861INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
862INITIALIZE_PASS_END(IRTranslatorLegacy, DEBUG_TYPE,
863 "IRTranslator LLVM IR -> MI", false, false)
864
865static void reportTranslationError(MachineFunction &MF,
866 OptimizationRemarkEmitter &ORE,
867 OptimizationRemarkMissed &R) {
868 MF.getProperties().setFailedISel();
869 bool IsGlobalISelAbortEnabled =
870 MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
871
872 // Print the function name explicitly if we don't have a debug location (which
873 // makes the diagnostic less useful) or if we're going to emit a raw error.
874 if (!R.getLocation().isValid() || IsGlobalISelAbortEnabled)
875 R << (" (in function: " + MF.getName() + ")").str();
876
877 if (IsGlobalISelAbortEnabled)
878 report_fatal_error(reason: Twine(R.getMsg()));
879 else
880 ORE.emit(OptDiag&: R);
881}
882
883IRTranslatorLegacy::IRTranslatorLegacy(CodeGenOptLevel OptLevel)
884 : MachineFunctionPass(ID), OptLevel(OptLevel),
885 Impl(std::make_unique<IRTranslatorImpl>(args&: OptLevel)) {}
886
887IRTranslatorLegacy::~IRTranslatorLegacy() = default;
888
889#ifndef NDEBUG
890namespace {
891/// Verify that every instruction created has the same DILocation as the
892/// instruction being translated.
893class DILocationVerifier : public GISelChangeObserver {
894 const Instruction *CurrInst = nullptr;
895
896public:
897 DILocationVerifier() = default;
898 ~DILocationVerifier() override = default;
899
900 const Instruction *getCurrentInst() const { return CurrInst; }
901 void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
902
903 void erasingInstr(MachineInstr &MI) override {}
904 void changingInstr(MachineInstr &MI) override {}
905 void changedInstr(MachineInstr &MI) override {}
906
907 void createdInstr(MachineInstr &MI) override {
908 assert(getCurrentInst() && "Inserted instruction without a current MI");
909
910 // Only print the check message if we're actually checking it.
911#ifndef NDEBUG
912 LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
913 << " was copied to " << MI);
914#endif
915 // We allow insts in the entry block to have no debug loc because
916 // they could have originated from constants, and we don't want a jumpy
917 // debug experience.
918 assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
919 (MI.getParent()->isEntryBlock() && !MI.getDebugLoc()) ||
920 (MI.isDebugInstr())) &&
921 "Line info was not transferred to all instructions");
922 }
923};
924} // namespace
925#endif // ifndef NDEBUG
926
927void IRTranslatorLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
928 AU.addRequired<StackProtector>();
929 AU.addRequired<TargetPassConfig>();
930 AU.addRequired<GISelCSEAnalysisWrapperPass>();
931 if (OptLevel != CodeGenOptLevel::None) {
932 AU.addRequired<AssumptionCacheTracker>();
933 AU.addRequired<BranchProbabilityInfoWrapperPass>();
934 AU.addRequired<AAResultsWrapperPass>();
935 }
936 AU.addRequired<TargetLibraryInfoWrapperPass>();
937 AU.addPreserved<TargetLibraryInfoWrapperPass>();
938 AU.addRequired<LibcallLoweringInfoWrapper>();
939
940 getSelectionDAGFallbackAnalysisUsage(AU);
941 MachineFunctionPass::getAnalysisUsage(AU);
942}
943
944IRTranslatorImpl::ValueToVRegInfo::VRegListT &
945IRTranslatorImpl::allocateVRegs(const Value &Val) {
946 auto VRegsIt = VMap.findVRegs(V: Val);
947 if (VRegsIt != VMap.vregs_end())
948 return *VRegsIt->second;
949 auto *Regs = VMap.getVRegs(V: Val);
950 if (!Val.getType()->isAggregateType()) {
951 Regs->push_back(Elt: 0);
952 return *Regs;
953 }
954
955 auto *Offsets = VMap.getOffsets(V: Val);
956 SmallVector<LLT, 4> SplitTys;
957 computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueLLTs&: SplitTys,
958 FixedOffsets: Offsets->empty() ? Offsets : nullptr);
959 for (unsigned i = 0; i < SplitTys.size(); ++i)
960 Regs->push_back(Elt: 0);
961 return *Regs;
962}
963
964ArrayRef<Register> IRTranslatorImpl::getOrCreateVRegs(const Value &Val) {
965 auto VRegsIt = VMap.findVRegs(V: Val);
966 if (VRegsIt != VMap.vregs_end())
967 return *VRegsIt->second;
968
969 if (Val.getType()->isVoidTy())
970 return *VMap.getVRegs(V: Val);
971
972 // Create entry for this type.
973 auto *VRegs = VMap.getVRegs(V: Val);
974
975 if (!Val.getType()->isTokenTy())
976 assert(Val.getType()->isSized() &&
977 "Don't know how to create an empty vreg");
978
979 // Fast-path values that lower to a single vreg.
980 if (!Val.getType()->isAggregateType()) {
981 LLT Ty = getLLTForType(Ty&: *Val.getType(), DL: *DL);
982 VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty));
983 if (isa<Constant>(Val)) {
984 bool Success = translate(C: cast<Constant>(Val), Reg: VRegs->front());
985 if (!Success) {
986 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
987 MF->getFunction().getSubprogram(),
988 &MF->getFunction().getEntryBlock());
989 R << "unable to translate constant: " << ore::NV("Type", Val.getType());
990 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
991 }
992 }
993 return *VRegs;
994 }
995
996 SmallVector<LLT, 4> SplitTys;
997 auto *Offsets = VMap.getOffsets(V: Val);
998 computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueLLTs&: SplitTys,
999 FixedOffsets: Offsets->empty() ? Offsets : nullptr);
1000
1001 if (!isa<Constant>(Val)) {
1002 for (auto Ty : SplitTys)
1003 VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty));
1004 return *VRegs;
1005 }
1006
1007 // UndefValue, ConstantAggregateZero
1008 auto &C = cast<Constant>(Val);
1009 unsigned Idx = 0;
1010 while (auto Elt = C.getAggregateElement(Elt: Idx++)) {
1011 auto EltRegs = getOrCreateVRegs(Val: *Elt);
1012 llvm::append_range(C&: *VRegs, R&: EltRegs);
1013 }
1014
1015 return *VRegs;
1016}
1017
1018int IRTranslatorImpl::getOrCreateFrameIndex(const AllocaInst &AI) {
1019 auto [MapEntry, Inserted] = FrameIndices.try_emplace(Key: &AI);
1020 if (!Inserted)
1021 return MapEntry->second;
1022
1023 TypeSize TySize = AI.getAllocationSize(DL: *DL).value_or(u: TypeSize::getZero());
1024 uint64_t Size = TySize.getKnownMinValue();
1025
1026 // Always allocate at least one byte.
1027 Size = std::max<uint64_t>(a: Size, b: 1u);
1028
1029 int &FI = MapEntry->second;
1030 FI = MF->getFrameInfo().CreateStackObject(Size, Alignment: AI.getAlign(), isSpillSlot: false, Alloca: &AI);
1031
1032 // Scalable vectors and structures that contain scalable vectors may
1033 // need a special StackID to distinguish them from other (fixed size)
1034 // stack objects.
1035 if (TySize.isScalable()) {
1036 auto StackID =
1037 MF->getSubtarget().getFrameLowering()->getStackIDForScalableVectors();
1038 MF->getFrameInfo().setStackID(ObjectIdx: FI, ID: StackID);
1039 }
1040
1041 return FI;
1042}
1043
1044Align IRTranslatorImpl::getMemOpAlign(const Instruction &I) {
1045 if (const StoreInst *SI = dyn_cast<StoreInst>(Val: &I))
1046 return SI->getAlign();
1047 if (const LoadInst *LI = dyn_cast<LoadInst>(Val: &I))
1048 return LI->getAlign();
1049 if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(Val: &I))
1050 return AI->getAlign();
1051 if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(Val: &I))
1052 return AI->getAlign();
1053
1054 OptimizationRemarkMissed R("gisel-ir-translator", "", &I);
1055 R << "unable to translate memop: " << ore::NV("Opcode", &I);
1056 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
1057 return Align(1);
1058}
1059
1060MachineBasicBlock &IRTranslatorImpl::getMBB(const BasicBlock &BB) {
1061 MachineBasicBlock *MBB = FuncInfo.getMBB(BB: &BB);
1062 assert(MBB && "BasicBlock was not encountered before");
1063 return *MBB;
1064}
1065
1066void IRTranslatorImpl::addMachineCFGPred(CFGEdge Edge,
1067 MachineBasicBlock *NewPred) {
1068 assert(NewPred && "new predecessor must be a real MachineBasicBlock");
1069 MachinePreds[Edge].push_back(Elt: NewPred);
1070}
1071
1072bool IRTranslatorImpl::translateBinaryOp(unsigned Opcode, const User &U,
1073 MachineIRBuilder &MIRBuilder) {
1074 if (!mayTranslateUserTypes(U))
1075 return false;
1076
1077 // Get or create a virtual register for each value.
1078 // Unless the value is a Constant => loadimm cst?
1079 // or inline constant each time?
1080 // Creation of a virtual register needs to have a size.
1081 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
1082 Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
1083 Register Res = getOrCreateVReg(Val: U);
1084 uint32_t Flags = 0;
1085 if (isa<Instruction>(Val: U)) {
1086 const Instruction &I = cast<Instruction>(Val: U);
1087 Flags = MachineInstr::copyFlagsFromInstruction(I);
1088 }
1089
1090 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0, Op1}, Flags);
1091 return true;
1092}
1093
1094bool IRTranslatorImpl::translateUnaryOp(unsigned Opcode, const User &U,
1095 MachineIRBuilder &MIRBuilder) {
1096 if (!mayTranslateUserTypes(U))
1097 return false;
1098
1099 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
1100 Register Res = getOrCreateVReg(Val: U);
1101 uint32_t Flags = 0;
1102 if (isa<Instruction>(Val: U)) {
1103 const Instruction &I = cast<Instruction>(Val: U);
1104 Flags = MachineInstr::copyFlagsFromInstruction(I);
1105 }
1106 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0}, Flags);
1107 return true;
1108}
1109
1110bool IRTranslatorImpl::translateFNeg(const User &U,
1111 MachineIRBuilder &MIRBuilder) {
1112 return translateUnaryOp(Opcode: TargetOpcode::G_FNEG, U, MIRBuilder);
1113}
1114
1115bool IRTranslatorImpl::translateCompare(const User &U,
1116 MachineIRBuilder &MIRBuilder) {
1117 if (!mayTranslateUserTypes(U))
1118 return false;
1119
1120 auto *CI = cast<CmpInst>(Val: &U);
1121 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
1122 Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
1123 Register Res = getOrCreateVReg(Val: U);
1124 CmpInst::Predicate Pred = CI->getPredicate();
1125 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: *CI);
1126 if (CmpInst::isIntPredicate(P: Pred))
1127 MIRBuilder.buildICmp(Pred, Res, Op0, Op1, Flags);
1128 else if (Pred == CmpInst::FCMP_FALSE)
1129 MIRBuilder.buildCopy(
1130 Res, Op: getOrCreateVReg(Val: *Constant::getNullValue(Ty: U.getType())));
1131 else if (Pred == CmpInst::FCMP_TRUE)
1132 MIRBuilder.buildCopy(
1133 Res, Op: getOrCreateVReg(Val: *Constant::getAllOnesValue(Ty: U.getType())));
1134 else
1135 MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
1136
1137 return true;
1138}
1139
1140bool IRTranslatorImpl::translateRet(const User &U,
1141 MachineIRBuilder &MIRBuilder) {
1142 const ReturnInst &RI = cast<ReturnInst>(Val: U);
1143 const Value *Ret = RI.getReturnValue();
1144 if (Ret && DL->getTypeStoreSize(Ty: Ret->getType()).isZero())
1145 Ret = nullptr;
1146
1147 ArrayRef<Register> VRegs;
1148 if (Ret)
1149 VRegs = getOrCreateVRegs(Val: *Ret);
1150
1151 Register SwiftErrorVReg = 0;
1152 if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
1153 SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
1154 &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
1155 }
1156
1157 // The target may mess up with the insertion point, but
1158 // this is not important as a return is the last instruction
1159 // of the block anyway.
1160 return CLI->lowerReturn(MIRBuilder, Val: Ret, VRegs, FLI&: FuncInfo, SwiftErrorVReg);
1161}
1162
1163void IRTranslatorImpl::emitBranchForMergedCondition(
1164 const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
1165 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
1166 BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
1167 // If the leaf of the tree is a comparison, merge the condition into
1168 // the caseblock.
1169 if (const CmpInst *BOp = dyn_cast<CmpInst>(Val: Cond)) {
1170 CmpInst::Predicate Condition;
1171 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Val: Cond)) {
1172 Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
1173 } else {
1174 const FCmpInst *FC = cast<FCmpInst>(Val: Cond);
1175 Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
1176 }
1177
1178 SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(i_nocapture: 0),
1179 BOp->getOperand(i_nocapture: 1), nullptr, TBB, FBB, CurBB,
1180 CurBuilder->getDebugLoc(), TProb, FProb);
1181 SL->SwitchCases.push_back(x: CB);
1182 return;
1183 }
1184
1185 // Create a CaseBlock record representing this branch.
1186 CmpInst::Predicate Pred = InvertCond ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ;
1187 SwitchCG::CaseBlock CB(
1188 Pred, false, Cond, ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
1189 nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
1190 SL->SwitchCases.push_back(x: CB);
1191}
1192
1193static bool isValInBlock(const Value *V, const BasicBlock *BB) {
1194 if (const Instruction *I = dyn_cast<Instruction>(Val: V))
1195 return I->getParent() == BB;
1196 return true;
1197}
1198
1199void IRTranslatorImpl::findMergedConditions(
1200 const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
1201 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
1202 Instruction::BinaryOps Opc, BranchProbability TProb,
1203 BranchProbability FProb, bool InvertCond) {
1204 using namespace PatternMatch;
1205 assert((Opc == Instruction::And || Opc == Instruction::Or) &&
1206 "Expected Opc to be AND/OR");
1207 // Skip over not part of the tree and remember to invert op and operands at
1208 // next level.
1209 Value *NotCond;
1210 if (match(V: Cond, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: NotCond)))) &&
1211 isValInBlock(V: NotCond, BB: CurBB->getBasicBlock())) {
1212 findMergedConditions(Cond: NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
1213 InvertCond: !InvertCond);
1214 return;
1215 }
1216
1217 const Instruction *BOp = dyn_cast<Instruction>(Val: Cond);
1218 const Value *BOpOp0, *BOpOp1;
1219 // Compute the effective opcode for Cond, taking into account whether it needs
1220 // to be inverted, e.g.
1221 // and (not (or A, B)), C
1222 // gets lowered as
1223 // and (and (not A, not B), C)
1224 Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
1225 if (BOp) {
1226 BOpc = match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
1227 ? Instruction::And
1228 : (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
1229 ? Instruction::Or
1230 : (Instruction::BinaryOps)0);
1231 if (InvertCond) {
1232 if (BOpc == Instruction::And)
1233 BOpc = Instruction::Or;
1234 else if (BOpc == Instruction::Or)
1235 BOpc = Instruction::And;
1236 }
1237 }
1238
1239 // If this node is not part of the or/and tree, emit it as a branch.
1240 // Note that all nodes in the tree should have same opcode.
1241 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
1242 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
1243 !isValInBlock(V: BOpOp0, BB: CurBB->getBasicBlock()) ||
1244 !isValInBlock(V: BOpOp1, BB: CurBB->getBasicBlock())) {
1245 emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
1246 InvertCond);
1247 return;
1248 }
1249
1250 // Create TmpBB after CurBB.
1251 MachineFunction::iterator BBI(CurBB);
1252 MachineBasicBlock *TmpBB =
1253 MF->CreateMachineBasicBlock(BB: CurBB->getBasicBlock());
1254 CurBB->getParent()->insert(MBBI: ++BBI, MBB: TmpBB);
1255
1256 if (Opc == Instruction::Or) {
1257 // Codegen X | Y as:
1258 // BB1:
1259 // jmp_if_X TBB
1260 // jmp TmpBB
1261 // TmpBB:
1262 // jmp_if_Y TBB
1263 // jmp FBB
1264 //
1265
1266 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1267 // The requirement is that
1268 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1269 // = TrueProb for original BB.
1270 // Assuming the original probabilities are A and B, one choice is to set
1271 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1272 // A/(1+B) and 2B/(1+B). This choice assumes that
1273 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1274 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1275 // TmpBB, but the math is more complicated.
1276
1277 auto NewTrueProb = TProb / 2;
1278 auto NewFalseProb = TProb / 2 + FProb;
1279 // Emit the LHS condition.
1280 findMergedConditions(Cond: BOpOp0, TBB, FBB: TmpBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
1281 FProb: NewFalseProb, InvertCond);
1282
1283 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1284 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1285 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
1286 // Emit the RHS condition into TmpBB.
1287 findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
1288 FProb: Probs[1], InvertCond);
1289 } else {
1290 assert(Opc == Instruction::And && "Unknown merge op!");
1291 // Codegen X & Y as:
1292 // BB1:
1293 // jmp_if_X TmpBB
1294 // jmp FBB
1295 // TmpBB:
1296 // jmp_if_Y TBB
1297 // jmp FBB
1298 //
1299 // This requires creation of TmpBB after CurBB.
1300
1301 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1302 // The requirement is that
1303 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1304 // = FalseProb for original BB.
1305 // Assuming the original probabilities are A and B, one choice is to set
1306 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1307 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1308 // TrueProb for BB1 * FalseProb for TmpBB.
1309
1310 auto NewTrueProb = TProb + FProb / 2;
1311 auto NewFalseProb = FProb / 2;
1312 // Emit the LHS condition.
1313 findMergedConditions(Cond: BOpOp0, TBB: TmpBB, FBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
1314 FProb: NewFalseProb, InvertCond);
1315
1316 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1317 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1318 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
1319 // Emit the RHS condition into TmpBB.
1320 findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
1321 FProb: Probs[1], InvertCond);
1322 }
1323}
1324
1325bool IRTranslatorImpl::shouldEmitAsBranches(
1326 const std::vector<SwitchCG::CaseBlock> &Cases) {
1327 // For multiple cases, it's better to emit as branches.
1328 if (Cases.size() != 2)
1329 return true;
1330
1331 // If this is two comparisons of the same values or'd or and'd together, they
1332 // will get folded into a single comparison, so don't emit two blocks.
1333 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1334 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1335 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1336 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1337 return false;
1338 }
1339
1340 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1341 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1342 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1343 Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
1344 isa<Constant>(Val: Cases[0].CmpRHS) &&
1345 cast<Constant>(Val: Cases[0].CmpRHS)->isNullValue()) {
1346 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
1347 Cases[0].TrueBB == Cases[1].ThisBB)
1348 return false;
1349 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
1350 Cases[0].FalseBB == Cases[1].ThisBB)
1351 return false;
1352 }
1353
1354 return true;
1355}
1356
1357bool IRTranslatorImpl::translateUncondBr(const User &U,
1358 MachineIRBuilder &MIRBuilder) {
1359 const UncondBrInst &BrInst = cast<UncondBrInst>(Val: U);
1360 auto &CurMBB = MIRBuilder.getMBB();
1361 auto *Succ0MBB = &getMBB(BB: *BrInst.getSuccessor(i: 0));
1362
1363 // If the unconditional target is the layout successor, fallthrough.
1364 if (OptLevel == CodeGenOptLevel::None || !CurMBB.isLayoutSuccessor(MBB: Succ0MBB))
1365 MIRBuilder.buildBr(Dest&: *Succ0MBB);
1366
1367 // Link successors.
1368 for (const BasicBlock *Succ : successors(I: &BrInst))
1369 CurMBB.addSuccessor(Succ: &getMBB(BB: *Succ));
1370 return true;
1371}
1372
1373bool IRTranslatorImpl::translateCondBr(const User &U,
1374 MachineIRBuilder &MIRBuilder) {
1375 const CondBrInst &BrInst = cast<CondBrInst>(Val: U);
1376 auto &CurMBB = MIRBuilder.getMBB();
1377 auto *Succ0MBB = &getMBB(BB: *BrInst.getSuccessor(i: 0));
1378
1379 // If this condition is one of the special cases we handle, do special stuff
1380 // now.
1381 const Value *CondVal = BrInst.getCondition();
1382 MachineBasicBlock *Succ1MBB = &getMBB(BB: *BrInst.getSuccessor(i: 1));
1383
1384 // If this is a series of conditions that are or'd or and'd together, emit
1385 // this as a sequence of branches instead of setcc's with and/or operations.
1386 // As long as jumps are not expensive (exceptions for multi-use logic ops,
1387 // unpredictable branches, and vector extracts because those jumps are likely
1388 // expensive for any target), this should improve performance.
1389 // For example, instead of something like:
1390 // cmp A, B
1391 // C = seteq
1392 // cmp D, E
1393 // F = setle
1394 // or C, F
1395 // jnz foo
1396 // Emit:
1397 // cmp A, B
1398 // je foo
1399 // cmp D, E
1400 // jle foo
1401 using namespace PatternMatch;
1402 const Instruction *CondI = dyn_cast<Instruction>(Val: CondVal);
1403 if (!TLI->isJumpExpensive() && CondI && CondI->hasOneUse() &&
1404 !BrInst.hasMetadata(KindID: LLVMContext::MD_unpredictable)) {
1405 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
1406 Value *Vec;
1407 const Value *BOp0, *BOp1;
1408 if (match(V: CondI, P: m_LogicalAnd(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
1409 Opcode = Instruction::And;
1410 else if (match(V: CondI, P: m_LogicalOr(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
1411 Opcode = Instruction::Or;
1412
1413 if (Opcode && !(match(V: BOp0, P: m_ExtractElt(Val: m_Value(V&: Vec), Idx: m_Value())) &&
1414 match(V: BOp1, P: m_ExtractElt(Val: m_Specific(V: Vec), Idx: m_Value())))) {
1415 findMergedConditions(Cond: CondI, TBB: Succ0MBB, FBB: Succ1MBB, CurBB: &CurMBB, SwitchBB: &CurMBB, Opc: Opcode,
1416 TProb: getEdgeProbability(Src: &CurMBB, Dst: Succ0MBB),
1417 FProb: getEdgeProbability(Src: &CurMBB, Dst: Succ1MBB),
1418 /*InvertCond=*/false);
1419 assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
1420
1421 // Allow some cases to be rejected.
1422 if (shouldEmitAsBranches(Cases: SL->SwitchCases)) {
1423 // Emit the branch for this block.
1424 emitSwitchCase(CB&: SL->SwitchCases[0], SwitchBB: &CurMBB, MIB&: *CurBuilder);
1425 SL->SwitchCases.erase(position: SL->SwitchCases.begin());
1426 return true;
1427 }
1428
1429 // Okay, we decided not to do this, remove any inserted MBB's and clear
1430 // SwitchCases.
1431 for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
1432 MF->erase(MBBI: SL->SwitchCases[I].ThisBB);
1433
1434 SL->SwitchCases.clear();
1435 }
1436 }
1437
1438 // Create a CaseBlock record representing this branch.
1439 SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
1440 ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
1441 nullptr, Succ0MBB, Succ1MBB, &CurMBB,
1442 CurBuilder->getDebugLoc());
1443
1444 // Use emitSwitchCase to actually insert the fast branch sequence for this
1445 // cond branch.
1446 emitSwitchCase(CB, SwitchBB: &CurMBB, MIB&: *CurBuilder);
1447 return true;
1448}
1449
1450void IRTranslatorImpl::addSuccessorWithProb(MachineBasicBlock *Src,
1451 MachineBasicBlock *Dst,
1452 BranchProbability Prob) {
1453 if (!FuncInfo.BPI) {
1454 Src->addSuccessorWithoutProb(Succ: Dst);
1455 return;
1456 }
1457 if (Prob.isUnknown())
1458 Prob = getEdgeProbability(Src, Dst);
1459 Src->addSuccessor(Succ: Dst, Prob);
1460}
1461
1462BranchProbability
1463IRTranslatorImpl::getEdgeProbability(const MachineBasicBlock *Src,
1464 const MachineBasicBlock *Dst) const {
1465 const BasicBlock *SrcBB = Src->getBasicBlock();
1466 const BasicBlock *DstBB = Dst->getBasicBlock();
1467 if (!FuncInfo.BPI) {
1468 // If BPI is not available, set the default probability as 1 / N, where N is
1469 // the number of successors.
1470 auto SuccSize = std::max<uint32_t>(a: succ_size(BB: SrcBB), b: 1);
1471 return BranchProbability(1, SuccSize);
1472 }
1473 return FuncInfo.BPI->getEdgeProbability(Src: SrcBB, Dst: DstBB);
1474}
1475
1476bool IRTranslatorImpl::translateSwitch(const User &U, MachineIRBuilder &MIB) {
1477 using namespace SwitchCG;
1478 // Extract cases from the switch.
1479 const SwitchInst &SI = cast<SwitchInst>(Val: U);
1480 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1481 CaseClusterVector Clusters;
1482 Clusters.reserve(n: SI.getNumCases());
1483 for (const auto &I : SI.cases()) {
1484 MachineBasicBlock *Succ = &getMBB(BB: *I.getCaseSuccessor());
1485 assert(Succ && "Could not find successor mbb in mapping");
1486 const ConstantInt *CaseVal = I.getCaseValue();
1487 BranchProbability Prob =
1488 BPI ? BPI->getEdgeProbability(Src: SI.getParent(), IndexInSuccessors: I.getSuccessorIndex())
1489 : BranchProbability(1, SI.getNumCases() + 1);
1490 Clusters.push_back(x: CaseCluster::range(Low: CaseVal, High: CaseVal, MBB: Succ, Prob));
1491 }
1492
1493 MachineBasicBlock *DefaultMBB = &getMBB(BB: *SI.getDefaultDest());
1494
1495 // Cluster adjacent cases with the same destination. We do this at all
1496 // optimization levels because it's cheap to do and will make codegen faster
1497 // if there are many clusters.
1498 sortAndRangeify(Clusters);
1499
1500 MachineBasicBlock *SwitchMBB = &getMBB(BB: *SI.getParent());
1501
1502 // If there is only the default destination, jump there directly.
1503 if (Clusters.empty()) {
1504 SwitchMBB->addSuccessor(Succ: DefaultMBB);
1505 if (DefaultMBB != SwitchMBB->getNextNode())
1506 MIB.buildBr(Dest&: *DefaultMBB);
1507 return true;
1508 }
1509
1510 SL->findJumpTables(Clusters, SI: &SI, SL: std::nullopt, DefaultMBB, PSI: nullptr, BFI: nullptr);
1511 SL->findBitTestClusters(Clusters, SI: &SI);
1512
1513 LLVM_DEBUG({
1514 dbgs() << "Case clusters: ";
1515 for (const CaseCluster &C : Clusters) {
1516 if (C.Kind == CC_JumpTable)
1517 dbgs() << "JT:";
1518 if (C.Kind == CC_BitTests)
1519 dbgs() << "BT:";
1520
1521 C.Low->getValue().print(dbgs(), true);
1522 if (C.Low != C.High) {
1523 dbgs() << '-';
1524 C.High->getValue().print(dbgs(), true);
1525 }
1526 dbgs() << ' ';
1527 }
1528 dbgs() << '\n';
1529 });
1530
1531 assert(!Clusters.empty());
1532 SwitchWorkList WorkList;
1533 CaseClusterIt First = Clusters.begin();
1534 CaseClusterIt Last = Clusters.end() - 1;
1535 auto DefaultProb = getEdgeProbability(Src: SwitchMBB, Dst: DefaultMBB);
1536 WorkList.push_back(Elt: {.MBB: SwitchMBB, .FirstCluster: First, .LastCluster: Last, .GE: nullptr, .LT: nullptr, .DefaultProb: DefaultProb});
1537
1538 while (!WorkList.empty()) {
1539 SwitchWorkListItem W = WorkList.pop_back_val();
1540
1541 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
1542 // For optimized builds, lower large range as a balanced binary tree.
1543 if (NumClusters > 3 &&
1544 MF->getTarget().getOptLevel() != CodeGenOptLevel::None &&
1545 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
1546 splitWorkItem(WorkList, W, Cond: SI.getCondition(), SwitchMBB, MIB);
1547 continue;
1548 }
1549
1550 if (!lowerSwitchWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
1551 return false;
1552 }
1553 return true;
1554}
1555
1556void IRTranslatorImpl::splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
1557 const SwitchCG::SwitchWorkListItem &W,
1558 Value *Cond, MachineBasicBlock *SwitchMBB,
1559 MachineIRBuilder &MIB) {
1560 using namespace SwitchCG;
1561 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
1562 "Clusters not sorted?");
1563 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
1564
1565 auto [LastLeft, FirstRight, LeftProb, RightProb] =
1566 SL->computeSplitWorkItemInfo(W);
1567
1568 // Use the first element on the right as pivot since we will make less-than
1569 // comparisons against it.
1570 CaseClusterIt PivotCluster = FirstRight;
1571 assert(PivotCluster > W.FirstCluster);
1572 assert(PivotCluster <= W.LastCluster);
1573
1574 CaseClusterIt FirstLeft = W.FirstCluster;
1575 CaseClusterIt LastRight = W.LastCluster;
1576
1577 const ConstantInt *Pivot = PivotCluster->Low;
1578
1579 // New blocks will be inserted immediately after the current one.
1580 MachineFunction::iterator BBI(W.MBB);
1581 ++BBI;
1582
1583 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
1584 // we can branch to its destination directly if it's squeezed exactly in
1585 // between the known lower bound and Pivot - 1.
1586 MachineBasicBlock *LeftMBB;
1587 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
1588 FirstLeft->Low == W.GE &&
1589 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
1590 LeftMBB = FirstLeft->MBB;
1591 } else {
1592 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
1593 FuncInfo.MF->insert(MBBI: BBI, MBB: LeftMBB);
1594 WorkList.push_back(
1595 Elt: {.MBB: LeftMBB, .FirstCluster: FirstLeft, .LastCluster: LastLeft, .GE: W.GE, .LT: Pivot, .DefaultProb: W.DefaultProb / 2});
1596 }
1597
1598 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
1599 // single cluster, RHS.Low == Pivot, and we can branch to its destination
1600 // directly if RHS.High equals the current upper bound.
1601 MachineBasicBlock *RightMBB;
1602 if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT &&
1603 (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
1604 RightMBB = FirstRight->MBB;
1605 } else {
1606 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
1607 FuncInfo.MF->insert(MBBI: BBI, MBB: RightMBB);
1608 WorkList.push_back(
1609 Elt: {.MBB: RightMBB, .FirstCluster: FirstRight, .LastCluster: LastRight, .GE: Pivot, .LT: W.LT, .DefaultProb: W.DefaultProb / 2});
1610 }
1611
1612 // Create the CaseBlock record that will be used to lower the branch.
1613 CaseBlock CB(ICmpInst::Predicate::ICMP_SLT, false, Cond, Pivot, nullptr,
1614 LeftMBB, RightMBB, W.MBB, MIB.getDebugLoc(), LeftProb,
1615 RightProb);
1616
1617 if (W.MBB == SwitchMBB)
1618 emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
1619 else
1620 SL->SwitchCases.push_back(x: CB);
1621}
1622
1623void IRTranslatorImpl::emitJumpTable(SwitchCG::JumpTable &JT,
1624 MachineBasicBlock *MBB) {
1625 // Emit the code for the jump table
1626 assert(JT.Reg && "Should lower JT Header first!");
1627 MachineIRBuilder MIB(*MBB->getParent());
1628 MIB.setMBB(*MBB);
1629 MIB.setDebugLoc(CurBuilder->getDebugLoc());
1630
1631 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
1632 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1633
1634 auto Table = MIB.buildJumpTable(PtrTy, JTI: JT.JTI);
1635 MIB.buildBrJT(TablePtr: Table.getReg(Idx: 0), JTI: JT.JTI, IndexReg: JT.Reg);
1636}
1637
1638bool IRTranslatorImpl::emitJumpTableHeader(SwitchCG::JumpTable &JT,
1639 SwitchCG::JumpTableHeader &JTH,
1640 MachineBasicBlock *HeaderBB) {
1641 MachineIRBuilder MIB(*HeaderBB->getParent());
1642 MIB.setMBB(*HeaderBB);
1643 MIB.setDebugLoc(CurBuilder->getDebugLoc());
1644
1645 const Value &SValue = *JTH.SValue;
1646 // Subtract the lowest switch case value from the value being switched on.
1647 const LLT SwitchTy = getLLTForType(Ty&: *SValue.getType(), DL: *DL);
1648 Register SwitchOpReg = getOrCreateVReg(Val: SValue);
1649 auto FirstCst = MIB.buildConstant(Res: SwitchTy, Val: JTH.First);
1650 auto Sub = MIB.buildSub(Dst: {SwitchTy}, Src0: SwitchOpReg, Src1: FirstCst);
1651
1652 // This value may be smaller or larger than the target's pointer type, and
1653 // therefore require extension or truncating.
1654 auto *PtrIRTy = PointerType::getUnqual(C&: SValue.getContext());
1655 const LLT PtrScalarTy = LLT::integer(SizeInBits: DL->getTypeSizeInBits(Ty: PtrIRTy));
1656 auto Index = MIB.buildZExtOrTrunc(Res: PtrScalarTy, Op: Sub);
1657
1658 JT.Reg = Index.getReg(Idx: 0);
1659
1660 if (JTH.FallthroughUnreachable) {
1661 if (JT.MBB != HeaderBB->getNextNode())
1662 MIB.buildBr(Dest&: *JT.MBB);
1663 return true;
1664 }
1665
1666 // Emit the range check for the jump table, and branch to the default block
1667 // for the switch statement if the value being switched on exceeds the
1668 // largest case in the switch.
1669 auto Cst = getOrCreateVReg(
1670 Val: *ConstantInt::get(Ty: SValue.getType(), V: JTH.Last - JTH.First));
1671 auto Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_UGT, Res: LLT::integer(SizeInBits: 1), Op0: Sub, Op1: Cst);
1672
1673 auto BrCond = MIB.buildBrCond(Tst: Cmp.getReg(Idx: 0), Dest&: *JT.Default);
1674
1675 // Avoid emitting unnecessary branches to the next block.
1676 if (JT.MBB != HeaderBB->getNextNode())
1677 BrCond = MIB.buildBr(Dest&: *JT.MBB);
1678 return true;
1679}
1680
1681void IRTranslatorImpl::emitSwitchCase(SwitchCG::CaseBlock &CB,
1682 MachineBasicBlock *SwitchBB,
1683 MachineIRBuilder &MIB) {
1684 Register CondLHS = getOrCreateVReg(Val: *CB.CmpLHS);
1685 Register Cond;
1686 DebugLoc OldDbgLoc = MIB.getDebugLoc();
1687 MIB.setDebugLoc(CB.DbgLoc);
1688 MIB.setMBB(*CB.ThisBB);
1689
1690 if (CB.PredInfo.NoCmp) {
1691 // Branch or fall through to TrueBB.
1692 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
1693 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
1694 NewPred: CB.ThisBB);
1695 CB.ThisBB->normalizeSuccProbs();
1696 if (CB.TrueBB != CB.ThisBB->getNextNode())
1697 MIB.buildBr(Dest&: *CB.TrueBB);
1698 MIB.setDebugLoc(OldDbgLoc);
1699 return;
1700 }
1701
1702 const LLT i1Ty = LLT::integer(SizeInBits: 1);
1703 // Build the compare.
1704 if (!CB.CmpMHS) {
1705 const auto *CI = dyn_cast<ConstantInt>(Val: CB.CmpRHS);
1706 // For conditional branch lowering, we might try to do something silly like
1707 // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
1708 // just re-use the existing condition vreg.
1709 if (MRI->getType(Reg: CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
1710 CB.PredInfo.Pred == CmpInst::ICMP_EQ) {
1711 Cond = CondLHS;
1712 } else {
1713 Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
1714 if (CmpInst::isFPPredicate(P: CB.PredInfo.Pred))
1715 Cond =
1716 MIB.buildFCmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
1717 else
1718 Cond =
1719 MIB.buildICmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
1720 }
1721 } else {
1722 assert(CB.PredInfo.Pred == CmpInst::ICMP_SLE &&
1723 "Can only handle SLE ranges");
1724
1725 const APInt& Low = cast<ConstantInt>(Val: CB.CmpLHS)->getValue();
1726 const APInt& High = cast<ConstantInt>(Val: CB.CmpRHS)->getValue();
1727
1728 Register CmpOpReg = getOrCreateVReg(Val: *CB.CmpMHS);
1729 if (cast<ConstantInt>(Val: CB.CmpLHS)->isMinValue(IsSigned: true)) {
1730 Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
1731 Cond =
1732 MIB.buildICmp(Pred: CmpInst::ICMP_SLE, Res: i1Ty, Op0: CmpOpReg, Op1: CondRHS).getReg(Idx: 0);
1733 } else {
1734 const LLT CmpTy = MRI->getType(Reg: CmpOpReg);
1735 auto Sub = MIB.buildSub(Dst: {CmpTy}, Src0: CmpOpReg, Src1: CondLHS);
1736 auto Diff = MIB.buildConstant(Res: CmpTy, Val: High - Low);
1737 Cond = MIB.buildICmp(Pred: CmpInst::ICMP_ULE, Res: i1Ty, Op0: Sub, Op1: Diff).getReg(Idx: 0);
1738 }
1739 }
1740
1741 // Update successor info
1742 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
1743
1744 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
1745 NewPred: CB.ThisBB);
1746
1747 // TrueBB and FalseBB are always different unless the incoming IR is
1748 // degenerate. This only happens when running llc on weird IR.
1749 if (CB.TrueBB != CB.FalseBB)
1750 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.FalseBB, Prob: CB.FalseProb);
1751 CB.ThisBB->normalizeSuccProbs();
1752
1753 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
1754 NewPred: CB.ThisBB);
1755
1756 MIB.buildBrCond(Tst: Cond, Dest&: *CB.TrueBB);
1757 MIB.buildBr(Dest&: *CB.FalseBB);
1758 MIB.setDebugLoc(OldDbgLoc);
1759}
1760
1761bool IRTranslatorImpl::lowerJumpTableWorkItem(
1762 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
1763 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1764 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
1765 BranchProbability UnhandledProbs, SwitchCG::CaseClusterIt I,
1766 MachineBasicBlock *Fallthrough, bool FallthroughUnreachable) {
1767 using namespace SwitchCG;
1768 MachineFunction *CurMF = SwitchMBB->getParent();
1769 // FIXME: Optimize away range check based on pivot comparisons.
1770 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
1771 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
1772 BranchProbability DefaultProb = W.DefaultProb;
1773
1774 // The jump block hasn't been inserted yet; insert it here.
1775 MachineBasicBlock *JumpMBB = JT->MBB;
1776 CurMF->insert(MBBI: BBI, MBB: JumpMBB);
1777
1778 // Since the jump table block is separate from the switch block, we need
1779 // to keep track of it as a machine predecessor to the default block,
1780 // otherwise we lose the phi edges.
1781 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1782 NewPred: CurMBB);
1783 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1784 NewPred: JumpMBB);
1785
1786 auto JumpProb = I->Prob;
1787 auto FallthroughProb = UnhandledProbs;
1788
1789 // If the default statement is a target of the jump table, we evenly
1790 // distribute the default probability to successors of CurMBB. Also
1791 // update the probability on the edge from JumpMBB to Fallthrough.
1792 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
1793 SE = JumpMBB->succ_end();
1794 SI != SE; ++SI) {
1795 if (*SI == DefaultMBB) {
1796 JumpProb += DefaultProb / 2;
1797 FallthroughProb -= DefaultProb / 2;
1798 JumpMBB->setSuccProbability(I: SI, Prob: DefaultProb / 2);
1799 JumpMBB->normalizeSuccProbs();
1800 } else {
1801 // Also record edges from the jump table block to it's successors.
1802 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
1803 NewPred: JumpMBB);
1804 }
1805 }
1806
1807 if (FallthroughUnreachable)
1808 JTH->FallthroughUnreachable = true;
1809
1810 if (!JTH->FallthroughUnreachable)
1811 addSuccessorWithProb(Src: CurMBB, Dst: Fallthrough, Prob: FallthroughProb);
1812 addSuccessorWithProb(Src: CurMBB, Dst: JumpMBB, Prob: JumpProb);
1813 CurMBB->normalizeSuccProbs();
1814
1815 // The jump table header will be inserted in our current block, do the
1816 // range check, and fall through to our fallthrough block.
1817 JTH->HeaderBB = CurMBB;
1818 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
1819
1820 // If we're in the right place, emit the jump table header right now.
1821 if (CurMBB == SwitchMBB) {
1822 if (!emitJumpTableHeader(JT&: *JT, JTH&: *JTH, HeaderBB: CurMBB))
1823 return false;
1824 JTH->Emitted = true;
1825 }
1826 return true;
1827}
1828bool IRTranslatorImpl::lowerSwitchRangeWorkItem(
1829 SwitchCG::CaseClusterIt I, Value *Cond, MachineBasicBlock *Fallthrough,
1830 bool FallthroughUnreachable, BranchProbability UnhandledProbs,
1831 MachineBasicBlock *CurMBB, MachineIRBuilder &MIB,
1832 MachineBasicBlock *SwitchMBB) {
1833 using namespace SwitchCG;
1834 const Value *RHS, *LHS, *MHS;
1835 CmpInst::Predicate Pred;
1836 if (I->Low == I->High) {
1837 // Check Cond == I->Low.
1838 Pred = CmpInst::ICMP_EQ;
1839 LHS = Cond;
1840 RHS = I->Low;
1841 MHS = nullptr;
1842 } else {
1843 // Check I->Low <= Cond <= I->High.
1844 Pred = CmpInst::ICMP_SLE;
1845 LHS = I->Low;
1846 MHS = Cond;
1847 RHS = I->High;
1848 }
1849
1850 // If Fallthrough is unreachable, fold away the comparison.
1851 // The false probability is the sum of all unhandled cases.
1852 CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
1853 CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
1854
1855 emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
1856 return true;
1857}
1858
1859void IRTranslatorImpl::emitBitTestHeader(SwitchCG::BitTestBlock &B,
1860 MachineBasicBlock *SwitchBB) {
1861 MachineIRBuilder &MIB = *CurBuilder;
1862 MIB.setMBB(*SwitchBB);
1863
1864 // Subtract the minimum value.
1865 Register SwitchOpReg = getOrCreateVReg(Val: *B.SValue);
1866
1867 LLT SwitchOpTy = MRI->getType(Reg: SwitchOpReg);
1868 Register MinValReg = MIB.buildConstant(Res: SwitchOpTy, Val: B.First).getReg(Idx: 0);
1869 auto RangeSub = MIB.buildSub(Dst: SwitchOpTy, Src0: SwitchOpReg, Src1: MinValReg);
1870
1871 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
1872 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1873
1874 LLT MaskTy = SwitchOpTy;
1875 if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
1876 !llvm::has_single_bit<uint32_t>(Value: MaskTy.getSizeInBits()))
1877 MaskTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
1878 else {
1879 // Ensure that the type will fit the mask value.
1880 for (const SwitchCG::BitTestCase &Case : B.Cases) {
1881 if (!isUIntN(N: SwitchOpTy.getSizeInBits(), x: Case.Mask)) {
1882 // Switch table case range are encoded into series of masks.
1883 // Just use pointer type, it's guaranteed to fit.
1884 MaskTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
1885 break;
1886 }
1887 }
1888 }
1889 Register SubReg = RangeSub.getReg(Idx: 0);
1890 if (SwitchOpTy != MaskTy)
1891 SubReg = MIB.buildZExtOrTrunc(Res: MaskTy, Op: SubReg).getReg(Idx: 0);
1892
1893 B.RegVT = getMVTForLLT(Ty: MaskTy);
1894 B.Reg = SubReg;
1895
1896 MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1897
1898 if (!B.FallthroughUnreachable)
1899 addSuccessorWithProb(Src: SwitchBB, Dst: B.Default, Prob: B.DefaultProb);
1900 addSuccessorWithProb(Src: SwitchBB, Dst: MBB, Prob: B.Prob);
1901
1902 SwitchBB->normalizeSuccProbs();
1903
1904 if (!B.FallthroughUnreachable) {
1905 // Conditional branch to the default block.
1906 auto RangeCst = MIB.buildConstant(Res: SwitchOpTy, Val: B.Range);
1907 auto RangeCmp = MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: LLT::integer(SizeInBits: 1),
1908 Op0: RangeSub, Op1: RangeCst);
1909 MIB.buildBrCond(Tst: RangeCmp, Dest&: *B.Default);
1910 }
1911
1912 // Avoid emitting unnecessary branches to the next block.
1913 if (MBB != SwitchBB->getNextNode())
1914 MIB.buildBr(Dest&: *MBB);
1915}
1916
1917void IRTranslatorImpl::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1918 MachineBasicBlock *NextMBB,
1919 BranchProbability BranchProbToNext,
1920 Register Reg, SwitchCG::BitTestCase &B,
1921 MachineBasicBlock *SwitchBB) {
1922 MachineIRBuilder &MIB = *CurBuilder;
1923 MIB.setMBB(*SwitchBB);
1924
1925 LLT SwitchTy = getLLTForMVT(Ty: BB.RegVT);
1926 Register Cmp;
1927 unsigned PopCount = llvm::popcount(Value: B.Mask);
1928 if (PopCount == 1) {
1929 // Testing for a single bit; just compare the shift count with what it
1930 // would need to be to shift a 1 bit in that position.
1931 auto MaskTrailingZeros =
1932 MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_zero(Val: B.Mask));
1933 Cmp = MIB.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: LLT::integer(SizeInBits: 1), Op0: Reg,
1934 Op1: MaskTrailingZeros)
1935 .getReg(Idx: 0);
1936 } else if (PopCount == BB.Range) {
1937 // There is only one zero bit in the range, test for it directly.
1938 auto MaskTrailingOnes =
1939 MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_one(Value: B.Mask));
1940 Cmp =
1941 MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::integer(SizeInBits: 1), Op0: Reg, Op1: MaskTrailingOnes)
1942 .getReg(Idx: 0);
1943 } else {
1944 // Make desired shift.
1945 auto CstOne = MIB.buildConstant(Res: SwitchTy, Val: 1);
1946 auto SwitchVal = MIB.buildShl(Dst: SwitchTy, Src0: CstOne, Src1: Reg);
1947
1948 // Emit bit tests and jumps.
1949 auto CstMask = MIB.buildConstant(Res: SwitchTy, Val: B.Mask);
1950 auto AndOp = MIB.buildAnd(Dst: SwitchTy, Src0: SwitchVal, Src1: CstMask);
1951 auto CstZero = MIB.buildConstant(Res: SwitchTy, Val: 0);
1952 Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::integer(SizeInBits: 1), Op0: AndOp, Op1: CstZero)
1953 .getReg(Idx: 0);
1954 }
1955
1956 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1957 addSuccessorWithProb(Src: SwitchBB, Dst: B.TargetBB, Prob: B.ExtraProb);
1958 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1959 addSuccessorWithProb(Src: SwitchBB, Dst: NextMBB, Prob: BranchProbToNext);
1960 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1961 // one as they are relative probabilities (and thus work more like weights),
1962 // and hence we need to normalize them to let the sum of them become one.
1963 SwitchBB->normalizeSuccProbs();
1964
1965 // Record the fact that the IR edge from the header to the bit test target
1966 // will go through our new block. Neeeded for PHIs to have nodes added.
1967 addMachineCFGPred(Edge: {BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1968 NewPred: SwitchBB);
1969
1970 MIB.buildBrCond(Tst: Cmp, Dest&: *B.TargetBB);
1971
1972 // Avoid emitting unnecessary branches to the next block.
1973 if (NextMBB != SwitchBB->getNextNode())
1974 MIB.buildBr(Dest&: *NextMBB);
1975}
1976
1977bool IRTranslatorImpl::lowerBitTestWorkItem(
1978 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
1979 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1980 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
1981 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1982 SwitchCG::CaseClusterIt I, MachineBasicBlock *Fallthrough,
1983 bool FallthroughUnreachable) {
1984 using namespace SwitchCG;
1985 MachineFunction *CurMF = SwitchMBB->getParent();
1986 // FIXME: Optimize away range check based on pivot comparisons.
1987 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
1988 // The bit test blocks haven't been inserted yet; insert them here.
1989 for (BitTestCase &BTC : BTB->Cases)
1990 CurMF->insert(MBBI: BBI, MBB: BTC.ThisBB);
1991
1992 // Fill in fields of the BitTestBlock.
1993 BTB->Parent = CurMBB;
1994 BTB->Default = Fallthrough;
1995
1996 BTB->DefaultProb = UnhandledProbs;
1997 // If the cases in bit test don't form a contiguous range, we evenly
1998 // distribute the probability on the edge to Fallthrough to two
1999 // successors of CurMBB.
2000 if (!BTB->ContiguousRange) {
2001 BTB->Prob += DefaultProb / 2;
2002 BTB->DefaultProb -= DefaultProb / 2;
2003 }
2004
2005 if (FallthroughUnreachable)
2006 BTB->FallthroughUnreachable = true;
2007
2008 // If we're in the right place, emit the bit test header right now.
2009 if (CurMBB == SwitchMBB) {
2010 emitBitTestHeader(B&: *BTB, SwitchBB: SwitchMBB);
2011 BTB->Emitted = true;
2012 }
2013 return true;
2014}
2015
2016bool IRTranslatorImpl::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
2017 Value *Cond,
2018 MachineBasicBlock *SwitchMBB,
2019 MachineBasicBlock *DefaultMBB,
2020 MachineIRBuilder &MIB) {
2021 using namespace SwitchCG;
2022 MachineFunction *CurMF = FuncInfo.MF;
2023 MachineBasicBlock *NextMBB = nullptr;
2024 MachineFunction::iterator BBI(W.MBB);
2025 if (++BBI != FuncInfo.MF->end())
2026 NextMBB = &*BBI;
2027
2028 if (EnableOpts) {
2029 // Here, we order cases by probability so the most likely case will be
2030 // checked first. However, two clusters can have the same probability in
2031 // which case their relative ordering is non-deterministic. So we use Low
2032 // as a tie-breaker as clusters are guaranteed to never overlap.
2033 llvm::sort(Start: W.FirstCluster, End: W.LastCluster + 1,
2034 Comp: [](const CaseCluster &a, const CaseCluster &b) {
2035 return a.Prob != b.Prob
2036 ? a.Prob > b.Prob
2037 : a.Low->getValue().slt(RHS: b.Low->getValue());
2038 });
2039
2040 // Rearrange the case blocks so that the last one falls through if possible
2041 // without changing the order of probabilities.
2042 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
2043 --I;
2044 if (I->Prob > W.LastCluster->Prob)
2045 break;
2046 if (I->Kind == CC_Range && I->MBB == NextMBB) {
2047 std::swap(a&: *I, b&: *W.LastCluster);
2048 break;
2049 }
2050 }
2051 }
2052
2053 // Compute total probability.
2054 BranchProbability DefaultProb = W.DefaultProb;
2055 BranchProbability UnhandledProbs = DefaultProb;
2056 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
2057 UnhandledProbs += I->Prob;
2058
2059 MachineBasicBlock *CurMBB = W.MBB;
2060 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
2061 bool FallthroughUnreachable = false;
2062 MachineBasicBlock *Fallthrough;
2063 if (I == W.LastCluster) {
2064 // For the last cluster, fall through to the default destination.
2065 Fallthrough = DefaultMBB;
2066 FallthroughUnreachable = isa<UnreachableInst>(
2067 Val: DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
2068 } else {
2069 Fallthrough = CurMF->CreateMachineBasicBlock(BB: CurMBB->getBasicBlock());
2070 CurMF->insert(MBBI: BBI, MBB: Fallthrough);
2071 }
2072 UnhandledProbs -= I->Prob;
2073
2074 switch (I->Kind) {
2075 case CC_BitTests: {
2076 if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
2077 DefaultProb, UnhandledProbs, I, Fallthrough,
2078 FallthroughUnreachable)) {
2079 LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
2080 return false;
2081 }
2082 break;
2083 }
2084
2085 case CC_JumpTable: {
2086 if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
2087 UnhandledProbs, I, Fallthrough,
2088 FallthroughUnreachable)) {
2089 LLVM_DEBUG(dbgs() << "Failed to lower jump table");
2090 return false;
2091 }
2092 break;
2093 }
2094 case CC_Range: {
2095 if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
2096 FallthroughUnreachable, UnhandledProbs,
2097 CurMBB, MIB, SwitchMBB)) {
2098 LLVM_DEBUG(dbgs() << "Failed to lower switch range");
2099 return false;
2100 }
2101 break;
2102 }
2103 }
2104 CurMBB = Fallthrough;
2105 }
2106
2107 return true;
2108}
2109
2110bool IRTranslatorImpl::translateIndirectBr(const User &U,
2111 MachineIRBuilder &MIRBuilder) {
2112 const IndirectBrInst &BrInst = cast<IndirectBrInst>(Val: U);
2113
2114 const Register Tgt = getOrCreateVReg(Val: *BrInst.getAddress());
2115 MIRBuilder.buildBrIndirect(Tgt);
2116
2117 // Link successors.
2118 SmallPtrSet<const BasicBlock *, 32> AddedSuccessors;
2119 MachineBasicBlock &CurBB = MIRBuilder.getMBB();
2120 for (const BasicBlock *Succ : successors(I: &BrInst)) {
2121 // It's legal for indirectbr instructions to have duplicate blocks in the
2122 // destination list. We don't allow this in MIR. Skip anything that's
2123 // already a successor.
2124 if (!AddedSuccessors.insert(Ptr: Succ).second)
2125 continue;
2126 CurBB.addSuccessor(Succ: &getMBB(BB: *Succ));
2127 }
2128
2129 return true;
2130}
2131
2132static bool isSwiftError(const Value *V) {
2133 if (auto Arg = dyn_cast<Argument>(Val: V))
2134 return Arg->hasSwiftErrorAttr();
2135 if (auto AI = dyn_cast<AllocaInst>(Val: V))
2136 return AI->isSwiftError();
2137 return false;
2138}
2139
2140bool IRTranslatorImpl::translateLoad(const User &U,
2141 MachineIRBuilder &MIRBuilder) {
2142 const LoadInst &LI = cast<LoadInst>(Val: U);
2143 TypeSize StoreSize = DL->getTypeStoreSize(Ty: LI.getType());
2144 if (StoreSize.isZero())
2145 return true;
2146
2147 ArrayRef<Register> Regs = getOrCreateVRegs(Val: LI);
2148 Register Base = getOrCreateVReg(Val: *LI.getPointerOperand());
2149 AAMDNodes AAInfo = LI.getAAMetadata();
2150
2151 const Value *Ptr = LI.getPointerOperand();
2152
2153 if (CLI->supportSwiftError() && isSwiftError(V: Ptr)) {
2154 assert(Regs.size() == 1 && "swifterror should be single pointer");
2155 Register VReg =
2156 SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);
2157 MIRBuilder.buildCopy(Res: Regs[0], Op: VReg);
2158 return true;
2159 }
2160
2161 MachineMemOperand::Flags Flags =
2162 TLI->getLoadMemOperandFlags(LI, DL: *DL, AC, LibInfo, OptLevel);
2163 if (AA && !(Flags & MachineMemOperand::MOInvariant)) {
2164 if (AA->pointsToConstantMemory(
2165 Loc: MemoryLocation(Ptr, LocationSize::precise(Value: StoreSize), AAInfo))) {
2166 Flags |= MachineMemOperand::MOInvariant;
2167 }
2168 }
2169
2170 // Fast-path the common single-register load.
2171 if (Regs.size() == 1) {
2172 auto *MMO = MF->getMachineMemOperand(
2173 PtrInfo: MachinePointerInfo(LI.getPointerOperand()), F: Flags,
2174 MemTy: MRI->getType(Reg: Regs[0]), BaseAlignment: getMemOpAlign(I: LI),
2175 Metadata: MMOMetadata(AAInfo, LI.getMetadata(KindID: LLVMContext::MD_range)),
2176 SSID: LI.getSyncScopeID(), Ordering: LI.getOrdering());
2177 MIRBuilder.buildLoad(Res: Regs[0], Addr: Base, MMO&: *MMO);
2178 return true;
2179 }
2180
2181 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: LI);
2182 Type *OffsetIRTy = DL->getIndexType(PtrTy: Ptr->getType());
2183 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2184 for (unsigned i = 0; i < Regs.size(); ++i) {
2185 Register Addr;
2186 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i]);
2187
2188 MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i]);
2189 Align BaseAlign = getMemOpAlign(I: LI);
2190 auto *MMO =
2191 MF->getMachineMemOperand(PtrInfo: Ptr, F: Flags, MemTy: MRI->getType(Reg: Regs[i]),
2192 BaseAlignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]), Metadata: AAInfo,
2193 SSID: LI.getSyncScopeID(), Ordering: LI.getOrdering());
2194 MIRBuilder.buildLoad(Res: Regs[i], Addr, MMO&: *MMO);
2195 }
2196
2197 return true;
2198}
2199
2200bool IRTranslatorImpl::translateStore(const User &U,
2201 MachineIRBuilder &MIRBuilder) {
2202 const StoreInst &SI = cast<StoreInst>(Val: U);
2203 if (DL->getTypeStoreSize(Ty: SI.getValueOperand()->getType()).isZero())
2204 return true;
2205
2206 ArrayRef<Register> Vals = getOrCreateVRegs(Val: *SI.getValueOperand());
2207 Register Base = getOrCreateVReg(Val: *SI.getPointerOperand());
2208
2209 if (CLI->supportSwiftError() && isSwiftError(V: SI.getPointerOperand())) {
2210 assert(Vals.size() == 1 && "swifterror should be single pointer");
2211
2212 Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
2213 SI.getPointerOperand());
2214 MIRBuilder.buildCopy(Res: VReg, Op: Vals[0]);
2215 return true;
2216 }
2217
2218 MachineMemOperand::Flags Flags = TLI->getStoreMemOperandFlags(SI, DL: *DL);
2219 // Fast-path the common single-register store.
2220 if (Vals.size() == 1) {
2221 auto *MMO = MF->getMachineMemOperand(
2222 PtrInfo: MachinePointerInfo(SI.getPointerOperand()), F: Flags,
2223 MemTy: MRI->getType(Reg: Vals[0]), BaseAlignment: getMemOpAlign(I: SI), Metadata: SI.getAAMetadata(),
2224 SSID: SI.getSyncScopeID(), Ordering: SI.getOrdering());
2225 MIRBuilder.buildStore(Val: Vals[0], Addr: Base, MMO&: *MMO);
2226 return true;
2227 }
2228
2229 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *SI.getValueOperand());
2230 Type *OffsetIRTy = DL->getIndexType(PtrTy: SI.getPointerOperandType());
2231 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2232 for (unsigned i = 0; i < Vals.size(); ++i) {
2233 Register Addr;
2234 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i]);
2235
2236 MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i]);
2237 Align BaseAlign = getMemOpAlign(I: SI);
2238 auto *MMO = MF->getMachineMemOperand(PtrInfo: Ptr, F: Flags, MemTy: MRI->getType(Reg: Vals[i]),
2239 BaseAlignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]),
2240 Metadata: SI.getAAMetadata(),
2241 SSID: SI.getSyncScopeID(), Ordering: SI.getOrdering());
2242 MIRBuilder.buildStore(Val: Vals[i], Addr, MMO&: *MMO);
2243 }
2244 return true;
2245}
2246
2247static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
2248 const Value *Src = U.getOperand(i: 0);
2249 Type *Int32Ty = Type::getInt32Ty(C&: U.getContext());
2250
2251 // getIndexedOffsetInType is designed for GEPs, so the first index is the
2252 // usual array element rather than looking into the actual aggregate.
2253 SmallVector<Value *, 1> Indices;
2254 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: 0));
2255
2256 if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: &U)) {
2257 for (auto Idx : EVI->indices())
2258 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
2259 } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(Val: &U)) {
2260 for (auto Idx : IVI->indices())
2261 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
2262 } else {
2263 llvm::append_range(C&: Indices, R: drop_begin(RangeOrContainer: U.operands()));
2264 }
2265
2266 return static_cast<uint64_t>(
2267 DL.getIndexedOffsetInType(ElemTy: Src->getType(), Indices));
2268}
2269
2270bool IRTranslatorImpl::translateExtractValue(const User &U,
2271 MachineIRBuilder &MIRBuilder) {
2272 const Value *Src = U.getOperand(i: 0);
2273 uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
2274 ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
2275 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *Src);
2276 unsigned Idx = llvm::lower_bound(Range&: Offsets, Value&: Offset) - Offsets.begin();
2277 auto &DstRegs = allocateVRegs(Val: U);
2278
2279 for (unsigned i = 0; i < DstRegs.size(); ++i)
2280 DstRegs[i] = SrcRegs[Idx++];
2281
2282 return true;
2283}
2284
2285bool IRTranslatorImpl::translateInsertValue(const User &U,
2286 MachineIRBuilder &MIRBuilder) {
2287 const Value *Src = U.getOperand(i: 0);
2288 uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
2289 auto &DstRegs = allocateVRegs(Val: U);
2290 ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(V: U);
2291 ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
2292 ArrayRef<Register> InsertedRegs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
2293 auto *InsertedIt = InsertedRegs.begin();
2294
2295 for (unsigned i = 0; i < DstRegs.size(); ++i) {
2296 if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
2297 DstRegs[i] = *InsertedIt++;
2298 else
2299 DstRegs[i] = SrcRegs[i];
2300 }
2301
2302 return true;
2303}
2304
2305bool IRTranslatorImpl::translateSelect(const User &U,
2306 MachineIRBuilder &MIRBuilder) {
2307 Register Tst = getOrCreateVReg(Val: *U.getOperand(i: 0));
2308 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: U);
2309 ArrayRef<Register> Op0Regs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
2310 ArrayRef<Register> Op1Regs = getOrCreateVRegs(Val: *U.getOperand(i: 2));
2311
2312 uint32_t Flags = 0;
2313 if (const SelectInst *SI = dyn_cast<SelectInst>(Val: &U))
2314 Flags = MachineInstr::copyFlagsFromInstruction(I: *SI);
2315
2316 for (unsigned i = 0; i < ResRegs.size(); ++i) {
2317 MIRBuilder.buildSelect(Res: ResRegs[i], Tst, Op0: Op0Regs[i], Op1: Op1Regs[i], Flags);
2318 }
2319
2320 return true;
2321}
2322
2323bool IRTranslatorImpl::translateCopy(const User &U, const Value &V,
2324 MachineIRBuilder &MIRBuilder) {
2325 return translateCopy(U, Src: getOrCreateVReg(Val: V), MIRBuilder);
2326}
2327
2328bool IRTranslatorImpl::translateCopy(const User &U, Register Src,
2329 MachineIRBuilder &MIRBuilder) {
2330 auto &Regs = *VMap.getVRegs(V: U);
2331 if (Regs.empty()) {
2332 Regs.push_back(Elt: Src);
2333 } else {
2334 // If we already assigned a vreg for this instruction, we can't change that.
2335 // Emit a copy to satisfy the users we already emitted.
2336 MIRBuilder.buildCopy(Res: Regs[0], Op: Src);
2337 }
2338 return true;
2339}
2340
2341bool IRTranslatorImpl::translateBitCast(const User &U,
2342 MachineIRBuilder &MIRBuilder) {
2343 Type *SrcTy = U.getOperand(i: 0)->getType();
2344 Type *DstTy = U.getType();
2345
2346 // If we're bitcasting to the source type, we can reuse the source vreg.
2347 if (getLLTForType(Ty&: *SrcTy, DL: *DL) == getLLTForType(Ty&: *DstTy, DL: *DL)) {
2348 // If the source is a ConstantInt then it was probably created by
2349 // ConstantHoisting and we should leave it alone.
2350 if (isa<ConstantInt>(Val: U.getOperand(i: 0)))
2351 return translateCast(Opcode: TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
2352 MIRBuilder);
2353 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
2354 }
2355
2356 // The IR only allows pointer/non-pointer bitcasts with byte types, but
2357 // G_BITCAST can't convert between pointers and other types. Go through an
2358 // integer with the pointer's shape instead: `bitcast <2 x b32> to ptr`
2359 // becomes a G_BITCAST to i64 and a G_INTTOPTR.
2360 if (SrcTy->isPtrOrPtrVectorTy() != DstTy->isPtrOrPtrVectorTy()) {
2361 assert((SrcTy->isByteOrByteVectorTy() || DstTy->isByteOrByteVectorTy()) &&
2362 "only byte types can be bitcast to or from pointers");
2363 Type *PtrIRTy = SrcTy->isPtrOrPtrVectorTy() ? SrcTy : DstTy;
2364 LLT IntTy = getLLTForType(Ty&: *DL->getIntPtrType(PtrIRTy), DL: *DL);
2365 Register Src = getOrCreateVReg(Val: *U.getOperand(i: 0));
2366 Register Dst = getOrCreateVReg(Val: U);
2367 if (MRI->getType(Reg: Src) != IntTy && MRI->getType(Reg: Dst) != IntTy)
2368 Src = MIRBuilder.buildCast(Dst: IntTy, Src).getReg(Idx: 0);
2369 MIRBuilder.buildCast(Dst, Src);
2370 return true;
2371 }
2372
2373 return translateCast(Opcode: TargetOpcode::G_BITCAST, U, MIRBuilder);
2374}
2375
2376bool IRTranslatorImpl::translateCast(unsigned Opcode, const User &U,
2377 MachineIRBuilder &MIRBuilder) {
2378 if (!mayTranslateUserTypes(U))
2379 return false;
2380
2381 uint32_t Flags = 0;
2382 if (const Instruction *I = dyn_cast<Instruction>(Val: &U))
2383 Flags = MachineInstr::copyFlagsFromInstruction(I: *I);
2384
2385 Register Op = getOrCreateVReg(Val: *U.getOperand(i: 0));
2386 Register Res = getOrCreateVReg(Val: U);
2387 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op}, Flags);
2388 return true;
2389}
2390
2391bool IRTranslatorImpl::translateGetElementPtr(const User &U,
2392 MachineIRBuilder &MIRBuilder) {
2393 Value &Op0 = *U.getOperand(i: 0);
2394 Register BaseReg = getOrCreateVReg(Val: Op0);
2395 Type *PtrIRTy = Op0.getType();
2396 LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
2397 Type *OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
2398 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2399
2400 uint32_t PtrAddFlags = 0;
2401 // Each PtrAdd generated to implement the GEP inherits its nuw, nusw, inbounds
2402 // flags.
2403 if (const Instruction *I = dyn_cast<Instruction>(Val: &U))
2404 PtrAddFlags = MachineInstr::copyFlagsFromInstruction(I: *I);
2405
2406 auto PtrAddFlagsWithConst = [&](int64_t Offset) {
2407 // For nusw/inbounds GEP with an offset that is nonnegative when interpreted
2408 // as signed, assume there is no unsigned overflow.
2409 if (Offset >= 0 && (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap))
2410 return PtrAddFlags | MachineInstr::MIFlag::NoUWrap;
2411 return PtrAddFlags;
2412 };
2413
2414 // Normalize Vector GEP - all scalar operands should be converted to the
2415 // splat vector.
2416 unsigned VectorWidth = 0;
2417
2418 // True if we should use a splat vector; using VectorWidth alone is not
2419 // sufficient.
2420 bool WantSplatVector = false;
2421 if (auto *VT = dyn_cast<VectorType>(Val: U.getType())) {
2422 VectorWidth = cast<FixedVectorType>(Val: VT)->getNumElements();
2423 // We don't produce 1 x N vectors; those are treated as scalars.
2424 WantSplatVector = VectorWidth > 1;
2425 }
2426
2427 if (cast<GEPOperator>(Val: U).hasAllZeroIndices())
2428 return translateCopy(U, Src: BaseReg, MIRBuilder);
2429
2430 // We might need to splat the base pointer into a vector if the offsets
2431 // are vectors.
2432 if (WantSplatVector && !PtrTy.isVector()) {
2433 BaseReg = MIRBuilder
2434 .buildSplatBuildVector(Res: LLT::fixed_vector(NumElements: VectorWidth, ScalarTy: PtrTy),
2435 Src: BaseReg)
2436 .getReg(Idx: 0);
2437 PtrIRTy = FixedVectorType::get(ElementType: PtrIRTy, NumElts: VectorWidth);
2438 PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
2439 OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
2440 OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2441 }
2442
2443 int64_t Offset = 0;
2444 for (gep_type_iterator GTI = gep_type_begin(GEP: &U), E = gep_type_end(GEP: &U);
2445 GTI != E; ++GTI) {
2446 const Value *Idx = GTI.getOperand();
2447 if (StructType *StTy = GTI.getStructTypeOrNull()) {
2448 unsigned Field = cast<Constant>(Val: Idx)->getUniqueInteger().getZExtValue();
2449 Offset += DL->getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
2450 continue;
2451 } else {
2452 uint64_t ElementSize = GTI.getSequentialElementStride(DL: *DL);
2453
2454 // If this is a scalar constant or a splat vector of constants,
2455 // handle it quickly.
2456 if (const auto *CI = dyn_cast<ConstantInt>(Val: Idx)) {
2457 if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {
2458 Offset += ElementSize * *Val;
2459 continue;
2460 }
2461 }
2462
2463 if (Offset != 0) {
2464 auto OffsetMIB = MIRBuilder.buildConstant(Res: {OffsetTy}, Val: Offset);
2465 BaseReg = MIRBuilder
2466 .buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0),
2467 Flags: PtrAddFlagsWithConst(Offset))
2468 .getReg(Idx: 0);
2469 Offset = 0;
2470 }
2471
2472 Register IdxReg = getOrCreateVReg(Val: *Idx);
2473 LLT IdxTy = MRI->getType(Reg: IdxReg);
2474 if (IdxTy != OffsetTy) {
2475 if (!IdxTy.isVector() && WantSplatVector) {
2476 IdxReg = MIRBuilder
2477 .buildSplatBuildVector(Res: OffsetTy.changeElementType(NewEltTy: IdxTy),
2478 Src: IdxReg)
2479 .getReg(Idx: 0);
2480 }
2481
2482 IdxReg = MIRBuilder.buildSExtOrTrunc(Res: OffsetTy, Op: IdxReg).getReg(Idx: 0);
2483 }
2484
2485 // N = N + Idx * ElementSize;
2486 // Avoid doing it for ElementSize of 1.
2487 Register GepOffsetReg;
2488 if (ElementSize != 1) {
2489 auto ElementSizeMIB = MIRBuilder.buildConstant(
2490 Res: getLLTForType(Ty&: *OffsetIRTy, DL: *DL), Val: ElementSize);
2491
2492 // The multiplication is NUW if the GEP is NUW and NSW if the GEP is
2493 // NUSW.
2494 uint32_t ScaleFlags = PtrAddFlags & MachineInstr::MIFlag::NoUWrap;
2495 if (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap)
2496 ScaleFlags |= MachineInstr::MIFlag::NoSWrap;
2497
2498 GepOffsetReg =
2499 MIRBuilder.buildMul(Dst: OffsetTy, Src0: IdxReg, Src1: ElementSizeMIB, Flags: ScaleFlags)
2500 .getReg(Idx: 0);
2501 } else {
2502 GepOffsetReg = IdxReg;
2503 }
2504
2505 BaseReg =
2506 MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: GepOffsetReg, Flags: PtrAddFlags)
2507 .getReg(Idx: 0);
2508 }
2509 }
2510
2511 if (Offset != 0) {
2512 auto OffsetMIB =
2513 MIRBuilder.buildConstant(Res: OffsetTy, Val: Offset);
2514
2515 MIRBuilder.buildPtrAdd(Res: getOrCreateVReg(Val: U), Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0),
2516 Flags: PtrAddFlagsWithConst(Offset));
2517 return true;
2518 }
2519
2520 return translateCopy(U, Src: BaseReg, MIRBuilder);
2521}
2522
2523bool IRTranslatorImpl::translateMemFunc(const CallInst &CI,
2524 MachineIRBuilder &MIRBuilder,
2525 unsigned Opcode) {
2526 const Value *SrcPtr = CI.getArgOperand(i: 1);
2527 // If the source is undef, then just emit a nop.
2528 if (isa<UndefValue>(Val: SrcPtr))
2529 return true;
2530
2531 SmallVector<Register, 3> SrcRegs;
2532
2533 unsigned MinPtrSize = UINT_MAX;
2534 for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(x: AI) != AE; ++AI) {
2535 Register SrcReg = getOrCreateVReg(Val: **AI);
2536 LLT SrcTy = MRI->getType(Reg: SrcReg);
2537 if (SrcTy.isPointer())
2538 MinPtrSize = std::min<unsigned>(a: SrcTy.getSizeInBits(), b: MinPtrSize);
2539 SrcRegs.push_back(Elt: SrcReg);
2540 }
2541
2542 LLT SizeTy = LLT::integer(SizeInBits: MinPtrSize);
2543
2544 // The size operand should be the minimum of the pointer sizes.
2545 Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
2546 if (MRI->getType(Reg: SizeOpReg) != SizeTy)
2547 SizeOpReg = MIRBuilder.buildZExtOrTrunc(Res: SizeTy, Op: SizeOpReg).getReg(Idx: 0);
2548
2549 auto ICall = MIRBuilder.buildInstr(Opcode);
2550 for (Register SrcReg : SrcRegs)
2551 ICall.addUse(RegNo: SrcReg);
2552
2553 Align DstAlign;
2554 Align SrcAlign;
2555 unsigned IsVol =
2556 cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1))->getZExtValue();
2557
2558 ConstantInt *CopySize = nullptr;
2559
2560 if (auto *MCI = dyn_cast<MemCpyInst>(Val: &CI)) {
2561 DstAlign = MCI->getDestAlign().valueOrOne();
2562 SrcAlign = MCI->getSourceAlign().valueOrOne();
2563 CopySize = dyn_cast<ConstantInt>(Val: MCI->getArgOperand(i: 2));
2564 } else if (auto *MMI = dyn_cast<MemMoveInst>(Val: &CI)) {
2565 DstAlign = MMI->getDestAlign().valueOrOne();
2566 SrcAlign = MMI->getSourceAlign().valueOrOne();
2567 CopySize = dyn_cast<ConstantInt>(Val: MMI->getArgOperand(i: 2));
2568 } else {
2569 auto *MSI = cast<MemSetInst>(Val: &CI);
2570 DstAlign = MSI->getDestAlign().valueOrOne();
2571 }
2572
2573 if (Opcode != TargetOpcode::G_MEMCPY_INLINE &&
2574 Opcode != TargetOpcode::G_MEMSET_INLINE) {
2575 // We need to propagate the tail call flag from the IR inst as an argument.
2576 // Otherwise, we have to pessimize and assume later that we cannot tail call
2577 // any memory intrinsics.
2578 ICall.addImm(Val: CI.isTailCall() ? 1 : 0);
2579 }
2580
2581 // Create mem operands to store the alignment and volatile info.
2582 MachineMemOperand::Flags LoadFlags = MachineMemOperand::MOLoad;
2583 MachineMemOperand::Flags StoreFlags = MachineMemOperand::MOStore;
2584 if (IsVol) {
2585 LoadFlags |= MachineMemOperand::MOVolatile;
2586 StoreFlags |= MachineMemOperand::MOVolatile;
2587 }
2588
2589 AAMDNodes AAInfo = CI.getAAMetadata();
2590 if (AA && CopySize &&
2591 AA->pointsToConstantMemory(Loc: MemoryLocation(
2592 SrcPtr, LocationSize::precise(Value: CopySize->getZExtValue()), AAInfo))) {
2593 LoadFlags |= MachineMemOperand::MOInvariant;
2594
2595 // FIXME: pointsToConstantMemory probably does not imply dereferenceable,
2596 // but the previous usage implied it did. Probably should check
2597 // isDereferenceableAndAlignedPointer.
2598 LoadFlags |= MachineMemOperand::MODereferenceable;
2599 }
2600
2601 ICall.addMemOperand(
2602 MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(CI.getArgOperand(i: 0)),
2603 F: StoreFlags, Size: 1, BaseAlignment: DstAlign, Metadata: AAInfo));
2604 if (Opcode != TargetOpcode::G_MEMSET &&
2605 Opcode != TargetOpcode::G_MEMSET_INLINE)
2606 ICall.addMemOperand(MMO: MF->getMachineMemOperand(
2607 PtrInfo: MachinePointerInfo(SrcPtr), F: LoadFlags, Size: 1, BaseAlignment: SrcAlign, Metadata: AAInfo));
2608
2609 return true;
2610}
2611
2612bool IRTranslatorImpl::translateTrap(const CallInst &CI,
2613 MachineIRBuilder &MIRBuilder,
2614 unsigned Opcode) {
2615 StringRef TrapFuncName =
2616 CI.getAttributes().getFnAttr(Kind: "trap-func-name").getValueAsString();
2617 if (TrapFuncName.empty()) {
2618 if (Opcode == TargetOpcode::G_UBSANTRAP) {
2619 uint64_t Code = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 0))->getZExtValue();
2620 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: ArrayRef<llvm::SrcOp>{Code});
2621 } else {
2622 MIRBuilder.buildInstr(Opcode);
2623 }
2624 return true;
2625 }
2626
2627 CallLowering::CallLoweringInfo Info;
2628 if (Opcode == TargetOpcode::G_UBSANTRAP)
2629 Info.OrigArgs.push_back(Elt: {getOrCreateVRegs(Val: *CI.getArgOperand(i: 0)),
2630 CI.getArgOperand(i: 0)->getType(), 0});
2631
2632 Info.Callee = MachineOperand::CreateES(SymName: TrapFuncName.data());
2633 Info.CB = &CI;
2634 Info.OrigRet = {Register(), Type::getVoidTy(C&: CI.getContext()), 0};
2635 return CLI->lowerCall(MIRBuilder, Info);
2636}
2637
2638bool IRTranslatorImpl::translateVectorInterleave2Intrinsic(
2639 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
2640 assert(CI.getIntrinsicID() == Intrinsic::vector_interleave2 &&
2641 "This function can only be called on the interleave2 intrinsic!");
2642 // Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG).
2643 Register Op0 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
2644 Register Op1 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1));
2645 Register Res = getOrCreateVReg(Val: CI);
2646
2647 LLT OpTy = MRI->getType(Reg: Op0);
2648 MIRBuilder.buildShuffleVector(Res, Src1: Op0, Src2: Op1,
2649 Mask: createInterleaveMask(VF: OpTy.getNumElements(), NumVecs: 2));
2650
2651 return true;
2652}
2653
2654bool IRTranslatorImpl::translateVectorDeinterleave2Intrinsic(
2655 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
2656 assert(CI.getIntrinsicID() == Intrinsic::vector_deinterleave2 &&
2657 "This function can only be called on the deinterleave2 intrinsic!");
2658 // Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to
2659 // SelectionDAG).
2660 Register Op = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
2661 auto Undef = MIRBuilder.buildUndef(Res: MRI->getType(Reg: Op));
2662 ArrayRef<Register> Res = getOrCreateVRegs(Val: CI);
2663
2664 LLT ResTy = MRI->getType(Reg: Res[0]);
2665 if (ResTy.isScalar()) {
2666 MIRBuilder.buildExtractVectorElementConstant(Res: Res[0], Val: Op, Idx: 0);
2667 MIRBuilder.buildExtractVectorElementConstant(Res: Res[1], Val: Op, Idx: 1);
2668
2669 return true;
2670 }
2671
2672 assert(ResTy.isVector() && "Expected vector result type");
2673 MIRBuilder.buildShuffleVector(Res: Res[0], Src1: Op, Src2: Undef,
2674 Mask: createStrideMask(Start: 0, Stride: 2, VF: ResTy.getNumElements()));
2675 MIRBuilder.buildShuffleVector(Res: Res[1], Src1: Op, Src2: Undef,
2676 Mask: createStrideMask(Start: 1, Stride: 2, VF: ResTy.getNumElements()));
2677
2678 return true;
2679}
2680
2681void IRTranslatorImpl::getStackGuard(Register DstReg,
2682 MachineIRBuilder &MIRBuilder) {
2683 Value *Global =
2684 TLI->getSDagStackGuard(M: *MF->getFunction().getParent(), Libcalls: *Libcalls);
2685 if (!Global) {
2686 LLVMContext &Ctx = MIRBuilder.getContext();
2687 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
2688 MIRBuilder.buildUndef(Res: DstReg);
2689 return;
2690 }
2691
2692 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
2693 MRI->setRegClass(Reg: DstReg,
2694 RC: TII.getRegClass(MCID: TII.get(Opcode: TargetOpcode::LOAD_STACK_GUARD), OpNum: 0));
2695 auto MIB =
2696 MIRBuilder.buildInstr(Opc: TargetOpcode::LOAD_STACK_GUARD, DstOps: {DstReg}, SrcOps: {});
2697
2698 unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
2699 LLT PtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL->getPointerSizeInBits(AS: AddrSpace));
2700
2701 MachinePointerInfo MPInfo(Global);
2702 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2703 MachineMemOperand::MODereferenceable;
2704 MachineMemOperand *MemRef = MF->getMachineMemOperand(
2705 PtrInfo: MPInfo, F: Flags, MemTy: PtrTy, BaseAlignment: DL->getPointerABIAlignment(AS: AddrSpace));
2706 MIB.setMemRefs({MemRef});
2707}
2708
2709bool IRTranslatorImpl::translateOverflowIntrinsic(
2710 const CallInst &CI, unsigned Op, MachineIRBuilder &MIRBuilder) {
2711 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: CI);
2712 MIRBuilder.buildInstr(
2713 Opc: Op, DstOps: {ResRegs[0], ResRegs[1]},
2714 SrcOps: {getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)), getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1))});
2715
2716 return true;
2717}
2718
2719bool IRTranslatorImpl::translateFixedPointIntrinsic(
2720 unsigned Op, const CallInst &CI, MachineIRBuilder &MIRBuilder) {
2721 Register Dst = getOrCreateVReg(Val: CI);
2722 Register Src0 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
2723 Register Src1 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1));
2724 uint64_t Scale = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
2725 MIRBuilder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: { Src0, Src1, Scale });
2726 return true;
2727}
2728
2729unsigned IRTranslatorImpl::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
2730 switch (ID) {
2731 default:
2732 break;
2733 case Intrinsic::acos:
2734 return TargetOpcode::G_FACOS;
2735 case Intrinsic::asin:
2736 return TargetOpcode::G_FASIN;
2737 case Intrinsic::atan:
2738 return TargetOpcode::G_FATAN;
2739 case Intrinsic::atan2:
2740 return TargetOpcode::G_FATAN2;
2741 case Intrinsic::bswap:
2742 return TargetOpcode::G_BSWAP;
2743 case Intrinsic::bitreverse:
2744 return TargetOpcode::G_BITREVERSE;
2745 case Intrinsic::clmul:
2746 return TargetOpcode::G_CLMUL;
2747 case Intrinsic::fshl:
2748 return TargetOpcode::G_FSHL;
2749 case Intrinsic::fshr:
2750 return TargetOpcode::G_FSHR;
2751 case Intrinsic::ceil:
2752 return TargetOpcode::G_FCEIL;
2753 case Intrinsic::cos:
2754 return TargetOpcode::G_FCOS;
2755 case Intrinsic::cosh:
2756 return TargetOpcode::G_FCOSH;
2757 case Intrinsic::ctpop:
2758 return TargetOpcode::G_CTPOP;
2759 case Intrinsic::smulh:
2760 return TargetOpcode::G_SMULH;
2761 case Intrinsic::umulh:
2762 return TargetOpcode::G_UMULH;
2763 case Intrinsic::exp:
2764 return TargetOpcode::G_FEXP;
2765 case Intrinsic::exp2:
2766 return TargetOpcode::G_FEXP2;
2767 case Intrinsic::exp10:
2768 return TargetOpcode::G_FEXP10;
2769 case Intrinsic::fabs:
2770 return TargetOpcode::G_FABS;
2771 case Intrinsic::copysign:
2772 return TargetOpcode::G_FCOPYSIGN;
2773 case Intrinsic::minnum:
2774 return TargetOpcode::G_FMINNUM;
2775 case Intrinsic::maxnum:
2776 return TargetOpcode::G_FMAXNUM;
2777 case Intrinsic::minimum:
2778 return TargetOpcode::G_FMINIMUM;
2779 case Intrinsic::maximum:
2780 return TargetOpcode::G_FMAXIMUM;
2781 case Intrinsic::minimumnum:
2782 return TargetOpcode::G_FMINIMUMNUM;
2783 case Intrinsic::maximumnum:
2784 return TargetOpcode::G_FMAXIMUMNUM;
2785 case Intrinsic::canonicalize:
2786 return TargetOpcode::G_FCANONICALIZE;
2787 case Intrinsic::floor:
2788 return TargetOpcode::G_FFLOOR;
2789 case Intrinsic::fma:
2790 return TargetOpcode::G_FMA;
2791 case Intrinsic::log:
2792 return TargetOpcode::G_FLOG;
2793 case Intrinsic::log2:
2794 return TargetOpcode::G_FLOG2;
2795 case Intrinsic::log10:
2796 return TargetOpcode::G_FLOG10;
2797 case Intrinsic::ldexp:
2798 return TargetOpcode::G_FLDEXP;
2799 case Intrinsic::nearbyint:
2800 return TargetOpcode::G_FNEARBYINT;
2801 case Intrinsic::pow:
2802 return TargetOpcode::G_FPOW;
2803 case Intrinsic::powi:
2804 return TargetOpcode::G_FPOWI;
2805 case Intrinsic::rint:
2806 return TargetOpcode::G_FRINT;
2807 case Intrinsic::round:
2808 return TargetOpcode::G_INTRINSIC_ROUND;
2809 case Intrinsic::roundeven:
2810 return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
2811 case Intrinsic::sin:
2812 return TargetOpcode::G_FSIN;
2813 case Intrinsic::sinh:
2814 return TargetOpcode::G_FSINH;
2815 case Intrinsic::sqrt:
2816 return TargetOpcode::G_FSQRT;
2817 case Intrinsic::tan:
2818 return TargetOpcode::G_FTAN;
2819 case Intrinsic::tanh:
2820 return TargetOpcode::G_FTANH;
2821 case Intrinsic::trunc:
2822 return TargetOpcode::G_INTRINSIC_TRUNC;
2823 case Intrinsic::readcyclecounter:
2824 return TargetOpcode::G_READCYCLECOUNTER;
2825 case Intrinsic::readsteadycounter:
2826 return TargetOpcode::G_READSTEADYCOUNTER;
2827 case Intrinsic::ptrmask:
2828 return TargetOpcode::G_PTRMASK;
2829 case Intrinsic::lrint:
2830 return TargetOpcode::G_INTRINSIC_LRINT;
2831 case Intrinsic::llrint:
2832 return TargetOpcode::G_INTRINSIC_LLRINT;
2833 // FADD/FMUL require checking the FMF, so are handled elsewhere.
2834 case Intrinsic::vector_reduce_fmin:
2835 return TargetOpcode::G_VECREDUCE_FMIN;
2836 case Intrinsic::vector_reduce_fmax:
2837 return TargetOpcode::G_VECREDUCE_FMAX;
2838 case Intrinsic::vector_reduce_fminimum:
2839 return TargetOpcode::G_VECREDUCE_FMINIMUM;
2840 case Intrinsic::vector_reduce_fmaximum:
2841 return TargetOpcode::G_VECREDUCE_FMAXIMUM;
2842 case Intrinsic::vector_reduce_fminimumnum:
2843 return TargetOpcode::G_VECREDUCE_FMINIMUMNUM;
2844 case Intrinsic::vector_reduce_fmaximumnum:
2845 return TargetOpcode::G_VECREDUCE_FMAXIMUMNUM;
2846 case Intrinsic::vector_reduce_add:
2847 return TargetOpcode::G_VECREDUCE_ADD;
2848 case Intrinsic::vector_reduce_mul:
2849 return TargetOpcode::G_VECREDUCE_MUL;
2850 case Intrinsic::vector_reduce_and:
2851 return TargetOpcode::G_VECREDUCE_AND;
2852 case Intrinsic::vector_reduce_or:
2853 return TargetOpcode::G_VECREDUCE_OR;
2854 case Intrinsic::vector_reduce_xor:
2855 return TargetOpcode::G_VECREDUCE_XOR;
2856 case Intrinsic::vector_reduce_smax:
2857 return TargetOpcode::G_VECREDUCE_SMAX;
2858 case Intrinsic::vector_reduce_smin:
2859 return TargetOpcode::G_VECREDUCE_SMIN;
2860 case Intrinsic::vector_reduce_umax:
2861 return TargetOpcode::G_VECREDUCE_UMAX;
2862 case Intrinsic::vector_reduce_umin:
2863 return TargetOpcode::G_VECREDUCE_UMIN;
2864 case Intrinsic::experimental_vector_compress:
2865 return TargetOpcode::G_VECTOR_COMPRESS;
2866 case Intrinsic::lround:
2867 return TargetOpcode::G_LROUND;
2868 case Intrinsic::llround:
2869 return TargetOpcode::G_LLROUND;
2870 case Intrinsic::get_fpenv:
2871 return TargetOpcode::G_GET_FPENV;
2872 case Intrinsic::get_fpmode:
2873 return TargetOpcode::G_GET_FPMODE;
2874 }
2875 return Intrinsic::not_intrinsic;
2876}
2877
2878bool IRTranslatorImpl::translateSimpleIntrinsic(const CallInst &CI,
2879 Intrinsic::ID ID,
2880 MachineIRBuilder &MIRBuilder) {
2881
2882 unsigned Op = getSimpleIntrinsicOpcode(ID);
2883
2884 // Is this a simple intrinsic?
2885 if (Op == Intrinsic::not_intrinsic)
2886 return false;
2887
2888 // Yes. Let's translate it.
2889 SmallVector<llvm::SrcOp, 4> VRegs;
2890 for (const auto &Arg : CI.args())
2891 VRegs.push_back(Elt: getOrCreateVReg(Val: *Arg));
2892
2893 MIRBuilder.buildInstr(Opc: Op, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: VRegs,
2894 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2895 return true;
2896}
2897
2898// TODO: Include ConstainedOps.def when all strict instructions are defined.
2899static unsigned getConstrainedOpcode(Intrinsic::ID ID) {
2900 switch (ID) {
2901 case Intrinsic::experimental_constrained_fadd:
2902 return TargetOpcode::G_STRICT_FADD;
2903 case Intrinsic::experimental_constrained_fsub:
2904 return TargetOpcode::G_STRICT_FSUB;
2905 case Intrinsic::experimental_constrained_fmul:
2906 return TargetOpcode::G_STRICT_FMUL;
2907 case Intrinsic::experimental_constrained_fdiv:
2908 return TargetOpcode::G_STRICT_FDIV;
2909 case Intrinsic::experimental_constrained_frem:
2910 return TargetOpcode::G_STRICT_FREM;
2911 case Intrinsic::experimental_constrained_fma:
2912 return TargetOpcode::G_STRICT_FMA;
2913 case Intrinsic::experimental_constrained_sqrt:
2914 return TargetOpcode::G_STRICT_FSQRT;
2915 case Intrinsic::experimental_constrained_ldexp:
2916 return TargetOpcode::G_STRICT_FLDEXP;
2917 case Intrinsic::experimental_constrained_fcmp:
2918 return TargetOpcode::G_STRICT_FCMP;
2919 case Intrinsic::experimental_constrained_fcmps:
2920 return TargetOpcode::G_STRICT_FCMPS;
2921 default:
2922 return 0;
2923 }
2924}
2925
2926bool IRTranslatorImpl::translateConstrainedFPIntrinsic(
2927 const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
2928 fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
2929
2930 unsigned Opcode = getConstrainedOpcode(ID: FPI.getIntrinsicID());
2931 if (!Opcode)
2932 return false;
2933
2934 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: FPI);
2935 if (EB == fp::ExceptionBehavior::ebIgnore)
2936 Flags |= MachineInstr::NoFPExcept;
2937
2938 if (Opcode == TargetOpcode::G_STRICT_FCMP ||
2939 Opcode == TargetOpcode::G_STRICT_FCMPS) {
2940 auto *FPCmp = cast<ConstrainedFPCmpIntrinsic>(Val: &FPI);
2941 Register Operand0 = getOrCreateVReg(Val: *FPCmp->getArgOperand(i: 0));
2942 Register Operand1 = getOrCreateVReg(Val: *FPCmp->getArgOperand(i: 1));
2943 Register Result = getOrCreateVReg(Val: FPI);
2944 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Result}, SrcOps: {}, Flags)
2945 .addPredicate(Pred: FPCmp->getPredicate())
2946 .addUse(RegNo: Operand0)
2947 .addUse(RegNo: Operand1);
2948 return true;
2949 }
2950
2951 SmallVector<llvm::SrcOp, 4> VRegs;
2952 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
2953 VRegs.push_back(Elt: getOrCreateVReg(Val: *FPI.getArgOperand(i: I)));
2954
2955 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: FPI)}, SrcOps: VRegs, Flags);
2956 return true;
2957}
2958
2959std::optional<MCRegister> IRTranslatorImpl::getArgPhysReg(Argument &Arg) {
2960 auto VRegs = getOrCreateVRegs(Val: Arg);
2961 if (VRegs.size() != 1)
2962 return std::nullopt;
2963
2964 // Arguments are lowered as a copy of a livein physical register.
2965 auto *VRegDef = MF->getRegInfo().getVRegDef(Reg: VRegs[0]);
2966 if (!VRegDef || !VRegDef->isCopy())
2967 return std::nullopt;
2968 return VRegDef->getOperand(i: 1).getReg().asMCReg();
2969}
2970
2971bool IRTranslatorImpl::translateIfEntryValueArgument(
2972 bool isDeclare, Value *Val, const DILocalVariable *Var,
2973 const DIExpression *Expr, const DebugLoc &DL,
2974 MachineIRBuilder &MIRBuilder) {
2975 auto *Arg = dyn_cast<Argument>(Val);
2976 if (!Arg)
2977 return false;
2978
2979 if (!Expr->isEntryValue())
2980 return false;
2981
2982 std::optional<MCRegister> PhysReg = getArgPhysReg(Arg&: *Arg);
2983 if (!PhysReg) {
2984 LLVM_DEBUG(dbgs() << "Dropping dbg." << (isDeclare ? "declare" : "value")
2985 << ": expression is entry_value but "
2986 << "couldn't find a physical register\n");
2987 LLVM_DEBUG(dbgs() << *Var << "\n");
2988 return true;
2989 }
2990
2991 if (isDeclare) {
2992 // Append an op deref to account for the fact that this is a dbg_declare.
2993 Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
2994 MF->setVariableDbgInfo(Var, Expr, Reg: *PhysReg, Loc: DL);
2995 } else {
2996 MIRBuilder.buildDirectDbgValue(Reg: *PhysReg, Variable: Var, Expr);
2997 }
2998
2999 return true;
3000}
3001
3002static unsigned getConvOpcode(Intrinsic::ID ID) {
3003 switch (ID) {
3004 default:
3005 llvm_unreachable("Unexpected intrinsic");
3006 case Intrinsic::experimental_convergence_anchor:
3007 return TargetOpcode::CONVERGENCECTRL_ANCHOR;
3008 case Intrinsic::experimental_convergence_entry:
3009 return TargetOpcode::CONVERGENCECTRL_ENTRY;
3010 case Intrinsic::experimental_convergence_loop:
3011 return TargetOpcode::CONVERGENCECTRL_LOOP;
3012 }
3013}
3014
3015bool IRTranslatorImpl::translateConvergenceControlIntrinsic(
3016 const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
3017 MachineInstrBuilder MIB = MIRBuilder.buildInstr(Opcode: getConvOpcode(ID));
3018 Register OutputReg = getOrCreateConvergenceTokenVReg(Token: CI);
3019 MIB.addDef(RegNo: OutputReg);
3020
3021 if (ID == Intrinsic::experimental_convergence_loop) {
3022 auto Bundle = CI.getOperandBundle(ID: LLVMContext::OB_convergencectrl);
3023 assert(Bundle && "Expected a convergence control token.");
3024 Register InputReg =
3025 getOrCreateConvergenceTokenVReg(Token: *Bundle->Inputs[0].get());
3026 MIB.addUse(RegNo: InputReg);
3027 }
3028
3029 return true;
3030}
3031
3032bool IRTranslatorImpl::translateKnownIntrinsic(const CallInst &CI,
3033 Intrinsic::ID ID,
3034 MachineIRBuilder &MIRBuilder) {
3035 if (auto *MI = dyn_cast<AnyMemIntrinsic>(Val: &CI)) {
3036 if (ORE->enabled()) {
3037 if (MemoryOpRemark::canHandle(I: MI, TLI: *LibInfo)) {
3038 MemoryOpRemark R(*ORE, "gisel-ir-translator-memsize", *DL, *LibInfo);
3039 R.visit(I: MI);
3040 }
3041 }
3042 }
3043
3044 // If this is a simple intrinsic (that is, we just need to add a def of
3045 // a vreg, and uses for each arg operand, then translate it.
3046 if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
3047 return true;
3048
3049 switch (ID) {
3050 default:
3051 break;
3052 case Intrinsic::lifetime_start:
3053 case Intrinsic::lifetime_end: {
3054 // No stack colouring in O0, discard region information.
3055 if (MF->getTarget().getOptLevel() == CodeGenOptLevel::None ||
3056 MF->getFunction().hasOptNone())
3057 return true;
3058
3059 unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
3060 : TargetOpcode::LIFETIME_END;
3061
3062 const AllocaInst *AI = dyn_cast<AllocaInst>(Val: CI.getArgOperand(i: 0));
3063 if (!AI || !AI->isStaticAlloca())
3064 return true;
3065
3066 MIRBuilder.buildInstr(Opcode: Op).addFrameIndex(Idx: getOrCreateFrameIndex(AI: *AI));
3067 return true;
3068 }
3069 case Intrinsic::fake_use: {
3070 SmallVector<llvm::SrcOp, 4> VRegs;
3071 for (const auto &Arg : CI.args())
3072 llvm::append_range(C&: VRegs, R: getOrCreateVRegs(Val: *Arg));
3073 MIRBuilder.buildInstr(Opc: TargetOpcode::FAKE_USE, DstOps: {}, SrcOps: VRegs);
3074 MF->setHasFakeUses(true);
3075 return true;
3076 }
3077 case Intrinsic::dbg_declare: {
3078 const DbgDeclareInst &DI = cast<DbgDeclareInst>(Val: CI);
3079 assert(DI.getVariable() && "Missing variable");
3080 translateDbgDeclareRecord(Address: DI.getAddress(), HasArgList: DI.hasArgList(), Variable: DI.getVariable(),
3081 Expression: DI.getExpression(), DL: DI.getDebugLoc(), MIRBuilder);
3082 return true;
3083 }
3084 case Intrinsic::dbg_label: {
3085 const DbgLabelInst &DI = cast<DbgLabelInst>(Val: CI);
3086 assert(DI.getLabel() && "Missing label");
3087
3088 assert(DI.getLabel()->isValidLocationForIntrinsic(
3089 MIRBuilder.getDebugLoc()) &&
3090 "Expected inlined-at fields to agree");
3091
3092 MIRBuilder.buildDbgLabel(Label: DI.getLabel());
3093 return true;
3094 }
3095 case Intrinsic::vaend:
3096 // No target I know of cares about va_end. Certainly no in-tree target
3097 // does. Simplest intrinsic ever!
3098 return true;
3099 case Intrinsic::vastart: {
3100 Value *Ptr = CI.getArgOperand(i: 0);
3101 unsigned ListSize = TLI->getVaListSizeInBits(DL: *DL) / 8;
3102 Align Alignment = getKnownAlignment(V: Ptr, DL: *DL);
3103
3104 MIRBuilder.buildInstr(Opc: TargetOpcode::G_VASTART, DstOps: {}, SrcOps: {getOrCreateVReg(Val: *Ptr)})
3105 .addMemOperand(MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Ptr),
3106 F: MachineMemOperand::MOStore,
3107 Size: ListSize, BaseAlignment: Alignment));
3108 return true;
3109 }
3110 case Intrinsic::dbg_assign:
3111 // A dbg.assign is a dbg.value with more information about stack locations,
3112 // typically produced during optimisation of variables with leaked
3113 // addresses. We can treat it like a normal dbg_value intrinsic here; to
3114 // benefit from the full analysis of stack/SSA locations, GlobalISel would
3115 // need to register for and use the AssignmentTrackingAnalysis pass.
3116 [[fallthrough]];
3117 case Intrinsic::dbg_value: {
3118 // This form of DBG_VALUE is target-independent.
3119 const DbgValueInst &DI = cast<DbgValueInst>(Val: CI);
3120 translateDbgValueRecord(V: DI.getValue(), HasArgList: DI.hasArgList(), Variable: DI.getVariable(),
3121 Expression: DI.getExpression(), DL: DI.getDebugLoc(), MIRBuilder);
3122 return true;
3123 }
3124 case Intrinsic::uadd_with_overflow:
3125 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UADDO, MIRBuilder);
3126 case Intrinsic::sadd_with_overflow:
3127 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SADDO, MIRBuilder);
3128 case Intrinsic::usub_with_overflow:
3129 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_USUBO, MIRBuilder);
3130 case Intrinsic::ssub_with_overflow:
3131 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SSUBO, MIRBuilder);
3132 case Intrinsic::umul_with_overflow:
3133 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UMULO, MIRBuilder);
3134 case Intrinsic::smul_with_overflow:
3135 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SMULO, MIRBuilder);
3136 case Intrinsic::uadd_sat:
3137 return translateBinaryOp(Opcode: TargetOpcode::G_UADDSAT, U: CI, MIRBuilder);
3138 case Intrinsic::sadd_sat:
3139 return translateBinaryOp(Opcode: TargetOpcode::G_SADDSAT, U: CI, MIRBuilder);
3140 case Intrinsic::usub_sat:
3141 return translateBinaryOp(Opcode: TargetOpcode::G_USUBSAT, U: CI, MIRBuilder);
3142 case Intrinsic::ssub_sat:
3143 return translateBinaryOp(Opcode: TargetOpcode::G_SSUBSAT, U: CI, MIRBuilder);
3144 case Intrinsic::ushl_sat:
3145 return translateBinaryOp(Opcode: TargetOpcode::G_USHLSAT, U: CI, MIRBuilder);
3146 case Intrinsic::sshl_sat:
3147 return translateBinaryOp(Opcode: TargetOpcode::G_SSHLSAT, U: CI, MIRBuilder);
3148 case Intrinsic::umin:
3149 return translateBinaryOp(Opcode: TargetOpcode::G_UMIN, U: CI, MIRBuilder);
3150 case Intrinsic::umax:
3151 return translateBinaryOp(Opcode: TargetOpcode::G_UMAX, U: CI, MIRBuilder);
3152 case Intrinsic::smin:
3153 return translateBinaryOp(Opcode: TargetOpcode::G_SMIN, U: CI, MIRBuilder);
3154 case Intrinsic::smax:
3155 return translateBinaryOp(Opcode: TargetOpcode::G_SMAX, U: CI, MIRBuilder);
3156 case Intrinsic::abs:
3157 // TODO: Preserve "int min is poison" arg in GMIR?
3158 return translateUnaryOp(Opcode: TargetOpcode::G_ABS, U: CI, MIRBuilder);
3159 case Intrinsic::smul_fix:
3160 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIX, CI, MIRBuilder);
3161 case Intrinsic::umul_fix:
3162 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIX, CI, MIRBuilder);
3163 case Intrinsic::smul_fix_sat:
3164 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
3165 case Intrinsic::umul_fix_sat:
3166 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
3167 case Intrinsic::sdiv_fix:
3168 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
3169 case Intrinsic::udiv_fix:
3170 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
3171 case Intrinsic::sdiv_fix_sat:
3172 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
3173 case Intrinsic::udiv_fix_sat:
3174 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
3175 case Intrinsic::fmuladd: {
3176 Register Dst = getOrCreateVReg(Val: CI);
3177 Register Op0 = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
3178 Register Op1 = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
3179 Register Op2 = getOrCreateVReg(Val: *CI.getArgOperand(i: 2));
3180 if (TLI->isFMAFasterThanFMulAndFAdd(MF: *MF,
3181 TLI->getValueType(DL: *DL, Ty: CI.getType()))) {
3182 // TODO: Revisit this to see if we should move this part of the
3183 // lowering to the combiner.
3184 MIRBuilder.buildFMA(Dst, Src0: Op0, Src1: Op1, Src2: Op2,
3185 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3186 } else {
3187 LLT Ty = getLLTForType(Ty&: *CI.getType(), DL: *DL);
3188 auto FMul = MIRBuilder.buildFMul(
3189 Dst: Ty, Src0: Op0, Src1: Op1, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3190 MIRBuilder.buildFAdd(Dst, Src0: FMul, Src1: Op2,
3191 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3192 }
3193 return true;
3194 }
3195 case Intrinsic::frexp: {
3196 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
3197 MIRBuilder.buildFFrexp(Fract: VRegs[0], Exp: VRegs[1],
3198 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
3199 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3200 return true;
3201 }
3202 case Intrinsic::modf: {
3203 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
3204 MIRBuilder.buildModf(Fract: VRegs[0], Int: VRegs[1],
3205 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
3206 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3207 return true;
3208 }
3209 case Intrinsic::sincos: {
3210 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
3211 MIRBuilder.buildFSincos(Sin: VRegs[0], Cos: VRegs[1],
3212 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
3213 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3214 return true;
3215 }
3216 case Intrinsic::fptosi_sat:
3217 MIRBuilder.buildFPTOSI_SAT(Dst: getOrCreateVReg(Val: CI),
3218 Src0: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
3219 return true;
3220 case Intrinsic::fptoui_sat:
3221 MIRBuilder.buildFPTOUI_SAT(Dst: getOrCreateVReg(Val: CI),
3222 Src0: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
3223 return true;
3224 case Intrinsic::memcpy_inline:
3225 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY_INLINE);
3226 case Intrinsic::memcpy:
3227 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY);
3228 case Intrinsic::memmove:
3229 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMMOVE);
3230 case Intrinsic::memset:
3231 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMSET);
3232 case Intrinsic::memset_inline:
3233 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMSET_INLINE);
3234 case Intrinsic::eh_typeid_for: {
3235 GlobalValue *GV = ExtractTypeInfo(V: CI.getArgOperand(i: 0));
3236 Register Reg = getOrCreateVReg(Val: CI);
3237 unsigned TypeID = MF->getTypeIDFor(TI: GV);
3238 MIRBuilder.buildConstant(Res: Reg, Val: TypeID);
3239 return true;
3240 }
3241 case Intrinsic::objectsize:
3242 llvm_unreachable("llvm.objectsize.* should have been lowered already");
3243
3244 case Intrinsic::is_constant:
3245 llvm_unreachable("llvm.is.constant.* should have been lowered already");
3246
3247 case Intrinsic::stackguard:
3248 getStackGuard(DstReg: getOrCreateVReg(Val: CI), MIRBuilder);
3249 return true;
3250 case Intrinsic::stackprotector: {
3251 LLT PtrTy = getLLTForType(Ty&: *CI.getArgOperand(i: 0)->getType(), DL: *DL);
3252 Register GuardVal;
3253 if (TLI->useLoadStackGuardNode(M: *CI.getModule())) {
3254 GuardVal = MRI->createGenericVirtualRegister(Ty: PtrTy);
3255 getStackGuard(DstReg: GuardVal, MIRBuilder);
3256 } else
3257 GuardVal = getOrCreateVReg(Val: *CI.getArgOperand(i: 0)); // The guard's value.
3258
3259 AllocaInst *Slot = cast<AllocaInst>(Val: CI.getArgOperand(i: 1));
3260 int FI = getOrCreateFrameIndex(AI: *Slot);
3261 MF->getFrameInfo().setStackProtectorIndex(FI);
3262
3263 MIRBuilder.buildStore(
3264 Val: GuardVal, Addr: getOrCreateVReg(Val: *Slot),
3265 MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI),
3266 F: MachineMemOperand::MOStore |
3267 MachineMemOperand::MOVolatile,
3268 MemTy: PtrTy, BaseAlignment: Align(8)));
3269 return true;
3270 }
3271 case Intrinsic::stacksave: {
3272 MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKSAVE, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {});
3273 return true;
3274 }
3275 case Intrinsic::stackrestore: {
3276 MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKRESTORE, DstOps: {},
3277 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
3278 return true;
3279 }
3280 case Intrinsic::cttz:
3281 case Intrinsic::ctlz: {
3282 ConstantInt *Cst = cast<ConstantInt>(Val: CI.getArgOperand(i: 1));
3283 bool isTrailing = ID == Intrinsic::cttz;
3284 unsigned Opcode = isTrailing ? Cst->isZero()
3285 ? TargetOpcode::G_CTTZ
3286 : TargetOpcode::G_CTTZ_ZERO_POISON
3287 : Cst->isZero() ? TargetOpcode::G_CTLZ
3288 : TargetOpcode::G_CTLZ_ZERO_POISON;
3289 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: CI)},
3290 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
3291 return true;
3292 }
3293 case Intrinsic::invariant_start: {
3294 MIRBuilder.buildUndef(Res: getOrCreateVReg(Val: CI));
3295 return true;
3296 }
3297 case Intrinsic::invariant_end:
3298 return true;
3299 case Intrinsic::expect:
3300 case Intrinsic::expect_with_probability:
3301 case Intrinsic::annotation:
3302 case Intrinsic::ptr_annotation:
3303 case Intrinsic::launder_invariant_group:
3304 case Intrinsic::threadlocal_address: {
3305 // Drop the intrinsic, but forward the value.
3306 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: CI),
3307 Op: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
3308 return true;
3309 }
3310 case Intrinsic::assume:
3311 case Intrinsic::experimental_noalias_scope_decl:
3312 case Intrinsic::var_annotation:
3313 case Intrinsic::sideeffect:
3314 // Discard annotate attributes, assumptions, and artificial side-effects.
3315 return true;
3316 case Intrinsic::read_volatile_register:
3317 case Intrinsic::read_register: {
3318 Value *Arg = CI.getArgOperand(i: 0);
3319 MIRBuilder
3320 .buildInstr(Opc: TargetOpcode::G_READ_REGISTER, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {})
3321 .addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()));
3322 return true;
3323 }
3324 case Intrinsic::write_register: {
3325 Value *Arg = CI.getArgOperand(i: 0);
3326 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_WRITE_REGISTER)
3327 .addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()))
3328 .addUse(RegNo: getOrCreateVReg(Val: *CI.getArgOperand(i: 1)));
3329 return true;
3330 }
3331 case Intrinsic::localescape: {
3332 MachineBasicBlock &EntryMBB = MF->front();
3333 StringRef EscapedName = GlobalValue::dropLLVMManglingEscape(Name: MF->getName());
3334
3335 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
3336 // is the same on all targets.
3337 for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
3338 Value *Arg = CI.getArgOperand(i: Idx)->stripPointerCasts();
3339 if (isa<ConstantPointerNull>(Val: Arg))
3340 continue; // Skip null pointers. They represent a hole in index space.
3341
3342 int FI = getOrCreateFrameIndex(AI: *cast<AllocaInst>(Val: Arg));
3343 MCSymbol *FrameAllocSym =
3344 MF->getContext().getOrCreateFrameAllocSymbol(FuncName: EscapedName, Idx);
3345
3346 // This should be inserted at the start of the entry block.
3347 auto LocalEscape =
3348 MIRBuilder.buildInstrNoInsert(Opcode: TargetOpcode::LOCAL_ESCAPE)
3349 .addSym(Sym: FrameAllocSym)
3350 .addFrameIndex(Idx: FI);
3351
3352 EntryMBB.insert(I: EntryMBB.begin(), MI: LocalEscape);
3353 }
3354
3355 return true;
3356 }
3357 case Intrinsic::vector_reduce_fadd:
3358 case Intrinsic::vector_reduce_fmul: {
3359 // Need to check for the reassoc flag to decide whether we want a
3360 // sequential reduction opcode or not.
3361 Register Dst = getOrCreateVReg(Val: CI);
3362 Register ScalarSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
3363 Register VecSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
3364 unsigned Opc = 0;
3365 if (!CI.hasAllowReassoc()) {
3366 // The sequential ordering case.
3367 Opc = ID == Intrinsic::vector_reduce_fadd
3368 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
3369 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
3370 if (!MRI->getType(Reg: VecSrc).isVector())
3371 Opc = ID == Intrinsic::vector_reduce_fadd ? TargetOpcode::G_FADD
3372 : TargetOpcode::G_FMUL;
3373 MIRBuilder.buildInstr(Opc, DstOps: {Dst}, SrcOps: {ScalarSrc, VecSrc},
3374 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3375 return true;
3376 }
3377 // We split the operation into a separate G_FADD/G_FMUL + the reduce,
3378 // since the associativity doesn't matter.
3379 unsigned ScalarOpc;
3380 if (ID == Intrinsic::vector_reduce_fadd) {
3381 Opc = TargetOpcode::G_VECREDUCE_FADD;
3382 ScalarOpc = TargetOpcode::G_FADD;
3383 } else {
3384 Opc = TargetOpcode::G_VECREDUCE_FMUL;
3385 ScalarOpc = TargetOpcode::G_FMUL;
3386 }
3387 LLT DstTy = MRI->getType(Reg: Dst);
3388 auto Rdx = MIRBuilder.buildInstr(
3389 Opc, DstOps: {DstTy}, SrcOps: {VecSrc}, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3390 MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {Dst}, SrcOps: {ScalarSrc, Rdx},
3391 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3392
3393 return true;
3394 }
3395 case Intrinsic::trap:
3396 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_TRAP);
3397 case Intrinsic::debugtrap:
3398 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_DEBUGTRAP);
3399 case Intrinsic::ubsantrap:
3400 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_UBSANTRAP);
3401 case Intrinsic::allow_runtime_check:
3402 case Intrinsic::allow_ubsan_check:
3403 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: CI),
3404 Op: getOrCreateVReg(Val: *ConstantInt::getTrue(Ty: CI.getType())));
3405 return true;
3406 case Intrinsic::amdgcn_cs_chain:
3407 case Intrinsic::amdgcn_call_whole_wave:
3408 return translateCallBase(CB: CI, MIRBuilder);
3409 case Intrinsic::fptrunc_round: {
3410 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: CI);
3411
3412 // Convert the metadata argument to a constant integer
3413 Metadata *MD = cast<MetadataAsValue>(Val: CI.getArgOperand(i: 1))->getMetadata();
3414 std::optional<RoundingMode> RoundMode =
3415 convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
3416
3417 // Add the Rounding mode as an integer
3418 MIRBuilder
3419 .buildInstr(Opc: TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
3420 DstOps: {getOrCreateVReg(Val: CI)},
3421 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))}, Flags)
3422 .addImm(Val: (int)*RoundMode);
3423
3424 return true;
3425 }
3426 case Intrinsic::is_fpclass: {
3427 Value *FpValue = CI.getOperand(i_nocapture: 0);
3428 ConstantInt *TestMaskValue = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1));
3429
3430 MIRBuilder
3431 .buildInstr(Opc: TargetOpcode::G_IS_FPCLASS, DstOps: {getOrCreateVReg(Val: CI)},
3432 SrcOps: {getOrCreateVReg(Val: *FpValue)})
3433 .addImm(Val: TestMaskValue->getZExtValue());
3434
3435 return true;
3436 }
3437 case Intrinsic::set_fpenv: {
3438 Value *FPEnv = CI.getOperand(i_nocapture: 0);
3439 MIRBuilder.buildSetFPEnv(Src: getOrCreateVReg(Val: *FPEnv));
3440 return true;
3441 }
3442 case Intrinsic::reset_fpenv:
3443 MIRBuilder.buildResetFPEnv();
3444 return true;
3445 case Intrinsic::set_fpmode: {
3446 Value *FPState = CI.getOperand(i_nocapture: 0);
3447 MIRBuilder.buildSetFPMode(Src: getOrCreateVReg(Val: *FPState));
3448 return true;
3449 }
3450 case Intrinsic::reset_fpmode:
3451 MIRBuilder.buildResetFPMode();
3452 return true;
3453 case Intrinsic::get_rounding:
3454 MIRBuilder.buildGetRounding(Dst: getOrCreateVReg(Val: CI));
3455 return true;
3456 case Intrinsic::set_rounding:
3457 MIRBuilder.buildSetRounding(Src: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)));
3458 return true;
3459 case Intrinsic::vscale: {
3460 MIRBuilder.buildVScale(Res: getOrCreateVReg(Val: CI), MinElts: 1);
3461 return true;
3462 }
3463 case Intrinsic::scmp:
3464 MIRBuilder.buildSCmp(Res: getOrCreateVReg(Val: CI),
3465 Op0: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)),
3466 Op1: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1)));
3467 return true;
3468 case Intrinsic::ucmp:
3469 MIRBuilder.buildUCmp(Res: getOrCreateVReg(Val: CI),
3470 Op0: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)),
3471 Op1: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1)));
3472 return true;
3473 case Intrinsic::vector_extract:
3474 return translateExtractVector(U: CI, MIRBuilder);
3475 case Intrinsic::vector_insert:
3476 return translateInsertVector(U: CI, MIRBuilder);
3477 case Intrinsic::stepvector: {
3478 MIRBuilder.buildStepVector(Res: getOrCreateVReg(Val: CI), Step: 1);
3479 return true;
3480 }
3481 case Intrinsic::prefetch: {
3482 Value *Addr = CI.getOperand(i_nocapture: 0);
3483 unsigned RW = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1))->getZExtValue();
3484 unsigned Locality = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
3485 unsigned CacheType = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 3))->getZExtValue();
3486
3487 auto Flags = RW ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
3488 auto &MMO = *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Addr), F: Flags,
3489 MemTy: LLT(), BaseAlignment: Align());
3490
3491 MIRBuilder.buildPrefetch(Addr: getOrCreateVReg(Val: *Addr), RW, Locality, CacheType,
3492 MMO);
3493
3494 return true;
3495 }
3496
3497 case Intrinsic::speculative_load: {
3498 // Only the pointer operand is needed at codegen; the remaining arguments
3499 // carry IR-level semantics only.
3500 const Value *Ptr = CI.getArgOperand(i: 0);
3501 Register Dst = getOrCreateVReg(Val: CI);
3502 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad;
3503 Flags |= TLI->getTargetMMOFlags(I: CI);
3504 if (CI.hasMetadata(KindID: LLVMContext::MD_nontemporal))
3505 Flags |= MachineMemOperand::MONonTemporal;
3506 if (CI.hasMetadata(KindID: LLVMContext::MD_invariant_load))
3507 Flags |= MachineMemOperand::MOInvariant;
3508 auto *MMO = MF->getMachineMemOperand(
3509 PtrInfo: MachinePointerInfo(Ptr), F: Flags, MemTy: MRI->getType(Reg: Dst),
3510 BaseAlignment: CI.getParamAlign(ArgNo: 0).valueOrOne(), Metadata: MMOMetadata(CI.getAAMetadata()));
3511 MIRBuilder.buildLoad(Res: Dst, Addr: getOrCreateVReg(Val: *Ptr), MMO&: *MMO);
3512 return true;
3513 }
3514
3515 case Intrinsic::vector_interleave2:
3516 case Intrinsic::vector_deinterleave2: {
3517 // Both intrinsics have at least one operand.
3518 Value *Op0 = CI.getOperand(i_nocapture: 0);
3519 LLT ResTy = getLLTForType(Ty&: *Op0->getType(), DL: MIRBuilder.getDataLayout());
3520 if (!ResTy.isFixedVector())
3521 return false;
3522
3523 if (CI.getIntrinsicID() == Intrinsic::vector_interleave2)
3524 return translateVectorInterleave2Intrinsic(CI, MIRBuilder);
3525
3526 return translateVectorDeinterleave2Intrinsic(CI, MIRBuilder);
3527 }
3528
3529#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
3530 case Intrinsic::INTRINSIC:
3531#include "llvm/IR/ConstrainedOps.def"
3532 return translateConstrainedFPIntrinsic(FPI: cast<ConstrainedFPIntrinsic>(Val: CI),
3533 MIRBuilder);
3534 case Intrinsic::experimental_convergence_anchor:
3535 case Intrinsic::experimental_convergence_entry:
3536 case Intrinsic::experimental_convergence_loop:
3537 return translateConvergenceControlIntrinsic(CI, ID, MIRBuilder);
3538 case Intrinsic::reloc_none: {
3539 Metadata *MD = cast<MetadataAsValue>(Val: CI.getArgOperand(i: 0))->getMetadata();
3540 StringRef SymbolName = cast<MDString>(Val: MD)->getString();
3541 MIRBuilder.buildInstr(Opcode: TargetOpcode::RELOC_NONE)
3542 .addExternalSymbol(FnName: SymbolName.data());
3543 return true;
3544 }
3545 }
3546 return false;
3547}
3548
3549bool IRTranslatorImpl::translateInlineAsm(const CallBase &CB,
3550 MachineIRBuilder &MIRBuilder) {
3551 if (!mayTranslateUserTypes(U: CB))
3552 return false;
3553
3554 const InlineAsmLowering *ALI = MF->getSubtarget().getInlineAsmLowering();
3555
3556 if (!ALI) {
3557 LLVM_DEBUG(
3558 dbgs() << "Inline asm lowering is not supported for this target yet\n");
3559 return false;
3560 }
3561
3562 return ALI->lowerInlineAsm(
3563 MIRBuilder, CB, GetOrCreateVRegs: [&](const Value &Val) { return getOrCreateVRegs(Val); });
3564}
3565
3566bool IRTranslatorImpl::translateCallBase(const CallBase &CB,
3567 MachineIRBuilder &MIRBuilder) {
3568 ArrayRef<Register> Res = getOrCreateVRegs(Val: CB);
3569
3570 SmallVector<ArrayRef<Register>, 8> Args;
3571 Register SwiftInVReg = 0;
3572 Register SwiftErrorVReg = 0;
3573 for (const auto &Arg : CB.args()) {
3574 if (CLI->supportSwiftError() && isSwiftError(V: Arg)) {
3575 assert(SwiftInVReg == 0 && "Expected only one swift error argument");
3576 LLT Ty = getLLTForType(Ty&: *Arg->getType(), DL: *DL);
3577 SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
3578 MIRBuilder.buildCopy(Res: SwiftInVReg, Op: SwiftError.getOrCreateVRegUseAt(
3579 &CB, &MIRBuilder.getMBB(), Arg));
3580 Args.emplace_back(Args: ArrayRef(SwiftInVReg));
3581 SwiftErrorVReg =
3582 SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
3583 continue;
3584 }
3585 Args.push_back(Elt: getOrCreateVRegs(Val: *Arg));
3586 }
3587
3588 if (auto *CI = dyn_cast<CallInst>(Val: &CB)) {
3589 if (ORE->enabled()) {
3590 if (MemoryOpRemark::canHandle(I: CI, TLI: *LibInfo)) {
3591 MemoryOpRemark R(*ORE, "gisel-ir-translator-memsize", *DL, *LibInfo);
3592 R.visit(I: CI);
3593 }
3594 }
3595 }
3596
3597 std::optional<CallLowering::PtrAuthInfo> PAI;
3598 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_ptrauth)) {
3599 // Functions should never be ptrauth-called directly.
3600 assert(!CB.getCalledFunction() && "invalid direct ptrauth call");
3601
3602 const Value *Key = Bundle->Inputs[0];
3603 const Value *Discriminator = Bundle->Inputs[1];
3604
3605 // Look through ptrauth constants to try to eliminate the matching bundle
3606 // and turn this into a direct call with no ptrauth.
3607 // CallLowering will use the raw pointer if it doesn't find the PAI.
3608 const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(Val: CB.getCalledOperand());
3609 if (!CalleeCPA || !isa<Function>(Val: CalleeCPA->getPointer()) ||
3610 !CalleeCPA->isKnownCompatibleWith(Key, Discriminator, DL: *DL)) {
3611 // If we can't make it direct, package the bundle into PAI.
3612 Register DiscReg = getOrCreateVReg(Val: *Discriminator);
3613 PAI = CallLowering::PtrAuthInfo{.Key: cast<ConstantInt>(Val: Key)->getZExtValue(),
3614 .Discriminator: DiscReg};
3615 }
3616 }
3617
3618 Register ConvergenceCtrlToken = 0;
3619 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
3620 const auto &Token = *Bundle->Inputs[0].get();
3621 ConvergenceCtrlToken = getOrCreateConvergenceTokenVReg(Token);
3622 }
3623
3624 // We don't set HasCalls on MFI here yet because call lowering may decide to
3625 // optimize into tail calls. Instead, we defer that to selection where a final
3626 // scan is done to check if any instructions are calls.
3627 bool Success = CLI->lowerCall(
3628 MIRBuilder, Call: CB, ResRegs: Res, ArgRegs: Args, SwiftErrorVReg, PAI, ConvergenceCtrlToken,
3629 GetCalleeReg: [&]() { return getOrCreateVReg(Val: *CB.getCalledOperand()); });
3630
3631 // Check if we just inserted a tail call.
3632 if (Success) {
3633 assert(!HasTailCall && "Can't tail call return twice from block?");
3634 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
3635 HasTailCall = TII->isTailCall(Inst: *std::prev(x: MIRBuilder.getInsertPt()));
3636 }
3637
3638 return Success;
3639}
3640
3641bool IRTranslatorImpl::translateCall(const User &U,
3642 MachineIRBuilder &MIRBuilder) {
3643 if (!mayTranslateUserTypes(U))
3644 return false;
3645
3646 const CallInst &CI = cast<CallInst>(Val: U);
3647 const Function *F = CI.getCalledFunction();
3648
3649 // FIXME: support Windows dllimport function calls and calls through
3650 // weak symbols.
3651 if (F && (F->hasDLLImportStorageClass() ||
3652 (MF->getTarget().getTargetTriple().isOSWindows() &&
3653 F->hasExternalWeakLinkage())))
3654 return false;
3655
3656 // FIXME: support control flow guard targets.
3657 if (CI.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
3658 return false;
3659
3660 // FIXME: support statepoints and related.
3661 if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(Val: U))
3662 return false;
3663
3664 if (CI.isInlineAsm())
3665 return translateInlineAsm(CB: CI, MIRBuilder);
3666
3667 Intrinsic::ID ID = F ? F->getIntrinsicID() : Intrinsic::not_intrinsic;
3668 if (!F || ID == Intrinsic::not_intrinsic) {
3669 if (translateCallBase(CB: CI, MIRBuilder)) {
3670 diagnoseDontCall(CI);
3671 return true;
3672 }
3673 return false;
3674 }
3675
3676 assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
3677
3678 if (!MF->getSubtarget().isIntrinsicSupported(IntrinsicID: ID))
3679 return handleUnsupportedIntrinsic(CB: CI, ID, MIRBuilder);
3680
3681 if (translateKnownIntrinsic(CI, ID, MIRBuilder))
3682 return true;
3683
3684 SmallVector<TargetLowering::IntrinsicInfo> Infos;
3685 TLI->getTgtMemIntrinsic(Infos, I: CI, MF&: *MF, Intrinsic: ID);
3686
3687 return translateIntrinsic(CB: CI, ID, MIRBuilder, TgtMemIntrinsicInfos: Infos);
3688}
3689
3690bool IRTranslatorImpl::handleUnsupportedIntrinsic(
3691 const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
3692 const Function &F = MF->getFunction();
3693 F.getContext().diagnose(
3694 DI: DiagnosticInfoUnsupportedTargetIntrinsic(F, ID, CB.getDebugLoc()));
3695
3696 if (!CB.getType()->isVoidTy()) {
3697 for (Register Reg : getOrCreateVRegs(Val: CB))
3698 MIRBuilder.buildUndef(Res: Reg);
3699 }
3700
3701 return true;
3702}
3703
3704/// Translate a call or callbr to an intrinsic.
3705bool IRTranslatorImpl::translateIntrinsic(
3706 const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
3707 ArrayRef<TargetLowering::IntrinsicInfo> TgtMemIntrinsicInfos) {
3708 if (!MF->getSubtarget().isIntrinsicSupported(IntrinsicID: ID))
3709 return handleUnsupportedIntrinsic(CB, ID, MIRBuilder);
3710
3711 ArrayRef<Register> ResultRegs;
3712 if (!CB.getType()->isVoidTy())
3713 ResultRegs = getOrCreateVRegs(Val: CB);
3714
3715 // Ignore the callsite attributes. Backend code is most likely not expecting
3716 // an intrinsic to sometimes have side effects and sometimes not.
3717 MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, Res: ResultRegs);
3718 if (isa<FPMathOperator>(Val: CB))
3719 MIB->copyIRFlags(I: CB);
3720
3721 for (const auto &Arg : enumerate(First: CB.args())) {
3722 // If this is required to be an immediate, don't materialize it in a
3723 // register.
3724 if (CB.paramHasAttr(ArgNo: Arg.index(), Kind: Attribute::ImmArg)) {
3725 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Arg.value())) {
3726 // imm arguments are more convenient than cimm (and realistically
3727 // probably sufficient), so use them.
3728 assert(CI->getBitWidth() <= 64 &&
3729 "large intrinsic immediates not handled");
3730 MIB.addImm(Val: CI->getSExtValue());
3731 } else {
3732 MIB.addFPImm(Val: cast<ConstantFP>(Val: Arg.value()));
3733 }
3734 } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Val: Arg.value())) {
3735 auto *MD = MDVal->getMetadata();
3736 auto *MDN = dyn_cast<MDNode>(Val: MD);
3737 if (!MDN) {
3738 if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(Val: MD))
3739 MDN = MDNode::get(Context&: MF->getFunction().getContext(), MDs: ConstMD);
3740 else // This was probably an MDString.
3741 return false;
3742 }
3743 MIB.addMetadata(MD: MDN);
3744 } else {
3745 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: *Arg.value());
3746 if (VRegs.size() > 1)
3747 return false;
3748 MIB.addUse(RegNo: VRegs[0]);
3749 }
3750 }
3751
3752 // Add MachineMemOperands for each memory access described by the target.
3753 for (const auto &Info : TgtMemIntrinsicInfos) {
3754 Align Alignment = Info.align.value_or(
3755 u: DL->getABITypeAlign(Ty: Info.memVT.getTypeForEVT(Context&: CB.getContext())));
3756 LLT MemTy = Info.memVT.isSimple()
3757 ? getLLTForMVT(Ty: Info.memVT.getSimpleVT())
3758 : LLT::scalar(SizeInBits: Info.memVT.getStoreSizeInBits());
3759
3760 // TODO: We currently just fallback to address space 0 if
3761 // getTgtMemIntrinsic didn't yield anything useful.
3762 MachinePointerInfo MPI;
3763 if (Info.ptrVal) {
3764 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
3765 } else if (Info.fallbackAddressSpace) {
3766 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
3767 }
3768 MIB.addMemOperand(MMO: MF->getMachineMemOperand(
3769 PtrInfo: MPI, F: Info.flags, MemTy, BaseAlignment: Alignment, Metadata: CB.getAAMetadata(), SSID: Info.ssid,
3770 Ordering: Info.order, FailureOrdering: Info.failureOrder));
3771 }
3772
3773 if (CB.isConvergent()) {
3774 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
3775 auto *Token = Bundle->Inputs[0].get();
3776 Register TokenReg = getOrCreateVReg(Val: *Token);
3777 MIB.addUse(RegNo: TokenReg, Flags: RegState::Implicit);
3778 }
3779 }
3780
3781 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol))
3782 MIB->setDeactivationSymbol(MF&: *MF, DS: Bundle->Inputs[0].get());
3783
3784 return true;
3785}
3786
3787bool IRTranslatorImpl::findUnwindDestinations(
3788 const BasicBlock *EHPadBB, BranchProbability Prob,
3789 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
3790 &UnwindDests) {
3791 EHPersonality Personality = classifyEHPersonality(
3792 Pers: EHPadBB->getParent()->getFunction().getPersonalityFn());
3793 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
3794 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
3795 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
3796 bool IsWasmD = Personality == EHPersonality::Wasm_D;
3797 bool IsSEH = isAsynchronousEHPersonality(Pers: Personality);
3798
3799 if (IsWasmCXX || IsWasmD) {
3800 // Ignore this for now.
3801 return false;
3802 }
3803
3804 while (EHPadBB) {
3805 BasicBlock::const_iterator Pad = EHPadBB->getFirstNonPHIIt();
3806 BasicBlock *NewEHPadBB = nullptr;
3807 if (isa<LandingPadInst>(Val: Pad)) {
3808 // Stop on landingpads. They are not funclets.
3809 UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
3810 break;
3811 }
3812 if (isa<CleanupPadInst>(Val: Pad)) {
3813 // Stop on cleanup pads. Cleanups are always funclet entries for all known
3814 // personalities.
3815 UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
3816 UnwindDests.back().first->setIsEHScopeEntry();
3817 UnwindDests.back().first->setIsEHFuncletEntry();
3818 break;
3819 }
3820 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: Pad)) {
3821 // Add the catchpad handlers to the possible destinations.
3822 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
3823 UnwindDests.emplace_back(Args: &getMBB(BB: *CatchPadBB), Args&: Prob);
3824 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
3825 if (IsMSVCCXX || IsCoreCLR)
3826 UnwindDests.back().first->setIsEHFuncletEntry();
3827 if (!IsSEH)
3828 UnwindDests.back().first->setIsEHScopeEntry();
3829 }
3830 NewEHPadBB = CatchSwitch->getUnwindDest();
3831 } else {
3832 continue;
3833 }
3834
3835 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3836 if (BPI && NewEHPadBB)
3837 Prob *= BPI->getEdgeProbability(Src: EHPadBB, Dst: NewEHPadBB);
3838 EHPadBB = NewEHPadBB;
3839 }
3840 return true;
3841}
3842
3843bool IRTranslatorImpl::translateInvoke(const User &U,
3844 MachineIRBuilder &MIRBuilder) {
3845 const InvokeInst &I = cast<InvokeInst>(Val: U);
3846 MCContext &Context = MF->getContext();
3847
3848 const BasicBlock *ReturnBB = I.getSuccessor(i: 0);
3849 const BasicBlock *EHPadBB = I.getSuccessor(i: 1);
3850
3851 const Function *Fn = I.getCalledFunction();
3852
3853 // FIXME: support invoking patchpoint and statepoint intrinsics.
3854 if (Fn && Fn->isIntrinsic())
3855 return false;
3856
3857 // FIXME: support whatever these are.
3858 if (I.hasDeoptState())
3859 return false;
3860
3861 // FIXME: support control flow guard targets.
3862 if (I.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
3863 return false;
3864
3865 // FIXME: support Windows exception handling.
3866 if (!isa<LandingPadInst>(Val: EHPadBB->getFirstNonPHIIt()))
3867 return false;
3868
3869 // FIXME: support Windows dllimport function calls and calls through
3870 // weak symbols.
3871 if (Fn && (Fn->hasDLLImportStorageClass() ||
3872 (MF->getTarget().getTargetTriple().isOSWindows() &&
3873 Fn->hasExternalWeakLinkage())))
3874 return false;
3875
3876 bool LowerInlineAsm = I.isInlineAsm();
3877 bool NeedEHLabel = true;
3878
3879 // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
3880 // the region covered by the try.
3881 MCSymbol *BeginSymbol = nullptr;
3882 if (NeedEHLabel) {
3883 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_INVOKE_REGION_START);
3884 BeginSymbol = Context.createTempSymbol();
3885 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: BeginSymbol);
3886 }
3887
3888 if (LowerInlineAsm) {
3889 if (!translateInlineAsm(CB: I, MIRBuilder))
3890 return false;
3891 } else if (!translateCallBase(CB: I, MIRBuilder))
3892 return false;
3893
3894 MCSymbol *EndSymbol = nullptr;
3895 if (NeedEHLabel) {
3896 EndSymbol = Context.createTempSymbol();
3897 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: EndSymbol);
3898 }
3899
3900 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
3901 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3902 MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
3903 BranchProbability EHPadBBProb =
3904 BPI ? BPI->getEdgeProbability(Src: InvokeMBB->getBasicBlock(), Dst: EHPadBB)
3905 : BranchProbability::getZero();
3906
3907 if (!findUnwindDestinations(EHPadBB, Prob: EHPadBBProb, UnwindDests))
3908 return false;
3909
3910 MachineBasicBlock &EHPadMBB = getMBB(BB: *EHPadBB),
3911 &ReturnMBB = getMBB(BB: *ReturnBB);
3912 // Update successor info.
3913 addSuccessorWithProb(Src: InvokeMBB, Dst: &ReturnMBB);
3914 for (auto &UnwindDest : UnwindDests) {
3915 UnwindDest.first->setIsEHPad();
3916 addSuccessorWithProb(Src: InvokeMBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
3917 }
3918 InvokeMBB->normalizeSuccProbs();
3919
3920 if (NeedEHLabel) {
3921 assert(BeginSymbol && "Expected a begin symbol!");
3922 assert(EndSymbol && "Expected an end symbol!");
3923 MF->addInvoke(LandingPad: &EHPadMBB, BeginLabel: BeginSymbol, EndLabel: EndSymbol);
3924 }
3925
3926 MIRBuilder.buildBr(Dest&: ReturnMBB);
3927 return true;
3928}
3929
3930/// The intrinsics currently supported by callbr are implicit control flow
3931/// intrinsics such as amdgcn.kill.
3932bool IRTranslatorImpl::translateCallBr(const User &U,
3933 MachineIRBuilder &MIRBuilder) {
3934 if (!mayTranslateUserTypes(U))
3935 return false; // see translateCall
3936
3937 const CallBrInst &I = cast<CallBrInst>(Val: U);
3938 MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB();
3939
3940 Intrinsic::ID IID = I.getIntrinsicID();
3941 if (I.isInlineAsm()) {
3942 // FIXME: inline asm is not yet supported for callbr in GlobalISel. As soon
3943 // as we add support, we need to handle the indirect asm targets, see
3944 // SelectionDAGBuilder::visitCallBr().
3945 return false;
3946 }
3947 if (!translateIntrinsic(CB: I, ID: IID, MIRBuilder))
3948 return false;
3949
3950 // Retrieve successors.
3951 SmallPtrSet<BasicBlock *, 8> Dests = {I.getDefaultDest()};
3952 MachineBasicBlock *Return = &getMBB(BB: *I.getDefaultDest());
3953
3954 // Update successor info.
3955 addSuccessorWithProb(Src: CallBrMBB, Dst: Return, Prob: BranchProbability::getOne());
3956
3957 // Add indirect targets as successors. For intrinsic callbr, these represent
3958 // implicit control flow (e.g., the "kill" path for amdgcn.kill). We mark them
3959 // with setIsInlineAsmBrIndirectTarget so the machine verifier accepts them as
3960 // valid successors, even though they're not from inline asm.
3961 for (BasicBlock *Dest : I.getIndirectDests()) {
3962 MachineBasicBlock &Target = getMBB(BB: *Dest);
3963 Target.setIsInlineAsmBrIndirectTarget();
3964 Target.setLabelMustBeEmitted();
3965 // Don't add duplicate machine successors.
3966 if (Dests.insert(Ptr: Dest).second)
3967 addSuccessorWithProb(Src: CallBrMBB, Dst: &Target, Prob: BranchProbability::getZero());
3968 }
3969
3970 CallBrMBB->normalizeSuccProbs();
3971
3972 // Drop into default successor.
3973 MIRBuilder.buildBr(Dest&: *Return);
3974
3975 return true;
3976}
3977
3978bool IRTranslatorImpl::translateLandingPad(const User &U,
3979 MachineIRBuilder &MIRBuilder) {
3980 const LandingPadInst &LP = cast<LandingPadInst>(Val: U);
3981
3982 MachineBasicBlock &MBB = MIRBuilder.getMBB();
3983
3984 MBB.setIsEHPad();
3985
3986 // If there aren't registers to copy the values into (e.g., during SjLj
3987 // exceptions), then don't bother.
3988 const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
3989 if (TLI->getExceptionPointerRegister(EH: FuncInfo.ExceptionModel,
3990 PersonalityFn) == 0 &&
3991 TLI->getExceptionSelectorRegister(EH: FuncInfo.ExceptionModel,
3992 PersonalityFn) == 0)
3993 return true;
3994
3995 // If landingpad's return type is token type, we don't create DAG nodes
3996 // for its exception pointer and selector value. The extraction of exception
3997 // pointer or selector value from token type landingpads is not currently
3998 // supported.
3999 if (LP.getType()->isTokenTy())
4000 return true;
4001
4002 if (!isExceptionPointerAndSelectorType(Ty: LP.getType()))
4003 return false;
4004
4005 // Add a label to mark the beginning of the landing pad. Deletion of the
4006 // landing pad can thus be detected via the MachineModuleInfo.
4007 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL)
4008 .addSym(Sym: MF->addLandingPad(LandingPad: &MBB));
4009
4010 // If the unwinder does not preserve all registers, ensure that the
4011 // function marks the clobbered registers as used.
4012 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
4013 if (auto *RegMask = TRI.getCustomEHPadPreservedMask(MF: *MF))
4014 MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask);
4015
4016 LLT Ty = getLLTForType(Ty&: *LP.getType(), DL: *DL);
4017 Register Undef = MRI->createGenericVirtualRegister(Ty);
4018 MIRBuilder.buildUndef(Res: Undef);
4019
4020 SmallVector<LLT, 2> Tys;
4021 for (Type *Ty : cast<StructType>(Val: LP.getType())->elements())
4022 Tys.push_back(Elt: getLLTForType(Ty&: *Ty, DL: *DL));
4023 assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
4024
4025 // Mark exception register as live in.
4026 Register ExceptionReg =
4027 TLI->getExceptionPointerRegister(EH: FuncInfo.ExceptionModel, PersonalityFn);
4028 if (!ExceptionReg)
4029 return false;
4030
4031 MBB.addLiveIn(PhysReg: ExceptionReg);
4032 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: LP);
4033 MIRBuilder.buildCopy(Res: ResRegs[0], Op: ExceptionReg);
4034
4035 Register SelectorReg =
4036 TLI->getExceptionSelectorRegister(EH: FuncInfo.ExceptionModel, PersonalityFn);
4037 if (!SelectorReg)
4038 return false;
4039
4040 MBB.addLiveIn(PhysReg: SelectorReg);
4041 Register PtrVReg = MRI->createGenericVirtualRegister(Ty: Tys[0]);
4042 MIRBuilder.buildCopy(Res: PtrVReg, Op: SelectorReg);
4043 MIRBuilder.buildCast(Dst: ResRegs[1], Src: PtrVReg);
4044
4045 return true;
4046}
4047
4048bool IRTranslatorImpl::translateAlloca(const User &U,
4049 MachineIRBuilder &MIRBuilder) {
4050 auto &AI = cast<AllocaInst>(Val: U);
4051
4052 if (AI.isSwiftError())
4053 return true;
4054
4055 if (AI.isStaticAlloca()) {
4056 Register Res = getOrCreateVReg(Val: AI);
4057 int FI = getOrCreateFrameIndex(AI);
4058 MIRBuilder.buildFrameIndex(Res, Idx: FI);
4059 return true;
4060 }
4061
4062 // FIXME: support stack probing for Windows.
4063 if (MF->getTarget().getTargetTriple().isOSWindows())
4064 return false;
4065
4066 // Now we're in the harder dynamic case.
4067 Register NumElts = getOrCreateVReg(Val: *AI.getArraySize());
4068 Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
4069 LLT IntPtrTy = getLLTForType(Ty&: *IntPtrIRTy, DL: *DL);
4070 if (MRI->getType(Reg: NumElts) != IntPtrTy) {
4071 Register ExtElts = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
4072 MIRBuilder.buildZExtOrTrunc(Res: ExtElts, Op: NumElts);
4073 NumElts = ExtElts;
4074 }
4075
4076 TypeSize TySize = AI.getAllocationBaseSize(DL: *DL);
4077
4078 Register AllocSize = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
4079 Register TySizeReg;
4080 if (TySize.isScalable()) {
4081 // For scalable types, use vscale * min_value
4082 TySizeReg = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
4083 MIRBuilder.buildVScale(Res: TySizeReg, MinElts: TySize.getKnownMinValue());
4084 } else {
4085 // For fixed types, use a constant
4086 TySizeReg =
4087 getOrCreateVReg(Val: *ConstantInt::get(Ty: IntPtrIRTy, V: TySize.getFixedValue()));
4088 }
4089 MIRBuilder.buildMul(Dst: AllocSize, Src0: NumElts, Src1: TySizeReg);
4090
4091 // Round the size of the allocation up to the stack alignment size
4092 // by add SA-1 to the size. This doesn't overflow because we're computing
4093 // an address inside an alloca.
4094 Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();
4095 auto SAMinusOne = MIRBuilder.buildConstant(Res: IntPtrTy, Val: StackAlign.value() - 1);
4096 auto AllocAdd = MIRBuilder.buildAdd(Dst: IntPtrTy, Src0: AllocSize, Src1: SAMinusOne,
4097 Flags: MachineInstr::NoUWrap);
4098 auto AlignCst =
4099 MIRBuilder.buildConstant(Res: IntPtrTy, Val: ~(uint64_t)(StackAlign.value() - 1));
4100 auto AlignedAlloc = MIRBuilder.buildAnd(Dst: IntPtrTy, Src0: AllocAdd, Src1: AlignCst);
4101
4102 Align Alignment = AI.getAlign();
4103 if (Alignment <= StackAlign)
4104 Alignment = Align(1);
4105 MIRBuilder.buildDynStackAlloc(Res: getOrCreateVReg(Val: AI), Size: AlignedAlloc, Alignment);
4106
4107 MF->getFrameInfo().CreateVariableSizedObject(Alignment, Alloca: &AI);
4108 assert(MF->getFrameInfo().hasVarSizedObjects());
4109 return true;
4110}
4111
4112bool IRTranslatorImpl::translateVAArg(const User &U,
4113 MachineIRBuilder &MIRBuilder) {
4114 // FIXME: We may need more info about the type. Because of how LLT works,
4115 // we're completely discarding the i64/double distinction here (amongst
4116 // others). Fortunately the ABIs I know of where that matters don't use va_arg
4117 // anyway but that's not guaranteed.
4118 MIRBuilder.buildInstr(Opc: TargetOpcode::G_VAARG, DstOps: {getOrCreateVReg(Val: U)},
4119 SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
4120 DL->getABITypeAlign(Ty: U.getType()).value()});
4121 return true;
4122}
4123
4124bool IRTranslatorImpl::translateUnreachable(const User &U,
4125 MachineIRBuilder &MIRBuilder) {
4126 auto &UI = cast<UnreachableInst>(Val: U);
4127 if (!UI.shouldLowerToTrap(TrapUnreachable: MF->getTarget().Options.TrapUnreachable,
4128 NoTrapAfterNoreturn: MF->getTarget().Options.NoTrapAfterNoreturn))
4129 return true;
4130
4131 MIRBuilder.buildTrap();
4132 return true;
4133}
4134
4135bool IRTranslatorImpl::translateInsertElement(const User &U,
4136 MachineIRBuilder &MIRBuilder) {
4137 // If it is a <1 x Ty> vector, use the scalar as it is
4138 // not a legal vector type in LLT.
4139 if (auto *FVT = dyn_cast<FixedVectorType>(Val: U.getType());
4140 FVT && FVT->getNumElements() == 1)
4141 return translateCopy(U, V: *U.getOperand(i: 1), MIRBuilder);
4142
4143 Register Res = getOrCreateVReg(Val: U);
4144 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
4145 Register Elt = getOrCreateVReg(Val: *U.getOperand(i: 1));
4146 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4147 Register Idx;
4148 if (auto *CI = dyn_cast<ConstantInt>(Val: U.getOperand(i: 2))) {
4149 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4150 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4151 auto *NewIdxCI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4152 Idx = getOrCreateVReg(Val: *NewIdxCI);
4153 }
4154 }
4155 if (!Idx)
4156 Idx = getOrCreateVReg(Val: *U.getOperand(i: 2));
4157 if (MRI->getType(Reg: Idx).getSizeInBits() != PreferredVecIdxWidth) {
4158 const LLT VecIdxTy =
4159 MRI->getType(Reg: Idx).changeElementSize(NewEltSize: PreferredVecIdxWidth);
4160 Idx = MIRBuilder.buildZExtOrTrunc(Res: VecIdxTy, Op: Idx).getReg(Idx: 0);
4161 }
4162 MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
4163 return true;
4164}
4165
4166bool IRTranslatorImpl::translateInsertVector(const User &U,
4167 MachineIRBuilder &MIRBuilder) {
4168 Register Dst = getOrCreateVReg(Val: U);
4169 Register Vec = getOrCreateVReg(Val: *U.getOperand(i: 0));
4170 Register Elt = getOrCreateVReg(Val: *U.getOperand(i: 1));
4171
4172 ConstantInt *CI = cast<ConstantInt>(Val: U.getOperand(i: 2));
4173 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4174
4175 // Resize Index to preferred index width.
4176 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4177 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4178 CI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4179 }
4180
4181 // If it is a <1 x Ty> vector, we have to use other means.
4182 if (auto *ResultType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 1)->getType());
4183 ResultType && ResultType->getNumElements() == 1) {
4184 if (auto *InputType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType());
4185 InputType && InputType->getNumElements() == 1) {
4186 // We are inserting an illegal fixed vector into an illegal
4187 // fixed vector, use the scalar as it is not a legal vector type
4188 // in LLT.
4189 return translateCopy(U, Src: Vec, MIRBuilder);
4190 }
4191 if (isa<FixedVectorType>(Val: U.getOperand(i: 0)->getType())) {
4192 // We are inserting an illegal fixed vector into a legal fixed
4193 // vector, use the scalar as it is not a legal vector type in
4194 // LLT.
4195 Register Idx = getOrCreateVReg(Val: *CI);
4196 MIRBuilder.buildInsertVectorElement(Res: Dst, Val: Vec, Elt, Idx);
4197 return true;
4198 }
4199 if (isa<ScalableVectorType>(Val: U.getOperand(i: 0)->getType())) {
4200 // We are inserting an illegal fixed vector into a scalable
4201 // vector, use a scalar element insert.
4202 LLT VecIdxTy = LLT::integer(SizeInBits: PreferredVecIdxWidth);
4203 Register Idx = getOrCreateVReg(Val: *CI);
4204 auto ScaledIndex = MIRBuilder.buildMul(
4205 Dst: VecIdxTy, Src0: MIRBuilder.buildVScale(Res: VecIdxTy, MinElts: 1), Src1: Idx);
4206 MIRBuilder.buildInsertVectorElement(Res: Dst, Val: Vec, Elt, Idx: ScaledIndex);
4207 return true;
4208 }
4209 }
4210
4211 MIRBuilder.buildInsertSubvector(Res: Dst, Src0: Vec, Src1: Elt, Index: CI->getZExtValue());
4212 return true;
4213}
4214
4215bool IRTranslatorImpl::translateExtractElement(const User &U,
4216 MachineIRBuilder &MIRBuilder) {
4217 // If it is a <1 x Ty> vector, use the scalar as it is
4218 // not a legal vector type in LLT.
4219 if (const FixedVectorType *FVT =
4220 dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType()))
4221 if (FVT->getNumElements() == 1)
4222 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
4223
4224 Register Res = getOrCreateVReg(Val: U);
4225 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
4226 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4227 Register Idx;
4228 if (auto *CI = dyn_cast<ConstantInt>(Val: U.getOperand(i: 1))) {
4229 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4230 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4231 auto *NewIdxCI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4232 Idx = getOrCreateVReg(Val: *NewIdxCI);
4233 }
4234 }
4235 if (!Idx)
4236 Idx = getOrCreateVReg(Val: *U.getOperand(i: 1));
4237 if (MRI->getType(Reg: Idx).getSizeInBits() != PreferredVecIdxWidth) {
4238 const LLT VecIdxTy =
4239 MRI->getType(Reg: Idx).changeElementSize(NewEltSize: PreferredVecIdxWidth);
4240 Idx = MIRBuilder.buildZExtOrTrunc(Res: VecIdxTy, Op: Idx).getReg(Idx: 0);
4241 }
4242 MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
4243 return true;
4244}
4245
4246bool IRTranslatorImpl::translateExtractVector(const User &U,
4247 MachineIRBuilder &MIRBuilder) {
4248 Register Res = getOrCreateVReg(Val: U);
4249 Register Vec = getOrCreateVReg(Val: *U.getOperand(i: 0));
4250 ConstantInt *CI = cast<ConstantInt>(Val: U.getOperand(i: 1));
4251 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4252
4253 // Resize Index to preferred index width.
4254 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4255 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4256 CI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4257 }
4258
4259 // If it is a <1 x Ty> vector, we have to use other means.
4260 if (auto *ResultType = dyn_cast<FixedVectorType>(Val: U.getType());
4261 ResultType && ResultType->getNumElements() == 1) {
4262 if (auto *InputType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType());
4263 InputType && InputType->getNumElements() == 1) {
4264 // We are extracting an illegal fixed vector from an illegal fixed vector,
4265 // use the scalar as it is not a legal vector type in LLT.
4266 return translateCopy(U, Src: Vec, MIRBuilder);
4267 }
4268 if (isa<FixedVectorType>(Val: U.getOperand(i: 0)->getType())) {
4269 // We are extracting an illegal fixed vector from a legal fixed
4270 // vector, use the scalar as it is not a legal vector type in
4271 // LLT.
4272 Register Idx = getOrCreateVReg(Val: *CI);
4273 MIRBuilder.buildExtractVectorElement(Res, Val: Vec, Idx);
4274 return true;
4275 }
4276 if (isa<ScalableVectorType>(Val: U.getOperand(i: 0)->getType())) {
4277 // We are extracting an illegal fixed vector from a scalable
4278 // vector, use a scalar element extract.
4279 LLT VecIdxTy = LLT::integer(SizeInBits: PreferredVecIdxWidth);
4280 Register Idx = getOrCreateVReg(Val: *CI);
4281 auto ScaledIndex = MIRBuilder.buildMul(
4282 Dst: VecIdxTy, Src0: MIRBuilder.buildVScale(Res: VecIdxTy, MinElts: 1), Src1: Idx);
4283 MIRBuilder.buildExtractVectorElement(Res, Val: Vec, Idx: ScaledIndex);
4284 return true;
4285 }
4286 }
4287
4288 MIRBuilder.buildExtractSubvector(Res, Src: Vec, Index: CI->getZExtValue());
4289 return true;
4290}
4291
4292bool IRTranslatorImpl::translateShuffleVector(const User &U,
4293 MachineIRBuilder &MIRBuilder) {
4294 // A ShuffleVector that operates on scalable vectors is a splat vector where
4295 // the value of the splat vector is the 0th element of the first operand,
4296 // since the index mask operand is the zeroinitializer (undef and
4297 // poison are treated as zeroinitializer here).
4298 if (U.getOperand(i: 0)->getType()->isScalableTy()) {
4299 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
4300 auto SplatVal = MIRBuilder.buildExtractVectorElementConstant(
4301 Res: MRI->getType(Reg: Val).getElementType(), Val, Idx: 0);
4302 MIRBuilder.buildSplatVector(Res: getOrCreateVReg(Val: U), Val: SplatVal);
4303 return true;
4304 }
4305
4306 ArrayRef<int> Mask;
4307 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: &U))
4308 Mask = SVI->getShuffleMask();
4309 else
4310 Mask = cast<ConstantExpr>(Val: U).getShuffleMask();
4311
4312 // As GISel does not represent <1 x > vectors as a separate type from scalars,
4313 // we transform shuffle_vector with a scalar output to an
4314 // ExtractVectorElement. If the input type is also scalar it becomes a Copy.
4315 unsigned DstElts = cast<FixedVectorType>(Val: U.getType())->getNumElements();
4316 unsigned SrcElts =
4317 cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType())->getNumElements();
4318 if (DstElts == 1) {
4319 unsigned M = Mask[0];
4320 if (SrcElts == 1) {
4321 if (M == 0 || M == 1)
4322 return translateCopy(U, V: *U.getOperand(i: M), MIRBuilder);
4323 MIRBuilder.buildUndef(Res: getOrCreateVReg(Val: U));
4324 } else {
4325 Register Dst = getOrCreateVReg(Val: U);
4326 if (M < SrcElts) {
4327 MIRBuilder.buildExtractVectorElementConstant(
4328 Res: Dst, Val: getOrCreateVReg(Val: *U.getOperand(i: 0)), Idx: M);
4329 } else if (M < SrcElts * 2) {
4330 MIRBuilder.buildExtractVectorElementConstant(
4331 Res: Dst, Val: getOrCreateVReg(Val: *U.getOperand(i: 1)), Idx: M - SrcElts);
4332 } else {
4333 MIRBuilder.buildUndef(Res: Dst);
4334 }
4335 }
4336 return true;
4337 }
4338
4339 // A single element src is transformed to a build_vector.
4340 if (SrcElts == 1) {
4341 SmallVector<Register> Ops;
4342 Register Undef;
4343 for (int M : Mask) {
4344 LLT SrcTy = getLLTForType(Ty&: *U.getOperand(i: 0)->getType(), DL: *DL);
4345 if (M == 0 || M == 1) {
4346 Ops.push_back(Elt: getOrCreateVReg(Val: *U.getOperand(i: M)));
4347 } else {
4348 if (!Undef.isValid()) {
4349 Undef = MRI->createGenericVirtualRegister(Ty: SrcTy);
4350 MIRBuilder.buildUndef(Res: Undef);
4351 }
4352 Ops.push_back(Elt: Undef);
4353 }
4354 }
4355 MIRBuilder.buildBuildVector(Res: getOrCreateVReg(Val: U), Ops);
4356 return true;
4357 }
4358
4359 ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
4360 MIRBuilder
4361 .buildInstr(Opc: TargetOpcode::G_SHUFFLE_VECTOR, DstOps: {getOrCreateVReg(Val: U)},
4362 SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
4363 getOrCreateVReg(Val: *U.getOperand(i: 1))})
4364 .addShuffleMask(Val: MaskAlloc);
4365 return true;
4366}
4367
4368bool IRTranslatorImpl::translateBitInsert(const User &U,
4369 MachineIRBuilder &MIRBuilder) {
4370 Register Res = getOrCreateVReg(Val: U);
4371 Register Base = getOrCreateVReg(Val: *U.getOperand(i: 0));
4372 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 1));
4373 Register Offset = getOrCreateVReg(Val: *U.getOperand(i: 2));
4374 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4375 LLT BaseTy = MRI.getType(Reg: Base);
4376 LLT ValTy = MRI.getType(Reg: Val);
4377
4378 assert(BaseTy.getSizeInBits() >= ValTy.getSizeInBits() &&
4379 "bitinsert val wider than base should be rejected by verifier");
4380
4381 // If Val is a floating-point type, bitcast it to an integer of the same
4382 // size so buildZExtOrTrunc can safely extend or truncate it.
4383 if (ValTy.isFloat()) {
4384 ValTy = LLT::scalar(SizeInBits: ValTy.getSizeInBits());
4385 Val = MIRBuilder.buildBitcast(Dst: ValTy, Src: Val).getReg(Idx: 0);
4386 } else if (ValTy.isPointer()) {
4387 ValTy = LLT::scalar(SizeInBits: ValTy.getSizeInBits());
4388 Val = MIRBuilder.buildPtrToInt(Dst: ValTy, Src: Val).getReg(Idx: 0);
4389 }
4390
4391 // Convert Offset to the target's preferred shift amount type.
4392 LLT ShiftAmtTy = TLI->getPreferredShiftAmountTy(ShiftValueTy: BaseTy);
4393 Register LegalOffset =
4394 MIRBuilder.buildZExtOrTrunc(Res: ShiftAmtTy, Op: Offset).getReg(Idx: 0);
4395
4396 // Truncate or extend Val to BaseTy so only the inserted bit range remains.
4397 Register ExtVal = MIRBuilder.buildZExtOrTrunc(Res: BaseTy, Op: Val).getReg(Idx: 0);
4398
4399 unsigned BaseBitWidth = BaseTy.getSizeInBits();
4400 unsigned ValBitWidth = ValTy.getSizeInBits();
4401 APInt InsertMask = APInt::getLowBitsSet(numBits: BaseBitWidth, loBitsSet: ValBitWidth);
4402 Register MaskConst = MIRBuilder.buildConstant(Res: BaseTy, Val: InsertMask).getReg(Idx: 0);
4403 Register ShiftedMask =
4404 MIRBuilder.buildShl(Dst: BaseTy, Src0: MaskConst, Src1: LegalOffset).getReg(Idx: 0);
4405 Register ClearMask = MIRBuilder.buildNot(Dst: BaseTy, Src0: ShiftedMask).getReg(Idx: 0);
4406 Register ClearedBase = MIRBuilder.buildAnd(Dst: BaseTy, Src0: Base, Src1: ClearMask).getReg(Idx: 0);
4407 Register ShiftedVal =
4408 MIRBuilder.buildShl(Dst: BaseTy, Src0: ExtVal, Src1: LegalOffset).getReg(Idx: 0);
4409 MIRBuilder.buildOr(Dst: Res, Src0: ClearedBase, Src1: ShiftedVal);
4410 return true;
4411}
4412
4413bool IRTranslatorImpl::translateBitExtract(const User &U,
4414 MachineIRBuilder &MIRBuilder) {
4415 Register Res = getOrCreateVReg(Val: U);
4416 Register Src = getOrCreateVReg(Val: *U.getOperand(i: 0));
4417 Register Offset = getOrCreateVReg(Val: *U.getOperand(i: 1));
4418 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4419 LLT SrcTy = MRI.getType(Reg: Src);
4420 LLT ResTy = MRI.getType(Reg: Res);
4421
4422 assert(ResTy.getSizeInBits() <= SrcTy.getSizeInBits() &&
4423 "bitextract result wider than source should be rejected by verifier");
4424
4425 // Convert Offset to the target's preferred shift amount type.
4426 LLT ShiftAmtTy = TLI->getPreferredShiftAmountTy(ShiftValueTy: SrcTy);
4427 Register LegalOffset =
4428 MIRBuilder.buildZExtOrTrunc(Res: ShiftAmtTy, Op: Offset).getReg(Idx: 0);
4429
4430 // Shift right by Offset to bring the target field down to bit 0.
4431 Register Shifted = MIRBuilder.buildLShr(Dst: SrcTy, Src0: Src, Src1: LegalOffset).getReg(Idx: 0);
4432
4433 if (ResTy.isFloat()) {
4434 // Drop into the integer domain to safely handle the size conversion
4435 LLT IntResTy = LLT::scalar(SizeInBits: ResTy.getSizeInBits());
4436 Register IntRes = MRI.createGenericVirtualRegister(Ty: IntResTy);
4437
4438 if (SrcTy == IntResTy)
4439 MIRBuilder.buildCopy(Res: IntRes, Op: Shifted);
4440 else
4441 MIRBuilder.buildTrunc(Res: IntRes, Op: Shifted);
4442
4443 // Bitcast the raw integer bits back into the requested floating-point
4444 // register
4445 MIRBuilder.buildBitcast(Dst: Res, Src: IntRes);
4446 } else if (ResTy.isPointer()) {
4447 // Drop into the integer domain to safely handle the size conversion
4448 LLT IntResTy = LLT::scalar(SizeInBits: ResTy.getSizeInBits());
4449 Register IntRes = MRI.createGenericVirtualRegister(Ty: IntResTy);
4450
4451 if (SrcTy == IntResTy)
4452 MIRBuilder.buildCopy(Res: IntRes, Op: Shifted);
4453 else
4454 MIRBuilder.buildTrunc(Res: IntRes, Op: Shifted);
4455
4456 MIRBuilder.buildIntToPtr(Dst: Res, Src: IntRes);
4457 } else {
4458 // Normal integer path
4459 if (SrcTy == ResTy)
4460 MIRBuilder.buildCopy(Res, Op: Shifted);
4461 else
4462 MIRBuilder.buildTrunc(Res, Op: Shifted);
4463 }
4464
4465 return true;
4466}
4467
4468bool IRTranslatorImpl::translatePHI(const User &U,
4469 MachineIRBuilder &MIRBuilder) {
4470 const PHINode &PI = cast<PHINode>(Val: U);
4471
4472 SmallVector<MachineInstr *, 4> Insts;
4473 for (auto Reg : getOrCreateVRegs(Val: PI)) {
4474 auto MIB = MIRBuilder.buildInstr(Opc: TargetOpcode::G_PHI, DstOps: {Reg}, SrcOps: {});
4475 Insts.push_back(Elt: MIB.getInstr());
4476 }
4477
4478 PendingPHIs.emplace_back(Args: &PI, Args: std::move(Insts));
4479 return true;
4480}
4481
4482bool IRTranslatorImpl::translateAtomicCmpXchg(const User &U,
4483 MachineIRBuilder &MIRBuilder) {
4484 const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(Val: U);
4485
4486 auto Flags = TLI->getAtomicMemOperandFlags(AI: I, DL: *DL);
4487
4488 auto Res = getOrCreateVRegs(Val: I);
4489 Register OldValRes = Res[0];
4490 Register SuccessRes = Res[1];
4491 Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
4492 Register Cmp = getOrCreateVReg(Val: *I.getCompareOperand());
4493 Register NewVal = getOrCreateVReg(Val: *I.getNewValOperand());
4494
4495 MIRBuilder.buildAtomicCmpXchgWithSuccess(
4496 OldValRes, SuccessRes, Addr, CmpVal: Cmp, NewVal,
4497 MMO&: *MF->getMachineMemOperand(
4498 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, MemTy: MRI->getType(Reg: Cmp),
4499 BaseAlignment: getMemOpAlign(I), Metadata: I.getAAMetadata(), SSID: I.getSyncScopeID(),
4500 Ordering: I.getSuccessOrdering(), FailureOrdering: I.getFailureOrdering()));
4501 return true;
4502}
4503
4504bool IRTranslatorImpl::translateAtomicRMW(const User &U,
4505 MachineIRBuilder &MIRBuilder) {
4506 if (!mayTranslateUserTypes(U))
4507 return false;
4508
4509 const AtomicRMWInst &I = cast<AtomicRMWInst>(Val: U);
4510 auto Flags = TLI->getAtomicMemOperandFlags(AI: I, DL: *DL);
4511
4512 Register Res = getOrCreateVReg(Val: I);
4513 Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
4514 Register Val = getOrCreateVReg(Val: *I.getValOperand());
4515
4516 unsigned Opcode = 0;
4517 switch (I.getOperation()) {
4518 default:
4519 return false;
4520 case AtomicRMWInst::Xchg:
4521 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
4522 break;
4523 case AtomicRMWInst::Add:
4524 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
4525 break;
4526 case AtomicRMWInst::Sub:
4527 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
4528 break;
4529 case AtomicRMWInst::And:
4530 Opcode = TargetOpcode::G_ATOMICRMW_AND;
4531 break;
4532 case AtomicRMWInst::Nand:
4533 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
4534 break;
4535 case AtomicRMWInst::Or:
4536 Opcode = TargetOpcode::G_ATOMICRMW_OR;
4537 break;
4538 case AtomicRMWInst::Xor:
4539 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
4540 break;
4541 case AtomicRMWInst::Max:
4542 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
4543 break;
4544 case AtomicRMWInst::Min:
4545 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
4546 break;
4547 case AtomicRMWInst::UMax:
4548 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
4549 break;
4550 case AtomicRMWInst::UMin:
4551 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
4552 break;
4553 case AtomicRMWInst::FAdd:
4554 Opcode = TargetOpcode::G_ATOMICRMW_FADD;
4555 break;
4556 case AtomicRMWInst::FSub:
4557 Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
4558 break;
4559 case AtomicRMWInst::FMax:
4560 Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
4561 break;
4562 case AtomicRMWInst::FMin:
4563 Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
4564 break;
4565 case AtomicRMWInst::FMaximum:
4566 Opcode = TargetOpcode::G_ATOMICRMW_FMAXIMUM;
4567 break;
4568 case AtomicRMWInst::FMinimum:
4569 Opcode = TargetOpcode::G_ATOMICRMW_FMINIMUM;
4570 break;
4571 case AtomicRMWInst::FMaximumNum:
4572 Opcode = TargetOpcode::G_ATOMICRMW_FMAXIMUMNUM;
4573 break;
4574 case AtomicRMWInst::FMinimumNum:
4575 Opcode = TargetOpcode::G_ATOMICRMW_FMINIMUMNUM;
4576 break;
4577 case AtomicRMWInst::UIncWrap:
4578 Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
4579 break;
4580 case AtomicRMWInst::UDecWrap:
4581 Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
4582 break;
4583 case AtomicRMWInst::USubCond:
4584 Opcode = TargetOpcode::G_ATOMICRMW_USUB_COND;
4585 break;
4586 case AtomicRMWInst::USubSat:
4587 Opcode = TargetOpcode::G_ATOMICRMW_USUB_SAT;
4588 break;
4589 }
4590
4591 MIRBuilder.buildAtomicRMW(
4592 Opcode, OldValRes: Res, Addr, Val,
4593 MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(I.getPointerOperand()),
4594 F: Flags, MemTy: MRI->getType(Reg: Val), BaseAlignment: getMemOpAlign(I),
4595 Metadata: I.getAAMetadata(), SSID: I.getSyncScopeID(),
4596 Ordering: I.getOrdering()));
4597 return true;
4598}
4599
4600bool IRTranslatorImpl::translateFence(const User &U,
4601 MachineIRBuilder &MIRBuilder) {
4602 const FenceInst &Fence = cast<FenceInst>(Val: U);
4603 MIRBuilder.buildFence(Ordering: static_cast<unsigned>(Fence.getOrdering()),
4604 Scope: Fence.getSyncScopeID());
4605 return true;
4606}
4607
4608bool IRTranslatorImpl::translateFreeze(const User &U,
4609 MachineIRBuilder &MIRBuilder) {
4610 const ArrayRef<Register> DstRegs = getOrCreateVRegs(Val: U);
4611 const ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *U.getOperand(i: 0));
4612
4613 assert(DstRegs.size() == SrcRegs.size() &&
4614 "Freeze with different source and destination type?");
4615
4616 for (unsigned I = 0; I < DstRegs.size(); ++I) {
4617 MIRBuilder.buildFreeze(Dst: DstRegs[I], Src: SrcRegs[I]);
4618 }
4619
4620 return true;
4621}
4622
4623void IRTranslatorImpl::finishPendingPhis() {
4624#ifndef NDEBUG
4625 DILocationVerifier Verifier;
4626 GISelObserverWrapper WrapperObserver(&Verifier);
4627 RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);
4628#endif // ifndef NDEBUG
4629 for (auto &Phi : PendingPHIs) {
4630 const PHINode *PI = Phi.first;
4631 if (PI->getType()->isEmptyTy())
4632 continue;
4633 ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
4634 MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
4635 EntryBuilder->setDebugLoc(PI->getDebugLoc());
4636#ifndef NDEBUG
4637 Verifier.setCurrentInst(PI);
4638#endif // ifndef NDEBUG
4639
4640 SmallPtrSet<const MachineBasicBlock *, 16> SeenPreds;
4641 for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
4642 auto IRPred = PI->getIncomingBlock(i);
4643 ArrayRef<Register> ValRegs = getOrCreateVRegs(Val: *PI->getIncomingValue(i));
4644 for (auto *Pred : getMachinePredBBs(Edge: {IRPred, PI->getParent()})) {
4645 if (SeenPreds.count(Ptr: Pred) || !PhiMBB->isPredecessor(MBB: Pred))
4646 continue;
4647 SeenPreds.insert(Ptr: Pred);
4648 for (unsigned j = 0; j < ValRegs.size(); ++j) {
4649 MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
4650 MIB.addUse(RegNo: ValRegs[j]);
4651 MIB.addMBB(MBB: Pred);
4652 }
4653 }
4654 }
4655 }
4656}
4657
4658void IRTranslatorImpl::translateDbgValueRecord(Value *V, bool HasArgList,
4659 const DILocalVariable *Variable,
4660 const DIExpression *Expression,
4661 const DebugLoc &DL,
4662 MachineIRBuilder &MIRBuilder) {
4663 assert(Variable->isValidLocationForIntrinsic(DL) &&
4664 "Expected inlined-at fields to agree");
4665 // Act as if we're handling a debug intrinsic.
4666 MIRBuilder.setDebugLoc(DL);
4667
4668 if (!V || HasArgList) {
4669 // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
4670 // terminate any prior location.
4671 MIRBuilder.buildIndirectDbgValue(Reg: 0, Variable, Expr: Expression);
4672 return;
4673 }
4674
4675 if (const auto *CI = dyn_cast<Constant>(Val: V)) {
4676 MIRBuilder.buildConstDbgValue(C: *CI, Variable, Expr: Expression);
4677 return;
4678 }
4679
4680 if (auto *AI = dyn_cast<AllocaInst>(Val: V);
4681 AI && AI->isStaticAlloca() && Expression->startsWithDeref()) {
4682 // If the value is an alloca and the expression starts with a
4683 // dereference, track a stack slot instead of a register, as registers
4684 // may be clobbered.
4685 auto ExprOperands = Expression->getElements();
4686 auto *ExprDerefRemoved =
4687 DIExpression::get(Context&: AI->getContext(), Elements: ExprOperands.drop_front());
4688 MIRBuilder.buildFIDbgValue(FI: getOrCreateFrameIndex(AI: *AI), Variable,
4689 Expr: ExprDerefRemoved);
4690 return;
4691 }
4692 if (translateIfEntryValueArgument(isDeclare: false, Val: V, Var: Variable, Expr: Expression, DL,
4693 MIRBuilder))
4694 return;
4695 for (Register Reg : getOrCreateVRegs(Val: *V)) {
4696 // FIXME: This does not handle register-indirect values at offset 0. The
4697 // direct/indirect thing shouldn't really be handled by something as
4698 // implicit as reg+noreg vs reg+imm in the first place, but it seems
4699 // pretty baked in right now.
4700 MIRBuilder.buildDirectDbgValue(Reg, Variable, Expr: Expression);
4701 }
4702}
4703
4704void IRTranslatorImpl::translateDbgDeclareRecord(
4705 Value *Address, bool HasArgList, const DILocalVariable *Variable,
4706 const DIExpression *Expression, const DebugLoc &DL,
4707 MachineIRBuilder &MIRBuilder) {
4708 if (!Address || isa<UndefValue>(Val: Address)) {
4709 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *Variable << "\n");
4710 return;
4711 }
4712
4713 assert(Variable->isValidLocationForIntrinsic(DL) &&
4714 "Expected inlined-at fields to agree");
4715 auto AI = dyn_cast<AllocaInst>(Val: Address);
4716 if (AI && AI->isStaticAlloca()) {
4717 // Static allocas are tracked at the MF level, no need for DBG_VALUE
4718 // instructions (in fact, they get ignored if they *do* exist).
4719 MF->setVariableDbgInfo(Var: Variable, Expr: Expression,
4720 Slot: getOrCreateFrameIndex(AI: *AI), Loc: DL);
4721 return;
4722 }
4723
4724 if (translateIfEntryValueArgument(isDeclare: true, Val: Address, Var: Variable,
4725 Expr: Expression, DL,
4726 MIRBuilder))
4727 return;
4728
4729 // A dbg.declare describes the address of a source variable, so lower it
4730 // into an indirect DBG_VALUE.
4731 MIRBuilder.setDebugLoc(DL);
4732 MIRBuilder.buildIndirectDbgValue(Reg: getOrCreateVReg(Val: *Address), Variable,
4733 Expr: Expression);
4734}
4735
4736void IRTranslatorImpl::translateDbgInfo(const Instruction &Inst,
4737 MachineIRBuilder &MIRBuilder) {
4738 for (DbgRecord &DR : Inst.getDbgRecordRange()) {
4739 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
4740 MIRBuilder.setDebugLoc(DLR->getDebugLoc());
4741 assert(DLR->getLabel() && "Missing label");
4742 assert(DLR->getLabel()->isValidLocationForIntrinsic(
4743 MIRBuilder.getDebugLoc()) &&
4744 "Expected inlined-at fields to agree");
4745 MIRBuilder.buildDbgLabel(Label: DLR->getLabel());
4746 continue;
4747 }
4748 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
4749 const DILocalVariable *Variable = DVR.getVariable();
4750 const DIExpression *Expression = DVR.getExpression();
4751 Value *V = DVR.getVariableLocationOp(OpIdx: 0);
4752 if (DVR.isDbgDeclare())
4753 translateDbgDeclareRecord(Address: V, HasArgList: DVR.hasArgList(), Variable, Expression,
4754 DL: DVR.getDebugLoc(), MIRBuilder);
4755 else
4756 translateDbgValueRecord(V, HasArgList: DVR.hasArgList(), Variable, Expression,
4757 DL: DVR.getDebugLoc(), MIRBuilder);
4758 }
4759}
4760
4761bool IRTranslatorImpl::translate(const Instruction &Inst) {
4762 CurBuilder->setDebugLoc(Inst.getDebugLoc());
4763 CurBuilder->setPCSections(Inst.getMetadata(KindID: LLVMContext::MD_pcsections));
4764 CurBuilder->setMMRAMetadata(Inst.getMetadata(KindID: LLVMContext::MD_mmra));
4765
4766 if (TLI->fallBackToDAGISel(Inst))
4767 return false;
4768
4769 switch (Inst.getOpcode()) {
4770#define HANDLE_INST(NUM, OPCODE, CLASS) \
4771 case Instruction::OPCODE: \
4772 return translate##OPCODE(Inst, *CurBuilder.get());
4773#include "llvm/IR/Instruction.def"
4774 default:
4775 return false;
4776 }
4777}
4778
4779bool IRTranslatorImpl::translate(const Constant &C, Register Reg) {
4780 // We only emit constants into the entry block from here. To prevent jumpy
4781 // debug behaviour remove debug line.
4782 if (auto CurrInstDL = CurBuilder->getDL())
4783 EntryBuilder->setDebugLoc(DebugLoc());
4784
4785 if (auto CI = dyn_cast<ConstantInt>(Val: &C)) {
4786 // buildConstant expects a to-be-splatted scalar ConstantInt.
4787 if (isa<VectorType>(Val: CI->getType()))
4788 CI = ConstantInt::get(Context&: CI->getContext(), V: CI->getValue());
4789 EntryBuilder->buildConstant(Res: Reg, Val: *CI);
4790 } else if (auto CB = dyn_cast<ConstantByte>(Val: &C)) {
4791 // Byte constants share G_CONSTANT with integers; the destination Reg's
4792 // LLT (an integer LLT, see getLLTForType) determines vector splatting.
4793 EntryBuilder->buildConstant(Res: Reg, Val: CB->getValue());
4794 } else if (auto CF = dyn_cast<ConstantFP>(Val: &C)) {
4795 // buildFConstant expects a to-be-splatted scalar ConstantFP.
4796 if (isa<VectorType>(Val: CF->getType()))
4797 CF = ConstantFP::get(Context&: CF->getContext(), V: CF->getValue());
4798 EntryBuilder->buildFConstant(Res: Reg, Val: *CF);
4799 } else if (isa<UndefValue>(Val: C))
4800 EntryBuilder->buildUndef(Res: Reg);
4801 else if (isa<ConstantPointerNull>(Val: C))
4802 EntryBuilder->buildConstant(Res: Reg, Val: 0);
4803 else if (auto GV = dyn_cast<GlobalValue>(Val: &C))
4804 EntryBuilder->buildGlobalValue(Res: Reg, GV);
4805 else if (auto CPA = dyn_cast<ConstantPtrAuth>(Val: &C)) {
4806 Register Addr = getOrCreateVReg(Val: *CPA->getPointer());
4807 Register AddrDisc = getOrCreateVReg(Val: *CPA->getAddrDiscriminator());
4808 EntryBuilder->buildConstantPtrAuth(Res: Reg, CPA, Addr, AddrDisc);
4809 } else if (auto CAZ = dyn_cast<ConstantAggregateZero>(Val: &C)) {
4810 Constant &Elt = *CAZ->getElementValue(Idx: 0u);
4811 if (isa<ScalableVectorType>(Val: CAZ->getType())) {
4812 EntryBuilder->buildSplatVector(Res: Reg, Val: getOrCreateVReg(Val: Elt));
4813 return true;
4814 }
4815 // Return the scalar if it is a <1 x Ty> vector.
4816 unsigned NumElts = CAZ->getElementCount().getFixedValue();
4817 if (NumElts == 1)
4818 return translateCopy(U: C, V: Elt, MIRBuilder&: *EntryBuilder);
4819 // All elements are zero so we can just use the first one.
4820 EntryBuilder->buildSplatBuildVector(Res: Reg, Src: getOrCreateVReg(Val: Elt));
4821 } else if (auto CV = dyn_cast<ConstantDataVector>(Val: &C)) {
4822 // Return the scalar if it is a <1 x Ty> vector.
4823 if (CV->getNumElements() == 1)
4824 return translateCopy(U: C, V: *CV->getElementAsConstant(i: 0), MIRBuilder&: *EntryBuilder);
4825 SmallVector<Register, 4> Ops;
4826 for (unsigned i = 0; i < CV->getNumElements(); ++i) {
4827 Constant &Elt = *CV->getElementAsConstant(i);
4828 Ops.push_back(Elt: getOrCreateVReg(Val: Elt));
4829 }
4830 EntryBuilder->buildBuildVector(Res: Reg, Ops);
4831 } else if (auto CE = dyn_cast<ConstantExpr>(Val: &C)) {
4832 switch(CE->getOpcode()) {
4833#define HANDLE_INST(NUM, OPCODE, CLASS) \
4834 case Instruction::OPCODE: \
4835 return translate##OPCODE(*CE, *EntryBuilder.get());
4836#include "llvm/IR/Instruction.def"
4837 default:
4838 return false;
4839 }
4840 } else if (auto CV = dyn_cast<ConstantVector>(Val: &C)) {
4841 if (CV->getNumOperands() == 1)
4842 return translateCopy(U: C, V: *CV->getOperand(i_nocapture: 0), MIRBuilder&: *EntryBuilder);
4843 SmallVector<Register, 4> Ops;
4844 for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
4845 Ops.push_back(Elt: getOrCreateVReg(Val: *CV->getOperand(i_nocapture: i)));
4846 }
4847 EntryBuilder->buildBuildVector(Res: Reg, Ops);
4848 } else if (auto *BA = dyn_cast<BlockAddress>(Val: &C)) {
4849 EntryBuilder->buildBlockAddress(Res: Reg, BA);
4850 } else
4851 return false;
4852
4853 return true;
4854}
4855
4856bool IRTranslatorImpl::mayTranslateUserTypes(const User &U) const {
4857 const TargetMachine &TM = TLI->getTargetMachine();
4858 if (LLT::getUseExtended())
4859 return true;
4860
4861 // BF16 cannot currently be represented by default LLT. To avoid miscompiles
4862 // we prevent any instructions using them by default in all targets that do
4863 // not explicitly enable it via LLT::setUseExtended(true).
4864 // SPIRV target is exception.
4865 return TM.getTargetTriple().isSPIRV() ||
4866 (!U.getType()->getScalarType()->isBFloatTy() &&
4867 !any_of(Range: U.operands(), P: [](Value *V) {
4868 return V->getType()->getScalarType()->isBFloatTy();
4869 }));
4870}
4871
4872bool IRTranslatorImpl::finalizeBasicBlock(const BasicBlock &BB,
4873 MachineBasicBlock &MBB) {
4874 for (auto &BTB : SL->BitTestCases) {
4875 // Emit header first, if it wasn't already emitted.
4876 if (!BTB.Emitted)
4877 emitBitTestHeader(B&: BTB, SwitchBB: BTB.Parent);
4878
4879 BranchProbability UnhandledProb = BTB.Prob;
4880 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
4881 UnhandledProb -= BTB.Cases[j].ExtraProb;
4882 // Set the current basic block to the mbb we wish to insert the code into
4883 MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
4884 // If all cases cover a contiguous range, it is not necessary to jump to
4885 // the default block after the last bit test fails. This is because the
4886 // range check during bit test header creation has guaranteed that every
4887 // case here doesn't go outside the range. In this case, there is no need
4888 // to perform the last bit test, as it will always be true. Instead, make
4889 // the second-to-last bit-test fall through to the target of the last bit
4890 // test, and delete the last bit test.
4891
4892 MachineBasicBlock *NextMBB;
4893 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
4894 // Second-to-last bit-test with contiguous range: fall through to the
4895 // target of the final bit test.
4896 NextMBB = BTB.Cases[j + 1].TargetBB;
4897 } else if (j + 1 == ej) {
4898 // For the last bit test, fall through to Default.
4899 NextMBB = BTB.Default;
4900 } else {
4901 // Otherwise, fall through to the next bit test.
4902 NextMBB = BTB.Cases[j + 1].ThisBB;
4903 }
4904
4905 emitBitTestCase(BB&: BTB, NextMBB, BranchProbToNext: UnhandledProb, Reg: BTB.Reg, B&: BTB.Cases[j], SwitchBB: MBB);
4906
4907 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
4908 // We need to record the replacement phi edge here that normally
4909 // happens in emitBitTestCase before we delete the case, otherwise the
4910 // phi edge will be lost.
4911 addMachineCFGPred(Edge: {BTB.Parent->getBasicBlock(),
4912 BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
4913 NewPred: MBB);
4914 // Since we're not going to use the final bit test, remove it.
4915 BTB.Cases.pop_back();
4916 break;
4917 }
4918 }
4919 // This is "default" BB. We have two jumps to it. From "header" BB and from
4920 // last "case" BB, unless the latter was skipped.
4921 CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
4922 BTB.Default->getBasicBlock()};
4923 addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Parent);
4924 if (!BTB.ContiguousRange) {
4925 addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Cases.back().ThisBB);
4926 }
4927 }
4928 SL->BitTestCases.clear();
4929
4930 for (auto &JTCase : SL->JTCases) {
4931 // Emit header first, if it wasn't already emitted.
4932 if (!JTCase.first.Emitted)
4933 emitJumpTableHeader(JT&: JTCase.second, JTH&: JTCase.first, HeaderBB: JTCase.first.HeaderBB);
4934
4935 emitJumpTable(JT&: JTCase.second, MBB: JTCase.second.MBB);
4936 }
4937 SL->JTCases.clear();
4938
4939 for (auto &SwCase : SL->SwitchCases)
4940 emitSwitchCase(CB&: SwCase, SwitchBB: &CurBuilder->getMBB(), MIB&: *CurBuilder);
4941 SL->SwitchCases.clear();
4942
4943 // Check if we need to generate stack-protector guard checks.
4944 if (SPInfo->shouldEmitSDCheck(BB)) {
4945 bool FunctionBasedInstrumentation =
4946 TLI->getSSPStackGuardCheck(M: *MF->getFunction().getParent(), Libcalls: *Libcalls);
4947 SPDescriptor.initialize(BB: &BB, MBB: &MBB, FunctionBasedInstrumentation);
4948 }
4949 // Handle stack protector.
4950 if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
4951 LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
4952 return false;
4953 } else if (SPDescriptor.shouldEmitStackProtector()) {
4954 MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
4955 MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
4956
4957 // Find the split point to split the parent mbb. At the same time copy all
4958 // physical registers used in the tail of parent mbb into virtual registers
4959 // before the split point and back into physical registers after the split
4960 // point. This prevents us needing to deal with Live-ins and many other
4961 // register allocation issues caused by us splitting the parent mbb. The
4962 // register allocator will clean up said virtual copies later on.
4963 MachineBasicBlock::iterator SplitPoint = findSplitPointForStackProtector(
4964 BB: ParentMBB, TII: *MF->getSubtarget().getInstrInfo());
4965
4966 // Splice the terminator of ParentMBB into SuccessMBB.
4967 SuccessMBB->splice(Where: SuccessMBB->end(), Other: ParentMBB, From: SplitPoint,
4968 To: ParentMBB->end());
4969
4970 // Add compare/jump on neq/jump to the parent BB.
4971 if (!emitSPDescriptorParent(SPD&: SPDescriptor, ParentBB: ParentMBB))
4972 return false;
4973
4974 // CodeGen Failure MBB if we have not codegened it yet.
4975 MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
4976 if (FailureMBB->empty()) {
4977 if (!emitSPDescriptorFailure(SPD&: SPDescriptor, FailureBB: FailureMBB))
4978 return false;
4979 }
4980
4981 // Clear the Per-BB State.
4982 SPDescriptor.resetPerBBState();
4983 }
4984 return true;
4985}
4986
4987bool IRTranslatorImpl::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
4988 MachineBasicBlock *ParentBB) {
4989 CurBuilder->setInsertPt(MBB&: *ParentBB, II: ParentBB->end());
4990 // First create the loads to the guard/stack slot for the comparison.
4991 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
4992 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
4993 LLT PtrMemTy = getLLTForMVT(Ty: TLI->getPointerMemTy(DL: *DL));
4994
4995 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
4996 int FI = MFI.getStackProtectorIndex();
4997
4998 Register Guard;
4999 Register StackSlotPtr = CurBuilder->buildFrameIndex(Res: PtrTy, Idx: FI).getReg(Idx: 0);
5000 const Module &M = *ParentBB->getParent()->getFunction().getParent();
5001 Align Align = DL->getPrefTypeAlign(Ty: PointerType::getUnqual(C&: M.getContext()));
5002
5003 // Generate code to load the content of the guard slot.
5004 Register GuardVal =
5005 CurBuilder
5006 ->buildLoad(Res: PtrMemTy, Addr: StackSlotPtr,
5007 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
5008 MMOFlags: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile)
5009 .getReg(Idx: 0);
5010
5011 // Retrieve guard check function, nullptr if instrumentation is inlined.
5012 if (const Function *GuardCheckFn = TLI->getSSPStackGuardCheck(M, Libcalls: *Libcalls)) {
5013 // This path is currently untestable on GlobalISel, since the only platform
5014 // that needs this seems to be Windows, and we fall back on that currently.
5015 // The code still lives here in case that changes.
5016 // Silence warning about unused variable until the code below that uses
5017 // 'GuardCheckFn' is enabled.
5018 (void)GuardCheckFn;
5019 return false;
5020#if 0
5021 // The target provides a guard check function to validate the guard value.
5022 // Generate a call to that function with the content of the guard slot as
5023 // argument.
5024 FunctionType *FnTy = GuardCheckFn->getFunctionType();
5025 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
5026 ISD::ArgFlagsTy Flags;
5027 if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
5028 Flags.setInReg();
5029 CallLowering::ArgInfo GuardArgInfo(
5030 {GuardVal, FnTy->getParamType(0), {Flags}});
5031
5032 CallLowering::CallLoweringInfo Info;
5033 Info.OrigArgs.push_back(GuardArgInfo);
5034 Info.CallConv = GuardCheckFn->getCallingConv();
5035 Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
5036 Info.OrigRet = {Register(), FnTy->getReturnType()};
5037 if (!CLI->lowerCall(MIRBuilder, Info)) {
5038 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
5039 return false;
5040 }
5041 return true;
5042#endif
5043 }
5044
5045 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
5046 // Otherwise, emit a volatile load to retrieve the stack guard value.
5047 if (TLI->useLoadStackGuardNode(M: *ParentBB->getBasicBlock()->getModule())) {
5048 Guard = MRI->createGenericVirtualRegister(Ty: PtrMemTy);
5049 getStackGuard(DstReg: Guard, MIRBuilder&: *CurBuilder);
5050 } else {
5051 // TODO: test using android subtarget when we support @llvm.thread.pointer.
5052 const Value *IRGuard = TLI->getSDagStackGuard(M, Libcalls: *Libcalls);
5053 Register GuardPtr = getOrCreateVReg(Val: *IRGuard);
5054
5055 Guard = CurBuilder
5056 ->buildLoad(Res: PtrMemTy, Addr: GuardPtr,
5057 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
5058 MMOFlags: MachineMemOperand::MOLoad |
5059 MachineMemOperand::MOVolatile)
5060 .getReg(Idx: 0);
5061 }
5062
5063 // Perform the comparison.
5064 auto Cmp =
5065 CurBuilder->buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::integer(SizeInBits: 1), Op0: Guard, Op1: GuardVal);
5066 // If the guard/stackslot do not equal, branch to failure MBB.
5067 CurBuilder->buildBrCond(Tst: Cmp, Dest&: *SPD.getFailureMBB());
5068 // Otherwise branch to success MBB.
5069 CurBuilder->buildBr(Dest&: *SPD.getSuccessMBB());
5070 return true;
5071}
5072
5073bool IRTranslatorImpl::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
5074 MachineBasicBlock *FailureBB) {
5075 const RTLIB::LibcallImpl LibcallImpl =
5076 Libcalls->getLibcallImpl(Call: RTLIB::STACKPROTECTOR_CHECK_FAIL);
5077 if (LibcallImpl == RTLIB::Unsupported)
5078 return false;
5079
5080 CurBuilder->setInsertPt(MBB&: *FailureBB, II: FailureBB->end());
5081
5082 CallLowering::CallLoweringInfo Info;
5083 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: LibcallImpl);
5084
5085 StringRef LibcallName =
5086 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LibcallImpl);
5087 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
5088 Info.OrigRet = {Register(), Type::getVoidTy(C&: MF->getFunction().getContext()),
5089 0};
5090 if (!CLI->lowerCall(MIRBuilder&: *CurBuilder, Info)) {
5091 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
5092 return false;
5093 }
5094
5095 // Emit a trap instruction if we are required to do so.
5096 const TargetOptions &TargetOpts = TLI->getTargetMachine().Options;
5097 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
5098 CurBuilder->buildInstr(Opcode: TargetOpcode::G_TRAP);
5099
5100 return true;
5101}
5102
5103void IRTranslatorImpl::finalizeFunction() {
5104 // Release the memory used by the different maps we
5105 // needed during the translation.
5106 PendingPHIs.clear();
5107 VMap.reset();
5108 FrameIndices.clear();
5109 MachinePreds.clear();
5110 // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
5111 // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
5112 // destroying it twice (in ~IRTranslator() and ~LLVMContext())
5113 EntryBuilder.reset();
5114 CurBuilder.reset();
5115 FuncInfo.clear();
5116 SPDescriptor.resetPerFunctionState();
5117}
5118
5119/// Returns true if a BasicBlock \p BB within a variadic function contains a
5120/// variadic musttail call.
5121static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
5122 if (!IsVarArg)
5123 return false;
5124
5125 // Walk the block backwards, because tail calls usually only appear at the end
5126 // of a block.
5127 return llvm::any_of(Range: llvm::reverse(C: BB), P: [](const Instruction &I) {
5128 const auto *CI = dyn_cast<CallInst>(Val: &I);
5129 return CI && CI->isMustTailCall();
5130 });
5131}
5132
5133bool IRTranslatorImpl::runOnMachineFunction(
5134 MachineFunction &CurMF, function_ref<GISelCSEInfo *()> GetCSEInfo,
5135 bool ShouldSkipOpts, function_ref<AAResults *()> GetAAResults,
5136 function_ref<BranchProbabilityInfo *()> GetBPI,
5137 function_ref<AssumptionCache *()> GetAC, TargetLibraryInfo *LibraryInfo,
5138 const LibcallLoweringInfo *LibcallInfo, SSPLayoutInfo *StackProtectorInfo) {
5139 MF = &CurMF;
5140 const Function &F = MF->getFunction();
5141 ORE = std::make_unique<OptimizationRemarkEmitter>(args: &F);
5142 CLI = MF->getSubtarget().getCallLowering();
5143 SPInfo = StackProtectorInfo;
5144
5145 if (CLI->fallBackToDAGISel(MF: *MF)) {
5146 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
5147 F.getSubprogram(), &F.getEntryBlock());
5148 R << "unable to lower function: "
5149 << ore::NV("Prototype", F.getFunctionType());
5150
5151 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5152 return false;
5153 }
5154
5155 // Set the CSEConfig and run the analysis.
5156 GISelCSEInfo *CSEInfo = nullptr;
5157
5158 bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
5159 ? EnableCSEInIRTranslator
5160 : true;
5161
5162 const TargetSubtargetInfo &Subtarget = MF->getSubtarget();
5163 TLI = Subtarget.getTargetLowering();
5164
5165 if (EnableCSE) {
5166 EntryBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
5167 CSEInfo = GetCSEInfo();
5168 EntryBuilder->setCSEInfo(CSEInfo);
5169 CurBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
5170 CurBuilder->setCSEInfo(CSEInfo);
5171 } else {
5172 EntryBuilder = std::make_unique<MachineIRBuilder>();
5173 CurBuilder = std::make_unique<MachineIRBuilder>();
5174 }
5175 CLI = Subtarget.getCallLowering();
5176 CurBuilder->setMF(*MF);
5177 EntryBuilder->setMF(*MF);
5178 MRI = &MF->getRegInfo();
5179 DL = &F.getDataLayout();
5180 const TargetMachine &TM = MF->getTarget();
5181 EnableOpts = OptLevel != CodeGenOptLevel::None && !ShouldSkipOpts;
5182 FuncInfo.MF = MF;
5183 // Prefer the "exception-model" module flag, else the TargetOptions default.
5184 FuncInfo.ExceptionModel = F.getParent()->getExceptionModel();
5185 if (FuncInfo.ExceptionModel == ExceptionHandling::Default)
5186 FuncInfo.ExceptionModel = TM.getExceptionModel();
5187 if (EnableOpts) {
5188 AA = GetAAResults();
5189 FuncInfo.BPI = GetBPI();
5190 AC = GetAC();
5191 } else {
5192 AA = nullptr;
5193 FuncInfo.BPI = nullptr;
5194 AC = nullptr;
5195 }
5196 LibInfo = LibraryInfo;
5197 Libcalls = LibcallInfo;
5198
5199 FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(MF&: *MF);
5200
5201 SL = std::make_unique<GISelSwitchLowering>(args: this, args&: FuncInfo);
5202 SL->init(tli: *TLI, tm: TM, dl: *DL);
5203
5204 assert(PendingPHIs.empty() && "stale PHIs");
5205
5206 // Targets which want to use big endian can enable it using
5207 // enableBigEndian()
5208 if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
5209 // Currently we don't properly handle big endian code.
5210 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
5211 F.getSubprogram(), &F.getEntryBlock());
5212 R << "unable to translate in big endian mode";
5213 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5214 return false;
5215 }
5216
5217 // Release the per-function state when we return, whether we succeeded or not.
5218 llvm::scope_exit FinalizeOnReturn([this]() { finalizeFunction(); });
5219
5220 // Setup a separate basic-block for the arguments and constants
5221 MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
5222 MF->push_back(MBB: EntryBB);
5223 EntryBuilder->setMBB(*EntryBB);
5224
5225 DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHIIt()->getDebugLoc();
5226 SwiftError.setFunction(CurMF);
5227 SwiftError.createEntriesInEntryBlock(DbgLoc);
5228
5229 bool IsVarArg = F.isVarArg();
5230 bool HasMustTailInVarArgFn = false;
5231 // Use arguments and instructions to estimate the number of mapped values and
5232 // virtual registers.
5233 unsigned NumValues = F.arg_size();
5234
5235 // Create all blocks, in IR order, to preserve the layout.
5236 FuncInfo.MBBMap.resize(N: F.getMaxBlockNumber());
5237 for (const BasicBlock &BB: F) {
5238 NumValues += BB.size();
5239 auto *&MBB = FuncInfo.MBBMap[BB.getNumber()];
5240
5241 MBB = MF->CreateMachineBasicBlock(BB: &BB);
5242 MF->push_back(MBB);
5243
5244 // Only mark the block if the BlockAddress actually has users. The
5245 // hasAddressTaken flag may be stale if the BlockAddress was optimized away
5246 // but the constant still exists in the uniquing table.
5247 if (BB.hasAddressTaken()) {
5248 if (BlockAddress *BA = BlockAddress::lookup(BB: &BB))
5249 if (!BA->hasZeroLiveUses())
5250 MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
5251 }
5252
5253 if (!HasMustTailInVarArgFn)
5254 HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
5255 }
5256
5257 VMap.reserveVRegs(NumValues);
5258 MRI->reserveVirtRegs(NumVirtRegs: NumValues);
5259
5260 MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
5261
5262 // Make our arguments/constants entry block fallthrough to the IR entry block.
5263 EntryBB->addSuccessor(Succ: &getMBB(BB: F.front()));
5264
5265 // Lower the actual args into this basic block.
5266 SmallVector<ArrayRef<Register>, 8> VRegArgs;
5267 for (const Argument &Arg: F.args()) {
5268 if (DL->getTypeStoreSize(Ty: Arg.getType()).isZero())
5269 continue; // Don't handle zero sized types.
5270 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: Arg);
5271 VRegArgs.push_back(Elt: VRegs);
5272
5273 if (CLI->supportSwiftError() && Arg.hasSwiftErrorAttr()) {
5274 assert(VRegs.size() == 1 && "Too many vregs for Swift error");
5275 SwiftError.setCurrentVReg(MBB: EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
5276 }
5277 }
5278
5279 if (!CLI->lowerFormalArguments(MIRBuilder&: *EntryBuilder, F, VRegs: VRegArgs, FLI&: FuncInfo)) {
5280 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
5281 F.getSubprogram(), &F.getEntryBlock());
5282 R << "unable to lower arguments: "
5283 << ore::NV("Prototype", F.getFunctionType());
5284 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5285 return false;
5286 }
5287
5288 // Need to visit defs before uses when translating instructions.
5289 GISelObserverWrapper WrapperObserver;
5290 if (EnableCSE && CSEInfo)
5291 WrapperObserver.addObserver(O: CSEInfo);
5292 {
5293 ReversePostOrderTraversal<const Function *> RPOT(&F);
5294#ifndef NDEBUG
5295 DILocationVerifier Verifier;
5296 WrapperObserver.addObserver(&Verifier);
5297#endif // ifndef NDEBUG
5298 RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);
5299 for (const BasicBlock *BB : RPOT) {
5300 MachineBasicBlock &MBB = getMBB(BB: *BB);
5301 // Set the insertion point of all the following translations to
5302 // the end of this basic block.
5303 CurBuilder->setMBB(MBB);
5304 HasTailCall = false;
5305 for (const Instruction &Inst : *BB) {
5306 // If we translated a tail call in the last step, then we know
5307 // everything after the call is either a return, or something that is
5308 // handled by the call itself. (E.g. a lifetime marker or assume
5309 // intrinsic.) In this case, we should stop translating the block and
5310 // move on.
5311 if (HasTailCall)
5312 break;
5313#ifndef NDEBUG
5314 Verifier.setCurrentInst(&Inst);
5315#endif // ifndef NDEBUG
5316
5317 // Translate any debug-info attached to the instruction.
5318 translateDbgInfo(Inst, MIRBuilder&: *CurBuilder);
5319
5320 if (translate(Inst))
5321 continue;
5322
5323 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
5324 Inst.getDebugLoc(), BB);
5325 R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
5326
5327 if (ORE->allowExtraAnalysis(PassName: "gisel-ir-translator")) {
5328 std::string InstStrStorage;
5329 raw_string_ostream InstStr(InstStrStorage);
5330 InstStr << Inst;
5331
5332 R << ": '" << InstStrStorage << "'";
5333 }
5334
5335 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5336 return false;
5337 }
5338
5339 if (!finalizeBasicBlock(BB: *BB, MBB)) {
5340 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
5341 BB->getTerminator()->getDebugLoc(), BB);
5342 R << "unable to translate basic block";
5343 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5344 return false;
5345 }
5346 }
5347#ifndef NDEBUG
5348 WrapperObserver.removeObserver(&Verifier);
5349#endif
5350 }
5351
5352 finishPendingPhis();
5353
5354 SwiftError.propagateVRegs();
5355
5356 // Merge the argument lowering and constants block with its single
5357 // successor, the LLVM-IR entry block. We want the basic block to
5358 // be maximal.
5359 assert(EntryBB->succ_size() == 1 &&
5360 "Custom BB used for lowering should have only one successor");
5361 // Get the successor of the current entry block.
5362 MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
5363 assert(NewEntryBB.pred_size() == 1 &&
5364 "LLVM-IR entry block has a predecessor!?");
5365 // Move all the instruction from the current entry block to the
5366 // new entry block.
5367 NewEntryBB.splice(Where: NewEntryBB.begin(), Other: EntryBB, From: EntryBB->begin(),
5368 To: EntryBB->end());
5369
5370 // Update the live-in information for the new entry block.
5371 for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
5372 NewEntryBB.addLiveIn(RegMaskPair: LiveIn);
5373 NewEntryBB.sortUniqueLiveIns();
5374
5375 // Get rid of the now empty basic block.
5376 EntryBB->removeSuccessor(Succ: &NewEntryBB);
5377 MF->remove(MBBI: EntryBB);
5378 MF->deleteMachineBasicBlock(MBB: EntryBB);
5379
5380 assert(&MF->front() == &NewEntryBB &&
5381 "New entry wasn't next in the list of basic block!");
5382
5383 // Initialize stack protector information.
5384 SPInfo->copyToMachineFrameInfo(MFI&: MF->getFrameInfo());
5385
5386 return false;
5387}
5388
5389bool IRTranslatorLegacy::runOnMachineFunction(MachineFunction &MF) {
5390 const TargetSubtargetInfo &Subtarget = MF.getSubtarget();
5391 Function &F = MF.getFunction();
5392
5393 bool ShouldSkipOpts = skipFunction(F: MF.getFunction());
5394 return Impl->runOnMachineFunction(
5395 CurMF&: MF,
5396 GetCSEInfo: [&]() {
5397 TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
5398 GISelCSEAnalysisWrapper &Wrapper =
5399 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
5400 return &Wrapper.get(CSEOpt: TPC.getCSEConfig());
5401 },
5402 ShouldSkipOpts,
5403 GetAAResults: [&]() { return &getAnalysis<AAResultsWrapperPass>().getAAResults(); },
5404 GetBPI: [&]() {
5405 return &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
5406 },
5407 GetAC: [&]() {
5408 return &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
5409 F&: MF.getFunction());
5410 },
5411 LibraryInfo: &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F: MF.getFunction()),
5412 LibcallInfo: &getAnalysis<LibcallLoweringInfoWrapper>().getLibcallLowering(
5413 M: *F.getParent(), Subtarget),
5414 StackProtectorInfo: &getAnalysis<StackProtector>().getLayoutInfo());
5415}
5416
5417IRTranslatorPass::IRTranslatorPass(CodeGenOptLevel OptLevel)
5418 : Impl(std::make_unique<IRTranslatorImpl>(args&: OptLevel)) {}
5419
5420IRTranslatorPass::~IRTranslatorPass() = default;
5421IRTranslatorPass::IRTranslatorPass(IRTranslatorPass &&) = default;
5422
5423PreservedAnalyses IRTranslatorPass::run(MachineFunction &MF,
5424 MachineFunctionAnalysisManager &MFAM) {
5425 const TargetSubtargetInfo &Subtarget = MF.getSubtarget();
5426 Function &F = MF.getFunction();
5427
5428 bool ShouldSkipOpts = MF.getFunction().hasOptNone() ||
5429 shouldSkipOptimizationForOptBisect(IR: MF.getFunction());
5430 auto &FAM = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
5431 .getManager();
5432 auto &MAMProxy =
5433 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF);
5434 const ModuleLibcallLoweringInfo *MLLI =
5435 MAMProxy.getCachedResult<LibcallLoweringModuleAnalysis>(IR&: *F.getParent());
5436 if (!MLLI)
5437 reportFatalUsageError(
5438 reason: "LibcallLoweringModuleAnalysis must be available for IRTranslator");
5439 Impl->runOnMachineFunction(
5440 CurMF&: MF, GetCSEInfo: [&]() { return MFAM.getResult<GISelCSEAnalysis>(IR&: MF).get(); },
5441 ShouldSkipOpts, GetAAResults: [&]() { return &FAM.getResult<AAManager>(IR&: F); },
5442 GetBPI: [&]() { return &FAM.getResult<BranchProbabilityAnalysis>(IR&: F); },
5443 GetAC: [&]() { return &FAM.getResult<AssumptionAnalysis>(IR&: F); },
5444 LibraryInfo: &FAM.getResult<TargetLibraryAnalysis>(IR&: F),
5445 LibcallInfo: &getLibcallLowering(ModuleInfo: *MLLI, Subtarget),
5446 StackProtectorInfo: &FAM.getResult<SSPLayoutAnalysis>(IR&: F));
5447
5448 return getMachineFunctionPassPreservedAnalyses();
5449}
5450