1//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the IRTranslator class.
10//===----------------------------------------------------------------------===//
11
12#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
13#include "llvm/ADT/PostOrderIterator.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/Analysis/AliasAnalysis.h"
18#include "llvm/Analysis/AssumptionCache.h"
19#include "llvm/Analysis/BranchProbabilityInfo.h"
20#include "llvm/Analysis/Loads.h"
21#include "llvm/Analysis/OptimizationRemarkEmitter.h"
22#include "llvm/Analysis/ValueTracking.h"
23#include "llvm/Analysis/VectorUtils.h"
24#include "llvm/CodeGen/Analysis.h"
25#include "llvm/CodeGen/CodeGenCommonISel.h"
26#include "llvm/CodeGen/FunctionLoweringInfo.h"
27#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
28#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
29#include "llvm/CodeGen/GlobalISel/CallLowering.h"
30#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
31#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
32#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
33#include "llvm/CodeGen/LowLevelTypeUtils.h"
34#include "llvm/CodeGen/MachineBasicBlock.h"
35#include "llvm/CodeGen/MachineFrameInfo.h"
36#include "llvm/CodeGen/MachineFunction.h"
37#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
38#include "llvm/CodeGen/MachineInstrBuilder.h"
39#include "llvm/CodeGen/MachineMemOperand.h"
40#include "llvm/CodeGen/MachineModuleInfo.h"
41#include "llvm/CodeGen/MachineOperand.h"
42#include "llvm/CodeGen/MachinePassManager.h"
43#include "llvm/CodeGen/MachineRegisterInfo.h"
44#include "llvm/CodeGen/StackProtector.h"
45#include "llvm/CodeGen/SwiftErrorValueTracking.h"
46#include "llvm/CodeGen/SwitchLoweringUtils.h"
47#include "llvm/CodeGen/TargetFrameLowering.h"
48#include "llvm/CodeGen/TargetInstrInfo.h"
49#include "llvm/CodeGen/TargetLowering.h"
50#include "llvm/CodeGen/TargetOpcodes.h"
51#include "llvm/CodeGen/TargetPassConfig.h"
52#include "llvm/CodeGen/TargetRegisterInfo.h"
53#include "llvm/CodeGen/TargetSubtargetInfo.h"
54#include "llvm/CodeGenTypes/LowLevelType.h"
55#include "llvm/IR/Analysis.h"
56#include "llvm/IR/BasicBlock.h"
57#include "llvm/IR/CFG.h"
58#include "llvm/IR/Constant.h"
59#include "llvm/IR/Constants.h"
60#include "llvm/IR/DataLayout.h"
61#include "llvm/IR/DerivedTypes.h"
62#include "llvm/IR/DiagnosticInfo.h"
63#include "llvm/IR/Function.h"
64#include "llvm/IR/GetElementPtrTypeIterator.h"
65#include "llvm/IR/InlineAsm.h"
66#include "llvm/IR/InstrTypes.h"
67#include "llvm/IR/Instructions.h"
68#include "llvm/IR/IntrinsicInst.h"
69#include "llvm/IR/Intrinsics.h"
70#include "llvm/IR/IntrinsicsAMDGPU.h"
71#include "llvm/IR/LLVMContext.h"
72#include "llvm/IR/Metadata.h"
73#include "llvm/IR/Module.h"
74#include "llvm/IR/PatternMatch.h"
75#include "llvm/IR/Statepoint.h"
76#include "llvm/IR/Type.h"
77#include "llvm/IR/User.h"
78#include "llvm/IR/Value.h"
79#include "llvm/InitializePasses.h"
80#include "llvm/MC/MCContext.h"
81#include "llvm/Pass.h"
82#include "llvm/Support/Casting.h"
83#include "llvm/Support/CodeGen.h"
84#include "llvm/Support/Debug.h"
85#include "llvm/Support/ErrorHandling.h"
86#include "llvm/Support/MathExtras.h"
87#include "llvm/Support/raw_ostream.h"
88#include "llvm/Target/TargetMachine.h"
89#include "llvm/Transforms/Utils/Local.h"
90#include "llvm/Transforms/Utils/MemoryOpRemark.h"
91#include <algorithm>
92#include <cassert>
93#include <cstdint>
94#include <iterator>
95#include <optional>
96#include <string>
97#include <utility>
98#include <vector>
99
100#define DEBUG_TYPE "ir-translator"
101
102using namespace llvm;
103
104static cl::opt<bool>
105 EnableCSEInIRTranslator("enable-cse-in-ir-translator",
106 cl::desc("Should enable CSE in ir-translator"),
107 cl::init(Val: false));
108
109namespace llvm {
110
111class IRTranslatorImpl {
112 /// Interface used to lower the everything related to calls.
113 const CallLowering *CLI = nullptr;
114
115 SSPLayoutInfo *SPInfo = nullptr;
116
117 /// This class contains the mapping between the Values to vreg related data.
118 class ValueToVRegInfo {
119 public:
120 ValueToVRegInfo() = default;
121
122 using VRegListT = SmallVector<Register, 1>;
123 using OffsetListT = SmallVector<uint64_t, 1>;
124
125 using const_vreg_iterator =
126 DenseMap<const Value *, VRegListT *>::const_iterator;
127
128 inline const_vreg_iterator vregs_end() const { return ValToVRegs.end(); }
129
130 VRegListT *getVRegs(const Value &V) {
131 auto [It, Inserted] = ValToVRegs.try_emplace(Key: &V);
132 if (!Inserted)
133 return It->second;
134
135 // We placement new using our fast allocator since we never try to free
136 // the vectors until translation is finished.
137 It->second = new (VRegAlloc.Allocate()) VRegListT();
138 return It->second;
139 }
140
141 OffsetListT *getOffsets(const Value &V) {
142 assert(V.getType()->isAggregateType() &&
143 "Offsets are for aggregate values");
144 auto [It, Inserted] = TypeToOffsets.try_emplace(Key: V.getType());
145 if (!Inserted)
146 return It->second;
147
148 It->second = new (OffsetAlloc.Allocate()) OffsetListT();
149 return It->second;
150 }
151
152 const_vreg_iterator findVRegs(const Value &V) const {
153 return ValToVRegs.find(Val: &V);
154 }
155
156 bool contains(const Value &V) const { return ValToVRegs.contains(Val: &V); }
157
158 void reserveVRegs(unsigned NumValues) { ValToVRegs.reserve(NumEntries: NumValues); }
159
160 void reset() {
161 ValToVRegs.clear();
162 TypeToOffsets.clear();
163 VRegAlloc.DestroyAll();
164 OffsetAlloc.DestroyAll();
165 }
166
167 private:
168 SpecificBumpPtrAllocator<VRegListT> VRegAlloc;
169 SpecificBumpPtrAllocator<OffsetListT> OffsetAlloc;
170
171 // We store pointers to vectors here since references may be invalidated
172 // while we hold them if we stored the vectors directly.
173 DenseMap<const Value *, VRegListT *> ValToVRegs;
174 DenseMap<const Type *, OffsetListT *> TypeToOffsets;
175 };
176
177 /// Mapping of the values of the current LLVM IR function to the related
178 /// virtual registers and offsets.
179 ValueToVRegInfo VMap;
180
181 // One BasicBlock can be translated to multiple MachineBasicBlocks. For such
182 // BasicBlocks translated to multiple MachineBasicBlocks, MachinePreds retains
183 // a mapping between the edges arriving at the BasicBlock to the corresponding
184 // created MachineBasicBlocks. Some BasicBlocks that get translated to a
185 // single MachineBasicBlock may also end up in this Map.
186 using CFGEdge = std::pair<const BasicBlock *, const BasicBlock *>;
187 DenseMap<CFGEdge, SmallVector<MachineBasicBlock *, 1>> MachinePreds;
188
189 // List of stubbed PHI instructions, for values and basic blocks to be filled
190 // in once all MachineBasicBlocks have been created.
191 SmallVector<std::pair<const PHINode *, SmallVector<MachineInstr *, 1>>, 4>
192 PendingPHIs;
193
194 // Constant DbgValues that cannot be materialized as DBG_VALUEs
195 // because they are not an immediate or symbol reference.
196 SmallVector<std::pair<MachineInstr *, const Constant *>, 4>
197 PendingConstDbgValues;
198
199 /// Record of what frame index has been allocated to specified allocas for
200 /// this function.
201 DenseMap<const AllocaInst *, int> FrameIndices;
202
203 SwiftErrorValueTracking SwiftError;
204
205 /// \name Methods for translating form LLVM IR to MachineInstr.
206 /// \see ::translate for general information on the translate methods.
207 /// @{
208
209 /// Translate \p Inst into its corresponding MachineInstr instruction(s).
210 /// Insert the newly translated instruction(s) right where the CurBuilder
211 /// is set.
212 ///
213 /// The general algorithm is:
214 /// 1. Look for a virtual register for each operand or
215 /// create one.
216 /// 2 Update the VMap accordingly.
217 /// 2.alt. For constant arguments, if they are compile time constants,
218 /// produce an immediate in the right operand and do not touch
219 /// ValToReg. Actually we will go with a virtual register for each
220 /// constants because it may be expensive to actually materialize the
221 /// constant. Moreover, if the constant spans on several instructions,
222 /// CSE may not catch them.
223 /// => Update ValToVReg and remember that we saw a constant in Constants.
224 /// We will materialize all the constants in finalize.
225 /// Note: we would need to do something so that we can recognize such operand
226 /// as constants.
227 /// 3. Create the generic instruction.
228 ///
229 /// \return true if the translation succeeded.
230 bool translate(const Instruction &Inst);
231
232 /// Materialize \p C into virtual-register \p Reg. The generic instructions
233 /// performing this materialization will be inserted into the entry block of
234 /// the function.
235 ///
236 /// \return true if the materialization succeeded.
237 bool translate(const Constant &C, Register Reg);
238
239 /// Examine any debug-info attached to the instruction (in the form of
240 /// DbgRecords) and translate it.
241 void translateDbgInfo(const Instruction &Inst, MachineIRBuilder &MIRBuilder);
242
243 /// Translate a debug-info record of a dbg.value into a DBG_* instruction.
244 /// Pass in all the contents of the record, rather than relying on how it's
245 /// stored.
246 void translateDbgValueRecord(Value *V, const DILocalVariable *Variable,
247 const DIExpression *Expression,
248 const DebugLoc &DL,
249 MachineIRBuilder &MIRBuilder);
250
251 /// Translate a debug-info record of a variadic dbg.value (one using a
252 /// DIArgList) into a DBG_VALUE_LIST instruction.
253 void translateDbgValueListRecord(
254 iterator_range<DbgVariableRecord::location_op_iterator> Values,
255 const DILocalVariable *Variable, const DIExpression *Expression,
256 const DebugLoc &DL, MachineIRBuilder &MIRBuilder);
257
258 /// Translate a debug-info record of a dbg.declare into an indirect DBG_*
259 /// instruction. Pass in all the contents of the record, rather than relying
260 /// on how it's stored.
261 void translateDbgDeclareRecord(Value *Address, bool HasArgList,
262 const DILocalVariable *Variable,
263 const DIExpression *Expression,
264 const DebugLoc &DL,
265 MachineIRBuilder &MIRBuilder);
266
267 // Translate U as a copy of V.
268 bool translateCopy(const User &U, const Value &V,
269 MachineIRBuilder &MIRBuilder);
270 bool translateCopy(const User &U, Register Src, MachineIRBuilder &MIRBuilder);
271
272 /// Translate an LLVM bitcast into generic IR. Either a COPY or a G_BITCAST is
273 /// emitted.
274 bool translateBitCast(const User &U, MachineIRBuilder &MIRBuilder);
275
276 /// Translate an LLVM load instruction into generic IR.
277 bool translateLoad(const User &U, MachineIRBuilder &MIRBuilder);
278
279 /// Translate an LLVM store instruction into generic IR.
280 bool translateStore(const User &U, MachineIRBuilder &MIRBuilder);
281
282 /// Translate an LLVM string intrinsic (memcpy, memset, ...).
283 bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder,
284 unsigned Opcode);
285
286 /// Translate an LLVM trap intrinsic (trap, debugtrap, ubsantrap).
287 bool translateTrap(const CallInst &U, MachineIRBuilder &MIRBuilder,
288 unsigned Opcode);
289
290 // Translate @llvm.vector.interleave2 and
291 // @llvm.vector.deinterleave2 intrinsics for fixed-width vector
292 // types into vector shuffles.
293 bool translateVectorInterleave2Intrinsic(const CallInst &CI,
294 MachineIRBuilder &MIRBuilder);
295 bool translateVectorDeinterleave2Intrinsic(const CallInst &CI,
296 MachineIRBuilder &MIRBuilder);
297
298 void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder);
299
300 bool translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
301 MachineIRBuilder &MIRBuilder);
302 bool translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
303 MachineIRBuilder &MIRBuilder);
304
305 /// Helper function for translateSimpleIntrinsic.
306 /// \return The generic opcode for \p IntrinsicID if \p IntrinsicID is a
307 /// simple intrinsic (ceil, fabs, etc.). Otherwise, returns
308 /// Intrinsic::not_intrinsic.
309 unsigned getSimpleIntrinsicOpcode(Intrinsic::ID ID);
310
311 /// Translates the intrinsics defined in getSimpleIntrinsicOpcode.
312 /// \return true if the translation succeeded.
313 bool translateSimpleIntrinsic(const CallInst &CI, Intrinsic::ID ID,
314 MachineIRBuilder &MIRBuilder);
315
316 bool translateConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI,
317 MachineIRBuilder &MIRBuilder);
318
319 bool translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
320 MachineIRBuilder &MIRBuilder);
321
322 /// Returns the single livein physical register Arg was lowered to, if
323 /// possible.
324 std::optional<MCRegister> getArgPhysReg(Argument &Arg);
325
326 /// If debug-info targets an Argument and its expression is an EntryValue,
327 /// lower it as either an entry in the MF debug table (dbg.declare), or a
328 /// DBG_VALUE targeting the corresponding livein register for that Argument
329 /// (dbg.value).
330 bool translateIfEntryValueArgument(bool isDeclare, Value *Arg,
331 const DILocalVariable *Var,
332 const DIExpression *Expr,
333 const DebugLoc &DL,
334 MachineIRBuilder &MIRBuilder);
335
336 bool translateInlineAsm(const CallBase &CB, MachineIRBuilder &MIRBuilder);
337
338 /// Common code for translating normal calls or invokes.
339 bool translateCallBase(const CallBase &CB, MachineIRBuilder &MIRBuilder);
340
341 /// Translate call instruction.
342 /// \pre \p U is a call instruction.
343 bool translateCall(const User &U, MachineIRBuilder &MIRBuilder);
344
345 bool translateIntrinsic(
346 const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
347 ArrayRef<TargetLowering::IntrinsicInfo> TgtMemIntrinsicInfos = {});
348
349 /// Report an intrinsic the subtarget does not support and define its results
350 /// with G_IMPLICIT_DEF. Prevents creating a malformed MIR.
351 bool handleUnsupportedIntrinsic(const CallBase &CB, Intrinsic::ID ID,
352 MachineIRBuilder &MIRBuilder);
353
354 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
355 /// many places it could ultimately go. In the IR, we have a single unwind
356 /// destination, but in the machine CFG, we enumerate all the possible blocks.
357 /// This function skips over imaginary basic blocks that hold catchswitch
358 /// instructions, and finds all the "real" machine
359 /// basic block destinations. As those destinations may not be successors of
360 /// EHPadBB, here we also calculate the edge probability to those
361 /// destinations. The passed-in Prob is the edge probability to EHPadBB.
362 bool findUnwindDestinations(
363 const BasicBlock *EHPadBB, BranchProbability Prob,
364 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
365 &UnwindDests);
366
367 bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder);
368
369 bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder);
370
371 bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder);
372
373 /// Translate one of LLVM's cast instructions into MachineInstrs, with the
374 /// given generic Opcode.
375 bool translateCast(unsigned Opcode, const User &U,
376 MachineIRBuilder &MIRBuilder);
377
378 /// Translate a phi instruction.
379 bool translatePHI(const User &U, MachineIRBuilder &MIRBuilder);
380
381 /// Translate a comparison (icmp or fcmp) instruction or constant.
382 bool translateCompare(const User &U, MachineIRBuilder &MIRBuilder);
383
384 /// Translate an integer compare instruction (or constant).
385 bool translateICmp(const User &U, MachineIRBuilder &MIRBuilder) {
386 return translateCompare(U, MIRBuilder);
387 }
388
389 /// Translate a floating-point compare instruction (or constant).
390 bool translateFCmp(const User &U, MachineIRBuilder &MIRBuilder) {
391 return translateCompare(U, MIRBuilder);
392 }
393
394 /// Add remaining operands onto phis we've translated. Executed after all
395 /// MachineBasicBlocks for the function have been created.
396 void finishPendingPhis();
397
398 /// Describe any constants in PendingConstDbgValues that are
399 /// materialized as vregs.
400 void finishPendingConstDbgValues();
401
402 /// Translate \p Inst into a unary operation \p Opcode.
403 /// \pre \p U is a unary operation.
404 bool translateUnaryOp(unsigned Opcode, const User &U,
405 MachineIRBuilder &MIRBuilder);
406
407 /// Translate \p Inst into a binary operation \p Opcode.
408 /// \pre \p U is a binary operation.
409 bool translateBinaryOp(unsigned Opcode, const User &U,
410 MachineIRBuilder &MIRBuilder);
411
412 /// If the set of cases should be emitted as a series of branches, return
413 /// true. If we should emit this as a bunch of and/or'd together conditions,
414 /// return false.
415 bool shouldEmitAsBranches(const std::vector<SwitchCG::CaseBlock> &Cases);
416 /// Helper method for findMergedConditions.
417 /// This function emits a branch and is used at the leaves of an OR or an
418 /// AND operator tree.
419 void emitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB,
420 MachineBasicBlock *FBB,
421 MachineBasicBlock *CurBB,
422 MachineBasicBlock *SwitchBB,
423 BranchProbability TProb,
424 BranchProbability FProb, bool InvertCond);
425 /// Used during condbr translation to find trees of conditions that can be
426 /// optimized.
427 void findMergedConditions(const Value *Cond, MachineBasicBlock *TBB,
428 MachineBasicBlock *FBB, MachineBasicBlock *CurBB,
429 MachineBasicBlock *SwitchBB,
430 Instruction::BinaryOps Opc, BranchProbability TProb,
431 BranchProbability FProb, bool InvertCond);
432
433 /// Translate branch (br) instruction.
434 /// \pre \p U is a branch instruction.
435 bool translateUncondBr(const User &U, MachineIRBuilder &MIRBuilder);
436 bool translateCondBr(const User &U, MachineIRBuilder &MIRBuilder);
437
438 // Begin switch lowering functions.
439 bool emitJumpTableHeader(SwitchCG::JumpTable &JT,
440 SwitchCG::JumpTableHeader &JTH,
441 MachineBasicBlock *HeaderBB);
442 void emitJumpTable(SwitchCG::JumpTable &JT, MachineBasicBlock *MBB);
443
444 void emitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB,
445 MachineIRBuilder &MIB);
446
447 /// Generate for the BitTest header block, which precedes each sequence of
448 /// BitTestCases.
449 void emitBitTestHeader(SwitchCG::BitTestBlock &BTB,
450 MachineBasicBlock *SwitchMBB);
451 /// Generate code to produces one "bit test" for a given BitTestCase \p B.
452 void emitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB,
453 BranchProbability BranchProbToNext, Register Reg,
454 SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB);
455
456 void splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
457 const SwitchCG::SwitchWorkListItem &W, Value *Cond,
458 MachineBasicBlock *SwitchMBB, MachineIRBuilder &MIB);
459
460 bool lowerJumpTableWorkItem(
461 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
462 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
463 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
464 BranchProbability UnhandledProbs, SwitchCG::CaseClusterIt I,
465 MachineBasicBlock *Fallthrough, bool FallthroughUnreachable);
466
467 bool lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I, Value *Cond,
468 MachineBasicBlock *Fallthrough,
469 bool FallthroughUnreachable,
470 BranchProbability UnhandledProbs,
471 MachineBasicBlock *CurMBB,
472 MachineIRBuilder &MIB,
473 MachineBasicBlock *SwitchMBB);
474
475 bool lowerBitTestWorkItem(
476 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
477 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
478 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
479 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
480 SwitchCG::CaseClusterIt I, MachineBasicBlock *Fallthrough,
481 bool FallthroughUnreachable);
482
483 bool lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W, Value *Cond,
484 MachineBasicBlock *SwitchMBB,
485 MachineBasicBlock *DefaultMBB,
486 MachineIRBuilder &MIB);
487
488 bool translateSwitch(const User &U, MachineIRBuilder &MIRBuilder);
489 // End switch lowering section.
490
491 bool translateIndirectBr(const User &U, MachineIRBuilder &MIRBuilder);
492
493 bool translateExtractValue(const User &U, MachineIRBuilder &MIRBuilder);
494
495 bool translateInsertValue(const User &U, MachineIRBuilder &MIRBuilder);
496
497 bool translateSelect(const User &U, MachineIRBuilder &MIRBuilder);
498
499 bool translateGetElementPtr(const User &U, MachineIRBuilder &MIRBuilder);
500
501 bool translateAlloca(const User &U, MachineIRBuilder &MIRBuilder);
502
503 /// Translate return (ret) instruction.
504 /// The target needs to implement CallLowering::lowerReturn for
505 /// this to succeed.
506 /// \pre \p U is a return instruction.
507 bool translateRet(const User &U, MachineIRBuilder &MIRBuilder);
508
509 bool translateFNeg(const User &U, MachineIRBuilder &MIRBuilder);
510
511 bool translateAdd(const User &U, MachineIRBuilder &MIRBuilder) {
512 return translateBinaryOp(Opcode: TargetOpcode::G_ADD, U, MIRBuilder);
513 }
514 bool translateSub(const User &U, MachineIRBuilder &MIRBuilder) {
515 return translateBinaryOp(Opcode: TargetOpcode::G_SUB, U, MIRBuilder);
516 }
517 bool translateAnd(const User &U, MachineIRBuilder &MIRBuilder) {
518 return translateBinaryOp(Opcode: TargetOpcode::G_AND, U, MIRBuilder);
519 }
520 bool translateMul(const User &U, MachineIRBuilder &MIRBuilder) {
521 return translateBinaryOp(Opcode: TargetOpcode::G_MUL, U, MIRBuilder);
522 }
523 bool translateOr(const User &U, MachineIRBuilder &MIRBuilder) {
524 return translateBinaryOp(Opcode: TargetOpcode::G_OR, U, MIRBuilder);
525 }
526 bool translateXor(const User &U, MachineIRBuilder &MIRBuilder) {
527 return translateBinaryOp(Opcode: TargetOpcode::G_XOR, U, MIRBuilder);
528 }
529
530 bool translateUDiv(const User &U, MachineIRBuilder &MIRBuilder) {
531 return translateBinaryOp(Opcode: TargetOpcode::G_UDIV, U, MIRBuilder);
532 }
533 bool translateSDiv(const User &U, MachineIRBuilder &MIRBuilder) {
534 return translateBinaryOp(Opcode: TargetOpcode::G_SDIV, U, MIRBuilder);
535 }
536 bool translateURem(const User &U, MachineIRBuilder &MIRBuilder) {
537 return translateBinaryOp(Opcode: TargetOpcode::G_UREM, U, MIRBuilder);
538 }
539 bool translateSRem(const User &U, MachineIRBuilder &MIRBuilder) {
540 return translateBinaryOp(Opcode: TargetOpcode::G_SREM, U, MIRBuilder);
541 }
542 bool translateIntToPtr(const User &U, MachineIRBuilder &MIRBuilder) {
543 return translateCast(Opcode: TargetOpcode::G_INTTOPTR, U, MIRBuilder);
544 }
545 bool translatePtrToInt(const User &U, MachineIRBuilder &MIRBuilder) {
546 return translateCast(Opcode: TargetOpcode::G_PTRTOINT, U, MIRBuilder);
547 }
548 bool translatePtrToAddr(const User &U, MachineIRBuilder &MIRBuilder) {
549 // FIXME: this is not correct for pointers with addr width != pointer width
550 return translatePtrToInt(U, MIRBuilder);
551 }
552 bool translateTrunc(const User &U, MachineIRBuilder &MIRBuilder) {
553 return translateCast(Opcode: TargetOpcode::G_TRUNC, U, MIRBuilder);
554 }
555 bool translateFPTrunc(const User &U, MachineIRBuilder &MIRBuilder) {
556 return translateCast(Opcode: TargetOpcode::G_FPTRUNC, U, MIRBuilder);
557 }
558 bool translateFPExt(const User &U, MachineIRBuilder &MIRBuilder) {
559 return translateCast(Opcode: TargetOpcode::G_FPEXT, U, MIRBuilder);
560 }
561 bool translateFPToUI(const User &U, MachineIRBuilder &MIRBuilder) {
562 return translateCast(Opcode: TargetOpcode::G_FPTOUI, U, MIRBuilder);
563 }
564 bool translateFPToSI(const User &U, MachineIRBuilder &MIRBuilder) {
565 return translateCast(Opcode: TargetOpcode::G_FPTOSI, U, MIRBuilder);
566 }
567 bool translateUIToFP(const User &U, MachineIRBuilder &MIRBuilder) {
568 return translateCast(Opcode: TargetOpcode::G_UITOFP, U, MIRBuilder);
569 }
570 bool translateSIToFP(const User &U, MachineIRBuilder &MIRBuilder) {
571 return translateCast(Opcode: TargetOpcode::G_SITOFP, U, MIRBuilder);
572 }
573 bool translateUnreachable(const User &U, MachineIRBuilder &MIRBuilder);
574
575 bool translateSExt(const User &U, MachineIRBuilder &MIRBuilder) {
576 return translateCast(Opcode: TargetOpcode::G_SEXT, U, MIRBuilder);
577 }
578
579 bool translateZExt(const User &U, MachineIRBuilder &MIRBuilder) {
580 return translateCast(Opcode: TargetOpcode::G_ZEXT, U, MIRBuilder);
581 }
582
583 bool translateShl(const User &U, MachineIRBuilder &MIRBuilder) {
584 return translateBinaryOp(Opcode: TargetOpcode::G_SHL, U, MIRBuilder);
585 }
586 bool translateLShr(const User &U, MachineIRBuilder &MIRBuilder) {
587 return translateBinaryOp(Opcode: TargetOpcode::G_LSHR, U, MIRBuilder);
588 }
589 bool translateAShr(const User &U, MachineIRBuilder &MIRBuilder) {
590 return translateBinaryOp(Opcode: TargetOpcode::G_ASHR, U, MIRBuilder);
591 }
592
593 bool translateFAdd(const User &U, MachineIRBuilder &MIRBuilder) {
594 return translateBinaryOp(Opcode: TargetOpcode::G_FADD, U, MIRBuilder);
595 }
596 bool translateFSub(const User &U, MachineIRBuilder &MIRBuilder) {
597 return translateBinaryOp(Opcode: TargetOpcode::G_FSUB, U, MIRBuilder);
598 }
599 bool translateFMul(const User &U, MachineIRBuilder &MIRBuilder) {
600 return translateBinaryOp(Opcode: TargetOpcode::G_FMUL, U, MIRBuilder);
601 }
602 bool translateFDiv(const User &U, MachineIRBuilder &MIRBuilder) {
603 return translateBinaryOp(Opcode: TargetOpcode::G_FDIV, U, MIRBuilder);
604 }
605 bool translateFRem(const User &U, MachineIRBuilder &MIRBuilder) {
606 return translateBinaryOp(Opcode: TargetOpcode::G_FREM, U, MIRBuilder);
607 }
608
609 bool translateVAArg(const User &U, MachineIRBuilder &MIRBuilder);
610
611 bool translateInsertElement(const User &U, MachineIRBuilder &MIRBuilder);
612 bool translateInsertVector(const User &U, MachineIRBuilder &MIRBuilder);
613
614 bool translateExtractElement(const User &U, MachineIRBuilder &MIRBuilder);
615 bool translateExtractVector(const User &U, MachineIRBuilder &MIRBuilder);
616
617 bool translateShuffleVector(const User &U, MachineIRBuilder &MIRBuilder);
618
619 bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder);
620 bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder);
621 bool translateFence(const User &U, MachineIRBuilder &MIRBuilder);
622 bool translateFreeze(const User &U, MachineIRBuilder &MIRBuilder);
623
624 bool translateBitExtract(const User &U, MachineIRBuilder &MIRBuilder);
625 bool translateBitInsert(const User &U, MachineIRBuilder &MIRBuilder);
626
627 // Stubs to keep the compiler happy while we implement the rest of the
628 // translation.
629 bool translateResume(const User &U, MachineIRBuilder &MIRBuilder) {
630 return false;
631 }
632 bool translateCleanupRet(const User &U, MachineIRBuilder &MIRBuilder) {
633 return false;
634 }
635 bool translateCatchRet(const User &U, MachineIRBuilder &MIRBuilder) {
636 return false;
637 }
638 bool translateCatchSwitch(const User &U, MachineIRBuilder &MIRBuilder) {
639 return false;
640 }
641 bool translateAddrSpaceCast(const User &U, MachineIRBuilder &MIRBuilder) {
642 return translateCast(Opcode: TargetOpcode::G_ADDRSPACE_CAST, U, MIRBuilder);
643 }
644 bool translateCleanupPad(const User &U, MachineIRBuilder &MIRBuilder) {
645 return false;
646 }
647 bool translateCatchPad(const User &U, MachineIRBuilder &MIRBuilder) {
648 return false;
649 }
650 bool translateUserOp1(const User &U, MachineIRBuilder &MIRBuilder) {
651 return false;
652 }
653 bool translateUserOp2(const User &U, MachineIRBuilder &MIRBuilder) {
654 return false;
655 }
656
657 bool translateConvergenceControlIntrinsic(const CallInst &CI,
658 Intrinsic::ID ID,
659 MachineIRBuilder &MIRBuilder);
660
661 /// @}
662
663 // Builder for machine instruction a la IRBuilder.
664 // I.e., compared to regular MIBuilder, this one also inserts the instruction
665 // in the current block, it can creates block, etc., basically a kind of
666 // IRBuilder, but for Machine IR.
667 // CSEMIRBuilder CurBuilder;
668 std::unique_ptr<MachineIRBuilder> CurBuilder;
669
670 // Builder set to the entry block (just after ABI lowering instructions). Used
671 // as a convenient location for Constants.
672 // CSEMIRBuilder EntryBuilder;
673 std::unique_ptr<MachineIRBuilder> EntryBuilder;
674
675 // The MachineFunction currently being translated.
676 MachineFunction *MF = nullptr;
677
678 /// MachineRegisterInfo used to create virtual registers.
679 MachineRegisterInfo *MRI = nullptr;
680
681 const DataLayout *DL = nullptr;
682
683 CodeGenOptLevel OptLevel;
684
685 /// Current optimization remark emitter. Used to report failures.
686 std::unique_ptr<OptimizationRemarkEmitter> ORE;
687
688 AAResults *AA = nullptr;
689 AssumptionCache *AC = nullptr;
690 const TargetLibraryInfo *LibInfo = nullptr;
691 const LibcallLoweringInfo *Libcalls = nullptr;
692 const TargetLowering *TLI = nullptr;
693 FunctionLoweringInfo FuncInfo;
694
695 // True when either the Target Machine specifies no optimizations or the
696 // function has the optnone attribute.
697 bool EnableOpts = false;
698
699 /// True when the block contains a tail call. This allows the IRTranslator to
700 /// stop translating such blocks early.
701 bool HasTailCall = false;
702
703 StackProtectorDescriptor SPDescriptor;
704
705 bool mayTranslateUserTypes(const User &U) const;
706
707 /// Switch analysis and optimization.
708 class GISelSwitchLowering : public SwitchCG::SwitchLowering {
709 public:
710 GISelSwitchLowering(IRTranslatorImpl *irt, FunctionLoweringInfo &funcinfo)
711 : SwitchLowering(funcinfo), IRT(irt) {
712 assert(irt && "irt is null!");
713 }
714
715 void addSuccessorWithProb(
716 MachineBasicBlock *Src, MachineBasicBlock *Dst,
717 BranchProbability Prob = BranchProbability::getUnknown()) override {
718 IRT->addSuccessorWithProb(Src, Dst, Prob);
719 }
720
721 ~GISelSwitchLowering() override = default;
722
723 private:
724 IRTranslatorImpl *IRT;
725 };
726
727 std::unique_ptr<GISelSwitchLowering> SL;
728
729 // * Insert all the code needed to materialize the constants
730 // at the proper place. E.g., Entry block or dominator block
731 // of each constant depending on how fancy we want to be.
732 // * Clear the different maps.
733 void finalizeFunction();
734
735 // Processing steps done per block. E.g. emitting jump tables, stack
736 // protectors etc. Returns true if no errors, false if there was a problem
737 // that caused an abort.
738 bool finalizeBasicBlock(const BasicBlock &BB, MachineBasicBlock &MBB);
739
740 /// Codegen a new tail for a stack protector check ParentMBB which has had its
741 /// tail spliced into a stack protector check success bb.
742 ///
743 /// For a high level explanation of how this fits into the stack protector
744 /// generation see the comment on the declaration of class
745 /// StackProtectorDescriptor.
746 ///
747 /// \return true if there were no problems.
748 bool emitSPDescriptorParent(StackProtectorDescriptor &SPD,
749 MachineBasicBlock *ParentBB);
750
751 /// Codegen the failure basic block for a stack protector check.
752 ///
753 /// A failure stack protector machine basic block consists simply of a call to
754 /// __stack_chk_fail().
755 ///
756 /// For a high level explanation of how this fits into the stack protector
757 /// generation see the comment on the declaration of class
758 /// StackProtectorDescriptor.
759 ///
760 /// \return true if there were no problems.
761 bool emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
762 MachineBasicBlock *FailureBB);
763
764 /// Get the VRegs that represent \p Val.
765 /// Non-aggregate types have just one corresponding VReg and the list can be
766 /// used as a single "unsigned". Aggregates get flattened. If such VRegs do
767 /// not exist, they are created.
768 ArrayRef<Register> getOrCreateVRegs(const Value &Val);
769
770 Register getOrCreateVReg(const Value &Val) {
771 auto Regs = getOrCreateVRegs(Val);
772 if (Regs.empty())
773 return 0;
774 assert(Regs.size() == 1 &&
775 "attempt to get single VReg for aggregate or void");
776 return Regs[0];
777 }
778
779 Register getOrCreateConvergenceTokenVReg(const Value &Token) {
780 assert(Token.getType()->isTokenTy());
781 auto &Regs = *VMap.getVRegs(V: Token);
782 if (!Regs.empty()) {
783 assert(Regs.size() == 1 &&
784 "Expected a single register for convergence tokens.");
785 return Regs[0];
786 }
787
788 auto Reg = MRI->createGenericVirtualRegister(Ty: LLT::token());
789 Regs.push_back(Elt: Reg);
790 return Reg;
791 }
792
793 /// Allocate empty vregs for \p Val. For aggregate values, also populate
794 /// their offsets.
795 ValueToVRegInfo::VRegListT &allocateVRegs(const Value &Val);
796
797 /// Get the frame index that represents \p Val.
798 /// If such VReg does not exist, it is created.
799 int getOrCreateFrameIndex(const AllocaInst &AI);
800
801 /// Get the alignment of the given memory operation instruction. This will
802 /// either be the explicitly specified value or the ABI-required alignment for
803 /// the type being accessed (according to the Module's DataLayout).
804 Align getMemOpAlign(const Instruction &I);
805
806 /// Get the MachineBasicBlock that represents \p BB. Specifically, the block
807 /// returned will be the head of the translated block (suitable for branch
808 /// destinations).
809 MachineBasicBlock &getMBB(const BasicBlock &BB);
810
811 /// Record \p NewPred as a Machine predecessor to `Edge.second`, corresponding
812 /// to `Edge.first` at the IR level. This is used when IRTranslation creates
813 /// multiple MachineBasicBlocks for a given IR block and the CFG is no longer
814 /// represented simply by the IR-level CFG.
815 void addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred);
816
817 /// Returns the Machine IR predecessors for the given IR CFG edge. Usually
818 /// this is just the single MachineBasicBlock corresponding to the predecessor
819 /// in the IR. More complex lowering can result in multiple MachineBasicBlocks
820 /// preceding the original though (e.g. switch instructions).
821 SmallVector<MachineBasicBlock *, 1> getMachinePredBBs(CFGEdge Edge) {
822 auto RemappedEdge = MachinePreds.find(Val: Edge);
823 if (RemappedEdge != MachinePreds.end())
824 return RemappedEdge->second;
825 return SmallVector<MachineBasicBlock *, 4>(1, &getMBB(BB: *Edge.first));
826 }
827
828 /// Return branch probability calculated by BranchProbabilityInfo for IR
829 /// blocks.
830 BranchProbability getEdgeProbability(const MachineBasicBlock *Src,
831 const MachineBasicBlock *Dst) const;
832
833 void addSuccessorWithProb(
834 MachineBasicBlock *Src, MachineBasicBlock *Dst,
835 BranchProbability Prob = BranchProbability::getUnknown());
836
837public:
838 IRTranslatorImpl(CodeGenOptLevel OptLevel = CodeGenOptLevel::None)
839 : OptLevel(OptLevel) {}
840
841 // Algo:
842 // CallLowering = MF.subtarget.getCallLowering()
843 // F = MF.getParent()
844 // MIRBuilder.reset(MF)
845 // getMBB(F.getEntryBB())
846 // CallLowering->translateArguments(MIRBuilder, F, ValToVReg)
847 // for each bb in F
848 // getMBB(bb)
849 // for each inst in bb
850 // if (!translate(MIRBuilder, inst, ValToVReg, ConstantToSequence))
851 // reportFatalUsageError("Don't know how to translate input");
852 // finalize()
853 bool runOnMachineFunction(MachineFunction &MF,
854 function_ref<GISelCSEInfo *()> GetCSEInfo,
855 bool ShouldSkipOpts,
856 function_ref<AAResults *()> GetAAResults,
857 function_ref<BranchProbabilityInfo *()> GetBPI,
858 function_ref<AssumptionCache *()> GetAC,
859 TargetLibraryInfo *LibraryInfo,
860 const LibcallLoweringInfo *LibcallInfo,
861 SSPLayoutInfo *StackProtectorInfo);
862};
863
864} // namespace llvm
865
866char IRTranslatorLegacy::ID = 0;
867
868INITIALIZE_PASS_BEGIN(IRTranslatorLegacy, DEBUG_TYPE,
869 "IRTranslator LLVM IR -> MI", false, false)
870INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
871INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
872INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
873INITIALIZE_PASS_DEPENDENCY(StackProtector)
874INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
875INITIALIZE_PASS_END(IRTranslatorLegacy, DEBUG_TYPE,
876 "IRTranslator LLVM IR -> MI", false, false)
877
878static void reportTranslationError(MachineFunction &MF,
879 OptimizationRemarkEmitter &ORE,
880 OptimizationRemarkMissed &R) {
881 MF.getProperties().setFailedISel();
882 bool IsGlobalISelAbortEnabled =
883 MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
884
885 // Print the function name explicitly if we don't have a debug location (which
886 // makes the diagnostic less useful) or if we're going to emit a raw error.
887 if (!R.getLocation().isValid() || IsGlobalISelAbortEnabled)
888 R << (" (in function: " + MF.getName() + ")").str();
889
890 if (IsGlobalISelAbortEnabled)
891 report_fatal_error(reason: Twine(R.getMsg()));
892 else
893 ORE.emit(OptDiag&: R);
894}
895
896IRTranslatorLegacy::IRTranslatorLegacy(CodeGenOptLevel OptLevel)
897 : MachineFunctionPass(ID), OptLevel(OptLevel),
898 Impl(std::make_unique<IRTranslatorImpl>(args&: OptLevel)) {}
899
900IRTranslatorLegacy::~IRTranslatorLegacy() = default;
901
902#ifndef NDEBUG
903namespace {
904/// Verify that every instruction created has the same DILocation as the
905/// instruction being translated.
906class DILocationVerifier : public GISelChangeObserver {
907 const Instruction *CurrInst = nullptr;
908
909public:
910 DILocationVerifier() = default;
911 ~DILocationVerifier() override = default;
912
913 const Instruction *getCurrentInst() const { return CurrInst; }
914 void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
915
916 void erasingInstr(MachineInstr &MI) override {}
917 void changingInstr(MachineInstr &MI) override {}
918 void changedInstr(MachineInstr &MI) override {}
919
920 void createdInstr(MachineInstr &MI) override {
921 assert(getCurrentInst() && "Inserted instruction without a current MI");
922
923 // Only print the check message if we're actually checking it.
924#ifndef NDEBUG
925 LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
926 << " was copied to " << MI);
927#endif
928 // We allow insts in the entry block to have no debug loc because
929 // they could have originated from constants, and we don't want a jumpy
930 // debug experience.
931 assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
932 (MI.getParent()->isEntryBlock() && !MI.getDebugLoc()) ||
933 (MI.isDebugInstr())) &&
934 "Line info was not transferred to all instructions");
935 }
936};
937} // namespace
938#endif // ifndef NDEBUG
939
940void IRTranslatorLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
941 AU.addRequired<StackProtector>();
942 AU.addRequired<TargetPassConfig>();
943 AU.addRequired<GISelCSEAnalysisWrapperPass>();
944 if (OptLevel != CodeGenOptLevel::None) {
945 AU.addRequired<AssumptionCacheTracker>();
946 AU.addRequired<BranchProbabilityInfoWrapperPass>();
947 AU.addRequired<AAResultsWrapperPass>();
948 }
949 AU.addRequired<TargetLibraryInfoWrapperPass>();
950 AU.addPreserved<TargetLibraryInfoWrapperPass>();
951 AU.addRequired<LibcallLoweringInfoWrapper>();
952
953 getSelectionDAGFallbackAnalysisUsage(AU);
954 MachineFunctionPass::getAnalysisUsage(AU);
955}
956
957IRTranslatorImpl::ValueToVRegInfo::VRegListT &
958IRTranslatorImpl::allocateVRegs(const Value &Val) {
959 auto VRegsIt = VMap.findVRegs(V: Val);
960 if (VRegsIt != VMap.vregs_end())
961 return *VRegsIt->second;
962 auto *Regs = VMap.getVRegs(V: Val);
963 if (!Val.getType()->isAggregateType()) {
964 Regs->push_back(Elt: 0);
965 return *Regs;
966 }
967
968 auto *Offsets = VMap.getOffsets(V: Val);
969 SmallVector<LLT, 4> SplitTys;
970 computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueLLTs&: SplitTys,
971 FixedOffsets: Offsets->empty() ? Offsets : nullptr);
972 for (unsigned i = 0; i < SplitTys.size(); ++i)
973 Regs->push_back(Elt: 0);
974 return *Regs;
975}
976
977ArrayRef<Register> IRTranslatorImpl::getOrCreateVRegs(const Value &Val) {
978 auto VRegsIt = VMap.findVRegs(V: Val);
979 if (VRegsIt != VMap.vregs_end())
980 return *VRegsIt->second;
981
982 if (Val.getType()->isVoidTy())
983 return *VMap.getVRegs(V: Val);
984
985 // Create entry for this type.
986 auto *VRegs = VMap.getVRegs(V: Val);
987
988 if (!Val.getType()->isTokenTy())
989 assert(Val.getType()->isSized() &&
990 "Don't know how to create an empty vreg");
991
992 // Fast-path values that lower to a single vreg.
993 if (!Val.getType()->isAggregateType()) {
994 LLT Ty = getLLTForType(Ty&: *Val.getType(), DL: *DL);
995 VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty));
996 if (isa<Constant>(Val)) {
997 bool Success = translate(C: cast<Constant>(Val), Reg: VRegs->front());
998 if (!Success) {
999 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
1000 MF->getFunction().getSubprogram(),
1001 &MF->getFunction().getEntryBlock());
1002 R << "unable to translate constant: " << ore::NV("Type", Val.getType());
1003 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
1004 }
1005 }
1006 return *VRegs;
1007 }
1008
1009 SmallVector<LLT, 4> SplitTys;
1010 auto *Offsets = VMap.getOffsets(V: Val);
1011 computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueLLTs&: SplitTys,
1012 FixedOffsets: Offsets->empty() ? Offsets : nullptr);
1013
1014 if (!isa<Constant>(Val)) {
1015 for (auto Ty : SplitTys)
1016 VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty));
1017 return *VRegs;
1018 }
1019
1020 // UndefValue, ConstantAggregateZero
1021 auto &C = cast<Constant>(Val);
1022 unsigned Idx = 0;
1023 while (auto Elt = C.getAggregateElement(Elt: Idx++)) {
1024 auto EltRegs = getOrCreateVRegs(Val: *Elt);
1025 llvm::append_range(C&: *VRegs, R&: EltRegs);
1026 }
1027
1028 return *VRegs;
1029}
1030
1031int IRTranslatorImpl::getOrCreateFrameIndex(const AllocaInst &AI) {
1032 auto [MapEntry, Inserted] = FrameIndices.try_emplace(Key: &AI);
1033 if (!Inserted)
1034 return MapEntry->second;
1035
1036 TypeSize TySize = AI.getAllocationSize(DL: *DL).value_or(u: TypeSize::getZero());
1037 uint64_t Size = TySize.getKnownMinValue();
1038
1039 // Always allocate at least one byte.
1040 Size = std::max<uint64_t>(a: Size, b: 1u);
1041
1042 int &FI = MapEntry->second;
1043 FI = MF->getFrameInfo().CreateStackObject(Size, Alignment: AI.getAlign(), isSpillSlot: false, Alloca: &AI);
1044
1045 // Scalable vectors and structures that contain scalable vectors may
1046 // need a special StackID to distinguish them from other (fixed size)
1047 // stack objects.
1048 if (TySize.isScalable()) {
1049 auto StackID =
1050 MF->getSubtarget().getFrameLowering()->getStackIDForScalableVectors();
1051 MF->getFrameInfo().setStackID(ObjectIdx: FI, ID: StackID);
1052 }
1053
1054 return FI;
1055}
1056
1057Align IRTranslatorImpl::getMemOpAlign(const Instruction &I) {
1058 if (const StoreInst *SI = dyn_cast<StoreInst>(Val: &I))
1059 return SI->getAlign();
1060 if (const LoadInst *LI = dyn_cast<LoadInst>(Val: &I))
1061 return LI->getAlign();
1062 if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(Val: &I))
1063 return AI->getAlign();
1064 if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(Val: &I))
1065 return AI->getAlign();
1066
1067 OptimizationRemarkMissed R("gisel-ir-translator", "", &I);
1068 R << "unable to translate memop: " << ore::NV("Opcode", &I);
1069 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
1070 return Align(1);
1071}
1072
1073MachineBasicBlock &IRTranslatorImpl::getMBB(const BasicBlock &BB) {
1074 MachineBasicBlock *MBB = FuncInfo.getMBB(BB: &BB);
1075 assert(MBB && "BasicBlock was not encountered before");
1076 return *MBB;
1077}
1078
1079void IRTranslatorImpl::addMachineCFGPred(CFGEdge Edge,
1080 MachineBasicBlock *NewPred) {
1081 assert(NewPred && "new predecessor must be a real MachineBasicBlock");
1082 MachinePreds[Edge].push_back(Elt: NewPred);
1083}
1084
1085bool IRTranslatorImpl::translateBinaryOp(unsigned Opcode, const User &U,
1086 MachineIRBuilder &MIRBuilder) {
1087 if (!mayTranslateUserTypes(U))
1088 return false;
1089
1090 // Get or create a virtual register for each value.
1091 // Unless the value is a Constant => loadimm cst?
1092 // or inline constant each time?
1093 // Creation of a virtual register needs to have a size.
1094 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
1095 Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
1096 Register Res = getOrCreateVReg(Val: U);
1097 uint32_t Flags = 0;
1098 if (isa<Instruction>(Val: U)) {
1099 const Instruction &I = cast<Instruction>(Val: U);
1100 Flags = MachineInstr::copyFlagsFromInstruction(I);
1101 }
1102
1103 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0, Op1}, Flags);
1104 return true;
1105}
1106
1107bool IRTranslatorImpl::translateUnaryOp(unsigned Opcode, const User &U,
1108 MachineIRBuilder &MIRBuilder) {
1109 if (!mayTranslateUserTypes(U))
1110 return false;
1111
1112 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
1113 Register Res = getOrCreateVReg(Val: U);
1114 uint32_t Flags = 0;
1115 if (isa<Instruction>(Val: U)) {
1116 const Instruction &I = cast<Instruction>(Val: U);
1117 Flags = MachineInstr::copyFlagsFromInstruction(I);
1118 }
1119 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0}, Flags);
1120 return true;
1121}
1122
1123bool IRTranslatorImpl::translateFNeg(const User &U,
1124 MachineIRBuilder &MIRBuilder) {
1125 return translateUnaryOp(Opcode: TargetOpcode::G_FNEG, U, MIRBuilder);
1126}
1127
1128bool IRTranslatorImpl::translateCompare(const User &U,
1129 MachineIRBuilder &MIRBuilder) {
1130 if (!mayTranslateUserTypes(U))
1131 return false;
1132
1133 auto *CI = cast<CmpInst>(Val: &U);
1134 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
1135 Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
1136 Register Res = getOrCreateVReg(Val: U);
1137 CmpInst::Predicate Pred = CI->getPredicate();
1138 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: *CI);
1139 if (CmpInst::isIntPredicate(P: Pred))
1140 MIRBuilder.buildICmp(Pred, Res, Op0, Op1, Flags);
1141 else if (Pred == CmpInst::FCMP_FALSE)
1142 MIRBuilder.buildCopy(
1143 Res, Op: getOrCreateVReg(Val: *Constant::getNullValue(Ty: U.getType())));
1144 else if (Pred == CmpInst::FCMP_TRUE)
1145 MIRBuilder.buildCopy(
1146 Res, Op: getOrCreateVReg(Val: *Constant::getAllOnesValue(Ty: U.getType())));
1147 else
1148 MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
1149
1150 return true;
1151}
1152
1153bool IRTranslatorImpl::translateRet(const User &U,
1154 MachineIRBuilder &MIRBuilder) {
1155 const ReturnInst &RI = cast<ReturnInst>(Val: U);
1156 const Value *Ret = RI.getReturnValue();
1157 if (Ret && DL->getTypeStoreSize(Ty: Ret->getType()).isZero())
1158 Ret = nullptr;
1159
1160 ArrayRef<Register> VRegs;
1161 if (Ret)
1162 VRegs = getOrCreateVRegs(Val: *Ret);
1163
1164 Register SwiftErrorVReg = 0;
1165 if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
1166 SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
1167 &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
1168 }
1169
1170 // The target may mess up with the insertion point, but
1171 // this is not important as a return is the last instruction
1172 // of the block anyway.
1173 return CLI->lowerReturn(MIRBuilder, Val: Ret, VRegs, FLI&: FuncInfo, SwiftErrorVReg);
1174}
1175
1176void IRTranslatorImpl::emitBranchForMergedCondition(
1177 const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
1178 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
1179 BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
1180 // If the leaf of the tree is a comparison, merge the condition into
1181 // the caseblock.
1182 if (const CmpInst *BOp = dyn_cast<CmpInst>(Val: Cond)) {
1183 CmpInst::Predicate Condition;
1184 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Val: Cond)) {
1185 Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
1186 } else {
1187 const FCmpInst *FC = cast<FCmpInst>(Val: Cond);
1188 Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
1189 }
1190
1191 SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(i_nocapture: 0),
1192 BOp->getOperand(i_nocapture: 1), nullptr, TBB, FBB, CurBB,
1193 CurBuilder->getDebugLoc(), TProb, FProb);
1194 SL->SwitchCases.push_back(x: CB);
1195 return;
1196 }
1197
1198 // Create a CaseBlock record representing this branch.
1199 CmpInst::Predicate Pred = InvertCond ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ;
1200 SwitchCG::CaseBlock CB(
1201 Pred, false, Cond, ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
1202 nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
1203 SL->SwitchCases.push_back(x: CB);
1204}
1205
1206static bool isValInBlock(const Value *V, const BasicBlock *BB) {
1207 if (const Instruction *I = dyn_cast<Instruction>(Val: V))
1208 return I->getParent() == BB;
1209 return true;
1210}
1211
1212void IRTranslatorImpl::findMergedConditions(
1213 const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
1214 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
1215 Instruction::BinaryOps Opc, BranchProbability TProb,
1216 BranchProbability FProb, bool InvertCond) {
1217 using namespace PatternMatch;
1218 assert((Opc == Instruction::And || Opc == Instruction::Or) &&
1219 "Expected Opc to be AND/OR");
1220 // Skip over not part of the tree and remember to invert op and operands at
1221 // next level.
1222 Value *NotCond;
1223 if (match(V: Cond, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: NotCond)))) &&
1224 isValInBlock(V: NotCond, BB: CurBB->getBasicBlock())) {
1225 findMergedConditions(Cond: NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
1226 InvertCond: !InvertCond);
1227 return;
1228 }
1229
1230 const Instruction *BOp = dyn_cast<Instruction>(Val: Cond);
1231 const Value *BOpOp0, *BOpOp1;
1232 // Compute the effective opcode for Cond, taking into account whether it needs
1233 // to be inverted, e.g.
1234 // and (not (or A, B)), C
1235 // gets lowered as
1236 // and (and (not A, not B), C)
1237 Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
1238 if (BOp) {
1239 BOpc = match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
1240 ? Instruction::And
1241 : (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
1242 ? Instruction::Or
1243 : (Instruction::BinaryOps)0);
1244 if (InvertCond) {
1245 if (BOpc == Instruction::And)
1246 BOpc = Instruction::Or;
1247 else if (BOpc == Instruction::Or)
1248 BOpc = Instruction::And;
1249 }
1250 }
1251
1252 // If this node is not part of the or/and tree, emit it as a branch.
1253 // Note that all nodes in the tree should have same opcode.
1254 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
1255 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
1256 !isValInBlock(V: BOpOp0, BB: CurBB->getBasicBlock()) ||
1257 !isValInBlock(V: BOpOp1, BB: CurBB->getBasicBlock())) {
1258 emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
1259 InvertCond);
1260 return;
1261 }
1262
1263 // Create TmpBB after CurBB.
1264 MachineFunction::iterator BBI(CurBB);
1265 MachineBasicBlock *TmpBB =
1266 MF->CreateMachineBasicBlock(BB: CurBB->getBasicBlock());
1267 CurBB->getParent()->insert(MBBI: ++BBI, MBB: TmpBB);
1268
1269 if (Opc == Instruction::Or) {
1270 // Codegen X | Y as:
1271 // BB1:
1272 // jmp_if_X TBB
1273 // jmp TmpBB
1274 // TmpBB:
1275 // jmp_if_Y TBB
1276 // jmp FBB
1277 //
1278
1279 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1280 // The requirement is that
1281 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1282 // = TrueProb for original BB.
1283 // Assuming the original probabilities are A and B, one choice is to set
1284 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1285 // A/(1+B) and 2B/(1+B). This choice assumes that
1286 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1287 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1288 // TmpBB, but the math is more complicated.
1289
1290 auto NewTrueProb = TProb / 2;
1291 auto NewFalseProb = TProb / 2 + FProb;
1292 // Emit the LHS condition.
1293 findMergedConditions(Cond: BOpOp0, TBB, FBB: TmpBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
1294 FProb: NewFalseProb, InvertCond);
1295
1296 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1297 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1298 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
1299 // Emit the RHS condition into TmpBB.
1300 findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
1301 FProb: Probs[1], InvertCond);
1302 } else {
1303 assert(Opc == Instruction::And && "Unknown merge op!");
1304 // Codegen X & Y as:
1305 // BB1:
1306 // jmp_if_X TmpBB
1307 // jmp FBB
1308 // TmpBB:
1309 // jmp_if_Y TBB
1310 // jmp FBB
1311 //
1312 // This requires creation of TmpBB after CurBB.
1313
1314 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1315 // The requirement is that
1316 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1317 // = FalseProb for original BB.
1318 // Assuming the original probabilities are A and B, one choice is to set
1319 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1320 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1321 // TrueProb for BB1 * FalseProb for TmpBB.
1322
1323 auto NewTrueProb = TProb + FProb / 2;
1324 auto NewFalseProb = FProb / 2;
1325 // Emit the LHS condition.
1326 findMergedConditions(Cond: BOpOp0, TBB: TmpBB, FBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
1327 FProb: NewFalseProb, InvertCond);
1328
1329 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1330 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1331 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
1332 // Emit the RHS condition into TmpBB.
1333 findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
1334 FProb: Probs[1], InvertCond);
1335 }
1336}
1337
1338bool IRTranslatorImpl::shouldEmitAsBranches(
1339 const std::vector<SwitchCG::CaseBlock> &Cases) {
1340 // For multiple cases, it's better to emit as branches.
1341 if (Cases.size() != 2)
1342 return true;
1343
1344 // If this is two comparisons of the same values or'd or and'd together, they
1345 // will get folded into a single comparison, so don't emit two blocks.
1346 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1347 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1348 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1349 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1350 return false;
1351 }
1352
1353 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1354 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1355 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1356 Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
1357 isa<Constant>(Val: Cases[0].CmpRHS) &&
1358 cast<Constant>(Val: Cases[0].CmpRHS)->isNullValue()) {
1359 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
1360 Cases[0].TrueBB == Cases[1].ThisBB)
1361 return false;
1362 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
1363 Cases[0].FalseBB == Cases[1].ThisBB)
1364 return false;
1365 }
1366
1367 return true;
1368}
1369
1370bool IRTranslatorImpl::translateUncondBr(const User &U,
1371 MachineIRBuilder &MIRBuilder) {
1372 const UncondBrInst &BrInst = cast<UncondBrInst>(Val: U);
1373 auto &CurMBB = MIRBuilder.getMBB();
1374 auto *Succ0MBB = &getMBB(BB: *BrInst.getSuccessor(i: 0));
1375
1376 // If the unconditional target is the layout successor, fallthrough.
1377 if (OptLevel == CodeGenOptLevel::None || !CurMBB.isLayoutSuccessor(MBB: Succ0MBB))
1378 MIRBuilder.buildBr(Dest&: *Succ0MBB);
1379
1380 // Link successors.
1381 for (const BasicBlock *Succ : successors(I: &BrInst))
1382 CurMBB.addSuccessor(Succ: &getMBB(BB: *Succ));
1383 return true;
1384}
1385
1386bool IRTranslatorImpl::translateCondBr(const User &U,
1387 MachineIRBuilder &MIRBuilder) {
1388 const CondBrInst &BrInst = cast<CondBrInst>(Val: U);
1389 auto &CurMBB = MIRBuilder.getMBB();
1390 auto *Succ0MBB = &getMBB(BB: *BrInst.getSuccessor(i: 0));
1391
1392 // If this condition is one of the special cases we handle, do special stuff
1393 // now.
1394 const Value *CondVal = BrInst.getCondition();
1395 MachineBasicBlock *Succ1MBB = &getMBB(BB: *BrInst.getSuccessor(i: 1));
1396
1397 // If this is a series of conditions that are or'd or and'd together, emit
1398 // this as a sequence of branches instead of setcc's with and/or operations.
1399 // As long as jumps are not expensive (exceptions for multi-use logic ops,
1400 // unpredictable branches, and vector extracts because those jumps are likely
1401 // expensive for any target), this should improve performance.
1402 // For example, instead of something like:
1403 // cmp A, B
1404 // C = seteq
1405 // cmp D, E
1406 // F = setle
1407 // or C, F
1408 // jnz foo
1409 // Emit:
1410 // cmp A, B
1411 // je foo
1412 // cmp D, E
1413 // jle foo
1414 using namespace PatternMatch;
1415 const Instruction *CondI = dyn_cast<Instruction>(Val: CondVal);
1416 if (!TLI->isJumpExpensive() && CondI && CondI->hasOneUse() &&
1417 !BrInst.hasMetadata(KindID: LLVMContext::MD_unpredictable)) {
1418 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
1419 Value *Vec;
1420 const Value *BOp0, *BOp1;
1421 if (match(V: CondI, P: m_LogicalAnd(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
1422 Opcode = Instruction::And;
1423 else if (match(V: CondI, P: m_LogicalOr(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
1424 Opcode = Instruction::Or;
1425
1426 if (Opcode && !(match(V: BOp0, P: m_ExtractElt(Val: m_Value(V&: Vec), Idx: m_Value())) &&
1427 match(V: BOp1, P: m_ExtractElt(Val: m_Specific(V: Vec), Idx: m_Value())))) {
1428 findMergedConditions(Cond: CondI, TBB: Succ0MBB, FBB: Succ1MBB, CurBB: &CurMBB, SwitchBB: &CurMBB, Opc: Opcode,
1429 TProb: getEdgeProbability(Src: &CurMBB, Dst: Succ0MBB),
1430 FProb: getEdgeProbability(Src: &CurMBB, Dst: Succ1MBB),
1431 /*InvertCond=*/false);
1432 assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
1433
1434 // Allow some cases to be rejected.
1435 if (shouldEmitAsBranches(Cases: SL->SwitchCases)) {
1436 // Emit the branch for this block.
1437 emitSwitchCase(CB&: SL->SwitchCases[0], SwitchBB: &CurMBB, MIB&: *CurBuilder);
1438 SL->SwitchCases.erase(position: SL->SwitchCases.begin());
1439 return true;
1440 }
1441
1442 // Okay, we decided not to do this, remove any inserted MBB's and clear
1443 // SwitchCases.
1444 for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
1445 MF->erase(MBBI: SL->SwitchCases[I].ThisBB);
1446
1447 SL->SwitchCases.clear();
1448 }
1449 }
1450
1451 // Create a CaseBlock record representing this branch.
1452 SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
1453 ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
1454 nullptr, Succ0MBB, Succ1MBB, &CurMBB,
1455 CurBuilder->getDebugLoc());
1456
1457 // Use emitSwitchCase to actually insert the fast branch sequence for this
1458 // cond branch.
1459 emitSwitchCase(CB, SwitchBB: &CurMBB, MIB&: *CurBuilder);
1460 return true;
1461}
1462
1463void IRTranslatorImpl::addSuccessorWithProb(MachineBasicBlock *Src,
1464 MachineBasicBlock *Dst,
1465 BranchProbability Prob) {
1466 if (!FuncInfo.BPI) {
1467 Src->addSuccessorWithoutProb(Succ: Dst);
1468 return;
1469 }
1470 if (Prob.isUnknown())
1471 Prob = getEdgeProbability(Src, Dst);
1472 Src->addSuccessor(Succ: Dst, Prob);
1473}
1474
1475BranchProbability
1476IRTranslatorImpl::getEdgeProbability(const MachineBasicBlock *Src,
1477 const MachineBasicBlock *Dst) const {
1478 const BasicBlock *SrcBB = Src->getBasicBlock();
1479 const BasicBlock *DstBB = Dst->getBasicBlock();
1480 if (!FuncInfo.BPI) {
1481 // If BPI is not available, set the default probability as 1 / N, where N is
1482 // the number of successors.
1483 auto SuccSize = std::max<uint32_t>(a: succ_size(BB: SrcBB), b: 1);
1484 return BranchProbability(1, SuccSize);
1485 }
1486 return FuncInfo.BPI->getEdgeProbability(Src: SrcBB, Dst: DstBB);
1487}
1488
1489bool IRTranslatorImpl::translateSwitch(const User &U, MachineIRBuilder &MIB) {
1490 using namespace SwitchCG;
1491 // Extract cases from the switch.
1492 const SwitchInst &SI = cast<SwitchInst>(Val: U);
1493 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1494 CaseClusterVector Clusters;
1495 Clusters.reserve(n: SI.getNumCases());
1496 for (const auto &I : SI.cases()) {
1497 MachineBasicBlock *Succ = &getMBB(BB: *I.getCaseSuccessor());
1498 assert(Succ && "Could not find successor mbb in mapping");
1499 const ConstantInt *CaseVal = I.getCaseValue();
1500 BranchProbability Prob =
1501 BPI ? BPI->getEdgeProbability(Src: SI.getParent(), IndexInSuccessors: I.getSuccessorIndex())
1502 : BranchProbability(1, SI.getNumCases() + 1);
1503 Clusters.push_back(x: CaseCluster::range(Low: CaseVal, High: CaseVal, MBB: Succ, Prob));
1504 }
1505
1506 MachineBasicBlock *DefaultMBB = &getMBB(BB: *SI.getDefaultDest());
1507
1508 // Cluster adjacent cases with the same destination. We do this at all
1509 // optimization levels because it's cheap to do and will make codegen faster
1510 // if there are many clusters.
1511 sortAndRangeify(Clusters);
1512
1513 MachineBasicBlock *SwitchMBB = &getMBB(BB: *SI.getParent());
1514
1515 // If there is only the default destination, jump there directly.
1516 if (Clusters.empty()) {
1517 SwitchMBB->addSuccessor(Succ: DefaultMBB);
1518 if (DefaultMBB != SwitchMBB->getNextNode())
1519 MIB.buildBr(Dest&: *DefaultMBB);
1520 return true;
1521 }
1522
1523 SL->findJumpTables(Clusters, SI: &SI, SL: std::nullopt, DefaultMBB, PSI: nullptr, BFI: nullptr);
1524 SL->findBitTestClusters(Clusters, SI: &SI);
1525
1526 LLVM_DEBUG({
1527 dbgs() << "Case clusters: ";
1528 for (const CaseCluster &C : Clusters) {
1529 if (C.Kind == CC_JumpTable)
1530 dbgs() << "JT:";
1531 if (C.Kind == CC_BitTests)
1532 dbgs() << "BT:";
1533
1534 C.Low->getValue().print(dbgs(), true);
1535 if (C.Low != C.High) {
1536 dbgs() << '-';
1537 C.High->getValue().print(dbgs(), true);
1538 }
1539 dbgs() << ' ';
1540 }
1541 dbgs() << '\n';
1542 });
1543
1544 assert(!Clusters.empty());
1545 SwitchWorkList WorkList;
1546 CaseClusterIt First = Clusters.begin();
1547 CaseClusterIt Last = Clusters.end() - 1;
1548 auto DefaultProb = getEdgeProbability(Src: SwitchMBB, Dst: DefaultMBB);
1549 WorkList.push_back(Elt: {.MBB: SwitchMBB, .FirstCluster: First, .LastCluster: Last, .GE: nullptr, .LT: nullptr, .DefaultProb: DefaultProb});
1550
1551 while (!WorkList.empty()) {
1552 SwitchWorkListItem W = WorkList.pop_back_val();
1553
1554 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
1555 // For optimized builds, lower large range as a balanced binary tree.
1556 if (NumClusters > 3 &&
1557 MF->getTarget().getOptLevel() != CodeGenOptLevel::None &&
1558 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
1559 splitWorkItem(WorkList, W, Cond: SI.getCondition(), SwitchMBB, MIB);
1560 continue;
1561 }
1562
1563 if (!lowerSwitchWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
1564 return false;
1565 }
1566 return true;
1567}
1568
1569void IRTranslatorImpl::splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
1570 const SwitchCG::SwitchWorkListItem &W,
1571 Value *Cond, MachineBasicBlock *SwitchMBB,
1572 MachineIRBuilder &MIB) {
1573 using namespace SwitchCG;
1574 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
1575 "Clusters not sorted?");
1576 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
1577
1578 auto [LastLeft, FirstRight, LeftProb, RightProb] =
1579 SL->computeSplitWorkItemInfo(W);
1580
1581 // Use the first element on the right as pivot since we will make less-than
1582 // comparisons against it.
1583 CaseClusterIt PivotCluster = FirstRight;
1584 assert(PivotCluster > W.FirstCluster);
1585 assert(PivotCluster <= W.LastCluster);
1586
1587 CaseClusterIt FirstLeft = W.FirstCluster;
1588 CaseClusterIt LastRight = W.LastCluster;
1589
1590 const ConstantInt *Pivot = PivotCluster->Low;
1591
1592 // New blocks will be inserted immediately after the current one.
1593 MachineFunction::iterator BBI(W.MBB);
1594 ++BBI;
1595
1596 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
1597 // we can branch to its destination directly if it's squeezed exactly in
1598 // between the known lower bound and Pivot - 1.
1599 MachineBasicBlock *LeftMBB;
1600 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
1601 FirstLeft->Low == W.GE &&
1602 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
1603 LeftMBB = FirstLeft->MBB;
1604 } else {
1605 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
1606 FuncInfo.MF->insert(MBBI: BBI, MBB: LeftMBB);
1607 WorkList.push_back(
1608 Elt: {.MBB: LeftMBB, .FirstCluster: FirstLeft, .LastCluster: LastLeft, .GE: W.GE, .LT: Pivot, .DefaultProb: W.DefaultProb / 2});
1609 }
1610
1611 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
1612 // single cluster, RHS.Low == Pivot, and we can branch to its destination
1613 // directly if RHS.High equals the current upper bound.
1614 MachineBasicBlock *RightMBB;
1615 if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT &&
1616 (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
1617 RightMBB = FirstRight->MBB;
1618 } else {
1619 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
1620 FuncInfo.MF->insert(MBBI: BBI, MBB: RightMBB);
1621 WorkList.push_back(
1622 Elt: {.MBB: RightMBB, .FirstCluster: FirstRight, .LastCluster: LastRight, .GE: Pivot, .LT: W.LT, .DefaultProb: W.DefaultProb / 2});
1623 }
1624
1625 // Create the CaseBlock record that will be used to lower the branch.
1626 CaseBlock CB(ICmpInst::Predicate::ICMP_SLT, false, Cond, Pivot, nullptr,
1627 LeftMBB, RightMBB, W.MBB, MIB.getDebugLoc(), LeftProb,
1628 RightProb);
1629
1630 if (W.MBB == SwitchMBB)
1631 emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
1632 else
1633 SL->SwitchCases.push_back(x: CB);
1634}
1635
1636void IRTranslatorImpl::emitJumpTable(SwitchCG::JumpTable &JT,
1637 MachineBasicBlock *MBB) {
1638 // Emit the code for the jump table
1639 assert(JT.Reg && "Should lower JT Header first!");
1640 MachineIRBuilder MIB(*MBB->getParent());
1641 MIB.setMBB(*MBB);
1642 MIB.setDebugLoc(CurBuilder->getDebugLoc());
1643
1644 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
1645 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1646
1647 auto Table = MIB.buildJumpTable(PtrTy, JTI: JT.JTI);
1648 MIB.buildBrJT(TablePtr: Table.getReg(Idx: 0), JTI: JT.JTI, IndexReg: JT.Reg);
1649}
1650
1651bool IRTranslatorImpl::emitJumpTableHeader(SwitchCG::JumpTable &JT,
1652 SwitchCG::JumpTableHeader &JTH,
1653 MachineBasicBlock *HeaderBB) {
1654 MachineIRBuilder MIB(*HeaderBB->getParent());
1655 MIB.setMBB(*HeaderBB);
1656 MIB.setDebugLoc(CurBuilder->getDebugLoc());
1657
1658 const Value &SValue = *JTH.SValue;
1659 // Subtract the lowest switch case value from the value being switched on.
1660 const LLT SwitchTy = getLLTForType(Ty&: *SValue.getType(), DL: *DL);
1661 Register SwitchOpReg = getOrCreateVReg(Val: SValue);
1662 auto FirstCst = MIB.buildConstant(Res: SwitchTy, Val: JTH.First);
1663 auto Sub = MIB.buildSub(Dst: {SwitchTy}, Src0: SwitchOpReg, Src1: FirstCst);
1664
1665 // This value may be smaller or larger than the target's pointer type, and
1666 // therefore require extension or truncating.
1667 auto *PtrIRTy = PointerType::getUnqual(C&: SValue.getContext());
1668 const LLT PtrScalarTy = LLT::integer(SizeInBits: DL->getTypeSizeInBits(Ty: PtrIRTy));
1669 auto Index = MIB.buildZExtOrTrunc(Res: PtrScalarTy, Op: Sub);
1670
1671 JT.Reg = Index.getReg(Idx: 0);
1672
1673 if (JTH.FallthroughUnreachable) {
1674 if (JT.MBB != HeaderBB->getNextNode())
1675 MIB.buildBr(Dest&: *JT.MBB);
1676 return true;
1677 }
1678
1679 // Emit the range check for the jump table, and branch to the default block
1680 // for the switch statement if the value being switched on exceeds the
1681 // largest case in the switch.
1682 auto Cst = getOrCreateVReg(
1683 Val: *ConstantInt::get(Ty: SValue.getType(), V: JTH.Last - JTH.First));
1684 auto Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_UGT, Res: LLT::integer(SizeInBits: 1), Op0: Sub, Op1: Cst);
1685
1686 auto BrCond = MIB.buildBrCond(Tst: Cmp.getReg(Idx: 0), Dest&: *JT.Default);
1687
1688 // Avoid emitting unnecessary branches to the next block.
1689 if (JT.MBB != HeaderBB->getNextNode())
1690 BrCond = MIB.buildBr(Dest&: *JT.MBB);
1691 return true;
1692}
1693
1694void IRTranslatorImpl::emitSwitchCase(SwitchCG::CaseBlock &CB,
1695 MachineBasicBlock *SwitchBB,
1696 MachineIRBuilder &MIB) {
1697 Register CondLHS = getOrCreateVReg(Val: *CB.CmpLHS);
1698 Register Cond;
1699 DebugLoc OldDbgLoc = MIB.getDebugLoc();
1700 MIB.setDebugLoc(CB.DbgLoc);
1701 MIB.setMBB(*CB.ThisBB);
1702
1703 if (CB.PredInfo.NoCmp) {
1704 // Branch or fall through to TrueBB.
1705 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
1706 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
1707 NewPred: CB.ThisBB);
1708 CB.ThisBB->normalizeSuccProbs();
1709 if (CB.TrueBB != CB.ThisBB->getNextNode())
1710 MIB.buildBr(Dest&: *CB.TrueBB);
1711 MIB.setDebugLoc(OldDbgLoc);
1712 return;
1713 }
1714
1715 const LLT i1Ty = LLT::integer(SizeInBits: 1);
1716 // Build the compare.
1717 if (!CB.CmpMHS) {
1718 const auto *CI = dyn_cast<ConstantInt>(Val: CB.CmpRHS);
1719 // For conditional branch lowering, we might try to do something silly like
1720 // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
1721 // just re-use the existing condition vreg.
1722 if (MRI->getType(Reg: CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
1723 CB.PredInfo.Pred == CmpInst::ICMP_EQ) {
1724 Cond = CondLHS;
1725 } else {
1726 Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
1727 if (CmpInst::isFPPredicate(P: CB.PredInfo.Pred))
1728 Cond =
1729 MIB.buildFCmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
1730 else
1731 Cond =
1732 MIB.buildICmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
1733 }
1734 } else {
1735 assert(CB.PredInfo.Pred == CmpInst::ICMP_SLE &&
1736 "Can only handle SLE ranges");
1737
1738 const APInt& Low = cast<ConstantInt>(Val: CB.CmpLHS)->getValue();
1739 const APInt& High = cast<ConstantInt>(Val: CB.CmpRHS)->getValue();
1740
1741 Register CmpOpReg = getOrCreateVReg(Val: *CB.CmpMHS);
1742 if (cast<ConstantInt>(Val: CB.CmpLHS)->isMinValue(IsSigned: true)) {
1743 Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
1744 Cond =
1745 MIB.buildICmp(Pred: CmpInst::ICMP_SLE, Res: i1Ty, Op0: CmpOpReg, Op1: CondRHS).getReg(Idx: 0);
1746 } else {
1747 const LLT CmpTy = MRI->getType(Reg: CmpOpReg);
1748 auto Sub = MIB.buildSub(Dst: {CmpTy}, Src0: CmpOpReg, Src1: CondLHS);
1749 auto Diff = MIB.buildConstant(Res: CmpTy, Val: High - Low);
1750 Cond = MIB.buildICmp(Pred: CmpInst::ICMP_ULE, Res: i1Ty, Op0: Sub, Op1: Diff).getReg(Idx: 0);
1751 }
1752 }
1753
1754 // Update successor info
1755 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
1756
1757 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
1758 NewPred: CB.ThisBB);
1759
1760 // TrueBB and FalseBB are always different unless the incoming IR is
1761 // degenerate. This only happens when running llc on weird IR.
1762 if (CB.TrueBB != CB.FalseBB)
1763 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.FalseBB, Prob: CB.FalseProb);
1764 CB.ThisBB->normalizeSuccProbs();
1765
1766 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
1767 NewPred: CB.ThisBB);
1768
1769 MIB.buildBrCond(Tst: Cond, Dest&: *CB.TrueBB);
1770 MIB.buildBr(Dest&: *CB.FalseBB);
1771 MIB.setDebugLoc(OldDbgLoc);
1772}
1773
1774bool IRTranslatorImpl::lowerJumpTableWorkItem(
1775 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
1776 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1777 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
1778 BranchProbability UnhandledProbs, SwitchCG::CaseClusterIt I,
1779 MachineBasicBlock *Fallthrough, bool FallthroughUnreachable) {
1780 using namespace SwitchCG;
1781 MachineFunction *CurMF = SwitchMBB->getParent();
1782 // FIXME: Optimize away range check based on pivot comparisons.
1783 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
1784 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
1785 BranchProbability DefaultProb = W.DefaultProb;
1786
1787 // The jump block hasn't been inserted yet; insert it here.
1788 MachineBasicBlock *JumpMBB = JT->MBB;
1789 CurMF->insert(MBBI: BBI, MBB: JumpMBB);
1790
1791 // Since the jump table block is separate from the switch block, we need
1792 // to keep track of it as a machine predecessor to the default block,
1793 // otherwise we lose the phi edges.
1794 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1795 NewPred: CurMBB);
1796 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1797 NewPred: JumpMBB);
1798
1799 auto JumpProb = I->Prob;
1800 auto FallthroughProb = UnhandledProbs;
1801
1802 // If the default statement is a target of the jump table, we evenly
1803 // distribute the default probability to successors of CurMBB. Also
1804 // update the probability on the edge from JumpMBB to Fallthrough.
1805 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
1806 SE = JumpMBB->succ_end();
1807 SI != SE; ++SI) {
1808 if (*SI == DefaultMBB) {
1809 JumpProb += DefaultProb / 2;
1810 FallthroughProb -= DefaultProb / 2;
1811 JumpMBB->setSuccProbability(I: SI, Prob: DefaultProb / 2);
1812 JumpMBB->normalizeSuccProbs();
1813 } else {
1814 // Also record edges from the jump table block to it's successors.
1815 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
1816 NewPred: JumpMBB);
1817 }
1818 }
1819
1820 if (FallthroughUnreachable)
1821 JTH->FallthroughUnreachable = true;
1822
1823 if (!JTH->FallthroughUnreachable)
1824 addSuccessorWithProb(Src: CurMBB, Dst: Fallthrough, Prob: FallthroughProb);
1825 addSuccessorWithProb(Src: CurMBB, Dst: JumpMBB, Prob: JumpProb);
1826 CurMBB->normalizeSuccProbs();
1827
1828 // The jump table header will be inserted in our current block, do the
1829 // range check, and fall through to our fallthrough block.
1830 JTH->HeaderBB = CurMBB;
1831 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
1832
1833 // If we're in the right place, emit the jump table header right now.
1834 if (CurMBB == SwitchMBB) {
1835 if (!emitJumpTableHeader(JT&: *JT, JTH&: *JTH, HeaderBB: CurMBB))
1836 return false;
1837 JTH->Emitted = true;
1838 }
1839 return true;
1840}
1841bool IRTranslatorImpl::lowerSwitchRangeWorkItem(
1842 SwitchCG::CaseClusterIt I, Value *Cond, MachineBasicBlock *Fallthrough,
1843 bool FallthroughUnreachable, BranchProbability UnhandledProbs,
1844 MachineBasicBlock *CurMBB, MachineIRBuilder &MIB,
1845 MachineBasicBlock *SwitchMBB) {
1846 using namespace SwitchCG;
1847 const Value *RHS, *LHS, *MHS;
1848 CmpInst::Predicate Pred;
1849 if (I->Low == I->High) {
1850 // Check Cond == I->Low.
1851 Pred = CmpInst::ICMP_EQ;
1852 LHS = Cond;
1853 RHS = I->Low;
1854 MHS = nullptr;
1855 } else {
1856 // Check I->Low <= Cond <= I->High.
1857 Pred = CmpInst::ICMP_SLE;
1858 LHS = I->Low;
1859 MHS = Cond;
1860 RHS = I->High;
1861 }
1862
1863 // If Fallthrough is unreachable, fold away the comparison.
1864 // The false probability is the sum of all unhandled cases.
1865 CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
1866 CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
1867
1868 emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
1869 return true;
1870}
1871
1872void IRTranslatorImpl::emitBitTestHeader(SwitchCG::BitTestBlock &B,
1873 MachineBasicBlock *SwitchBB) {
1874 MachineIRBuilder &MIB = *CurBuilder;
1875 MIB.setMBB(*SwitchBB);
1876
1877 // Subtract the minimum value.
1878 Register SwitchOpReg = getOrCreateVReg(Val: *B.SValue);
1879
1880 LLT SwitchOpTy = MRI->getType(Reg: SwitchOpReg);
1881 Register MinValReg = MIB.buildConstant(Res: SwitchOpTy, Val: B.First).getReg(Idx: 0);
1882 auto RangeSub = MIB.buildSub(Dst: SwitchOpTy, Src0: SwitchOpReg, Src1: MinValReg);
1883
1884 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
1885 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1886
1887 LLT MaskTy = SwitchOpTy;
1888 if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
1889 !llvm::has_single_bit<uint32_t>(Value: MaskTy.getSizeInBits()))
1890 MaskTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
1891 else {
1892 // Ensure that the type will fit the mask value.
1893 for (const SwitchCG::BitTestCase &Case : B.Cases) {
1894 if (!isUIntN(N: SwitchOpTy.getSizeInBits(), x: Case.Mask)) {
1895 // Switch table case range are encoded into series of masks.
1896 // Just use pointer type, it's guaranteed to fit.
1897 MaskTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
1898 break;
1899 }
1900 }
1901 }
1902 Register SubReg = RangeSub.getReg(Idx: 0);
1903 if (SwitchOpTy != MaskTy)
1904 SubReg = MIB.buildZExtOrTrunc(Res: MaskTy, Op: SubReg).getReg(Idx: 0);
1905
1906 B.RegVT = getMVTForLLT(Ty: MaskTy);
1907 B.Reg = SubReg;
1908
1909 MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1910
1911 if (!B.FallthroughUnreachable)
1912 addSuccessorWithProb(Src: SwitchBB, Dst: B.Default, Prob: B.DefaultProb);
1913 addSuccessorWithProb(Src: SwitchBB, Dst: MBB, Prob: B.Prob);
1914
1915 SwitchBB->normalizeSuccProbs();
1916
1917 if (!B.FallthroughUnreachable) {
1918 // Conditional branch to the default block.
1919 auto RangeCst = MIB.buildConstant(Res: SwitchOpTy, Val: B.Range);
1920 auto RangeCmp = MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: LLT::integer(SizeInBits: 1),
1921 Op0: RangeSub, Op1: RangeCst);
1922 MIB.buildBrCond(Tst: RangeCmp, Dest&: *B.Default);
1923 }
1924
1925 // Avoid emitting unnecessary branches to the next block.
1926 if (MBB != SwitchBB->getNextNode())
1927 MIB.buildBr(Dest&: *MBB);
1928}
1929
1930void IRTranslatorImpl::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1931 MachineBasicBlock *NextMBB,
1932 BranchProbability BranchProbToNext,
1933 Register Reg, SwitchCG::BitTestCase &B,
1934 MachineBasicBlock *SwitchBB) {
1935 MachineIRBuilder &MIB = *CurBuilder;
1936 MIB.setMBB(*SwitchBB);
1937
1938 LLT SwitchTy = getLLTForMVT(Ty: BB.RegVT);
1939 Register Cmp;
1940 unsigned PopCount = llvm::popcount(Value: B.Mask);
1941 if (PopCount == 1) {
1942 // Testing for a single bit; just compare the shift count with what it
1943 // would need to be to shift a 1 bit in that position.
1944 auto MaskTrailingZeros =
1945 MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_zero(Val: B.Mask));
1946 Cmp = MIB.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: LLT::integer(SizeInBits: 1), Op0: Reg,
1947 Op1: MaskTrailingZeros)
1948 .getReg(Idx: 0);
1949 } else if (PopCount == BB.Range) {
1950 // There is only one zero bit in the range, test for it directly.
1951 auto MaskTrailingOnes =
1952 MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_one(Value: B.Mask));
1953 Cmp =
1954 MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::integer(SizeInBits: 1), Op0: Reg, Op1: MaskTrailingOnes)
1955 .getReg(Idx: 0);
1956 } else {
1957 // Make desired shift.
1958 auto CstOne = MIB.buildConstant(Res: SwitchTy, Val: 1);
1959 auto SwitchVal = MIB.buildShl(Dst: SwitchTy, Src0: CstOne, Src1: Reg);
1960
1961 // Emit bit tests and jumps.
1962 auto CstMask = MIB.buildConstant(Res: SwitchTy, Val: B.Mask);
1963 auto AndOp = MIB.buildAnd(Dst: SwitchTy, Src0: SwitchVal, Src1: CstMask);
1964 auto CstZero = MIB.buildConstant(Res: SwitchTy, Val: 0);
1965 Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::integer(SizeInBits: 1), Op0: AndOp, Op1: CstZero)
1966 .getReg(Idx: 0);
1967 }
1968
1969 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1970 addSuccessorWithProb(Src: SwitchBB, Dst: B.TargetBB, Prob: B.ExtraProb);
1971 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1972 addSuccessorWithProb(Src: SwitchBB, Dst: NextMBB, Prob: BranchProbToNext);
1973 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1974 // one as they are relative probabilities (and thus work more like weights),
1975 // and hence we need to normalize them to let the sum of them become one.
1976 SwitchBB->normalizeSuccProbs();
1977
1978 // Record the fact that the IR edge from the header to the bit test target
1979 // will go through our new block. Neeeded for PHIs to have nodes added.
1980 addMachineCFGPred(Edge: {BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1981 NewPred: SwitchBB);
1982
1983 MIB.buildBrCond(Tst: Cmp, Dest&: *B.TargetBB);
1984
1985 // Avoid emitting unnecessary branches to the next block.
1986 if (NextMBB != SwitchBB->getNextNode())
1987 MIB.buildBr(Dest&: *NextMBB);
1988}
1989
1990bool IRTranslatorImpl::lowerBitTestWorkItem(
1991 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
1992 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1993 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
1994 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1995 SwitchCG::CaseClusterIt I, MachineBasicBlock *Fallthrough,
1996 bool FallthroughUnreachable) {
1997 using namespace SwitchCG;
1998 MachineFunction *CurMF = SwitchMBB->getParent();
1999 // FIXME: Optimize away range check based on pivot comparisons.
2000 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
2001 // The bit test blocks haven't been inserted yet; insert them here.
2002 for (BitTestCase &BTC : BTB->Cases)
2003 CurMF->insert(MBBI: BBI, MBB: BTC.ThisBB);
2004
2005 // Fill in fields of the BitTestBlock.
2006 BTB->Parent = CurMBB;
2007 BTB->Default = Fallthrough;
2008
2009 BTB->DefaultProb = UnhandledProbs;
2010 // If the cases in bit test don't form a contiguous range, we evenly
2011 // distribute the probability on the edge to Fallthrough to two
2012 // successors of CurMBB.
2013 if (!BTB->ContiguousRange) {
2014 BTB->Prob += DefaultProb / 2;
2015 BTB->DefaultProb -= DefaultProb / 2;
2016 }
2017
2018 if (FallthroughUnreachable)
2019 BTB->FallthroughUnreachable = true;
2020
2021 // If we're in the right place, emit the bit test header right now.
2022 if (CurMBB == SwitchMBB) {
2023 emitBitTestHeader(B&: *BTB, SwitchBB: SwitchMBB);
2024 BTB->Emitted = true;
2025 }
2026 return true;
2027}
2028
2029bool IRTranslatorImpl::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
2030 Value *Cond,
2031 MachineBasicBlock *SwitchMBB,
2032 MachineBasicBlock *DefaultMBB,
2033 MachineIRBuilder &MIB) {
2034 using namespace SwitchCG;
2035 MachineFunction *CurMF = FuncInfo.MF;
2036 MachineBasicBlock *NextMBB = nullptr;
2037 MachineFunction::iterator BBI(W.MBB);
2038 if (++BBI != FuncInfo.MF->end())
2039 NextMBB = &*BBI;
2040
2041 if (EnableOpts) {
2042 // Here, we order cases by probability so the most likely case will be
2043 // checked first. However, two clusters can have the same probability in
2044 // which case their relative ordering is non-deterministic. So we use Low
2045 // as a tie-breaker as clusters are guaranteed to never overlap.
2046 llvm::sort(Start: W.FirstCluster, End: W.LastCluster + 1,
2047 Comp: [](const CaseCluster &a, const CaseCluster &b) {
2048 return a.Prob != b.Prob
2049 ? a.Prob > b.Prob
2050 : a.Low->getValue().slt(RHS: b.Low->getValue());
2051 });
2052
2053 // Rearrange the case blocks so that the last one falls through if possible
2054 // without changing the order of probabilities.
2055 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
2056 --I;
2057 if (I->Prob > W.LastCluster->Prob)
2058 break;
2059 if (I->Kind == CC_Range && I->MBB == NextMBB) {
2060 std::swap(a&: *I, b&: *W.LastCluster);
2061 break;
2062 }
2063 }
2064 }
2065
2066 // Compute total probability.
2067 BranchProbability DefaultProb = W.DefaultProb;
2068 BranchProbability UnhandledProbs = DefaultProb;
2069 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
2070 UnhandledProbs += I->Prob;
2071
2072 MachineBasicBlock *CurMBB = W.MBB;
2073 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
2074 bool FallthroughUnreachable = false;
2075 MachineBasicBlock *Fallthrough;
2076 if (I == W.LastCluster) {
2077 // For the last cluster, fall through to the default destination.
2078 Fallthrough = DefaultMBB;
2079 FallthroughUnreachable =
2080 isa<UnreachableInst>(Val: DefaultMBB->getBasicBlock()->getFirstNonPHIIt());
2081 } else {
2082 Fallthrough = CurMF->CreateMachineBasicBlock(BB: CurMBB->getBasicBlock());
2083 CurMF->insert(MBBI: BBI, MBB: Fallthrough);
2084 }
2085 UnhandledProbs -= I->Prob;
2086
2087 switch (I->Kind) {
2088 case CC_BitTests: {
2089 if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
2090 DefaultProb, UnhandledProbs, I, Fallthrough,
2091 FallthroughUnreachable)) {
2092 LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
2093 return false;
2094 }
2095 break;
2096 }
2097
2098 case CC_JumpTable: {
2099 if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
2100 UnhandledProbs, I, Fallthrough,
2101 FallthroughUnreachable)) {
2102 LLVM_DEBUG(dbgs() << "Failed to lower jump table");
2103 return false;
2104 }
2105 break;
2106 }
2107 case CC_Range: {
2108 if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
2109 FallthroughUnreachable, UnhandledProbs,
2110 CurMBB, MIB, SwitchMBB)) {
2111 LLVM_DEBUG(dbgs() << "Failed to lower switch range");
2112 return false;
2113 }
2114 break;
2115 }
2116 }
2117 CurMBB = Fallthrough;
2118 }
2119
2120 return true;
2121}
2122
2123bool IRTranslatorImpl::translateIndirectBr(const User &U,
2124 MachineIRBuilder &MIRBuilder) {
2125 const IndirectBrInst &BrInst = cast<IndirectBrInst>(Val: U);
2126
2127 const Register Tgt = getOrCreateVReg(Val: *BrInst.getAddress());
2128 MIRBuilder.buildBrIndirect(Tgt);
2129
2130 // Link successors.
2131 SmallPtrSet<const BasicBlock *, 32> AddedSuccessors;
2132 MachineBasicBlock &CurBB = MIRBuilder.getMBB();
2133 for (const BasicBlock *Succ : successors(I: &BrInst)) {
2134 // It's legal for indirectbr instructions to have duplicate blocks in the
2135 // destination list. We don't allow this in MIR. Skip anything that's
2136 // already a successor.
2137 if (!AddedSuccessors.insert(Ptr: Succ).second)
2138 continue;
2139 CurBB.addSuccessor(Succ: &getMBB(BB: *Succ));
2140 }
2141
2142 return true;
2143}
2144
2145static bool isSwiftError(const Value *V) {
2146 if (auto Arg = dyn_cast<Argument>(Val: V))
2147 return Arg->hasSwiftErrorAttr();
2148 if (auto AI = dyn_cast<AllocaInst>(Val: V))
2149 return AI->isSwiftError();
2150 return false;
2151}
2152
2153bool IRTranslatorImpl::translateLoad(const User &U,
2154 MachineIRBuilder &MIRBuilder) {
2155 const LoadInst &LI = cast<LoadInst>(Val: U);
2156 TypeSize StoreSize = DL->getTypeStoreSize(Ty: LI.getType());
2157 if (StoreSize.isZero())
2158 return true;
2159
2160 ArrayRef<Register> Regs = getOrCreateVRegs(Val: LI);
2161 Register Base = getOrCreateVReg(Val: *LI.getPointerOperand());
2162 AAMDNodes AAInfo = LI.getAAMetadata();
2163
2164 const Value *Ptr = LI.getPointerOperand();
2165
2166 if (CLI->supportSwiftError() && isSwiftError(V: Ptr)) {
2167 assert(Regs.size() == 1 && "swifterror should be single pointer");
2168 Register VReg =
2169 SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);
2170 MIRBuilder.buildCopy(Res: Regs[0], Op: VReg);
2171 return true;
2172 }
2173
2174 MachineMemOperand::Flags Flags =
2175 TLI->getLoadMemOperandFlags(LI, DL: *DL, AC, LibInfo, OptLevel);
2176 if (AA && !(Flags & MachineMemOperand::MOInvariant)) {
2177 if (AA->pointsToConstantMemory(
2178 Loc: MemoryLocation(Ptr, LocationSize::precise(Value: StoreSize), AAInfo))) {
2179 Flags |= MachineMemOperand::MOInvariant;
2180 }
2181 }
2182
2183 // Fast-path the common single-register load.
2184 if (Regs.size() == 1) {
2185 auto *MMO = MF->getMachineMemOperand(
2186 PtrInfo: MachinePointerInfo(LI.getPointerOperand()), F: Flags,
2187 MemTy: MRI->getType(Reg: Regs[0]), BaseAlignment: getMemOpAlign(I: LI),
2188 Metadata: MMOMetadata(AAInfo, LI.getMetadata(KindID: LLVMContext::MD_range)),
2189 SSID: LI.getSyncScopeID(), Ordering: LI.getOrdering());
2190 MIRBuilder.buildLoad(Res: Regs[0], Addr: Base, MMO&: *MMO);
2191 return true;
2192 }
2193
2194 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: LI);
2195 Type *OffsetIRTy = DL->getIndexType(PtrTy: Ptr->getType());
2196 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2197 for (unsigned i = 0; i < Regs.size(); ++i) {
2198 Register Addr;
2199 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i]);
2200
2201 MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i]);
2202 Align BaseAlign = getMemOpAlign(I: LI);
2203 auto *MMO =
2204 MF->getMachineMemOperand(PtrInfo: Ptr, F: Flags, MemTy: MRI->getType(Reg: Regs[i]),
2205 BaseAlignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]), Metadata: AAInfo,
2206 SSID: LI.getSyncScopeID(), Ordering: LI.getOrdering());
2207 MIRBuilder.buildLoad(Res: Regs[i], Addr, MMO&: *MMO);
2208 }
2209
2210 return true;
2211}
2212
2213bool IRTranslatorImpl::translateStore(const User &U,
2214 MachineIRBuilder &MIRBuilder) {
2215 const StoreInst &SI = cast<StoreInst>(Val: U);
2216 if (DL->getTypeStoreSize(Ty: SI.getValueOperand()->getType()).isZero())
2217 return true;
2218
2219 ArrayRef<Register> Vals = getOrCreateVRegs(Val: *SI.getValueOperand());
2220 Register Base = getOrCreateVReg(Val: *SI.getPointerOperand());
2221
2222 if (CLI->supportSwiftError() && isSwiftError(V: SI.getPointerOperand())) {
2223 assert(Vals.size() == 1 && "swifterror should be single pointer");
2224
2225 Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
2226 SI.getPointerOperand());
2227 MIRBuilder.buildCopy(Res: VReg, Op: Vals[0]);
2228 return true;
2229 }
2230
2231 MachineMemOperand::Flags Flags = TLI->getStoreMemOperandFlags(SI, DL: *DL);
2232 // Fast-path the common single-register store.
2233 if (Vals.size() == 1) {
2234 auto *MMO = MF->getMachineMemOperand(
2235 PtrInfo: MachinePointerInfo(SI.getPointerOperand()), F: Flags,
2236 MemTy: MRI->getType(Reg: Vals[0]), BaseAlignment: getMemOpAlign(I: SI), Metadata: SI.getAAMetadata(),
2237 SSID: SI.getSyncScopeID(), Ordering: SI.getOrdering());
2238 MIRBuilder.buildStore(Val: Vals[0], Addr: Base, MMO&: *MMO);
2239 return true;
2240 }
2241
2242 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *SI.getValueOperand());
2243 Type *OffsetIRTy = DL->getIndexType(PtrTy: SI.getPointerOperandType());
2244 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2245 for (unsigned i = 0; i < Vals.size(); ++i) {
2246 Register Addr;
2247 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i]);
2248
2249 MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i]);
2250 Align BaseAlign = getMemOpAlign(I: SI);
2251 auto *MMO = MF->getMachineMemOperand(PtrInfo: Ptr, F: Flags, MemTy: MRI->getType(Reg: Vals[i]),
2252 BaseAlignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]),
2253 Metadata: SI.getAAMetadata(),
2254 SSID: SI.getSyncScopeID(), Ordering: SI.getOrdering());
2255 MIRBuilder.buildStore(Val: Vals[i], Addr, MMO&: *MMO);
2256 }
2257 return true;
2258}
2259
2260static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
2261 const Value *Src = U.getOperand(i: 0);
2262 Type *Int32Ty = Type::getInt32Ty(C&: U.getContext());
2263
2264 // getIndexedOffsetInType is designed for GEPs, so the first index is the
2265 // usual array element rather than looking into the actual aggregate.
2266 SmallVector<Value *, 1> Indices;
2267 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: 0));
2268
2269 if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: &U)) {
2270 for (auto Idx : EVI->indices())
2271 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
2272 } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(Val: &U)) {
2273 for (auto Idx : IVI->indices())
2274 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
2275 } else {
2276 llvm::append_range(C&: Indices, R: drop_begin(RangeOrContainer: U.operands()));
2277 }
2278
2279 return static_cast<uint64_t>(
2280 DL.getIndexedOffsetInType(ElemTy: Src->getType(), Indices));
2281}
2282
2283bool IRTranslatorImpl::translateExtractValue(const User &U,
2284 MachineIRBuilder &MIRBuilder) {
2285 const Value *Src = U.getOperand(i: 0);
2286 uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
2287 ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
2288 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *Src);
2289 unsigned Idx = llvm::lower_bound(Range&: Offsets, Value&: Offset) - Offsets.begin();
2290 auto &DstRegs = allocateVRegs(Val: U);
2291
2292 for (unsigned i = 0; i < DstRegs.size(); ++i)
2293 DstRegs[i] = SrcRegs[Idx++];
2294
2295 return true;
2296}
2297
2298bool IRTranslatorImpl::translateInsertValue(const User &U,
2299 MachineIRBuilder &MIRBuilder) {
2300 const Value *Src = U.getOperand(i: 0);
2301 uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
2302 auto &DstRegs = allocateVRegs(Val: U);
2303 ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(V: U);
2304 ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
2305 ArrayRef<Register> InsertedRegs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
2306 auto *InsertedIt = InsertedRegs.begin();
2307
2308 for (unsigned i = 0; i < DstRegs.size(); ++i) {
2309 if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
2310 DstRegs[i] = *InsertedIt++;
2311 else
2312 DstRegs[i] = SrcRegs[i];
2313 }
2314
2315 return true;
2316}
2317
2318bool IRTranslatorImpl::translateSelect(const User &U,
2319 MachineIRBuilder &MIRBuilder) {
2320 Register Tst = getOrCreateVReg(Val: *U.getOperand(i: 0));
2321 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: U);
2322 ArrayRef<Register> Op0Regs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
2323 ArrayRef<Register> Op1Regs = getOrCreateVRegs(Val: *U.getOperand(i: 2));
2324
2325 uint32_t Flags = 0;
2326 if (const SelectInst *SI = dyn_cast<SelectInst>(Val: &U))
2327 Flags = MachineInstr::copyFlagsFromInstruction(I: *SI);
2328
2329 for (unsigned i = 0; i < ResRegs.size(); ++i) {
2330 MIRBuilder.buildSelect(Res: ResRegs[i], Tst, Op0: Op0Regs[i], Op1: Op1Regs[i], Flags);
2331 }
2332
2333 return true;
2334}
2335
2336bool IRTranslatorImpl::translateCopy(const User &U, const Value &V,
2337 MachineIRBuilder &MIRBuilder) {
2338 return translateCopy(U, Src: getOrCreateVReg(Val: V), MIRBuilder);
2339}
2340
2341bool IRTranslatorImpl::translateCopy(const User &U, Register Src,
2342 MachineIRBuilder &MIRBuilder) {
2343 auto &Regs = *VMap.getVRegs(V: U);
2344 if (Regs.empty()) {
2345 Regs.push_back(Elt: Src);
2346 } else {
2347 // If we already assigned a vreg for this instruction, we can't change that.
2348 // Emit a copy to satisfy the users we already emitted.
2349 MIRBuilder.buildCopy(Res: Regs[0], Op: Src);
2350 }
2351 return true;
2352}
2353
2354bool IRTranslatorImpl::translateBitCast(const User &U,
2355 MachineIRBuilder &MIRBuilder) {
2356 Type *SrcTy = U.getOperand(i: 0)->getType();
2357 Type *DstTy = U.getType();
2358
2359 // If we're bitcasting to the source type, we can reuse the source vreg.
2360 if (getLLTForType(Ty&: *SrcTy, DL: *DL) == getLLTForType(Ty&: *DstTy, DL: *DL)) {
2361 // If the source is a ConstantInt then it was probably created by
2362 // ConstantHoisting and we should leave it alone.
2363 if (isa<ConstantInt>(Val: U.getOperand(i: 0)))
2364 return translateCast(Opcode: TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
2365 MIRBuilder);
2366 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
2367 }
2368
2369 // The IR only allows pointer/non-pointer bitcasts with byte types, but
2370 // G_BITCAST can't convert between pointers and other types. Go through an
2371 // integer with the pointer's shape instead: `bitcast <2 x b32> to ptr`
2372 // becomes a G_BITCAST to i64 and a G_INTTOPTR.
2373 if (SrcTy->isPtrOrPtrVectorTy() != DstTy->isPtrOrPtrVectorTy()) {
2374 assert((SrcTy->isByteOrByteVectorTy() || DstTy->isByteOrByteVectorTy()) &&
2375 "only byte types can be bitcast to or from pointers");
2376 Type *PtrIRTy = SrcTy->isPtrOrPtrVectorTy() ? SrcTy : DstTy;
2377 LLT IntTy = getLLTForType(Ty&: *DL->getIntPtrType(PtrIRTy), DL: *DL);
2378 Register Src = getOrCreateVReg(Val: *U.getOperand(i: 0));
2379 Register Dst = getOrCreateVReg(Val: U);
2380 if (MRI->getType(Reg: Src) != IntTy && MRI->getType(Reg: Dst) != IntTy)
2381 Src = MIRBuilder.buildCast(Dst: IntTy, Src).getReg(Idx: 0);
2382 MIRBuilder.buildCast(Dst, Src);
2383 return true;
2384 }
2385
2386 return translateCast(Opcode: TargetOpcode::G_BITCAST, U, MIRBuilder);
2387}
2388
2389bool IRTranslatorImpl::translateCast(unsigned Opcode, const User &U,
2390 MachineIRBuilder &MIRBuilder) {
2391 if (!mayTranslateUserTypes(U))
2392 return false;
2393
2394 uint32_t Flags = 0;
2395 if (const Instruction *I = dyn_cast<Instruction>(Val: &U))
2396 Flags = MachineInstr::copyFlagsFromInstruction(I: *I);
2397
2398 Register Op = getOrCreateVReg(Val: *U.getOperand(i: 0));
2399 Register Res = getOrCreateVReg(Val: U);
2400 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op}, Flags);
2401 return true;
2402}
2403
2404bool IRTranslatorImpl::translateGetElementPtr(const User &U,
2405 MachineIRBuilder &MIRBuilder) {
2406 Value &Op0 = *U.getOperand(i: 0);
2407 Register BaseReg = getOrCreateVReg(Val: Op0);
2408 Type *PtrIRTy = Op0.getType();
2409 LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
2410 Type *OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
2411 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2412
2413 uint32_t PtrAddFlags = 0;
2414 // Each PtrAdd generated to implement the GEP inherits its nuw, nusw, inbounds
2415 // flags.
2416 if (const Instruction *I = dyn_cast<Instruction>(Val: &U))
2417 PtrAddFlags = MachineInstr::copyFlagsFromInstruction(I: *I);
2418
2419 auto PtrAddFlagsWithConst = [&](int64_t Offset) {
2420 // For nusw/inbounds GEP with an offset that is nonnegative when interpreted
2421 // as signed, assume there is no unsigned overflow.
2422 if (Offset >= 0 && (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap))
2423 return PtrAddFlags | MachineInstr::MIFlag::NoUWrap;
2424 return PtrAddFlags;
2425 };
2426
2427 // Normalize Vector GEP - all scalar operands should be converted to the
2428 // splat vector.
2429 unsigned VectorWidth = 0;
2430
2431 // True if we should use a splat vector; using VectorWidth alone is not
2432 // sufficient.
2433 bool WantSplatVector = false;
2434 if (auto *VT = dyn_cast<VectorType>(Val: U.getType())) {
2435 VectorWidth = cast<FixedVectorType>(Val: VT)->getNumElements();
2436 // We don't produce 1 x N vectors; those are treated as scalars.
2437 WantSplatVector = VectorWidth > 1;
2438 }
2439
2440 if (cast<GEPOperator>(Val: U).hasAllZeroIndices())
2441 return translateCopy(U, Src: BaseReg, MIRBuilder);
2442
2443 // We might need to splat the base pointer into a vector if the offsets
2444 // are vectors.
2445 if (WantSplatVector && !PtrTy.isVector()) {
2446 BaseReg = MIRBuilder
2447 .buildSplatBuildVector(Res: LLT::fixed_vector(NumElements: VectorWidth, ScalarTy: PtrTy),
2448 Src: BaseReg)
2449 .getReg(Idx: 0);
2450 PtrIRTy = FixedVectorType::get(ElementType: PtrIRTy, NumElts: VectorWidth);
2451 PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
2452 OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
2453 OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
2454 }
2455
2456 int64_t Offset = 0;
2457 for (gep_type_iterator GTI = gep_type_begin(GEP: &U), E = gep_type_end(GEP: &U);
2458 GTI != E; ++GTI) {
2459 const Value *Idx = GTI.getOperand();
2460 if (StructType *StTy = GTI.getStructTypeOrNull()) {
2461 unsigned Field = cast<Constant>(Val: Idx)->getUniqueInteger().getZExtValue();
2462 Offset += DL->getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
2463 continue;
2464 } else {
2465 uint64_t ElementSize = GTI.getSequentialElementStride(DL: *DL);
2466
2467 // If this is a scalar constant or a splat vector of constants,
2468 // handle it quickly.
2469 if (const auto *CI = dyn_cast<ConstantInt>(Val: Idx)) {
2470 if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {
2471 Offset += ElementSize * *Val;
2472 continue;
2473 }
2474 }
2475
2476 if (Offset != 0) {
2477 auto OffsetMIB = MIRBuilder.buildConstant(Res: {OffsetTy}, Val: Offset);
2478 BaseReg = MIRBuilder
2479 .buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0),
2480 Flags: PtrAddFlagsWithConst(Offset))
2481 .getReg(Idx: 0);
2482 Offset = 0;
2483 }
2484
2485 Register IdxReg = getOrCreateVReg(Val: *Idx);
2486 LLT IdxTy = MRI->getType(Reg: IdxReg);
2487 if (IdxTy != OffsetTy) {
2488 if (!IdxTy.isVector() && WantSplatVector) {
2489 IdxReg = MIRBuilder
2490 .buildSplatBuildVector(Res: OffsetTy.changeElementType(NewEltTy: IdxTy),
2491 Src: IdxReg)
2492 .getReg(Idx: 0);
2493 }
2494
2495 IdxReg = MIRBuilder.buildSExtOrTrunc(Res: OffsetTy, Op: IdxReg).getReg(Idx: 0);
2496 }
2497
2498 // N = N + Idx * ElementSize;
2499 // Avoid doing it for ElementSize of 1.
2500 Register GepOffsetReg;
2501 if (ElementSize != 1) {
2502 auto ElementSizeMIB = MIRBuilder.buildConstant(
2503 Res: getLLTForType(Ty&: *OffsetIRTy, DL: *DL), Val: ElementSize);
2504
2505 // The multiplication is NUW if the GEP is NUW and NSW if the GEP is
2506 // NUSW.
2507 uint32_t ScaleFlags = PtrAddFlags & MachineInstr::MIFlag::NoUWrap;
2508 if (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap)
2509 ScaleFlags |= MachineInstr::MIFlag::NoSWrap;
2510
2511 GepOffsetReg =
2512 MIRBuilder.buildMul(Dst: OffsetTy, Src0: IdxReg, Src1: ElementSizeMIB, Flags: ScaleFlags)
2513 .getReg(Idx: 0);
2514 } else {
2515 GepOffsetReg = IdxReg;
2516 }
2517
2518 BaseReg =
2519 MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: GepOffsetReg, Flags: PtrAddFlags)
2520 .getReg(Idx: 0);
2521 }
2522 }
2523
2524 if (Offset != 0) {
2525 auto OffsetMIB =
2526 MIRBuilder.buildConstant(Res: OffsetTy, Val: Offset);
2527
2528 MIRBuilder.buildPtrAdd(Res: getOrCreateVReg(Val: U), Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0),
2529 Flags: PtrAddFlagsWithConst(Offset));
2530 return true;
2531 }
2532
2533 return translateCopy(U, Src: BaseReg, MIRBuilder);
2534}
2535
2536bool IRTranslatorImpl::translateMemFunc(const CallInst &CI,
2537 MachineIRBuilder &MIRBuilder,
2538 unsigned Opcode) {
2539 const Value *SrcPtr = CI.getArgOperand(i: 1);
2540 // If the source is undef, then just emit a nop.
2541 if (isa<UndefValue>(Val: SrcPtr))
2542 return true;
2543
2544 SmallVector<Register, 3> SrcRegs;
2545
2546 unsigned MinPtrSize = UINT_MAX;
2547 for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(x: AI) != AE; ++AI) {
2548 Register SrcReg = getOrCreateVReg(Val: **AI);
2549 LLT SrcTy = MRI->getType(Reg: SrcReg);
2550 if (SrcTy.isPointer())
2551 MinPtrSize = std::min<unsigned>(a: SrcTy.getSizeInBits(), b: MinPtrSize);
2552 SrcRegs.push_back(Elt: SrcReg);
2553 }
2554
2555 LLT SizeTy = LLT::integer(SizeInBits: MinPtrSize);
2556
2557 // The size operand should be the minimum of the pointer sizes.
2558 Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
2559 if (MRI->getType(Reg: SizeOpReg) != SizeTy)
2560 SizeOpReg = MIRBuilder.buildZExtOrTrunc(Res: SizeTy, Op: SizeOpReg).getReg(Idx: 0);
2561
2562 auto ICall = MIRBuilder.buildInstr(Opcode);
2563 for (Register SrcReg : SrcRegs)
2564 ICall.addUse(RegNo: SrcReg);
2565
2566 Align DstAlign;
2567 Align SrcAlign;
2568 unsigned IsVol =
2569 cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1))->getZExtValue();
2570
2571 ConstantInt *CopySize = nullptr;
2572
2573 if (auto *MCI = dyn_cast<MemCpyInst>(Val: &CI)) {
2574 DstAlign = MCI->getDestAlign().valueOrOne();
2575 SrcAlign = MCI->getSourceAlign().valueOrOne();
2576 CopySize = dyn_cast<ConstantInt>(Val: MCI->getArgOperand(i: 2));
2577 } else if (auto *MMI = dyn_cast<MemMoveInst>(Val: &CI)) {
2578 DstAlign = MMI->getDestAlign().valueOrOne();
2579 SrcAlign = MMI->getSourceAlign().valueOrOne();
2580 CopySize = dyn_cast<ConstantInt>(Val: MMI->getArgOperand(i: 2));
2581 } else {
2582 auto *MSI = cast<MemSetInst>(Val: &CI);
2583 DstAlign = MSI->getDestAlign().valueOrOne();
2584 }
2585
2586 if (Opcode != TargetOpcode::G_MEMCPY_INLINE &&
2587 Opcode != TargetOpcode::G_MEMSET_INLINE) {
2588 // We need to propagate the tail call flag from the IR inst as an argument.
2589 // Otherwise, we have to pessimize and assume later that we cannot tail call
2590 // any memory intrinsics.
2591 ICall.addImm(Val: CI.isTailCall() ? 1 : 0);
2592 }
2593
2594 // Create mem operands to store the alignment and volatile info.
2595 MachineMemOperand::Flags LoadFlags = MachineMemOperand::MOLoad;
2596 MachineMemOperand::Flags StoreFlags = MachineMemOperand::MOStore;
2597 if (IsVol) {
2598 LoadFlags |= MachineMemOperand::MOVolatile;
2599 StoreFlags |= MachineMemOperand::MOVolatile;
2600 }
2601
2602 AAMDNodes AAInfo = CI.getAAMetadata();
2603 if (AA && CopySize &&
2604 AA->pointsToConstantMemory(Loc: MemoryLocation(
2605 SrcPtr, LocationSize::precise(Value: CopySize->getZExtValue()), AAInfo))) {
2606 LoadFlags |= MachineMemOperand::MOInvariant;
2607
2608 // FIXME: pointsToConstantMemory probably does not imply dereferenceable,
2609 // but the previous usage implied it did. Probably should check
2610 // isDereferenceableAndAlignedPointer.
2611 LoadFlags |= MachineMemOperand::MODereferenceable;
2612 }
2613
2614 ICall.addMemOperand(
2615 MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(CI.getArgOperand(i: 0)),
2616 F: StoreFlags, Size: 1, BaseAlignment: DstAlign, Metadata: AAInfo));
2617 if (Opcode != TargetOpcode::G_MEMSET &&
2618 Opcode != TargetOpcode::G_MEMSET_INLINE)
2619 ICall.addMemOperand(MMO: MF->getMachineMemOperand(
2620 PtrInfo: MachinePointerInfo(SrcPtr), F: LoadFlags, Size: 1, BaseAlignment: SrcAlign, Metadata: AAInfo));
2621
2622 return true;
2623}
2624
2625bool IRTranslatorImpl::translateTrap(const CallInst &CI,
2626 MachineIRBuilder &MIRBuilder,
2627 unsigned Opcode) {
2628 StringRef TrapFuncName =
2629 CI.getAttributes().getFnAttr(Kind: "trap-func-name").getValueAsString();
2630 if (TrapFuncName.empty()) {
2631 MachineInstrBuilder MIB;
2632 if (Opcode == TargetOpcode::G_UBSANTRAP) {
2633 uint64_t Code = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 0))->getZExtValue();
2634 MIB = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: ArrayRef<llvm::SrcOp>{Code});
2635 } else {
2636 MIB = MIRBuilder.buildInstr(Opcode);
2637 }
2638 if (CI.cannotMerge())
2639 MIB.setMIFlag(MachineInstr::MIFlag::NoMerge);
2640 return true;
2641 }
2642
2643 CallLowering::CallLoweringInfo Info;
2644 if (Opcode == TargetOpcode::G_UBSANTRAP)
2645 Info.OrigArgs.push_back(Elt: {getOrCreateVRegs(Val: *CI.getArgOperand(i: 0)),
2646 CI.getArgOperand(i: 0)->getType(), 0});
2647
2648 Info.Callee = MachineOperand::CreateES(SymName: TrapFuncName.data());
2649 Info.CB = &CI;
2650 Info.OrigRet = {Register(), Type::getVoidTy(C&: CI.getContext()), 0};
2651 Info.NoMerge = CI.cannotMerge();
2652 return CLI->lowerCall(MIRBuilder, Info);
2653}
2654
2655bool IRTranslatorImpl::translateVectorInterleave2Intrinsic(
2656 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
2657 assert(CI.getIntrinsicID() == Intrinsic::vector_interleave2 &&
2658 "This function can only be called on the interleave2 intrinsic!");
2659 // Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG).
2660 Register Op0 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
2661 Register Op1 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1));
2662 Register Res = getOrCreateVReg(Val: CI);
2663
2664 LLT OpTy = MRI->getType(Reg: Op0);
2665 MIRBuilder.buildShuffleVector(Res, Src1: Op0, Src2: Op1,
2666 Mask: createInterleaveMask(VF: OpTy.getNumElements(), NumVecs: 2));
2667
2668 return true;
2669}
2670
2671bool IRTranslatorImpl::translateVectorDeinterleave2Intrinsic(
2672 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
2673 assert(CI.getIntrinsicID() == Intrinsic::vector_deinterleave2 &&
2674 "This function can only be called on the deinterleave2 intrinsic!");
2675 // Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to
2676 // SelectionDAG).
2677 Register Op = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
2678 auto Undef = MIRBuilder.buildUndef(Res: MRI->getType(Reg: Op));
2679 ArrayRef<Register> Res = getOrCreateVRegs(Val: CI);
2680
2681 LLT ResTy = MRI->getType(Reg: Res[0]);
2682 if (ResTy.isScalar()) {
2683 MIRBuilder.buildExtractVectorElementConstant(Res: Res[0], Val: Op, Idx: 0);
2684 MIRBuilder.buildExtractVectorElementConstant(Res: Res[1], Val: Op, Idx: 1);
2685
2686 return true;
2687 }
2688
2689 assert(ResTy.isVector() && "Expected vector result type");
2690 MIRBuilder.buildShuffleVector(Res: Res[0], Src1: Op, Src2: Undef,
2691 Mask: createStrideMask(Start: 0, Stride: 2, VF: ResTy.getNumElements()));
2692 MIRBuilder.buildShuffleVector(Res: Res[1], Src1: Op, Src2: Undef,
2693 Mask: createStrideMask(Start: 1, Stride: 2, VF: ResTy.getNumElements()));
2694
2695 return true;
2696}
2697
2698void IRTranslatorImpl::getStackGuard(Register DstReg,
2699 MachineIRBuilder &MIRBuilder) {
2700 Value *Global =
2701 TLI->getSDagStackGuard(M: *MF->getFunction().getParent(), Libcalls: *Libcalls);
2702 if (!Global) {
2703 LLVMContext &Ctx = MIRBuilder.getContext();
2704 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
2705 MIRBuilder.buildUndef(Res: DstReg);
2706 return;
2707 }
2708
2709 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
2710 MRI->setRegClass(Reg: DstReg,
2711 RC: TII.getRegClass(MCID: TII.get(Opcode: TargetOpcode::LOAD_STACK_GUARD), OpNum: 0));
2712 auto MIB =
2713 MIRBuilder.buildInstr(Opc: TargetOpcode::LOAD_STACK_GUARD, DstOps: {DstReg}, SrcOps: {});
2714
2715 unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
2716 LLT PtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL->getPointerSizeInBits(AS: AddrSpace));
2717
2718 MachinePointerInfo MPInfo(Global);
2719 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2720 MachineMemOperand::MODereferenceable;
2721 MachineMemOperand *MemRef = MF->getMachineMemOperand(
2722 PtrInfo: MPInfo, F: Flags, MemTy: PtrTy, BaseAlignment: DL->getPointerABIAlignment(AS: AddrSpace));
2723 MIB.setMemRefs({MemRef});
2724}
2725
2726bool IRTranslatorImpl::translateOverflowIntrinsic(
2727 const CallInst &CI, unsigned Op, MachineIRBuilder &MIRBuilder) {
2728 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: CI);
2729 MIRBuilder.buildInstr(
2730 Opc: Op, DstOps: {ResRegs[0], ResRegs[1]},
2731 SrcOps: {getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)), getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1))});
2732
2733 return true;
2734}
2735
2736bool IRTranslatorImpl::translateFixedPointIntrinsic(
2737 unsigned Op, const CallInst &CI, MachineIRBuilder &MIRBuilder) {
2738 Register Dst = getOrCreateVReg(Val: CI);
2739 Register Src0 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
2740 Register Src1 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1));
2741 uint64_t Scale = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
2742 MIRBuilder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: { Src0, Src1, Scale });
2743 return true;
2744}
2745
2746unsigned IRTranslatorImpl::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
2747 switch (ID) {
2748 default:
2749 break;
2750 case Intrinsic::acos:
2751 return TargetOpcode::G_FACOS;
2752 case Intrinsic::asin:
2753 return TargetOpcode::G_FASIN;
2754 case Intrinsic::atan:
2755 return TargetOpcode::G_FATAN;
2756 case Intrinsic::atan2:
2757 return TargetOpcode::G_FATAN2;
2758 case Intrinsic::bswap:
2759 return TargetOpcode::G_BSWAP;
2760 case Intrinsic::bitreverse:
2761 return TargetOpcode::G_BITREVERSE;
2762 case Intrinsic::clmul:
2763 return TargetOpcode::G_CLMUL;
2764 case Intrinsic::fshl:
2765 return TargetOpcode::G_FSHL;
2766 case Intrinsic::fshr:
2767 return TargetOpcode::G_FSHR;
2768 case Intrinsic::ceil:
2769 return TargetOpcode::G_FCEIL;
2770 case Intrinsic::cos:
2771 return TargetOpcode::G_FCOS;
2772 case Intrinsic::cosh:
2773 return TargetOpcode::G_FCOSH;
2774 case Intrinsic::ctpop:
2775 return TargetOpcode::G_CTPOP;
2776 case Intrinsic::smulh:
2777 return TargetOpcode::G_SMULH;
2778 case Intrinsic::umulh:
2779 return TargetOpcode::G_UMULH;
2780 case Intrinsic::exp:
2781 return TargetOpcode::G_FEXP;
2782 case Intrinsic::exp2:
2783 return TargetOpcode::G_FEXP2;
2784 case Intrinsic::exp10:
2785 return TargetOpcode::G_FEXP10;
2786 case Intrinsic::fabs:
2787 return TargetOpcode::G_FABS;
2788 case Intrinsic::copysign:
2789 return TargetOpcode::G_FCOPYSIGN;
2790 case Intrinsic::minnum:
2791 return TargetOpcode::G_FMINNUM;
2792 case Intrinsic::maxnum:
2793 return TargetOpcode::G_FMAXNUM;
2794 case Intrinsic::minimum:
2795 return TargetOpcode::G_FMINIMUM;
2796 case Intrinsic::maximum:
2797 return TargetOpcode::G_FMAXIMUM;
2798 case Intrinsic::minimumnum:
2799 return TargetOpcode::G_FMINIMUMNUM;
2800 case Intrinsic::maximumnum:
2801 return TargetOpcode::G_FMAXIMUMNUM;
2802 case Intrinsic::canonicalize:
2803 return TargetOpcode::G_FCANONICALIZE;
2804 case Intrinsic::floor:
2805 return TargetOpcode::G_FFLOOR;
2806 case Intrinsic::fma:
2807 return TargetOpcode::G_FMA;
2808 case Intrinsic::log:
2809 return TargetOpcode::G_FLOG;
2810 case Intrinsic::log2:
2811 return TargetOpcode::G_FLOG2;
2812 case Intrinsic::log10:
2813 return TargetOpcode::G_FLOG10;
2814 case Intrinsic::ldexp:
2815 return TargetOpcode::G_FLDEXP;
2816 case Intrinsic::nearbyint:
2817 return TargetOpcode::G_FNEARBYINT;
2818 case Intrinsic::pow:
2819 return TargetOpcode::G_FPOW;
2820 case Intrinsic::powi:
2821 return TargetOpcode::G_FPOWI;
2822 case Intrinsic::rint:
2823 return TargetOpcode::G_FRINT;
2824 case Intrinsic::round:
2825 return TargetOpcode::G_INTRINSIC_ROUND;
2826 case Intrinsic::roundeven:
2827 return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
2828 case Intrinsic::sin:
2829 return TargetOpcode::G_FSIN;
2830 case Intrinsic::sinh:
2831 return TargetOpcode::G_FSINH;
2832 case Intrinsic::sqrt:
2833 return TargetOpcode::G_FSQRT;
2834 case Intrinsic::tan:
2835 return TargetOpcode::G_FTAN;
2836 case Intrinsic::tanh:
2837 return TargetOpcode::G_FTANH;
2838 case Intrinsic::trunc:
2839 return TargetOpcode::G_INTRINSIC_TRUNC;
2840 case Intrinsic::readcyclecounter:
2841 return TargetOpcode::G_READCYCLECOUNTER;
2842 case Intrinsic::readsteadycounter:
2843 return TargetOpcode::G_READSTEADYCOUNTER;
2844 case Intrinsic::ptrmask:
2845 return TargetOpcode::G_PTRMASK;
2846 case Intrinsic::lrint:
2847 return TargetOpcode::G_INTRINSIC_LRINT;
2848 case Intrinsic::llrint:
2849 return TargetOpcode::G_INTRINSIC_LLRINT;
2850 // FADD/FMUL require checking the FMF, so are handled elsewhere.
2851 case Intrinsic::vector_reduce_fmin:
2852 return TargetOpcode::G_VECREDUCE_FMIN;
2853 case Intrinsic::vector_reduce_fmax:
2854 return TargetOpcode::G_VECREDUCE_FMAX;
2855 case Intrinsic::vector_reduce_fminimum:
2856 return TargetOpcode::G_VECREDUCE_FMINIMUM;
2857 case Intrinsic::vector_reduce_fmaximum:
2858 return TargetOpcode::G_VECREDUCE_FMAXIMUM;
2859 case Intrinsic::vector_reduce_fminimumnum:
2860 return TargetOpcode::G_VECREDUCE_FMINIMUMNUM;
2861 case Intrinsic::vector_reduce_fmaximumnum:
2862 return TargetOpcode::G_VECREDUCE_FMAXIMUMNUM;
2863 case Intrinsic::vector_reduce_add:
2864 return TargetOpcode::G_VECREDUCE_ADD;
2865 case Intrinsic::vector_reduce_mul:
2866 return TargetOpcode::G_VECREDUCE_MUL;
2867 case Intrinsic::vector_reduce_and:
2868 return TargetOpcode::G_VECREDUCE_AND;
2869 case Intrinsic::vector_reduce_or:
2870 return TargetOpcode::G_VECREDUCE_OR;
2871 case Intrinsic::vector_reduce_xor:
2872 return TargetOpcode::G_VECREDUCE_XOR;
2873 case Intrinsic::vector_reduce_smax:
2874 return TargetOpcode::G_VECREDUCE_SMAX;
2875 case Intrinsic::vector_reduce_smin:
2876 return TargetOpcode::G_VECREDUCE_SMIN;
2877 case Intrinsic::vector_reduce_umax:
2878 return TargetOpcode::G_VECREDUCE_UMAX;
2879 case Intrinsic::vector_reduce_umin:
2880 return TargetOpcode::G_VECREDUCE_UMIN;
2881 case Intrinsic::experimental_vector_compress:
2882 return TargetOpcode::G_VECTOR_COMPRESS;
2883 case Intrinsic::lround:
2884 return TargetOpcode::G_LROUND;
2885 case Intrinsic::llround:
2886 return TargetOpcode::G_LLROUND;
2887 case Intrinsic::get_fpenv:
2888 return TargetOpcode::G_GET_FPENV;
2889 case Intrinsic::get_fpmode:
2890 return TargetOpcode::G_GET_FPMODE;
2891 }
2892 return Intrinsic::not_intrinsic;
2893}
2894
2895bool IRTranslatorImpl::translateSimpleIntrinsic(const CallInst &CI,
2896 Intrinsic::ID ID,
2897 MachineIRBuilder &MIRBuilder) {
2898
2899 unsigned Op = getSimpleIntrinsicOpcode(ID);
2900
2901 // Is this a simple intrinsic?
2902 if (Op == Intrinsic::not_intrinsic)
2903 return false;
2904
2905 // Yes. Let's translate it.
2906 SmallVector<llvm::SrcOp, 4> VRegs;
2907 for (const auto &Arg : CI.args())
2908 VRegs.push_back(Elt: getOrCreateVReg(Val: *Arg));
2909
2910 MIRBuilder.buildInstr(Opc: Op, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: VRegs,
2911 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2912 return true;
2913}
2914
2915// TODO: Include ConstainedOps.def when all strict instructions are defined.
2916static unsigned getConstrainedOpcode(Intrinsic::ID ID) {
2917 switch (ID) {
2918 case Intrinsic::experimental_constrained_fadd:
2919 return TargetOpcode::G_STRICT_FADD;
2920 case Intrinsic::experimental_constrained_fsub:
2921 return TargetOpcode::G_STRICT_FSUB;
2922 case Intrinsic::experimental_constrained_fmul:
2923 return TargetOpcode::G_STRICT_FMUL;
2924 case Intrinsic::experimental_constrained_fdiv:
2925 return TargetOpcode::G_STRICT_FDIV;
2926 case Intrinsic::experimental_constrained_frem:
2927 return TargetOpcode::G_STRICT_FREM;
2928 case Intrinsic::experimental_constrained_fma:
2929 return TargetOpcode::G_STRICT_FMA;
2930 case Intrinsic::experimental_constrained_sqrt:
2931 return TargetOpcode::G_STRICT_FSQRT;
2932 case Intrinsic::experimental_constrained_ldexp:
2933 return TargetOpcode::G_STRICT_FLDEXP;
2934 case Intrinsic::experimental_constrained_fcmp:
2935 return TargetOpcode::G_STRICT_FCMP;
2936 case Intrinsic::experimental_constrained_fcmps:
2937 return TargetOpcode::G_STRICT_FCMPS;
2938 default:
2939 return 0;
2940 }
2941}
2942
2943bool IRTranslatorImpl::translateConstrainedFPIntrinsic(
2944 const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
2945 fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
2946
2947 unsigned Opcode = getConstrainedOpcode(ID: FPI.getIntrinsicID());
2948 if (!Opcode)
2949 return false;
2950
2951 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: FPI);
2952 if (EB == fp::ExceptionBehavior::ebIgnore)
2953 Flags |= MachineInstr::NoFPExcept;
2954
2955 if (Opcode == TargetOpcode::G_STRICT_FCMP ||
2956 Opcode == TargetOpcode::G_STRICT_FCMPS) {
2957 auto *FPCmp = cast<ConstrainedFPCmpIntrinsic>(Val: &FPI);
2958 Register Operand0 = getOrCreateVReg(Val: *FPCmp->getArgOperand(i: 0));
2959 Register Operand1 = getOrCreateVReg(Val: *FPCmp->getArgOperand(i: 1));
2960 Register Result = getOrCreateVReg(Val: FPI);
2961 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Result}, SrcOps: {}, Flags)
2962 .addPredicate(Pred: FPCmp->getPredicate())
2963 .addUse(RegNo: Operand0)
2964 .addUse(RegNo: Operand1);
2965 return true;
2966 }
2967
2968 SmallVector<llvm::SrcOp, 4> VRegs;
2969 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
2970 VRegs.push_back(Elt: getOrCreateVReg(Val: *FPI.getArgOperand(i: I)));
2971
2972 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: FPI)}, SrcOps: VRegs, Flags);
2973 return true;
2974}
2975
2976std::optional<MCRegister> IRTranslatorImpl::getArgPhysReg(Argument &Arg) {
2977 auto VRegs = getOrCreateVRegs(Val: Arg);
2978 if (VRegs.size() != 1)
2979 return std::nullopt;
2980
2981 // Arguments are lowered as a copy of a livein physical register.
2982 auto *VRegDef = MF->getRegInfo().getVRegDef(Reg: VRegs[0]);
2983 if (!VRegDef || !VRegDef->isCopy())
2984 return std::nullopt;
2985 return VRegDef->getOperand(i: 1).getReg().asMCReg();
2986}
2987
2988bool IRTranslatorImpl::translateIfEntryValueArgument(
2989 bool isDeclare, Value *Val, const DILocalVariable *Var,
2990 const DIExpression *Expr, const DebugLoc &DL,
2991 MachineIRBuilder &MIRBuilder) {
2992 auto *Arg = dyn_cast<Argument>(Val);
2993 if (!Arg)
2994 return false;
2995
2996 if (!Expr->isEntryValue())
2997 return false;
2998
2999 std::optional<MCRegister> PhysReg = getArgPhysReg(Arg&: *Arg);
3000 if (!PhysReg) {
3001 LLVM_DEBUG(dbgs() << "Dropping dbg." << (isDeclare ? "declare" : "value")
3002 << ": expression is entry_value but "
3003 << "couldn't find a physical register\n");
3004 LLVM_DEBUG(dbgs() << *Var << "\n");
3005 return true;
3006 }
3007
3008 if (isDeclare) {
3009 // Append an op deref to account for the fact that this is a dbg_declare.
3010 Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
3011 MF->setVariableDbgInfo(Var, Expr, Reg: *PhysReg, Loc: DL);
3012 } else {
3013 MIRBuilder.buildDirectDbgValue(Reg: *PhysReg, Variable: Var, Expr);
3014 }
3015
3016 return true;
3017}
3018
3019static unsigned getConvOpcode(Intrinsic::ID ID) {
3020 switch (ID) {
3021 default:
3022 llvm_unreachable("Unexpected intrinsic");
3023 case Intrinsic::experimental_convergence_anchor:
3024 return TargetOpcode::CONVERGENCECTRL_ANCHOR;
3025 case Intrinsic::experimental_convergence_entry:
3026 return TargetOpcode::CONVERGENCECTRL_ENTRY;
3027 case Intrinsic::experimental_convergence_loop:
3028 return TargetOpcode::CONVERGENCECTRL_LOOP;
3029 }
3030}
3031
3032bool IRTranslatorImpl::translateConvergenceControlIntrinsic(
3033 const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
3034 MachineInstrBuilder MIB = MIRBuilder.buildInstr(Opcode: getConvOpcode(ID));
3035 Register OutputReg = getOrCreateConvergenceTokenVReg(Token: CI);
3036 MIB.addDef(RegNo: OutputReg);
3037
3038 if (ID == Intrinsic::experimental_convergence_loop) {
3039 auto Bundle = CI.getOperandBundle(ID: LLVMContext::OB_convergencectrl);
3040 assert(Bundle && "Expected a convergence control token.");
3041 Register InputReg =
3042 getOrCreateConvergenceTokenVReg(Token: *Bundle->Inputs[0].get());
3043 MIB.addUse(RegNo: InputReg);
3044 }
3045
3046 return true;
3047}
3048
3049bool IRTranslatorImpl::translateKnownIntrinsic(const CallInst &CI,
3050 Intrinsic::ID ID,
3051 MachineIRBuilder &MIRBuilder) {
3052 if (auto *MI = dyn_cast<AnyMemIntrinsic>(Val: &CI)) {
3053 if (ORE->enabled()) {
3054 if (MemoryOpRemark::canHandle(I: MI, TLI: *LibInfo)) {
3055 MemoryOpRemark R(*ORE, "gisel-ir-translator-memsize", *DL, *LibInfo);
3056 R.visit(I: MI);
3057 }
3058 }
3059 }
3060
3061 // If this is a simple intrinsic (that is, we just need to add a def of
3062 // a vreg, and uses for each arg operand, then translate it.
3063 if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
3064 return true;
3065
3066 switch (ID) {
3067 default:
3068 break;
3069 case Intrinsic::lifetime_start:
3070 case Intrinsic::lifetime_end: {
3071 // No stack colouring in O0, discard region information.
3072 if (MF->getTarget().getOptLevel() == CodeGenOptLevel::None ||
3073 MF->getFunction().hasOptNone())
3074 return true;
3075
3076 unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
3077 : TargetOpcode::LIFETIME_END;
3078
3079 const AllocaInst *AI = dyn_cast<AllocaInst>(Val: CI.getArgOperand(i: 0));
3080 if (!AI || !AI->isStaticAlloca())
3081 return true;
3082
3083 MIRBuilder.buildInstr(Opcode: Op).addFrameIndex(Idx: getOrCreateFrameIndex(AI: *AI));
3084 return true;
3085 }
3086 case Intrinsic::fake_use: {
3087 SmallVector<llvm::SrcOp, 4> VRegs;
3088 for (const auto &Arg : CI.args())
3089 llvm::append_range(C&: VRegs, R: getOrCreateVRegs(Val: *Arg));
3090 MIRBuilder.buildInstr(Opc: TargetOpcode::FAKE_USE, DstOps: {}, SrcOps: VRegs);
3091 MF->setHasFakeUses(true);
3092 return true;
3093 }
3094 case Intrinsic::vaend:
3095 // No target I know of cares about va_end. Certainly no in-tree target
3096 // does. Simplest intrinsic ever!
3097 return true;
3098 case Intrinsic::vastart: {
3099 Value *Ptr = CI.getArgOperand(i: 0);
3100 unsigned ListSize = TLI->getVaListSizeInBits(DL: *DL) / 8;
3101 Align Alignment = getKnownAlignment(V: Ptr, DL: *DL);
3102
3103 MIRBuilder.buildInstr(Opc: TargetOpcode::G_VASTART, DstOps: {}, SrcOps: {getOrCreateVReg(Val: *Ptr)})
3104 .addMemOperand(MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Ptr),
3105 F: MachineMemOperand::MOStore,
3106 Size: ListSize, BaseAlignment: Alignment));
3107 return true;
3108 }
3109 case Intrinsic::uadd_with_overflow:
3110 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UADDO, MIRBuilder);
3111 case Intrinsic::sadd_with_overflow:
3112 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SADDO, MIRBuilder);
3113 case Intrinsic::usub_with_overflow:
3114 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_USUBO, MIRBuilder);
3115 case Intrinsic::ssub_with_overflow:
3116 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SSUBO, MIRBuilder);
3117 case Intrinsic::umul_with_overflow:
3118 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UMULO, MIRBuilder);
3119 case Intrinsic::smul_with_overflow:
3120 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SMULO, MIRBuilder);
3121 case Intrinsic::uadd_sat:
3122 return translateBinaryOp(Opcode: TargetOpcode::G_UADDSAT, U: CI, MIRBuilder);
3123 case Intrinsic::sadd_sat:
3124 return translateBinaryOp(Opcode: TargetOpcode::G_SADDSAT, U: CI, MIRBuilder);
3125 case Intrinsic::usub_sat:
3126 return translateBinaryOp(Opcode: TargetOpcode::G_USUBSAT, U: CI, MIRBuilder);
3127 case Intrinsic::ssub_sat:
3128 return translateBinaryOp(Opcode: TargetOpcode::G_SSUBSAT, U: CI, MIRBuilder);
3129 case Intrinsic::ushl_sat:
3130 return translateBinaryOp(Opcode: TargetOpcode::G_USHLSAT, U: CI, MIRBuilder);
3131 case Intrinsic::sshl_sat:
3132 return translateBinaryOp(Opcode: TargetOpcode::G_SSHLSAT, U: CI, MIRBuilder);
3133 case Intrinsic::umin:
3134 return translateBinaryOp(Opcode: TargetOpcode::G_UMIN, U: CI, MIRBuilder);
3135 case Intrinsic::umax:
3136 return translateBinaryOp(Opcode: TargetOpcode::G_UMAX, U: CI, MIRBuilder);
3137 case Intrinsic::smin:
3138 return translateBinaryOp(Opcode: TargetOpcode::G_SMIN, U: CI, MIRBuilder);
3139 case Intrinsic::smax:
3140 return translateBinaryOp(Opcode: TargetOpcode::G_SMAX, U: CI, MIRBuilder);
3141 case Intrinsic::abs:
3142 // TODO: Preserve "int min is poison" arg in GMIR?
3143 return translateUnaryOp(Opcode: TargetOpcode::G_ABS, U: CI, MIRBuilder);
3144 case Intrinsic::smul_fix:
3145 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIX, CI, MIRBuilder);
3146 case Intrinsic::umul_fix:
3147 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIX, CI, MIRBuilder);
3148 case Intrinsic::smul_fix_sat:
3149 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
3150 case Intrinsic::umul_fix_sat:
3151 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
3152 case Intrinsic::sdiv_fix:
3153 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
3154 case Intrinsic::udiv_fix:
3155 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
3156 case Intrinsic::sdiv_fix_sat:
3157 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
3158 case Intrinsic::udiv_fix_sat:
3159 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
3160 case Intrinsic::fmuladd: {
3161 Register Dst = getOrCreateVReg(Val: CI);
3162 Register Op0 = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
3163 Register Op1 = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
3164 Register Op2 = getOrCreateVReg(Val: *CI.getArgOperand(i: 2));
3165 if (TLI->isFMAFasterThanFMulAndFAdd(MF: *MF,
3166 TLI->getValueType(DL: *DL, Ty: CI.getType()))) {
3167 // TODO: Revisit this to see if we should move this part of the
3168 // lowering to the combiner.
3169 MIRBuilder.buildFMA(Dst, Src0: Op0, Src1: Op1, Src2: Op2,
3170 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3171 } else {
3172 LLT Ty = getLLTForType(Ty&: *CI.getType(), DL: *DL);
3173 auto FMul = MIRBuilder.buildFMul(
3174 Dst: Ty, Src0: Op0, Src1: Op1, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3175 MIRBuilder.buildFAdd(Dst, Src0: FMul, Src1: Op2,
3176 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3177 }
3178 return true;
3179 }
3180 case Intrinsic::frexp: {
3181 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
3182 MIRBuilder.buildFFrexp(Fract: VRegs[0], Exp: VRegs[1],
3183 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
3184 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3185 return true;
3186 }
3187 case Intrinsic::modf: {
3188 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
3189 MIRBuilder.buildModf(Fract: VRegs[0], Int: VRegs[1],
3190 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
3191 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3192 return true;
3193 }
3194 case Intrinsic::sincos: {
3195 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
3196 MIRBuilder.buildFSincos(Sin: VRegs[0], Cos: VRegs[1],
3197 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
3198 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3199 return true;
3200 }
3201 case Intrinsic::fptosi_sat:
3202 MIRBuilder.buildFPTOSI_SAT(Dst: getOrCreateVReg(Val: CI),
3203 Src0: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
3204 return true;
3205 case Intrinsic::fptoui_sat:
3206 MIRBuilder.buildFPTOUI_SAT(Dst: getOrCreateVReg(Val: CI),
3207 Src0: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
3208 return true;
3209 case Intrinsic::memcpy_inline:
3210 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY_INLINE);
3211 case Intrinsic::memcpy:
3212 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY);
3213 case Intrinsic::memmove:
3214 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMMOVE);
3215 case Intrinsic::memset:
3216 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMSET);
3217 case Intrinsic::memset_inline:
3218 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMSET_INLINE);
3219 case Intrinsic::eh_typeid_for: {
3220 GlobalValue *GV = ExtractTypeInfo(V: CI.getArgOperand(i: 0));
3221 Register Reg = getOrCreateVReg(Val: CI);
3222 unsigned TypeID = MF->getTypeIDFor(TI: GV);
3223 MIRBuilder.buildConstant(Res: Reg, Val: TypeID);
3224 return true;
3225 }
3226 case Intrinsic::objectsize:
3227 llvm_unreachable("llvm.objectsize.* should have been lowered already");
3228
3229 case Intrinsic::is_constant:
3230 llvm_unreachable("llvm.is.constant.* should have been lowered already");
3231
3232 case Intrinsic::stackguard:
3233 getStackGuard(DstReg: getOrCreateVReg(Val: CI), MIRBuilder);
3234 return true;
3235 case Intrinsic::stackprotector: {
3236 LLT PtrTy = getLLTForType(Ty&: *CI.getArgOperand(i: 0)->getType(), DL: *DL);
3237 Register GuardVal;
3238 if (TLI->useLoadStackGuardNode(M: *CI.getModule())) {
3239 GuardVal = MRI->createGenericVirtualRegister(Ty: PtrTy);
3240 getStackGuard(DstReg: GuardVal, MIRBuilder);
3241 } else
3242 GuardVal = getOrCreateVReg(Val: *CI.getArgOperand(i: 0)); // The guard's value.
3243
3244 AllocaInst *Slot = cast<AllocaInst>(Val: CI.getArgOperand(i: 1));
3245 int FI = getOrCreateFrameIndex(AI: *Slot);
3246 MF->getFrameInfo().setStackProtectorIndex(FI);
3247
3248 MIRBuilder.buildStore(
3249 Val: GuardVal, Addr: getOrCreateVReg(Val: *Slot),
3250 MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI),
3251 F: MachineMemOperand::MOStore |
3252 MachineMemOperand::MOVolatile,
3253 MemTy: PtrTy, BaseAlignment: Align(8)));
3254 return true;
3255 }
3256 case Intrinsic::stacksave: {
3257 MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKSAVE, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {});
3258 return true;
3259 }
3260 case Intrinsic::stackrestore: {
3261 MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKRESTORE, DstOps: {},
3262 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
3263 return true;
3264 }
3265 case Intrinsic::cttz:
3266 case Intrinsic::ctlz: {
3267 ConstantInt *Cst = cast<ConstantInt>(Val: CI.getArgOperand(i: 1));
3268 bool isTrailing = ID == Intrinsic::cttz;
3269 unsigned Opcode = isTrailing ? Cst->isZero()
3270 ? TargetOpcode::G_CTTZ
3271 : TargetOpcode::G_CTTZ_ZERO_POISON
3272 : Cst->isZero() ? TargetOpcode::G_CTLZ
3273 : TargetOpcode::G_CTLZ_ZERO_POISON;
3274 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: CI)},
3275 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
3276 return true;
3277 }
3278 case Intrinsic::invariant_start: {
3279 MIRBuilder.buildUndef(Res: getOrCreateVReg(Val: CI));
3280 return true;
3281 }
3282 case Intrinsic::invariant_end:
3283 return true;
3284 case Intrinsic::expect:
3285 case Intrinsic::expect_with_probability:
3286 case Intrinsic::annotation:
3287 case Intrinsic::ptr_annotation:
3288 case Intrinsic::launder_invariant_group:
3289 case Intrinsic::threadlocal_address: {
3290 // Drop the intrinsic, but forward the value.
3291 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: CI),
3292 Op: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
3293 return true;
3294 }
3295 case Intrinsic::assume:
3296 case Intrinsic::experimental_noalias_scope_decl:
3297 case Intrinsic::var_annotation:
3298 case Intrinsic::sideeffect:
3299 // Discard annotate attributes, assumptions, and artificial side-effects.
3300 return true;
3301 case Intrinsic::read_volatile_register:
3302 case Intrinsic::read_register: {
3303 Value *Arg = CI.getArgOperand(i: 0);
3304 MIRBuilder
3305 .buildInstr(Opc: TargetOpcode::G_READ_REGISTER, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {})
3306 .addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()));
3307 return true;
3308 }
3309 case Intrinsic::write_register: {
3310 Value *Arg = CI.getArgOperand(i: 0);
3311 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_WRITE_REGISTER)
3312 .addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()))
3313 .addUse(RegNo: getOrCreateVReg(Val: *CI.getArgOperand(i: 1)));
3314 return true;
3315 }
3316 case Intrinsic::localescape: {
3317 MachineBasicBlock &EntryMBB = MF->front();
3318 StringRef EscapedName = GlobalValue::dropLLVMManglingEscape(Name: MF->getName());
3319
3320 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
3321 // is the same on all targets.
3322 for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
3323 Value *Arg = CI.getArgOperand(i: Idx)->stripPointerCasts();
3324 if (isa<ConstantPointerNull>(Val: Arg))
3325 continue; // Skip null pointers. They represent a hole in index space.
3326
3327 int FI = getOrCreateFrameIndex(AI: *cast<AllocaInst>(Val: Arg));
3328 MCSymbol *FrameAllocSym =
3329 MF->getContext().getOrCreateFrameAllocSymbol(FuncName: EscapedName, Idx);
3330
3331 // This should be inserted at the start of the entry block.
3332 auto LocalEscape =
3333 MIRBuilder.buildInstrNoInsert(Opcode: TargetOpcode::LOCAL_ESCAPE)
3334 .addSym(Sym: FrameAllocSym)
3335 .addFrameIndex(Idx: FI);
3336
3337 EntryMBB.insert(I: EntryMBB.begin(), MI: LocalEscape);
3338 }
3339
3340 return true;
3341 }
3342 case Intrinsic::vector_reduce_fadd:
3343 case Intrinsic::vector_reduce_fmul: {
3344 // Need to check for the reassoc flag to decide whether we want a
3345 // sequential reduction opcode or not.
3346 Register Dst = getOrCreateVReg(Val: CI);
3347 Register ScalarSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
3348 Register VecSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
3349 unsigned Opc = 0;
3350 if (!CI.hasAllowReassoc()) {
3351 // The sequential ordering case.
3352 Opc = ID == Intrinsic::vector_reduce_fadd
3353 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
3354 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
3355 if (!MRI->getType(Reg: VecSrc).isVector())
3356 Opc = ID == Intrinsic::vector_reduce_fadd ? TargetOpcode::G_FADD
3357 : TargetOpcode::G_FMUL;
3358 MIRBuilder.buildInstr(Opc, DstOps: {Dst}, SrcOps: {ScalarSrc, VecSrc},
3359 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3360 return true;
3361 }
3362 // We split the operation into a separate G_FADD/G_FMUL + the reduce,
3363 // since the associativity doesn't matter.
3364 unsigned ScalarOpc;
3365 if (ID == Intrinsic::vector_reduce_fadd) {
3366 Opc = TargetOpcode::G_VECREDUCE_FADD;
3367 ScalarOpc = TargetOpcode::G_FADD;
3368 } else {
3369 Opc = TargetOpcode::G_VECREDUCE_FMUL;
3370 ScalarOpc = TargetOpcode::G_FMUL;
3371 }
3372 LLT DstTy = MRI->getType(Reg: Dst);
3373 auto Rdx = MIRBuilder.buildInstr(
3374 Opc, DstOps: {DstTy}, SrcOps: {VecSrc}, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3375 MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {Dst}, SrcOps: {ScalarSrc, Rdx},
3376 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
3377
3378 return true;
3379 }
3380 case Intrinsic::trap:
3381 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_TRAP);
3382 case Intrinsic::debugtrap:
3383 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_DEBUGTRAP);
3384 case Intrinsic::ubsantrap:
3385 return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_UBSANTRAP);
3386 case Intrinsic::allow_runtime_check:
3387 case Intrinsic::allow_ubsan_check:
3388 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: CI),
3389 Op: getOrCreateVReg(Val: *ConstantInt::getTrue(Ty: CI.getType())));
3390 return true;
3391 case Intrinsic::amdgcn_cs_chain:
3392 case Intrinsic::amdgcn_call_whole_wave:
3393 return translateCallBase(CB: CI, MIRBuilder);
3394 case Intrinsic::fptrunc_round: {
3395 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: CI);
3396
3397 // Convert the metadata argument to a constant integer
3398 Metadata *MD = cast<MetadataAsValue>(Val: CI.getArgOperand(i: 1))->getMetadata();
3399 std::optional<RoundingMode> RoundMode =
3400 convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
3401
3402 // Add the Rounding mode as an integer
3403 MIRBuilder
3404 .buildInstr(Opc: TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
3405 DstOps: {getOrCreateVReg(Val: CI)},
3406 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))}, Flags)
3407 .addImm(Val: (int)*RoundMode);
3408
3409 return true;
3410 }
3411 case Intrinsic::is_fpclass: {
3412 Value *FpValue = CI.getOperand(i_nocapture: 0);
3413 ConstantInt *TestMaskValue = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1));
3414
3415 MIRBuilder
3416 .buildInstr(Opc: TargetOpcode::G_IS_FPCLASS, DstOps: {getOrCreateVReg(Val: CI)},
3417 SrcOps: {getOrCreateVReg(Val: *FpValue)})
3418 .addImm(Val: TestMaskValue->getZExtValue());
3419
3420 return true;
3421 }
3422 case Intrinsic::set_fpenv: {
3423 Value *FPEnv = CI.getOperand(i_nocapture: 0);
3424 MIRBuilder.buildSetFPEnv(Src: getOrCreateVReg(Val: *FPEnv));
3425 return true;
3426 }
3427 case Intrinsic::reset_fpenv:
3428 MIRBuilder.buildResetFPEnv();
3429 return true;
3430 case Intrinsic::set_fpmode: {
3431 Value *FPState = CI.getOperand(i_nocapture: 0);
3432 MIRBuilder.buildSetFPMode(Src: getOrCreateVReg(Val: *FPState));
3433 return true;
3434 }
3435 case Intrinsic::reset_fpmode:
3436 MIRBuilder.buildResetFPMode();
3437 return true;
3438 case Intrinsic::get_rounding:
3439 MIRBuilder.buildGetRounding(Dst: getOrCreateVReg(Val: CI));
3440 return true;
3441 case Intrinsic::set_rounding:
3442 MIRBuilder.buildSetRounding(Src: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)));
3443 return true;
3444 case Intrinsic::vscale: {
3445 MIRBuilder.buildVScale(Res: getOrCreateVReg(Val: CI), MinElts: 1);
3446 return true;
3447 }
3448 case Intrinsic::scmp:
3449 MIRBuilder.buildSCmp(Res: getOrCreateVReg(Val: CI),
3450 Op0: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)),
3451 Op1: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1)));
3452 return true;
3453 case Intrinsic::ucmp:
3454 MIRBuilder.buildUCmp(Res: getOrCreateVReg(Val: CI),
3455 Op0: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)),
3456 Op1: getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1)));
3457 return true;
3458 case Intrinsic::vector_extract:
3459 return translateExtractVector(U: CI, MIRBuilder);
3460 case Intrinsic::vector_insert:
3461 return translateInsertVector(U: CI, MIRBuilder);
3462 case Intrinsic::stepvector: {
3463 MIRBuilder.buildStepVector(Res: getOrCreateVReg(Val: CI), Step: 1);
3464 return true;
3465 }
3466 case Intrinsic::prefetch: {
3467 Value *Addr = CI.getOperand(i_nocapture: 0);
3468 unsigned RW = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1))->getZExtValue();
3469 unsigned Locality = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
3470 unsigned CacheType = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 3))->getZExtValue();
3471
3472 auto Flags = RW ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
3473 auto &MMO = *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Addr), F: Flags,
3474 MemTy: LLT(), BaseAlignment: Align());
3475
3476 MIRBuilder.buildPrefetch(Addr: getOrCreateVReg(Val: *Addr), RW, Locality, CacheType,
3477 MMO);
3478
3479 return true;
3480 }
3481
3482 case Intrinsic::speculative_load: {
3483 // Only the pointer operand is needed at codegen; the remaining arguments
3484 // carry IR-level semantics only.
3485 const Value *Ptr = CI.getArgOperand(i: 0);
3486 Register Dst = getOrCreateVReg(Val: CI);
3487 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad;
3488 Flags |= TLI->getTargetMMOFlags(I: CI);
3489 if (CI.hasMetadata(KindID: LLVMContext::MD_nontemporal))
3490 Flags |= MachineMemOperand::MONonTemporal;
3491 if (CI.hasMetadata(KindID: LLVMContext::MD_invariant_load))
3492 Flags |= MachineMemOperand::MOInvariant;
3493 auto *MMO = MF->getMachineMemOperand(
3494 PtrInfo: MachinePointerInfo(Ptr), F: Flags, MemTy: MRI->getType(Reg: Dst),
3495 BaseAlignment: CI.getParamAlign(ArgNo: 0).valueOrOne(), Metadata: MMOMetadata(CI.getAAMetadata()));
3496 MIRBuilder.buildLoad(Res: Dst, Addr: getOrCreateVReg(Val: *Ptr), MMO&: *MMO);
3497 return true;
3498 }
3499
3500 case Intrinsic::vector_interleave2:
3501 case Intrinsic::vector_deinterleave2: {
3502 // Both intrinsics have at least one operand.
3503 Value *Op0 = CI.getOperand(i_nocapture: 0);
3504 LLT ResTy = getLLTForType(Ty&: *Op0->getType(), DL: MIRBuilder.getDataLayout());
3505 if (!ResTy.isFixedVector())
3506 return false;
3507
3508 if (CI.getIntrinsicID() == Intrinsic::vector_interleave2)
3509 return translateVectorInterleave2Intrinsic(CI, MIRBuilder);
3510
3511 return translateVectorDeinterleave2Intrinsic(CI, MIRBuilder);
3512 }
3513
3514#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
3515 case Intrinsic::INTRINSIC:
3516#include "llvm/IR/ConstrainedOps.def"
3517 return translateConstrainedFPIntrinsic(FPI: cast<ConstrainedFPIntrinsic>(Val: CI),
3518 MIRBuilder);
3519 case Intrinsic::experimental_convergence_anchor:
3520 case Intrinsic::experimental_convergence_entry:
3521 case Intrinsic::experimental_convergence_loop:
3522 return translateConvergenceControlIntrinsic(CI, ID, MIRBuilder);
3523 case Intrinsic::reloc_none: {
3524 Metadata *MD = cast<MetadataAsValue>(Val: CI.getArgOperand(i: 0))->getMetadata();
3525 StringRef SymbolName = cast<MDString>(Val: MD)->getString();
3526 MIRBuilder.buildInstr(Opcode: TargetOpcode::RELOC_NONE)
3527 .addExternalSymbol(FnName: SymbolName.data());
3528 return true;
3529 }
3530 }
3531 return false;
3532}
3533
3534bool IRTranslatorImpl::translateInlineAsm(const CallBase &CB,
3535 MachineIRBuilder &MIRBuilder) {
3536 if (!mayTranslateUserTypes(U: CB))
3537 return false;
3538
3539 const InlineAsmLowering *ALI = MF->getSubtarget().getInlineAsmLowering();
3540
3541 if (!ALI) {
3542 LLVM_DEBUG(
3543 dbgs() << "Inline asm lowering is not supported for this target yet\n");
3544 return false;
3545 }
3546
3547 return ALI->lowerInlineAsm(
3548 MIRBuilder, CB, GetOrCreateVRegs: [&](const Value &Val) { return getOrCreateVRegs(Val); });
3549}
3550
3551bool IRTranslatorImpl::translateCallBase(const CallBase &CB,
3552 MachineIRBuilder &MIRBuilder) {
3553 ArrayRef<Register> Res = getOrCreateVRegs(Val: CB);
3554
3555 SmallVector<ArrayRef<Register>, 8> Args;
3556 Register SwiftInVReg = 0;
3557 Register SwiftErrorVReg = 0;
3558 for (const auto &Arg : CB.args()) {
3559 if (CLI->supportSwiftError() && isSwiftError(V: Arg)) {
3560 assert(SwiftInVReg == 0 && "Expected only one swift error argument");
3561 LLT Ty = getLLTForType(Ty&: *Arg->getType(), DL: *DL);
3562 SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
3563 MIRBuilder.buildCopy(Res: SwiftInVReg, Op: SwiftError.getOrCreateVRegUseAt(
3564 &CB, &MIRBuilder.getMBB(), Arg));
3565 Args.emplace_back(Args: ArrayRef(SwiftInVReg));
3566 SwiftErrorVReg =
3567 SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
3568 continue;
3569 }
3570 Args.push_back(Elt: getOrCreateVRegs(Val: *Arg));
3571 }
3572
3573 if (auto *CI = dyn_cast<CallInst>(Val: &CB)) {
3574 if (ORE->enabled()) {
3575 if (MemoryOpRemark::canHandle(I: CI, TLI: *LibInfo)) {
3576 MemoryOpRemark R(*ORE, "gisel-ir-translator-memsize", *DL, *LibInfo);
3577 R.visit(I: CI);
3578 }
3579 }
3580 }
3581
3582 std::optional<CallLowering::PtrAuthInfo> PAI;
3583 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_ptrauth)) {
3584 // Functions should never be ptrauth-called directly.
3585 assert(!CB.getCalledFunction() && "invalid direct ptrauth call");
3586
3587 const Value *Key = Bundle->Inputs[0];
3588 const Value *Discriminator = Bundle->Inputs[1];
3589
3590 // Look through ptrauth constants to try to eliminate the matching bundle
3591 // and turn this into a direct call with no ptrauth.
3592 // CallLowering will use the raw pointer if it doesn't find the PAI.
3593 const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(Val: CB.getCalledOperand());
3594 if (!CalleeCPA || !isa<Function>(Val: CalleeCPA->getPointer()) ||
3595 !CalleeCPA->isKnownCompatibleWith(Key, Discriminator, DL: *DL)) {
3596 // If we can't make it direct, package the bundle into PAI.
3597 Register DiscReg = getOrCreateVReg(Val: *Discriminator);
3598 PAI = CallLowering::PtrAuthInfo{.Key: cast<ConstantInt>(Val: Key)->getZExtValue(),
3599 .Discriminator: DiscReg};
3600 }
3601 }
3602
3603 Register ConvergenceCtrlToken = 0;
3604 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
3605 const auto &Token = *Bundle->Inputs[0].get();
3606 ConvergenceCtrlToken = getOrCreateConvergenceTokenVReg(Token);
3607 }
3608
3609 // We don't set HasCalls on MFI here yet because call lowering may decide to
3610 // optimize into tail calls. Instead, we defer that to selection where a final
3611 // scan is done to check if any instructions are calls.
3612 bool Success = CLI->lowerCall(
3613 MIRBuilder, Call: CB, ResRegs: Res, ArgRegs: Args, SwiftErrorVReg, PAI, ConvergenceCtrlToken,
3614 GetCalleeReg: [&]() { return getOrCreateVReg(Val: *CB.getCalledOperand()); });
3615
3616 // Check if we just inserted a tail call.
3617 if (Success) {
3618 assert(!HasTailCall && "Can't tail call return twice from block?");
3619 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
3620 HasTailCall = TII->isTailCall(Inst: *std::prev(x: MIRBuilder.getInsertPt()));
3621 }
3622
3623 return Success;
3624}
3625
3626bool IRTranslatorImpl::translateCall(const User &U,
3627 MachineIRBuilder &MIRBuilder) {
3628 if (!mayTranslateUserTypes(U))
3629 return false;
3630
3631 const CallInst &CI = cast<CallInst>(Val: U);
3632 const Function *F = CI.getCalledFunction();
3633
3634 // FIXME: support Windows dllimport function calls and calls through
3635 // weak symbols.
3636 if (F && (F->hasDLLImportStorageClass() ||
3637 (MF->getTarget().getTargetTriple().isOSWindowsOrUEFI() &&
3638 F->hasExternalWeakLinkage())))
3639 return false;
3640
3641 // FIXME: support control flow guard targets.
3642 if (CI.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
3643 return false;
3644
3645 // FIXME: support statepoints and related.
3646 if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(Val: U))
3647 return false;
3648
3649 if (CI.isInlineAsm())
3650 return translateInlineAsm(CB: CI, MIRBuilder);
3651
3652 Intrinsic::ID ID = F ? F->getIntrinsicID() : Intrinsic::not_intrinsic;
3653 if (!F || ID == Intrinsic::not_intrinsic) {
3654 if (translateCallBase(CB: CI, MIRBuilder)) {
3655 diagnoseDontCall(CI);
3656 return true;
3657 }
3658 return false;
3659 }
3660
3661 assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
3662
3663 if (!MF->getSubtarget().isIntrinsicSupported(IntrinsicID: ID))
3664 return handleUnsupportedIntrinsic(CB: CI, ID, MIRBuilder);
3665
3666 if (translateKnownIntrinsic(CI, ID, MIRBuilder))
3667 return true;
3668
3669 SmallVector<TargetLowering::IntrinsicInfo> Infos;
3670 TLI->getTgtMemIntrinsic(Infos, I: CI, MF&: *MF, Intrinsic: ID);
3671
3672 return translateIntrinsic(CB: CI, ID, MIRBuilder, TgtMemIntrinsicInfos: Infos);
3673}
3674
3675bool IRTranslatorImpl::handleUnsupportedIntrinsic(
3676 const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
3677 const Function &F = MF->getFunction();
3678 F.getContext().diagnose(
3679 DI: DiagnosticInfoUnsupportedTargetIntrinsic(F, ID, CB.getDebugLoc()));
3680
3681 if (!CB.getType()->isVoidTy()) {
3682 for (Register Reg : getOrCreateVRegs(Val: CB))
3683 MIRBuilder.buildUndef(Res: Reg);
3684 }
3685
3686 return true;
3687}
3688
3689/// Translate a call or callbr to an intrinsic.
3690bool IRTranslatorImpl::translateIntrinsic(
3691 const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
3692 ArrayRef<TargetLowering::IntrinsicInfo> TgtMemIntrinsicInfos) {
3693 if (!MF->getSubtarget().isIntrinsicSupported(IntrinsicID: ID))
3694 return handleUnsupportedIntrinsic(CB, ID, MIRBuilder);
3695
3696 ArrayRef<Register> ResultRegs;
3697 if (!CB.getType()->isVoidTy())
3698 ResultRegs = getOrCreateVRegs(Val: CB);
3699
3700 // Ignore the callsite attributes. Backend code is most likely not expecting
3701 // an intrinsic to sometimes have side effects and sometimes not.
3702 MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, Res: ResultRegs);
3703 if (isa<FPMathOperator>(Val: CB))
3704 MIB->copyIRFlags(I: CB);
3705
3706 for (const auto &Arg : enumerate(First: CB.args())) {
3707 // If this is required to be an immediate, don't materialize it in a
3708 // register.
3709 if (CB.paramHasAttr(ArgNo: Arg.index(), Kind: Attribute::ImmArg)) {
3710 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Arg.value())) {
3711 // imm arguments are more convenient than cimm (and realistically
3712 // probably sufficient), so use them.
3713 assert(CI->getBitWidth() <= 64 &&
3714 "large intrinsic immediates not handled");
3715 MIB.addImm(Val: CI->getSExtValue());
3716 } else {
3717 MIB.addFPImm(Val: cast<ConstantFP>(Val: Arg.value()));
3718 }
3719 } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Val: Arg.value())) {
3720 auto *MD = MDVal->getMetadata();
3721 auto *MDN = dyn_cast<MDNode>(Val: MD);
3722 if (!MDN) {
3723 if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(Val: MD))
3724 MDN = MDNode::get(Context&: MF->getFunction().getContext(), MDs: ConstMD);
3725 else // This was probably an MDString.
3726 return false;
3727 }
3728 MIB.addMetadata(MD: MDN);
3729 } else {
3730 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: *Arg.value());
3731 if (VRegs.size() > 1)
3732 return false;
3733 MIB.addUse(RegNo: VRegs[0]);
3734 }
3735 }
3736
3737 // Add MachineMemOperands for each memory access described by the target.
3738 for (const auto &Info : TgtMemIntrinsicInfos) {
3739 Align Alignment = Info.align.value_or(
3740 u: DL->getABITypeAlign(Ty: Info.memVT.getTypeForEVT(Context&: CB.getContext())));
3741 LLT MemTy = Info.memVT.isSimple()
3742 ? getLLTForMVT(Ty: Info.memVT.getSimpleVT())
3743 : LLT::scalar(SizeInBits: Info.memVT.getStoreSizeInBits());
3744
3745 // TODO: We currently just fallback to address space 0 if
3746 // getTgtMemIntrinsic didn't yield anything useful.
3747 MachinePointerInfo MPI;
3748 if (Info.ptrVal) {
3749 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
3750 } else if (Info.fallbackAddressSpace) {
3751 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
3752 }
3753 MIB.addMemOperand(MMO: MF->getMachineMemOperand(
3754 PtrInfo: MPI, F: Info.flags, MemTy, BaseAlignment: Alignment, Metadata: CB.getAAMetadata(), SSID: Info.ssid,
3755 Ordering: Info.order, FailureOrdering: Info.failureOrder));
3756 }
3757
3758 if (CB.isConvergent()) {
3759 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
3760 auto *Token = Bundle->Inputs[0].get();
3761 Register TokenReg = getOrCreateVReg(Val: *Token);
3762 MIB.addUse(RegNo: TokenReg, Flags: RegState::Implicit);
3763 }
3764 }
3765
3766 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol))
3767 MIB->setDeactivationSymbol(MF&: *MF, DS: Bundle->Inputs[0].get());
3768
3769 return true;
3770}
3771
3772bool IRTranslatorImpl::findUnwindDestinations(
3773 const BasicBlock *EHPadBB, BranchProbability Prob,
3774 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
3775 &UnwindDests) {
3776 EHPersonality Personality = classifyEHPersonality(
3777 Pers: EHPadBB->getParent()->getFunction().getPersonalityFn());
3778 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
3779 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
3780 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
3781 bool IsWasmD = Personality == EHPersonality::Wasm_D;
3782 bool IsSEH = isAsynchronousEHPersonality(Pers: Personality);
3783
3784 if (IsWasmCXX || IsWasmD) {
3785 // Ignore this for now.
3786 return false;
3787 }
3788
3789 while (EHPadBB) {
3790 BasicBlock::const_iterator Pad = EHPadBB->getFirstNonPHIIt();
3791 BasicBlock *NewEHPadBB = nullptr;
3792 if (isa<LandingPadInst>(Val: Pad)) {
3793 // Stop on landingpads. They are not funclets.
3794 UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
3795 break;
3796 }
3797 if (isa<CleanupPadInst>(Val: Pad)) {
3798 // Stop on cleanup pads. Cleanups are always funclet entries for all known
3799 // personalities.
3800 UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
3801 UnwindDests.back().first->setIsEHScopeEntry();
3802 UnwindDests.back().first->setIsEHFuncletEntry();
3803 break;
3804 }
3805 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: Pad)) {
3806 // Add the catchpad handlers to the possible destinations.
3807 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
3808 UnwindDests.emplace_back(Args: &getMBB(BB: *CatchPadBB), Args&: Prob);
3809 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
3810 if (IsMSVCCXX || IsCoreCLR)
3811 UnwindDests.back().first->setIsEHFuncletEntry();
3812 if (!IsSEH)
3813 UnwindDests.back().first->setIsEHScopeEntry();
3814 }
3815 NewEHPadBB = CatchSwitch->getUnwindDest();
3816 } else {
3817 continue;
3818 }
3819
3820 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3821 if (BPI && NewEHPadBB)
3822 Prob *= BPI->getEdgeProbability(Src: EHPadBB, Dst: NewEHPadBB);
3823 EHPadBB = NewEHPadBB;
3824 }
3825 return true;
3826}
3827
3828bool IRTranslatorImpl::translateInvoke(const User &U,
3829 MachineIRBuilder &MIRBuilder) {
3830 const InvokeInst &I = cast<InvokeInst>(Val: U);
3831 MCContext &Context = MF->getContext();
3832
3833 const BasicBlock *ReturnBB = I.getSuccessor(i: 0);
3834 const BasicBlock *EHPadBB = I.getSuccessor(i: 1);
3835
3836 const Function *Fn = I.getCalledFunction();
3837
3838 // FIXME: support invoking patchpoint and statepoint intrinsics.
3839 if (Fn && Fn->isIntrinsic())
3840 return false;
3841
3842 // FIXME: support whatever these are.
3843 if (I.hasDeoptState())
3844 return false;
3845
3846 // FIXME: support control flow guard targets.
3847 if (I.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
3848 return false;
3849
3850 // FIXME: support Windows exception handling.
3851 if (!isa<LandingPadInst>(Val: EHPadBB->getFirstNonPHIIt()))
3852 return false;
3853
3854 // FIXME: support Windows dllimport function calls and calls through
3855 // weak symbols.
3856 if (Fn && (Fn->hasDLLImportStorageClass() ||
3857 (MF->getTarget().getTargetTriple().isOSWindowsOrUEFI() &&
3858 Fn->hasExternalWeakLinkage())))
3859 return false;
3860
3861 bool LowerInlineAsm = I.isInlineAsm();
3862 bool NeedEHLabel = true;
3863
3864 // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
3865 // the region covered by the try.
3866 MCSymbol *BeginSymbol = nullptr;
3867 if (NeedEHLabel) {
3868 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_INVOKE_REGION_START);
3869 BeginSymbol = Context.createTempSymbol();
3870 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: BeginSymbol);
3871 }
3872
3873 if (LowerInlineAsm) {
3874 if (!translateInlineAsm(CB: I, MIRBuilder))
3875 return false;
3876 } else if (!translateCallBase(CB: I, MIRBuilder))
3877 return false;
3878
3879 MCSymbol *EndSymbol = nullptr;
3880 if (NeedEHLabel) {
3881 EndSymbol = Context.createTempSymbol();
3882 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: EndSymbol);
3883 }
3884
3885 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
3886 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3887 MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
3888 BranchProbability EHPadBBProb =
3889 BPI ? BPI->getEdgeProbability(Src: InvokeMBB->getBasicBlock(), Dst: EHPadBB)
3890 : BranchProbability::getZero();
3891
3892 if (!findUnwindDestinations(EHPadBB, Prob: EHPadBBProb, UnwindDests))
3893 return false;
3894
3895 MachineBasicBlock &EHPadMBB = getMBB(BB: *EHPadBB),
3896 &ReturnMBB = getMBB(BB: *ReturnBB);
3897 // Update successor info.
3898 addSuccessorWithProb(Src: InvokeMBB, Dst: &ReturnMBB);
3899 for (auto &UnwindDest : UnwindDests) {
3900 UnwindDest.first->setIsEHPad();
3901 addSuccessorWithProb(Src: InvokeMBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
3902 }
3903 InvokeMBB->normalizeSuccProbs();
3904
3905 if (NeedEHLabel) {
3906 assert(BeginSymbol && "Expected a begin symbol!");
3907 assert(EndSymbol && "Expected an end symbol!");
3908 MF->addInvoke(LandingPad: &EHPadMBB, BeginLabel: BeginSymbol, EndLabel: EndSymbol);
3909 }
3910
3911 MIRBuilder.buildBr(Dest&: ReturnMBB);
3912 return true;
3913}
3914
3915/// The intrinsics currently supported by callbr are implicit control flow
3916/// intrinsics such as amdgcn.kill.
3917bool IRTranslatorImpl::translateCallBr(const User &U,
3918 MachineIRBuilder &MIRBuilder) {
3919 if (!mayTranslateUserTypes(U))
3920 return false; // see translateCall
3921
3922 const CallBrInst &I = cast<CallBrInst>(Val: U);
3923 MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB();
3924
3925 Intrinsic::ID IID = I.getIntrinsicID();
3926 if (I.isInlineAsm()) {
3927 // FIXME: inline asm is not yet supported for callbr in GlobalISel. As soon
3928 // as we add support, we need to handle the indirect asm targets, see
3929 // SelectionDAGBuilder::visitCallBr().
3930 return false;
3931 }
3932 if (!translateIntrinsic(CB: I, ID: IID, MIRBuilder))
3933 return false;
3934
3935 // Retrieve successors.
3936 SmallPtrSet<BasicBlock *, 8> Dests = {I.getDefaultDest()};
3937 MachineBasicBlock *Return = &getMBB(BB: *I.getDefaultDest());
3938
3939 // Update successor info.
3940 addSuccessorWithProb(Src: CallBrMBB, Dst: Return, Prob: BranchProbability::getOne());
3941
3942 // Add indirect targets as successors. For intrinsic callbr, these represent
3943 // implicit control flow (e.g., the "kill" path for amdgcn.kill). We mark them
3944 // with setIsInlineAsmBrIndirectTarget so the machine verifier accepts them as
3945 // valid successors, even though they're not from inline asm.
3946 for (BasicBlock *Dest : I.getIndirectDests()) {
3947 MachineBasicBlock &Target = getMBB(BB: *Dest);
3948 Target.setIsInlineAsmBrIndirectTarget();
3949 Target.setLabelMustBeEmitted();
3950 // Don't add duplicate machine successors.
3951 if (Dests.insert(Ptr: Dest).second)
3952 addSuccessorWithProb(Src: CallBrMBB, Dst: &Target, Prob: BranchProbability::getZero());
3953 }
3954
3955 CallBrMBB->normalizeSuccProbs();
3956
3957 // Drop into default successor.
3958 MIRBuilder.buildBr(Dest&: *Return);
3959
3960 return true;
3961}
3962
3963bool IRTranslatorImpl::translateLandingPad(const User &U,
3964 MachineIRBuilder &MIRBuilder) {
3965 const LandingPadInst &LP = cast<LandingPadInst>(Val: U);
3966
3967 MachineBasicBlock &MBB = MIRBuilder.getMBB();
3968
3969 MBB.setIsEHPad();
3970
3971 // If there aren't registers to copy the values into (e.g., during SjLj
3972 // exceptions), then don't bother.
3973 const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
3974 if (TLI->getExceptionPointerRegister(EH: FuncInfo.ExceptionModel,
3975 PersonalityFn) == 0 &&
3976 TLI->getExceptionSelectorRegister(EH: FuncInfo.ExceptionModel,
3977 PersonalityFn) == 0)
3978 return true;
3979
3980 // If landingpad's return type is token type, we don't create DAG nodes
3981 // for its exception pointer and selector value. The extraction of exception
3982 // pointer or selector value from token type landingpads is not currently
3983 // supported.
3984 if (LP.getType()->isTokenTy())
3985 return true;
3986
3987 if (!isExceptionPointerAndSelectorType(Ty: LP.getType()))
3988 return false;
3989
3990 // Add a label to mark the beginning of the landing pad. Deletion of the
3991 // landing pad can thus be detected via the MachineModuleInfo.
3992 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL)
3993 .addSym(Sym: MF->addLandingPad(LandingPad: &MBB));
3994
3995 // If the unwinder does not preserve all registers, ensure that the
3996 // function marks the clobbered registers as used.
3997 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
3998 if (auto *RegMask = TRI.getCustomEHPadPreservedMask(MF: *MF))
3999 MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask);
4000
4001 LLT Ty = getLLTForType(Ty&: *LP.getType(), DL: *DL);
4002 Register Undef = MRI->createGenericVirtualRegister(Ty);
4003 MIRBuilder.buildUndef(Res: Undef);
4004
4005 SmallVector<LLT, 2> Tys;
4006 for (Type *Ty : cast<StructType>(Val: LP.getType())->elements())
4007 Tys.push_back(Elt: getLLTForType(Ty&: *Ty, DL: *DL));
4008 assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
4009
4010 // Mark exception register as live in.
4011 Register ExceptionReg =
4012 TLI->getExceptionPointerRegister(EH: FuncInfo.ExceptionModel, PersonalityFn);
4013 if (!ExceptionReg)
4014 return false;
4015
4016 MBB.addLiveIn(PhysReg: ExceptionReg);
4017 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: LP);
4018 MIRBuilder.buildCopy(Res: ResRegs[0], Op: ExceptionReg);
4019
4020 Register SelectorReg =
4021 TLI->getExceptionSelectorRegister(EH: FuncInfo.ExceptionModel, PersonalityFn);
4022 if (!SelectorReg)
4023 return false;
4024
4025 MBB.addLiveIn(PhysReg: SelectorReg);
4026 Register PtrVReg = MRI->createGenericVirtualRegister(Ty: Tys[0]);
4027 MIRBuilder.buildCopy(Res: PtrVReg, Op: SelectorReg);
4028 MIRBuilder.buildCast(Dst: ResRegs[1], Src: PtrVReg);
4029
4030 return true;
4031}
4032
4033bool IRTranslatorImpl::translateAlloca(const User &U,
4034 MachineIRBuilder &MIRBuilder) {
4035 auto &AI = cast<AllocaInst>(Val: U);
4036
4037 if (AI.isSwiftError())
4038 return true;
4039
4040 if (AI.isStaticAlloca()) {
4041 Register Res = getOrCreateVReg(Val: AI);
4042 int FI = getOrCreateFrameIndex(AI);
4043 auto MIB = MIRBuilder.buildFrameIndex(Res, Idx: FI);
4044 // The frame address may be materialized before the alloca's source line is
4045 // reached. Clear its debug location to avoid reporting that line early.
4046 if (AI.getDebugLoc())
4047 MIB->setDebugLoc(DebugLoc::getDropped());
4048 return true;
4049 }
4050
4051 // FIXME: support stack probing for Windows and UEFI.
4052 if (MF->getTarget().getTargetTriple().isOSWindowsOrUEFI())
4053 return false;
4054
4055 // Now we're in the harder dynamic case.
4056 Register NumElts = getOrCreateVReg(Val: *AI.getArraySize());
4057 Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
4058 LLT IntPtrTy = getLLTForType(Ty&: *IntPtrIRTy, DL: *DL);
4059 if (MRI->getType(Reg: NumElts) != IntPtrTy) {
4060 Register ExtElts = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
4061 MIRBuilder.buildZExtOrTrunc(Res: ExtElts, Op: NumElts);
4062 NumElts = ExtElts;
4063 }
4064
4065 TypeSize TySize = AI.getAllocationBaseSize(DL: *DL);
4066
4067 Register AllocSize = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
4068 Register TySizeReg;
4069 if (TySize.isScalable()) {
4070 // For scalable types, use vscale * min_value
4071 TySizeReg = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
4072 MIRBuilder.buildVScale(Res: TySizeReg, MinElts: TySize.getKnownMinValue());
4073 } else {
4074 // For fixed types, use a constant
4075 TySizeReg =
4076 getOrCreateVReg(Val: *ConstantInt::get(Ty: IntPtrIRTy, V: TySize.getFixedValue()));
4077 }
4078 MIRBuilder.buildMul(Dst: AllocSize, Src0: NumElts, Src1: TySizeReg);
4079
4080 // Round the size of the allocation up to the stack alignment size
4081 // by add SA-1 to the size. This doesn't overflow because we're computing
4082 // an address inside an alloca.
4083 Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();
4084 auto SAMinusOne = MIRBuilder.buildConstant(Res: IntPtrTy, Val: StackAlign.value() - 1);
4085 auto AllocAdd = MIRBuilder.buildAdd(Dst: IntPtrTy, Src0: AllocSize, Src1: SAMinusOne,
4086 Flags: MachineInstr::NoUWrap);
4087 auto AlignCst =
4088 MIRBuilder.buildConstant(Res: IntPtrTy, Val: ~(uint64_t)(StackAlign.value() - 1));
4089 auto AlignedAlloc = MIRBuilder.buildAnd(Dst: IntPtrTy, Src0: AllocAdd, Src1: AlignCst);
4090
4091 Align Alignment = AI.getAlign();
4092 if (Alignment <= StackAlign)
4093 Alignment = Align(1);
4094 MIRBuilder.buildDynStackAlloc(Res: getOrCreateVReg(Val: AI), Size: AlignedAlloc, Alignment);
4095
4096 MF->getFrameInfo().CreateVariableSizedObject(Alignment, Alloca: &AI);
4097 assert(MF->getFrameInfo().hasVarSizedObjects());
4098 return true;
4099}
4100
4101bool IRTranslatorImpl::translateVAArg(const User &U,
4102 MachineIRBuilder &MIRBuilder) {
4103 // FIXME: We may need more info about the type. Because of how LLT works,
4104 // we're completely discarding the i64/double distinction here (amongst
4105 // others). Fortunately the ABIs I know of where that matters don't use va_arg
4106 // anyway but that's not guaranteed.
4107 MIRBuilder.buildInstr(Opc: TargetOpcode::G_VAARG, DstOps: {getOrCreateVReg(Val: U)},
4108 SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
4109 DL->getABITypeAlign(Ty: U.getType()).value()});
4110 return true;
4111}
4112
4113bool IRTranslatorImpl::translateUnreachable(const User &U,
4114 MachineIRBuilder &MIRBuilder) {
4115 auto &UI = cast<UnreachableInst>(Val: U);
4116 if (!UI.shouldLowerToTrap(TrapUnreachable: MF->getTarget().Options.TrapUnreachable,
4117 NoTrapAfterNoreturn: MF->getTarget().Options.NoTrapAfterNoreturn))
4118 return true;
4119
4120 MIRBuilder.buildTrap();
4121 return true;
4122}
4123
4124bool IRTranslatorImpl::translateInsertElement(const User &U,
4125 MachineIRBuilder &MIRBuilder) {
4126 // If it is a <1 x Ty> vector, use the scalar as it is
4127 // not a legal vector type in LLT.
4128 if (auto *FVT = dyn_cast<FixedVectorType>(Val: U.getType());
4129 FVT && FVT->getNumElements() == 1)
4130 return translateCopy(U, V: *U.getOperand(i: 1), MIRBuilder);
4131
4132 Register Res = getOrCreateVReg(Val: U);
4133 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
4134 Register Elt = getOrCreateVReg(Val: *U.getOperand(i: 1));
4135 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4136 Register Idx;
4137 if (auto *CI = dyn_cast<ConstantInt>(Val: U.getOperand(i: 2))) {
4138 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4139 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4140 auto *NewIdxCI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4141 Idx = getOrCreateVReg(Val: *NewIdxCI);
4142 }
4143 }
4144 if (!Idx)
4145 Idx = getOrCreateVReg(Val: *U.getOperand(i: 2));
4146 if (MRI->getType(Reg: Idx).getSizeInBits() != PreferredVecIdxWidth) {
4147 const LLT VecIdxTy =
4148 MRI->getType(Reg: Idx).changeElementSize(NewEltSize: PreferredVecIdxWidth);
4149 Idx = MIRBuilder.buildZExtOrTrunc(Res: VecIdxTy, Op: Idx).getReg(Idx: 0);
4150 }
4151 MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
4152 return true;
4153}
4154
4155bool IRTranslatorImpl::translateInsertVector(const User &U,
4156 MachineIRBuilder &MIRBuilder) {
4157 Register Dst = getOrCreateVReg(Val: U);
4158 Register Vec = getOrCreateVReg(Val: *U.getOperand(i: 0));
4159 Register Elt = getOrCreateVReg(Val: *U.getOperand(i: 1));
4160
4161 ConstantInt *CI = cast<ConstantInt>(Val: U.getOperand(i: 2));
4162 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4163
4164 // Resize Index to preferred index width.
4165 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4166 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4167 CI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4168 }
4169
4170 // If it is a <1 x Ty> vector, we have to use other means.
4171 if (auto *ResultType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 1)->getType());
4172 ResultType && ResultType->getNumElements() == 1) {
4173 if (auto *InputType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType());
4174 InputType && InputType->getNumElements() == 1) {
4175 // We are inserting an illegal fixed vector into an illegal
4176 // fixed vector, use the scalar as it is not a legal vector type
4177 // in LLT. Since inserting a <1 x Ty> at index 0 replaces the whole
4178 // destination, the result is the source operand.
4179 return translateCopy(U, Src: Elt, MIRBuilder);
4180 }
4181 if (isa<FixedVectorType>(Val: U.getOperand(i: 0)->getType())) {
4182 // We are inserting an illegal fixed vector into a legal fixed
4183 // vector, use the scalar as it is not a legal vector type in
4184 // LLT.
4185 Register Idx = getOrCreateVReg(Val: *CI);
4186 MIRBuilder.buildInsertVectorElement(Res: Dst, Val: Vec, Elt, Idx);
4187 return true;
4188 }
4189 if (isa<ScalableVectorType>(Val: U.getOperand(i: 0)->getType())) {
4190 // We are inserting an illegal fixed vector into a scalable
4191 // vector, use a scalar element insert.
4192 LLT VecIdxTy = LLT::integer(SizeInBits: PreferredVecIdxWidth);
4193 Register Idx = getOrCreateVReg(Val: *CI);
4194 auto ScaledIndex = MIRBuilder.buildMul(
4195 Dst: VecIdxTy, Src0: MIRBuilder.buildVScale(Res: VecIdxTy, MinElts: 1), Src1: Idx);
4196 MIRBuilder.buildInsertVectorElement(Res: Dst, Val: Vec, Elt, Idx: ScaledIndex);
4197 return true;
4198 }
4199 }
4200
4201 MIRBuilder.buildInsertSubvector(Res: Dst, Src0: Vec, Src1: Elt, Index: CI->getZExtValue());
4202 return true;
4203}
4204
4205bool IRTranslatorImpl::translateExtractElement(const User &U,
4206 MachineIRBuilder &MIRBuilder) {
4207 // If it is a <1 x Ty> vector, use the scalar as it is
4208 // not a legal vector type in LLT.
4209 if (const FixedVectorType *FVT =
4210 dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType()))
4211 if (FVT->getNumElements() == 1)
4212 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
4213
4214 Register Res = getOrCreateVReg(Val: U);
4215 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
4216 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4217 Register Idx;
4218 if (auto *CI = dyn_cast<ConstantInt>(Val: U.getOperand(i: 1))) {
4219 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4220 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4221 auto *NewIdxCI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4222 Idx = getOrCreateVReg(Val: *NewIdxCI);
4223 }
4224 }
4225 if (!Idx)
4226 Idx = getOrCreateVReg(Val: *U.getOperand(i: 1));
4227 if (MRI->getType(Reg: Idx).getSizeInBits() != PreferredVecIdxWidth) {
4228 const LLT VecIdxTy =
4229 MRI->getType(Reg: Idx).changeElementSize(NewEltSize: PreferredVecIdxWidth);
4230 Idx = MIRBuilder.buildZExtOrTrunc(Res: VecIdxTy, Op: Idx).getReg(Idx: 0);
4231 }
4232 MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
4233 return true;
4234}
4235
4236bool IRTranslatorImpl::translateExtractVector(const User &U,
4237 MachineIRBuilder &MIRBuilder) {
4238 Register Res = getOrCreateVReg(Val: U);
4239 Register Vec = getOrCreateVReg(Val: *U.getOperand(i: 0));
4240 ConstantInt *CI = cast<ConstantInt>(Val: U.getOperand(i: 1));
4241 unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(DL: *DL);
4242
4243 // Resize Index to preferred index width.
4244 if (CI->getBitWidth() != PreferredVecIdxWidth) {
4245 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
4246 CI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
4247 }
4248
4249 // If it is a <1 x Ty> vector, we have to use other means.
4250 if (auto *ResultType = dyn_cast<FixedVectorType>(Val: U.getType());
4251 ResultType && ResultType->getNumElements() == 1) {
4252 if (auto *InputType = dyn_cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType());
4253 InputType && InputType->getNumElements() == 1) {
4254 // We are extracting an illegal fixed vector from an illegal fixed vector,
4255 // use the scalar as it is not a legal vector type in LLT.
4256 return translateCopy(U, Src: Vec, MIRBuilder);
4257 }
4258 if (isa<FixedVectorType>(Val: U.getOperand(i: 0)->getType())) {
4259 // We are extracting an illegal fixed vector from a legal fixed
4260 // vector, use the scalar as it is not a legal vector type in
4261 // LLT.
4262 Register Idx = getOrCreateVReg(Val: *CI);
4263 MIRBuilder.buildExtractVectorElement(Res, Val: Vec, Idx);
4264 return true;
4265 }
4266 if (isa<ScalableVectorType>(Val: U.getOperand(i: 0)->getType())) {
4267 // We are extracting an illegal fixed vector from a scalable
4268 // vector, use a scalar element extract.
4269 LLT VecIdxTy = LLT::integer(SizeInBits: PreferredVecIdxWidth);
4270 Register Idx = getOrCreateVReg(Val: *CI);
4271 auto ScaledIndex = MIRBuilder.buildMul(
4272 Dst: VecIdxTy, Src0: MIRBuilder.buildVScale(Res: VecIdxTy, MinElts: 1), Src1: Idx);
4273 MIRBuilder.buildExtractVectorElement(Res, Val: Vec, Idx: ScaledIndex);
4274 return true;
4275 }
4276 }
4277
4278 MIRBuilder.buildExtractSubvector(Res, Src: Vec, Index: CI->getZExtValue());
4279 return true;
4280}
4281
4282bool IRTranslatorImpl::translateShuffleVector(const User &U,
4283 MachineIRBuilder &MIRBuilder) {
4284 // A ShuffleVector that operates on scalable vectors is a splat vector where
4285 // the value of the splat vector is the 0th element of the first operand,
4286 // since the index mask operand is the zeroinitializer (undef and
4287 // poison are treated as zeroinitializer here).
4288 if (U.getOperand(i: 0)->getType()->isScalableTy()) {
4289 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
4290 auto SplatVal = MIRBuilder.buildExtractVectorElementConstant(
4291 Res: MRI->getType(Reg: Val).getElementType(), Val, Idx: 0);
4292 MIRBuilder.buildSplatVector(Res: getOrCreateVReg(Val: U), Val: SplatVal);
4293 return true;
4294 }
4295
4296 ArrayRef<int> Mask;
4297 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: &U))
4298 Mask = SVI->getShuffleMask();
4299 else
4300 Mask = cast<ConstantExpr>(Val: U).getShuffleMask();
4301
4302 // As GISel does not represent <1 x > vectors as a separate type from scalars,
4303 // we transform shuffle_vector with a scalar output to an
4304 // ExtractVectorElement. If the input type is also scalar it becomes a Copy.
4305 unsigned DstElts = cast<FixedVectorType>(Val: U.getType())->getNumElements();
4306 unsigned SrcElts =
4307 cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType())->getNumElements();
4308 if (DstElts == 1) {
4309 unsigned M = Mask[0];
4310 if (SrcElts == 1) {
4311 if (M == 0 || M == 1)
4312 return translateCopy(U, V: *U.getOperand(i: M), MIRBuilder);
4313 MIRBuilder.buildUndef(Res: getOrCreateVReg(Val: U));
4314 } else {
4315 Register Dst = getOrCreateVReg(Val: U);
4316 if (M < SrcElts) {
4317 MIRBuilder.buildExtractVectorElementConstant(
4318 Res: Dst, Val: getOrCreateVReg(Val: *U.getOperand(i: 0)), Idx: M);
4319 } else if (M < SrcElts * 2) {
4320 MIRBuilder.buildExtractVectorElementConstant(
4321 Res: Dst, Val: getOrCreateVReg(Val: *U.getOperand(i: 1)), Idx: M - SrcElts);
4322 } else {
4323 MIRBuilder.buildUndef(Res: Dst);
4324 }
4325 }
4326 return true;
4327 }
4328
4329 // A single element src is transformed to a build_vector.
4330 if (SrcElts == 1) {
4331 SmallVector<Register> Ops;
4332 Register Undef;
4333 for (int M : Mask) {
4334 LLT SrcTy = getLLTForType(Ty&: *U.getOperand(i: 0)->getType(), DL: *DL);
4335 if (M == 0 || M == 1) {
4336 Ops.push_back(Elt: getOrCreateVReg(Val: *U.getOperand(i: M)));
4337 } else {
4338 if (!Undef.isValid()) {
4339 Undef = MRI->createGenericVirtualRegister(Ty: SrcTy);
4340 MIRBuilder.buildUndef(Res: Undef);
4341 }
4342 Ops.push_back(Elt: Undef);
4343 }
4344 }
4345 MIRBuilder.buildBuildVector(Res: getOrCreateVReg(Val: U), Ops);
4346 return true;
4347 }
4348
4349 ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
4350 MIRBuilder
4351 .buildInstr(Opc: TargetOpcode::G_SHUFFLE_VECTOR, DstOps: {getOrCreateVReg(Val: U)},
4352 SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
4353 getOrCreateVReg(Val: *U.getOperand(i: 1))})
4354 .addShuffleMask(Val: MaskAlloc);
4355 return true;
4356}
4357
4358bool IRTranslatorImpl::translateBitInsert(const User &U,
4359 MachineIRBuilder &MIRBuilder) {
4360 Register Res = getOrCreateVReg(Val: U);
4361 Register Base = getOrCreateVReg(Val: *U.getOperand(i: 0));
4362 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 1));
4363 Register Offset = getOrCreateVReg(Val: *U.getOperand(i: 2));
4364 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4365 LLT BaseTy = MRI.getType(Reg: Base);
4366 LLT ValTy = MRI.getType(Reg: Val);
4367
4368 assert(BaseTy.getScalarSizeInBits() >= ValTy.getScalarSizeInBits() &&
4369 "bitinsert val wider than base should be rejected by verifier");
4370
4371 // If Val is a floating-point type, bitcast it to an integer of the same
4372 // size so buildZExtOrTrunc can safely extend or truncate it.
4373 if (ValTy.isFloatOrFloatVector()) {
4374 ValTy = ValTy.changeToInteger();
4375 Val = MIRBuilder.buildBitcast(Dst: ValTy, Src: Val).getReg(Idx: 0);
4376 } else if (ValTy.isPointerOrPointerVector()) {
4377 ValTy = ValTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: ValTy.getScalarSizeInBits()));
4378 Val = MIRBuilder.buildPtrToInt(Dst: ValTy, Src: Val).getReg(Idx: 0);
4379 }
4380
4381 // Convert Offset to the target's preferred shift amount type.
4382 LLT ShiftAmtTy = TLI->getPreferredShiftAmountTy(ShiftValueTy: BaseTy);
4383 Register LegalOffset =
4384 MIRBuilder.buildZExtOrTrunc(Res: ShiftAmtTy, Op: Offset).getReg(Idx: 0);
4385
4386 // Truncate or extend Val to BaseTy so only the inserted bit range remains.
4387 Register ExtVal = MIRBuilder.buildZExtOrTrunc(Res: BaseTy, Op: Val).getReg(Idx: 0);
4388
4389 unsigned BaseBitWidth = BaseTy.getScalarSizeInBits();
4390 unsigned ValBitWidth = ValTy.getScalarSizeInBits();
4391 APInt InsertMask = APInt::getLowBitsSet(numBits: BaseBitWidth, loBitsSet: ValBitWidth);
4392 Register MaskConst = MIRBuilder.buildConstant(Res: BaseTy, Val: InsertMask).getReg(Idx: 0);
4393 Register ShiftedMask =
4394 MIRBuilder.buildShl(Dst: BaseTy, Src0: MaskConst, Src1: LegalOffset).getReg(Idx: 0);
4395 Register ClearMask = MIRBuilder.buildNot(Dst: BaseTy, Src0: ShiftedMask).getReg(Idx: 0);
4396 Register ClearedBase = MIRBuilder.buildAnd(Dst: BaseTy, Src0: Base, Src1: ClearMask).getReg(Idx: 0);
4397 Register ShiftedVal =
4398 MIRBuilder.buildShl(Dst: BaseTy, Src0: ExtVal, Src1: LegalOffset).getReg(Idx: 0);
4399 MIRBuilder.buildOr(Dst: Res, Src0: ClearedBase, Src1: ShiftedVal);
4400 return true;
4401}
4402
4403bool IRTranslatorImpl::translateBitExtract(const User &U,
4404 MachineIRBuilder &MIRBuilder) {
4405 Register Res = getOrCreateVReg(Val: U);
4406 Register Src = getOrCreateVReg(Val: *U.getOperand(i: 0));
4407 Register Offset = getOrCreateVReg(Val: *U.getOperand(i: 1));
4408 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4409 LLT SrcTy = MRI.getType(Reg: Src);
4410 LLT ResTy = MRI.getType(Reg: Res);
4411
4412 assert(ResTy.getScalarSizeInBits() <= SrcTy.getScalarSizeInBits() &&
4413 "bitextract result wider than source should be rejected by verifier");
4414
4415 // Convert Offset to the target's preferred shift amount type.
4416 LLT ShiftAmtTy = TLI->getPreferredShiftAmountTy(ShiftValueTy: SrcTy);
4417 Register LegalOffset =
4418 MIRBuilder.buildZExtOrTrunc(Res: ShiftAmtTy, Op: Offset).getReg(Idx: 0);
4419
4420 // Shift right by Offset to bring the target field down to bit 0.
4421 Register Shifted = MIRBuilder.buildLShr(Dst: SrcTy, Src0: Src, Src1: LegalOffset).getReg(Idx: 0);
4422
4423 if (ResTy.isFloatOrFloatVector()) {
4424 // Drop into the integer domain to safely handle the size conversion
4425 LLT IntResTy = ResTy.changeToInteger();
4426 Register IntRes = MRI.createGenericVirtualRegister(Ty: IntResTy);
4427
4428 if (SrcTy == IntResTy)
4429 MIRBuilder.buildCopy(Res: IntRes, Op: Shifted);
4430 else
4431 MIRBuilder.buildTrunc(Res: IntRes, Op: Shifted);
4432
4433 // Bitcast the raw integer bits back into the requested floating-point
4434 // register
4435 MIRBuilder.buildBitcast(Dst: Res, Src: IntRes);
4436 } else if (ResTy.isPointerOrPointerVector()) {
4437 // Drop into the integer domain to safely handle the size conversion
4438 LLT IntResTy =
4439 ResTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: ResTy.getScalarSizeInBits()));
4440 Register IntRes = MRI.createGenericVirtualRegister(Ty: IntResTy);
4441
4442 if (SrcTy == IntResTy)
4443 MIRBuilder.buildCopy(Res: IntRes, Op: Shifted);
4444 else
4445 MIRBuilder.buildTrunc(Res: IntRes, Op: Shifted);
4446
4447 MIRBuilder.buildIntToPtr(Dst: Res, Src: IntRes);
4448 } else {
4449 // Normal integer path
4450 if (SrcTy == ResTy)
4451 MIRBuilder.buildCopy(Res, Op: Shifted);
4452 else
4453 MIRBuilder.buildTrunc(Res, Op: Shifted);
4454 }
4455
4456 return true;
4457}
4458
4459bool IRTranslatorImpl::translatePHI(const User &U,
4460 MachineIRBuilder &MIRBuilder) {
4461 const PHINode &PI = cast<PHINode>(Val: U);
4462
4463 SmallVector<MachineInstr *, 4> Insts;
4464 for (auto Reg : getOrCreateVRegs(Val: PI)) {
4465 auto MIB = MIRBuilder.buildInstr(Opc: TargetOpcode::G_PHI, DstOps: {Reg}, SrcOps: {});
4466 Insts.push_back(Elt: MIB.getInstr());
4467 }
4468
4469 PendingPHIs.emplace_back(Args: &PI, Args: std::move(Insts));
4470 return true;
4471}
4472
4473bool IRTranslatorImpl::translateAtomicCmpXchg(const User &U,
4474 MachineIRBuilder &MIRBuilder) {
4475 const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(Val: U);
4476
4477 auto Flags = TLI->getAtomicMemOperandFlags(AI: I, DL: *DL);
4478
4479 auto Res = getOrCreateVRegs(Val: I);
4480 Register OldValRes = Res[0];
4481 Register SuccessRes = Res[1];
4482 Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
4483 Register Cmp = getOrCreateVReg(Val: *I.getCompareOperand());
4484 Register NewVal = getOrCreateVReg(Val: *I.getNewValOperand());
4485
4486 MIRBuilder.buildAtomicCmpXchgWithSuccess(
4487 OldValRes, SuccessRes, Addr, CmpVal: Cmp, NewVal,
4488 MMO&: *MF->getMachineMemOperand(
4489 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, MemTy: MRI->getType(Reg: Cmp),
4490 BaseAlignment: getMemOpAlign(I), Metadata: I.getAAMetadata(), SSID: I.getSyncScopeID(),
4491 Ordering: I.getSuccessOrdering(), FailureOrdering: I.getFailureOrdering()));
4492 return true;
4493}
4494
4495bool IRTranslatorImpl::translateAtomicRMW(const User &U,
4496 MachineIRBuilder &MIRBuilder) {
4497 if (!mayTranslateUserTypes(U))
4498 return false;
4499
4500 const AtomicRMWInst &I = cast<AtomicRMWInst>(Val: U);
4501 auto Flags = TLI->getAtomicMemOperandFlags(AI: I, DL: *DL);
4502
4503 Register Res = getOrCreateVReg(Val: I);
4504 Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
4505 Register Val = getOrCreateVReg(Val: *I.getValOperand());
4506
4507 unsigned Opcode = 0;
4508 switch (I.getOperation()) {
4509 default:
4510 return false;
4511 case AtomicRMWInst::Xchg:
4512 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
4513 break;
4514 case AtomicRMWInst::Add:
4515 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
4516 break;
4517 case AtomicRMWInst::Sub:
4518 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
4519 break;
4520 case AtomicRMWInst::And:
4521 Opcode = TargetOpcode::G_ATOMICRMW_AND;
4522 break;
4523 case AtomicRMWInst::Nand:
4524 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
4525 break;
4526 case AtomicRMWInst::Or:
4527 Opcode = TargetOpcode::G_ATOMICRMW_OR;
4528 break;
4529 case AtomicRMWInst::Xor:
4530 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
4531 break;
4532 case AtomicRMWInst::Max:
4533 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
4534 break;
4535 case AtomicRMWInst::Min:
4536 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
4537 break;
4538 case AtomicRMWInst::UMax:
4539 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
4540 break;
4541 case AtomicRMWInst::UMin:
4542 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
4543 break;
4544 case AtomicRMWInst::FAdd:
4545 Opcode = TargetOpcode::G_ATOMICRMW_FADD;
4546 break;
4547 case AtomicRMWInst::FSub:
4548 Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
4549 break;
4550 case AtomicRMWInst::FMax:
4551 Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
4552 break;
4553 case AtomicRMWInst::FMin:
4554 Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
4555 break;
4556 case AtomicRMWInst::FMaximum:
4557 Opcode = TargetOpcode::G_ATOMICRMW_FMAXIMUM;
4558 break;
4559 case AtomicRMWInst::FMinimum:
4560 Opcode = TargetOpcode::G_ATOMICRMW_FMINIMUM;
4561 break;
4562 case AtomicRMWInst::FMaximumNum:
4563 Opcode = TargetOpcode::G_ATOMICRMW_FMAXIMUMNUM;
4564 break;
4565 case AtomicRMWInst::FMinimumNum:
4566 Opcode = TargetOpcode::G_ATOMICRMW_FMINIMUMNUM;
4567 break;
4568 case AtomicRMWInst::UIncWrap:
4569 Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
4570 break;
4571 case AtomicRMWInst::UDecWrap:
4572 Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
4573 break;
4574 case AtomicRMWInst::USubCond:
4575 Opcode = TargetOpcode::G_ATOMICRMW_USUB_COND;
4576 break;
4577 case AtomicRMWInst::USubSat:
4578 Opcode = TargetOpcode::G_ATOMICRMW_USUB_SAT;
4579 break;
4580 }
4581
4582 MIRBuilder.buildAtomicRMW(
4583 Opcode, OldValRes: Res, Addr, Val,
4584 MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(I.getPointerOperand()),
4585 F: Flags, MemTy: MRI->getType(Reg: Val), BaseAlignment: getMemOpAlign(I),
4586 Metadata: I.getAAMetadata(), SSID: I.getSyncScopeID(),
4587 Ordering: I.getOrdering()));
4588 return true;
4589}
4590
4591bool IRTranslatorImpl::translateFence(const User &U,
4592 MachineIRBuilder &MIRBuilder) {
4593 const FenceInst &Fence = cast<FenceInst>(Val: U);
4594 MIRBuilder.buildFence(Ordering: static_cast<unsigned>(Fence.getOrdering()),
4595 Scope: Fence.getSyncScopeID());
4596 return true;
4597}
4598
4599bool IRTranslatorImpl::translateFreeze(const User &U,
4600 MachineIRBuilder &MIRBuilder) {
4601 const ArrayRef<Register> DstRegs = getOrCreateVRegs(Val: U);
4602 const ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *U.getOperand(i: 0));
4603
4604 assert(DstRegs.size() == SrcRegs.size() &&
4605 "Freeze with different source and destination type?");
4606
4607 for (unsigned I = 0; I < DstRegs.size(); ++I) {
4608 MIRBuilder.buildFreeze(Dst: DstRegs[I], Src: SrcRegs[I]);
4609 }
4610
4611 return true;
4612}
4613
4614void IRTranslatorImpl::finishPendingConstDbgValues() {
4615 for (auto [MI, C] : PendingConstDbgValues) {
4616 auto VRegsIt = VMap.findVRegs(V: *C);
4617 if (VRegsIt == VMap.vregs_end() || VRegsIt->second->size() != 1)
4618 continue;
4619 // Constants are materialized in the entry block, so the vreg dominates
4620 // every debug value; where it is not live, the location is dropped later.
4621 MachineIRBuilder Builder(*MI);
4622 Builder.buildDirectDbgValue(Reg: VRegsIt->second->front(),
4623 Variable: MI->getDebugVariable(),
4624 Expr: MI->getDebugExpression());
4625 MI->eraseFromParent();
4626 }
4627 PendingConstDbgValues.clear();
4628}
4629
4630void IRTranslatorImpl::finishPendingPhis() {
4631#ifndef NDEBUG
4632 DILocationVerifier Verifier;
4633 GISelObserverWrapper WrapperObserver(&Verifier);
4634 RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);
4635#endif // ifndef NDEBUG
4636 for (auto &Phi : PendingPHIs) {
4637 const PHINode *PI = Phi.first;
4638 if (PI->getType()->isEmptyTy())
4639 continue;
4640 ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
4641 MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
4642 EntryBuilder->setDebugLoc(PI->getDebugLoc());
4643#ifndef NDEBUG
4644 Verifier.setCurrentInst(PI);
4645#endif // ifndef NDEBUG
4646
4647 SmallPtrSet<const MachineBasicBlock *, 16> SeenPreds;
4648 for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
4649 auto IRPred = PI->getIncomingBlock(i);
4650 ArrayRef<Register> ValRegs = getOrCreateVRegs(Val: *PI->getIncomingValue(i));
4651 for (auto *Pred : getMachinePredBBs(Edge: {IRPred, PI->getParent()})) {
4652 if (SeenPreds.count(Ptr: Pred) || !PhiMBB->isPredecessor(MBB: Pred))
4653 continue;
4654 SeenPreds.insert(Ptr: Pred);
4655 for (unsigned j = 0; j < ValRegs.size(); ++j) {
4656 MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
4657 MIB.addUse(RegNo: ValRegs[j]);
4658 MIB.addMBB(MBB: Pred);
4659 }
4660 }
4661 }
4662 }
4663}
4664
4665void IRTranslatorImpl::translateDbgValueRecord(Value *V,
4666 const DILocalVariable *Variable,
4667 const DIExpression *Expression,
4668 const DebugLoc &DL,
4669 MachineIRBuilder &MIRBuilder) {
4670 assert(Variable->isValidLocationForIntrinsic(DL) &&
4671 "Expected inlined-at fields to agree");
4672 // Act as if we're handling a debug intrinsic.
4673 MIRBuilder.setDebugLoc(DL);
4674
4675 if (!V) {
4676 // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
4677 // terminate any prior location.
4678 MIRBuilder.buildIndirectDbgValue(Reg: 0, Variable, Expr: Expression);
4679 return;
4680 }
4681
4682 if (const auto *CI = dyn_cast<Constant>(Val: V)) {
4683 MachineInstr *MI = MIRBuilder.buildConstDbgValue(C: *CI, Variable, Expr: Expression);
4684 if (MI->getDebugOperand(Index: 0).isReg() && !isa<UndefValue>(Val: CI))
4685 PendingConstDbgValues.emplace_back(Args&: MI, Args&: CI);
4686 return;
4687 }
4688
4689 if (auto *AI = dyn_cast<AllocaInst>(Val: V);
4690 AI && AI->isStaticAlloca() && Expression->startsWithDeref()) {
4691 // If the value is an alloca and the expression starts with a
4692 // dereference, track a stack slot instead of a register, as registers
4693 // may be clobbered.
4694 auto ExprOperands = Expression->getElements();
4695 auto *ExprDerefRemoved =
4696 DIExpression::get(Context&: AI->getContext(), Elements: ExprOperands.drop_front());
4697 MIRBuilder.buildFIDbgValue(FI: getOrCreateFrameIndex(AI: *AI), Variable,
4698 Expr: ExprDerefRemoved);
4699 return;
4700 }
4701 if (translateIfEntryValueArgument(isDeclare: false, Val: V, Var: Variable, Expr: Expression, DL,
4702 MIRBuilder))
4703 return;
4704 for (Register Reg : getOrCreateVRegs(Val: *V)) {
4705 // FIXME: This does not handle register-indirect values at offset 0. The
4706 // direct/indirect thing shouldn't really be handled by something as
4707 // implicit as reg+noreg vs reg+imm in the first place, but it seems
4708 // pretty baked in right now.
4709 MIRBuilder.buildDirectDbgValue(Reg, Variable, Expr: Expression);
4710 }
4711}
4712
4713void IRTranslatorImpl::translateDbgValueListRecord(
4714 iterator_range<DbgVariableRecord::location_op_iterator> Values,
4715 const DILocalVariable *Variable, const DIExpression *Expression,
4716 const DebugLoc &DL, MachineIRBuilder &MIRBuilder) {
4717 assert(Variable->isValidLocationForIntrinsic(DL) &&
4718 "Expected inlined-at fields to agree");
4719 MIRBuilder.setDebugLoc(DL);
4720
4721 auto GetLocationOperand = [&](Value *V) -> std::optional<MachineOperand> {
4722 if (!V || isa<UndefValue>(Val: V))
4723 return std::nullopt;
4724 if (auto *CI = dyn_cast<ConstantInt>(Val: V)) {
4725 if (CI->getBitWidth() > 64)
4726 return MachineOperand::CreateCImm(CI);
4727 return MachineOperand::CreateImm(
4728 Val: CI->getBitWidth() == 1 ? CI->getZExtValue() : CI->getSExtValue());
4729 }
4730 if (auto *CFP = dyn_cast<ConstantFP>(Val: V))
4731 return MachineOperand::CreateFPImm(CFP);
4732 if (isa<ConstantPointerNull>(Val: V))
4733 return MachineOperand::CreateImm(Val: 0);
4734 if (isa<Constant>(Val: V))
4735 return std::nullopt;
4736 ArrayRef<Register> Regs = getOrCreateVRegs(Val: *V);
4737 if (Regs.size() != 1)
4738 return std::nullopt;
4739 return MachineOperand::CreateReg(Reg: Regs.front(), /*isDef=*/false,
4740 /*isImp=*/false, /*isKill=*/false,
4741 /*isDead=*/false, /*isUndef=*/false,
4742 /*isEarlyClobber=*/false, /*SubReg=*/0,
4743 /*isDebug=*/true);
4744 };
4745
4746 SmallVector<MachineOperand, 4> MOs;
4747 for (Value *V : Values) {
4748 std::optional<MachineOperand> MO = GetLocationOperand(V);
4749 if (!MO) {
4750 // One of the location operands cannot be described, so produce an undef
4751 // DBG_VALUE to terminate any prior location.
4752 MIRBuilder.buildIndirectDbgValue(Reg: 0, Variable, Expr: Expression);
4753 return;
4754 }
4755 MOs.push_back(Elt: *MO);
4756 }
4757
4758 MIRBuilder.insertInstr(
4759 MIB: BuildMI(MF&: MIRBuilder.getMF(), DL,
4760 MCID: MIRBuilder.getTII().get(Opcode: TargetOpcode::DBG_VALUE_LIST),
4761 /*IsIndirect=*/false, MOs, Variable, Expr: Expression));
4762}
4763
4764void IRTranslatorImpl::translateDbgDeclareRecord(
4765 Value *Address, bool HasArgList, const DILocalVariable *Variable,
4766 const DIExpression *Expression, const DebugLoc &DL,
4767 MachineIRBuilder &MIRBuilder) {
4768 if (!Address || isa<UndefValue>(Val: Address)) {
4769 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *Variable << "\n");
4770 return;
4771 }
4772
4773 assert(Variable->isValidLocationForIntrinsic(DL) &&
4774 "Expected inlined-at fields to agree");
4775 auto AI = dyn_cast<AllocaInst>(Val: Address);
4776 if (AI && AI->isStaticAlloca()) {
4777 // Static allocas are tracked at the MF level, no need for DBG_VALUE
4778 // instructions (in fact, they get ignored if they *do* exist).
4779 MF->setVariableDbgInfo(Var: Variable, Expr: Expression,
4780 Slot: getOrCreateFrameIndex(AI: *AI), Loc: DL);
4781 return;
4782 }
4783
4784 if (translateIfEntryValueArgument(isDeclare: true, Val: Address, Var: Variable,
4785 Expr: Expression, DL,
4786 MIRBuilder))
4787 return;
4788
4789 // A dbg.declare describes the address of a source variable, so lower it
4790 // into an indirect DBG_VALUE.
4791 MIRBuilder.setDebugLoc(DL);
4792 MIRBuilder.buildIndirectDbgValue(Reg: getOrCreateVReg(Val: *Address), Variable,
4793 Expr: Expression);
4794}
4795
4796void IRTranslatorImpl::translateDbgInfo(const Instruction &Inst,
4797 MachineIRBuilder &MIRBuilder) {
4798 for (DbgRecord &DR : Inst.getDbgRecordRange()) {
4799 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
4800 MIRBuilder.setDebugLoc(DLR->getDebugLoc());
4801 assert(DLR->getLabel() && "Missing label");
4802 assert(DLR->getLabel()->isValidLocationForIntrinsic(
4803 MIRBuilder.getDebugLoc()) &&
4804 "Expected inlined-at fields to agree");
4805 MIRBuilder.buildDbgLabel(Label: DLR->getLabel());
4806 continue;
4807 }
4808 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
4809 const DILocalVariable *Variable = DVR.getVariable();
4810 const DIExpression *Expression = DVR.getExpression();
4811 Value *V = DVR.getVariableLocationOp(OpIdx: 0);
4812 if (DVR.isDbgDeclare())
4813 translateDbgDeclareRecord(Address: V, HasArgList: DVR.hasArgList(), Variable, Expression,
4814 DL: DVR.getDebugLoc(), MIRBuilder);
4815 else if (DVR.hasArgList())
4816 translateDbgValueListRecord(Values: DVR.location_ops(), Variable, Expression,
4817 DL: DVR.getDebugLoc(), MIRBuilder);
4818 else
4819 translateDbgValueRecord(V, Variable, Expression, DL: DVR.getDebugLoc(),
4820 MIRBuilder);
4821 }
4822}
4823
4824bool IRTranslatorImpl::translate(const Instruction &Inst) {
4825 CurBuilder->setDebugLoc(Inst.getDebugLoc());
4826 CurBuilder->setPCSections(Inst.getMetadata(KindID: LLVMContext::MD_pcsections));
4827 CurBuilder->setMMRAMetadata(Inst.getMetadata(KindID: LLVMContext::MD_mmra));
4828
4829 if (TLI->fallBackToDAGISel(Inst))
4830 return false;
4831
4832 switch (Inst.getOpcode()) {
4833#define HANDLE_INST(NUM, OPCODE, CLASS) \
4834 case Instruction::OPCODE: \
4835 return translate##OPCODE(Inst, *CurBuilder.get());
4836#include "llvm/IR/Instruction.def"
4837 default:
4838 return false;
4839 }
4840}
4841
4842bool IRTranslatorImpl::translate(const Constant &C, Register Reg) {
4843 // We only emit constants into the entry block from here. To prevent jumpy
4844 // debug behaviour remove debug line.
4845 if (auto CurrInstDL = CurBuilder->getDL())
4846 EntryBuilder->setDebugLoc(DebugLoc());
4847
4848 if (auto CI = dyn_cast<ConstantInt>(Val: &C)) {
4849 // buildConstant expects a to-be-splatted scalar ConstantInt.
4850 if (isa<VectorType>(Val: CI->getType()))
4851 CI = ConstantInt::get(Context&: CI->getContext(), V: CI->getValue());
4852 EntryBuilder->buildConstant(Res: Reg, Val: *CI);
4853 } else if (auto CB = dyn_cast<ConstantByte>(Val: &C)) {
4854 // Byte constants share G_CONSTANT with integers; the destination Reg's
4855 // LLT (an integer LLT, see getLLTForType) determines vector splatting.
4856 EntryBuilder->buildConstant(Res: Reg, Val: CB->getValue());
4857 } else if (auto CF = dyn_cast<ConstantFP>(Val: &C)) {
4858 // buildFConstant expects a to-be-splatted scalar ConstantFP.
4859 if (isa<VectorType>(Val: CF->getType()))
4860 CF = ConstantFP::get(Context&: CF->getContext(), V: CF->getValue());
4861 EntryBuilder->buildFConstant(Res: Reg, Val: *CF);
4862 } else if (isa<UndefValue>(Val: C))
4863 EntryBuilder->buildUndef(Res: Reg);
4864 else if (isa<ConstantPointerNull>(Val: C))
4865 EntryBuilder->buildConstant(Res: Reg, Val: 0);
4866 else if (auto GV = dyn_cast<GlobalValue>(Val: &C))
4867 EntryBuilder->buildGlobalValue(Res: Reg, GV);
4868 else if (auto CPA = dyn_cast<ConstantPtrAuth>(Val: &C)) {
4869 Register Addr = getOrCreateVReg(Val: *CPA->getPointer());
4870 Register AddrDisc = getOrCreateVReg(Val: *CPA->getAddrDiscriminator());
4871 EntryBuilder->buildConstantPtrAuth(Res: Reg, CPA, Addr, AddrDisc);
4872 } else if (auto CAZ = dyn_cast<ConstantAggregateZero>(Val: &C)) {
4873 Constant &Elt = *CAZ->getElementValue(Idx: 0u);
4874 if (isa<ScalableVectorType>(Val: CAZ->getType())) {
4875 EntryBuilder->buildSplatVector(Res: Reg, Val: getOrCreateVReg(Val: Elt));
4876 return true;
4877 }
4878 // Return the scalar if it is a <1 x Ty> vector.
4879 unsigned NumElts = CAZ->getElementCount().getFixedValue();
4880 if (NumElts == 1)
4881 return translateCopy(U: C, V: Elt, MIRBuilder&: *EntryBuilder);
4882 // All elements are zero so we can just use the first one.
4883 EntryBuilder->buildSplatBuildVector(Res: Reg, Src: getOrCreateVReg(Val: Elt));
4884 } else if (auto CV = dyn_cast<ConstantDataVector>(Val: &C)) {
4885 // Return the scalar if it is a <1 x Ty> vector.
4886 if (CV->getNumElements() == 1)
4887 return translateCopy(U: C, V: *CV->getElementAsConstant(i: 0), MIRBuilder&: *EntryBuilder);
4888 SmallVector<Register, 4> Ops;
4889 for (unsigned i = 0; i < CV->getNumElements(); ++i) {
4890 Constant &Elt = *CV->getElementAsConstant(i);
4891 Ops.push_back(Elt: getOrCreateVReg(Val: Elt));
4892 }
4893 EntryBuilder->buildBuildVector(Res: Reg, Ops);
4894 } else if (auto CE = dyn_cast<ConstantExpr>(Val: &C)) {
4895 switch(CE->getOpcode()) {
4896#define HANDLE_INST(NUM, OPCODE, CLASS) \
4897 case Instruction::OPCODE: \
4898 return translate##OPCODE(*CE, *EntryBuilder.get());
4899#include "llvm/IR/Instruction.def"
4900 default:
4901 return false;
4902 }
4903 } else if (auto CV = dyn_cast<ConstantVector>(Val: &C)) {
4904 if (CV->getNumOperands() == 1)
4905 return translateCopy(U: C, V: *CV->getOperand(i_nocapture: 0), MIRBuilder&: *EntryBuilder);
4906 SmallVector<Register, 4> Ops;
4907 for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
4908 Ops.push_back(Elt: getOrCreateVReg(Val: *CV->getOperand(i_nocapture: i)));
4909 }
4910 EntryBuilder->buildBuildVector(Res: Reg, Ops);
4911 } else if (auto *BA = dyn_cast<BlockAddress>(Val: &C)) {
4912 EntryBuilder->buildBlockAddress(Res: Reg, BA);
4913 } else
4914 return false;
4915
4916 return true;
4917}
4918
4919bool IRTranslatorImpl::mayTranslateUserTypes(const User &U) const {
4920 if (LLT::getUseExtended())
4921 return true;
4922
4923 // BF16 cannot currently be represented by default LLT. To avoid miscompiles
4924 // we prevent any instructions using them by default in all targets that do
4925 // not explicitly enable it via LLT::setUseExtended(true).
4926 return !U.getType()->getScalarType()->isBFloatTy() &&
4927 !any_of(Range: U.operands(), P: [](Value *V) {
4928 return V->getType()->getScalarType()->isBFloatTy();
4929 });
4930}
4931
4932bool IRTranslatorImpl::finalizeBasicBlock(const BasicBlock &BB,
4933 MachineBasicBlock &MBB) {
4934 for (auto &BTB : SL->BitTestCases) {
4935 // Emit header first, if it wasn't already emitted.
4936 if (!BTB.Emitted)
4937 emitBitTestHeader(B&: BTB, SwitchBB: BTB.Parent);
4938
4939 BranchProbability UnhandledProb = BTB.Prob;
4940 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
4941 UnhandledProb -= BTB.Cases[j].ExtraProb;
4942 // Set the current basic block to the mbb we wish to insert the code into
4943 MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
4944 // If all cases cover a contiguous range, it is not necessary to jump to
4945 // the default block after the last bit test fails. This is because the
4946 // range check during bit test header creation has guaranteed that every
4947 // case here doesn't go outside the range. In this case, there is no need
4948 // to perform the last bit test, as it will always be true. Instead, make
4949 // the second-to-last bit-test fall through to the target of the last bit
4950 // test, and delete the last bit test.
4951
4952 MachineBasicBlock *NextMBB;
4953 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
4954 // Second-to-last bit-test with contiguous range: fall through to the
4955 // target of the final bit test.
4956 NextMBB = BTB.Cases[j + 1].TargetBB;
4957 } else if (j + 1 == ej) {
4958 // For the last bit test, fall through to Default.
4959 NextMBB = BTB.Default;
4960 } else {
4961 // Otherwise, fall through to the next bit test.
4962 NextMBB = BTB.Cases[j + 1].ThisBB;
4963 }
4964
4965 emitBitTestCase(BB&: BTB, NextMBB, BranchProbToNext: UnhandledProb, Reg: BTB.Reg, B&: BTB.Cases[j], SwitchBB: MBB);
4966
4967 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
4968 // We need to record the replacement phi edge here that normally
4969 // happens in emitBitTestCase before we delete the case, otherwise the
4970 // phi edge will be lost.
4971 addMachineCFGPred(Edge: {BTB.Parent->getBasicBlock(),
4972 BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
4973 NewPred: MBB);
4974 // Since we're not going to use the final bit test, remove it.
4975 BTB.Cases.pop_back();
4976 break;
4977 }
4978 }
4979 // This is "default" BB. We have two jumps to it. From "header" BB and from
4980 // last "case" BB, unless the latter was skipped.
4981 CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
4982 BTB.Default->getBasicBlock()};
4983 addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Parent);
4984 if (!BTB.ContiguousRange) {
4985 addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Cases.back().ThisBB);
4986 }
4987 }
4988 SL->BitTestCases.clear();
4989
4990 for (auto &JTCase : SL->JTCases) {
4991 // Emit header first, if it wasn't already emitted.
4992 if (!JTCase.first.Emitted)
4993 emitJumpTableHeader(JT&: JTCase.second, JTH&: JTCase.first, HeaderBB: JTCase.first.HeaderBB);
4994
4995 emitJumpTable(JT&: JTCase.second, MBB: JTCase.second.MBB);
4996 }
4997 SL->JTCases.clear();
4998
4999 for (auto &SwCase : SL->SwitchCases)
5000 emitSwitchCase(CB&: SwCase, SwitchBB: &CurBuilder->getMBB(), MIB&: *CurBuilder);
5001 SL->SwitchCases.clear();
5002
5003 // Check if we need to generate stack-protector guard checks.
5004 if (SPInfo->shouldEmitSDCheck(BB)) {
5005 bool FunctionBasedInstrumentation =
5006 TLI->getSSPStackGuardCheck(M: *MF->getFunction().getParent(), Libcalls: *Libcalls);
5007 SPDescriptor.initialize(BB: &BB, MBB: &MBB, FunctionBasedInstrumentation);
5008 }
5009 // Handle stack protector.
5010 if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
5011 LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
5012 return false;
5013 } else if (SPDescriptor.shouldEmitStackProtector()) {
5014 MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
5015 MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
5016
5017 // Find the split point to split the parent mbb. At the same time copy all
5018 // physical registers used in the tail of parent mbb into virtual registers
5019 // before the split point and back into physical registers after the split
5020 // point. This prevents us needing to deal with Live-ins and many other
5021 // register allocation issues caused by us splitting the parent mbb. The
5022 // register allocator will clean up said virtual copies later on.
5023 MachineBasicBlock::iterator SplitPoint = findSplitPointForStackProtector(
5024 BB: ParentMBB, TII: *MF->getSubtarget().getInstrInfo());
5025
5026 // Splice the terminator of ParentMBB into SuccessMBB.
5027 SuccessMBB->splice(Where: SuccessMBB->end(), Other: ParentMBB, From: SplitPoint,
5028 To: ParentMBB->end());
5029
5030 // Add compare/jump on neq/jump to the parent BB.
5031 if (!emitSPDescriptorParent(SPD&: SPDescriptor, ParentBB: ParentMBB))
5032 return false;
5033
5034 // CodeGen Failure MBB if we have not codegened it yet.
5035 MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
5036 if (FailureMBB->empty()) {
5037 if (!emitSPDescriptorFailure(SPD&: SPDescriptor, FailureBB: FailureMBB))
5038 return false;
5039 }
5040
5041 // Clear the Per-BB State.
5042 SPDescriptor.resetPerBBState();
5043 }
5044 return true;
5045}
5046
5047bool IRTranslatorImpl::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
5048 MachineBasicBlock *ParentBB) {
5049 CurBuilder->setInsertPt(MBB&: *ParentBB, II: ParentBB->end());
5050 // First create the loads to the guard/stack slot for the comparison.
5051 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
5052 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
5053 LLT PtrMemTy = getLLTForMVT(Ty: TLI->getPointerMemTy(DL: *DL));
5054
5055 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
5056 int FI = MFI.getStackProtectorIndex();
5057
5058 Register Guard;
5059 Register StackSlotPtr = CurBuilder->buildFrameIndex(Res: PtrTy, Idx: FI).getReg(Idx: 0);
5060 const Module &M = *ParentBB->getParent()->getFunction().getParent();
5061 Align Align = DL->getPrefTypeAlign(Ty: PointerType::getUnqual(C&: M.getContext()));
5062
5063 // Generate code to load the content of the guard slot.
5064 Register GuardVal =
5065 CurBuilder
5066 ->buildLoad(Res: PtrMemTy, Addr: StackSlotPtr,
5067 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
5068 MMOFlags: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile)
5069 .getReg(Idx: 0);
5070
5071 // Retrieve guard check function, nullptr if instrumentation is inlined.
5072 if (const Function *GuardCheckFn = TLI->getSSPStackGuardCheck(M, Libcalls: *Libcalls)) {
5073 // This path is currently untestable on GlobalISel, since the only platform
5074 // that needs this seems to be Windows, and we fall back on that currently.
5075 // The code still lives here in case that changes.
5076 // Silence warning about unused variable until the code below that uses
5077 // 'GuardCheckFn' is enabled.
5078 (void)GuardCheckFn;
5079 return false;
5080#if 0
5081 // The target provides a guard check function to validate the guard value.
5082 // Generate a call to that function with the content of the guard slot as
5083 // argument.
5084 FunctionType *FnTy = GuardCheckFn->getFunctionType();
5085 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
5086 ISD::ArgFlagsTy Flags;
5087 if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
5088 Flags.setInReg();
5089 CallLowering::ArgInfo GuardArgInfo(
5090 {GuardVal, FnTy->getParamType(0), {Flags}});
5091
5092 CallLowering::CallLoweringInfo Info;
5093 Info.OrigArgs.push_back(GuardArgInfo);
5094 Info.CallConv = GuardCheckFn->getCallingConv();
5095 Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
5096 Info.OrigRet = {Register(), FnTy->getReturnType()};
5097 if (!CLI->lowerCall(MIRBuilder, Info)) {
5098 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
5099 return false;
5100 }
5101 return true;
5102#endif
5103 }
5104
5105 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
5106 // Otherwise, emit a volatile load to retrieve the stack guard value.
5107 if (TLI->useLoadStackGuardNode(M: *ParentBB->getBasicBlock()->getModule())) {
5108 Guard = MRI->createGenericVirtualRegister(Ty: PtrMemTy);
5109 getStackGuard(DstReg: Guard, MIRBuilder&: *CurBuilder);
5110 } else {
5111 // TODO: test using android subtarget when we support @llvm.thread.pointer.
5112 const Value *IRGuard = TLI->getSDagStackGuard(M, Libcalls: *Libcalls);
5113 if (IRGuard) {
5114 Register GuardPtr = getOrCreateVReg(Val: *IRGuard);
5115
5116 Guard = CurBuilder
5117 ->buildLoad(Res: PtrMemTy, Addr: GuardPtr,
5118 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
5119 MMOFlags: MachineMemOperand::MOLoad |
5120 MachineMemOperand::MOVolatile)
5121 .getReg(Idx: 0);
5122 } else {
5123 LLVMContext &Ctx = CurBuilder->getContext();
5124 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
5125 Guard = CurBuilder->buildUndef(Res: PtrMemTy).getReg(Idx: 0);
5126 }
5127 }
5128
5129 // Perform the comparison.
5130 auto Cmp =
5131 CurBuilder->buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::integer(SizeInBits: 1), Op0: Guard, Op1: GuardVal);
5132 // If the guard/stackslot do not equal, branch to failure MBB.
5133 CurBuilder->buildBrCond(Tst: Cmp, Dest&: *SPD.getFailureMBB());
5134 // Otherwise branch to success MBB.
5135 CurBuilder->buildBr(Dest&: *SPD.getSuccessMBB());
5136 return true;
5137}
5138
5139bool IRTranslatorImpl::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
5140 MachineBasicBlock *FailureBB) {
5141 const RTLIB::LibcallImpl LibcallImpl =
5142 Libcalls->getLibcallImpl(Call: RTLIB::STACKPROTECTOR_CHECK_FAIL);
5143 if (LibcallImpl == RTLIB::Unsupported)
5144 return false;
5145
5146 CurBuilder->setInsertPt(MBB&: *FailureBB, II: FailureBB->end());
5147
5148 CallLowering::CallLoweringInfo Info;
5149 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: LibcallImpl);
5150
5151 StringRef LibcallName =
5152 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LibcallImpl);
5153 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
5154 Info.OrigRet = {Register(), Type::getVoidTy(C&: MF->getFunction().getContext()),
5155 0};
5156 if (!CLI->lowerCall(MIRBuilder&: *CurBuilder, Info)) {
5157 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
5158 return false;
5159 }
5160
5161 // Emit a trap instruction if we are required to do so.
5162 const TargetOptions &TargetOpts = TLI->getTargetMachine().Options;
5163 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
5164 CurBuilder->buildInstr(Opcode: TargetOpcode::G_TRAP);
5165
5166 return true;
5167}
5168
5169void IRTranslatorImpl::finalizeFunction() {
5170 // Release the memory used by the different maps we
5171 // needed during the translation.
5172 PendingPHIs.clear();
5173 PendingConstDbgValues.clear();
5174 VMap.reset();
5175 FrameIndices.clear();
5176 MachinePreds.clear();
5177 // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
5178 // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
5179 // destroying it twice (in ~IRTranslator() and ~LLVMContext())
5180 EntryBuilder.reset();
5181 CurBuilder.reset();
5182 FuncInfo.clear();
5183 SPDescriptor.resetPerFunctionState();
5184}
5185
5186/// Returns true if a BasicBlock \p BB within a variadic function contains a
5187/// variadic musttail call.
5188static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
5189 if (!IsVarArg)
5190 return false;
5191
5192 // Walk the block backwards, because tail calls usually only appear at the end
5193 // of a block.
5194 return llvm::any_of(Range: llvm::reverse(C: BB), P: [](const Instruction &I) {
5195 const auto *CI = dyn_cast<CallInst>(Val: &I);
5196 return CI && CI->isMustTailCall();
5197 });
5198}
5199
5200bool IRTranslatorImpl::runOnMachineFunction(
5201 MachineFunction &CurMF, function_ref<GISelCSEInfo *()> GetCSEInfo,
5202 bool ShouldSkipOpts, function_ref<AAResults *()> GetAAResults,
5203 function_ref<BranchProbabilityInfo *()> GetBPI,
5204 function_ref<AssumptionCache *()> GetAC, TargetLibraryInfo *LibraryInfo,
5205 const LibcallLoweringInfo *LibcallInfo, SSPLayoutInfo *StackProtectorInfo) {
5206 MF = &CurMF;
5207 const Function &F = MF->getFunction();
5208 ORE = std::make_unique<OptimizationRemarkEmitter>(args: &F);
5209 CLI = MF->getSubtarget().getCallLowering();
5210 SPInfo = StackProtectorInfo;
5211
5212 if (CLI->fallBackToDAGISel(MF: *MF)) {
5213 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
5214 F.getSubprogram(), &F.getEntryBlock());
5215 R << "unable to lower function: "
5216 << ore::NV("Prototype", F.getFunctionType());
5217
5218 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5219 return false;
5220 }
5221
5222 // Set the CSEConfig and run the analysis.
5223 GISelCSEInfo *CSEInfo = nullptr;
5224
5225 bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
5226 ? EnableCSEInIRTranslator
5227 : true;
5228
5229 const TargetSubtargetInfo &Subtarget = MF->getSubtarget();
5230 TLI = Subtarget.getTargetLowering();
5231
5232 if (EnableCSE) {
5233 EntryBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
5234 CSEInfo = GetCSEInfo();
5235 EntryBuilder->setCSEInfo(CSEInfo);
5236 CurBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
5237 CurBuilder->setCSEInfo(CSEInfo);
5238 } else {
5239 EntryBuilder = std::make_unique<MachineIRBuilder>();
5240 CurBuilder = std::make_unique<MachineIRBuilder>();
5241 }
5242 CLI = Subtarget.getCallLowering();
5243 CurBuilder->setMF(*MF);
5244 EntryBuilder->setMF(*MF);
5245 MRI = &MF->getRegInfo();
5246 DL = &F.getDataLayout();
5247 const TargetMachine &TM = MF->getTarget();
5248 EnableOpts = OptLevel != CodeGenOptLevel::None && !ShouldSkipOpts;
5249 FuncInfo.MF = MF;
5250 FuncInfo.ExceptionModel = F.getParent()->getExceptionModel();
5251 if (EnableOpts) {
5252 AA = GetAAResults();
5253 FuncInfo.BPI = GetBPI();
5254 AC = GetAC();
5255 } else {
5256 AA = nullptr;
5257 FuncInfo.BPI = nullptr;
5258 AC = nullptr;
5259 }
5260 LibInfo = LibraryInfo;
5261 Libcalls = LibcallInfo;
5262
5263 FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(MF&: *MF);
5264
5265 SL = std::make_unique<GISelSwitchLowering>(args: this, args&: FuncInfo);
5266 SL->init(tli: *TLI, tm: TM, dl: *DL);
5267
5268 assert(PendingPHIs.empty() && "stale PHIs");
5269
5270 // Targets which want to use big endian can enable it using
5271 // enableBigEndian()
5272 if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
5273 // Currently we don't properly handle big endian code.
5274 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
5275 F.getSubprogram(), &F.getEntryBlock());
5276 R << "unable to translate in big endian mode";
5277 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5278 return false;
5279 }
5280
5281 // Release the per-function state when we return, whether we succeeded or not.
5282 llvm::scope_exit FinalizeOnReturn([this]() { finalizeFunction(); });
5283
5284 // Setup a separate basic-block for the arguments and constants
5285 MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
5286 MF->push_back(MBB: EntryBB);
5287 EntryBuilder->setMBB(*EntryBB);
5288
5289 DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHIIt()->getDebugLoc();
5290 SwiftError.setFunction(CurMF);
5291 SwiftError.createEntriesInEntryBlock(DbgLoc);
5292
5293 bool IsVarArg = F.isVarArg();
5294 bool HasMustTailInVarArgFn = false;
5295 // Use arguments and instructions to estimate the number of mapped values and
5296 // virtual registers.
5297 unsigned NumValues = F.arg_size();
5298
5299 // Create all blocks, in IR order, to preserve the layout.
5300 FuncInfo.MBBMap.resize(N: F.getMaxBlockNumber());
5301 for (const BasicBlock &BB: F) {
5302 NumValues += BB.size();
5303 auto *&MBB = FuncInfo.MBBMap[BB.getNumber()];
5304
5305 MBB = MF->CreateMachineBasicBlock(BB: &BB);
5306 MF->push_back(MBB);
5307
5308 // Only mark the block if the BlockAddress actually has users. The
5309 // hasAddressTaken flag may be stale if the BlockAddress was optimized away
5310 // but the constant still exists in the uniquing table.
5311 if (BB.hasAddressTaken()) {
5312 if (BlockAddress *BA = BlockAddress::lookup(BB: &BB))
5313 if (!BA->hasZeroLiveUses())
5314 MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
5315 }
5316
5317 if (!HasMustTailInVarArgFn)
5318 HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
5319 }
5320
5321 VMap.reserveVRegs(NumValues);
5322 MRI->reserveVirtRegs(NumVirtRegs: NumValues);
5323
5324 MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
5325
5326 // Make our arguments/constants entry block fallthrough to the IR entry block.
5327 EntryBB->addSuccessor(Succ: &getMBB(BB: F.front()));
5328
5329 // Lower the actual args into this basic block.
5330 SmallVector<ArrayRef<Register>, 8> VRegArgs;
5331 for (const Argument &Arg: F.args()) {
5332 if (DL->getTypeStoreSize(Ty: Arg.getType()).isZero())
5333 continue; // Don't handle zero sized types.
5334 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: Arg);
5335 VRegArgs.push_back(Elt: VRegs);
5336
5337 if (CLI->supportSwiftError() && Arg.hasSwiftErrorAttr()) {
5338 assert(VRegs.size() == 1 && "Too many vregs for Swift error");
5339 SwiftError.setCurrentVReg(MBB: EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
5340 }
5341 }
5342
5343 if (!CLI->lowerFormalArguments(MIRBuilder&: *EntryBuilder, F, VRegs: VRegArgs, FLI&: FuncInfo)) {
5344 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
5345 F.getSubprogram(), &F.getEntryBlock());
5346 R << "unable to lower arguments: "
5347 << ore::NV("Prototype", F.getFunctionType());
5348 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5349 return false;
5350 }
5351
5352 // Need to visit defs before uses when translating instructions.
5353 GISelObserverWrapper WrapperObserver;
5354 if (EnableCSE && CSEInfo)
5355 WrapperObserver.addObserver(O: CSEInfo);
5356 {
5357 ReversePostOrderTraversal<const Function *> RPOT(&F);
5358#ifndef NDEBUG
5359 DILocationVerifier Verifier;
5360 WrapperObserver.addObserver(&Verifier);
5361#endif // ifndef NDEBUG
5362 RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);
5363 for (const BasicBlock *BB : RPOT) {
5364 MachineBasicBlock &MBB = getMBB(BB: *BB);
5365 // Set the insertion point of all the following translations to
5366 // the end of this basic block.
5367 CurBuilder->setMBB(MBB);
5368 HasTailCall = false;
5369 for (const Instruction &Inst : *BB) {
5370 // If we translated a tail call in the last step, then we know
5371 // everything after the call is either a return, or something that is
5372 // handled by the call itself. (E.g. a lifetime marker or assume
5373 // intrinsic.) In this case, we should stop translating the block and
5374 // move on.
5375 if (HasTailCall)
5376 break;
5377#ifndef NDEBUG
5378 Verifier.setCurrentInst(&Inst);
5379#endif // ifndef NDEBUG
5380
5381 // Translate any debug-info attached to the instruction.
5382 translateDbgInfo(Inst, MIRBuilder&: *CurBuilder);
5383
5384 if (translate(Inst))
5385 continue;
5386
5387 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
5388 Inst.getDebugLoc(), BB);
5389 R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
5390
5391 if (ORE->allowExtraAnalysis(PassName: "gisel-ir-translator")) {
5392 std::string InstStrStorage;
5393 raw_string_ostream InstStr(InstStrStorage);
5394 InstStr << Inst;
5395
5396 R << ": '" << InstStrStorage << "'";
5397 }
5398
5399 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5400 return false;
5401 }
5402
5403 if (!finalizeBasicBlock(BB: *BB, MBB)) {
5404 OptimizationRemarkMissed R("gisel-ir-translator", "GISelFailure",
5405 BB->getTerminator()->getDebugLoc(), BB);
5406 R << "unable to translate basic block";
5407 reportTranslationError(MF&: *MF, ORE&: *ORE, R);
5408 return false;
5409 }
5410 }
5411#ifndef NDEBUG
5412 WrapperObserver.removeObserver(&Verifier);
5413#endif
5414 }
5415
5416 finishPendingPhis();
5417 finishPendingConstDbgValues();
5418
5419 SwiftError.propagateVRegs();
5420
5421 // Merge the argument lowering and constants block with its single
5422 // successor, the LLVM-IR entry block. We want the basic block to
5423 // be maximal.
5424 assert(EntryBB->succ_size() == 1 &&
5425 "Custom BB used for lowering should have only one successor");
5426 // Get the successor of the current entry block.
5427 MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
5428 assert(NewEntryBB.pred_size() == 1 &&
5429 "LLVM-IR entry block has a predecessor!?");
5430 // Move all the instruction from the current entry block to the
5431 // new entry block.
5432 NewEntryBB.splice(Where: NewEntryBB.begin(), Other: EntryBB, From: EntryBB->begin(),
5433 To: EntryBB->end());
5434
5435 // Update the live-in information for the new entry block.
5436 for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
5437 NewEntryBB.addLiveIn(RegMaskPair: LiveIn);
5438 NewEntryBB.sortUniqueLiveIns();
5439
5440 // Get rid of the now empty basic block.
5441 EntryBB->removeSuccessor(Succ: &NewEntryBB);
5442 MF->remove(MBBI: EntryBB);
5443 MF->deleteMachineBasicBlock(MBB: EntryBB);
5444
5445 assert(&MF->front() == &NewEntryBB &&
5446 "New entry wasn't next in the list of basic block!");
5447
5448 // Initialize stack protector information.
5449 SPInfo->copyToMachineFrameInfo(MFI&: MF->getFrameInfo());
5450
5451 return false;
5452}
5453
5454bool IRTranslatorLegacy::runOnMachineFunction(MachineFunction &MF) {
5455 const TargetSubtargetInfo &Subtarget = MF.getSubtarget();
5456 Function &F = MF.getFunction();
5457
5458 bool ShouldSkipOpts = skipFunction(F: MF.getFunction());
5459 return Impl->runOnMachineFunction(
5460 CurMF&: MF,
5461 GetCSEInfo: [&]() {
5462 TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
5463 GISelCSEAnalysisWrapper &Wrapper =
5464 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
5465 return &Wrapper.get(CSEOpt: TPC.getCSEConfig());
5466 },
5467 ShouldSkipOpts,
5468 GetAAResults: [&]() { return &getAnalysis<AAResultsWrapperPass>().getAAResults(); },
5469 GetBPI: [&]() {
5470 return &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
5471 },
5472 GetAC: [&]() {
5473 return &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
5474 F&: MF.getFunction());
5475 },
5476 LibraryInfo: &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F: MF.getFunction()),
5477 LibcallInfo: &getAnalysis<LibcallLoweringInfoWrapper>().getLibcallLowering(
5478 M: *F.getParent(), Subtarget),
5479 StackProtectorInfo: &getAnalysis<StackProtector>().getLayoutInfo());
5480}
5481
5482IRTranslatorPass::IRTranslatorPass(CodeGenOptLevel OptLevel)
5483 : Impl(std::make_unique<IRTranslatorImpl>(args&: OptLevel)) {}
5484
5485IRTranslatorPass::~IRTranslatorPass() = default;
5486IRTranslatorPass::IRTranslatorPass(IRTranslatorPass &&) = default;
5487
5488PreservedAnalyses IRTranslatorPass::run(MachineFunction &MF,
5489 MachineFunctionAnalysisManager &MFAM) {
5490 const TargetSubtargetInfo &Subtarget = MF.getSubtarget();
5491 Function &F = MF.getFunction();
5492
5493 bool ShouldSkipOpts = MF.getFunction().hasOptNone() ||
5494 shouldSkipOptimizationForOptBisect(IR: MF.getFunction());
5495 auto &FAM = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
5496 .getManager();
5497 auto &MAMProxy =
5498 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF);
5499 const ModuleLibcallLoweringInfo *MLLI =
5500 MAMProxy.getCachedResult<LibcallLoweringModuleAnalysis>(IR&: *F.getParent());
5501 if (!MLLI)
5502 reportFatalUsageError(
5503 reason: "LibcallLoweringModuleAnalysis must be available for IRTranslator");
5504 Impl->runOnMachineFunction(
5505 CurMF&: MF, GetCSEInfo: [&]() { return MFAM.getResult<GISelCSEAnalysis>(IR&: MF).get(); },
5506 ShouldSkipOpts, GetAAResults: [&]() { return &FAM.getResult<AAManager>(IR&: F); },
5507 GetBPI: [&]() { return &FAM.getResult<BranchProbabilityAnalysis>(IR&: F); },
5508 GetAC: [&]() { return &FAM.getResult<AssumptionAnalysis>(IR&: F); },
5509 LibraryInfo: &FAM.getResult<TargetLibraryAnalysis>(IR&: F),
5510 LibcallInfo: &getLibcallLowering(ModuleInfo: *MLLI, Subtarget),
5511 StackProtectorInfo: &FAM.getResult<SSPLayoutAnalysis>(IR&: F));
5512
5513 return getMachineFunctionPassPreservedAnalyses();
5514}
5515