1//===- FastISel.cpp - Implementation of the FastISel class ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the FastISel class.
10//
11// "Fast" instruction selection is designed to emit very poor code quickly.
12// Also, it is not designed to be able to do much lowering, so most illegal
13// types (e.g. i64 on 32-bit targets) and operations are not supported. It is
14// also not intended to be able to do much optimization, except in a few cases
15// where doing optimizations reduces overall compile time. For example, folding
16// constants into immediate fields is often done, because it's cheap and it
17// reduces the number of instructions later phases have to examine.
18//
19// "Fast" instruction selection is able to fail gracefully and transfer
20// control to the SelectionDAG selector for operations that it doesn't
21// support. In many cases, this allows us to avoid duplicating a lot of
22// the complicated lowering logic that SelectionDAG currently has.
23//
24// The intended use for "fast" instruction selection is "-O0" mode
25// compilation, where the quality of the generated code is irrelevant when
26// weighed against the speed at which the code can be generated. Also,
27// at -O0, the LLVM optimizers are not running, and this makes the
28// compile time of codegen a much higher portion of the overall compile
29// time. Despite its limitations, "fast" instruction selection is able to
30// handle enough code on its own to provide noticeable overall speedups
31// in -O0 compiles.
32//
33// Basic operations are supported in a target-independent way, by reading
34// the same instruction descriptions that the SelectionDAG selector reads,
35// and identifying simple arithmetic operations that can be directly selected
36// from simple operators. More complicated operations currently require
37// target-specific code.
38//
39//===----------------------------------------------------------------------===//
40
41#include "llvm/CodeGen/FastISel.h"
42#include "llvm/ADT/APFloat.h"
43#include "llvm/ADT/APSInt.h"
44#include "llvm/ADT/DenseMap.h"
45#include "llvm/ADT/SmallPtrSet.h"
46#include "llvm/ADT/SmallString.h"
47#include "llvm/ADT/SmallVector.h"
48#include "llvm/ADT/Statistic.h"
49#include "llvm/Analysis/BranchProbabilityInfo.h"
50#include "llvm/Analysis/TargetLibraryInfo.h"
51#include "llvm/CodeGen/Analysis.h"
52#include "llvm/CodeGen/FunctionLoweringInfo.h"
53#include "llvm/CodeGen/ISDOpcodes.h"
54#include "llvm/CodeGen/MachineBasicBlock.h"
55#include "llvm/CodeGen/MachineFrameInfo.h"
56#include "llvm/CodeGen/MachineInstr.h"
57#include "llvm/CodeGen/MachineInstrBuilder.h"
58#include "llvm/CodeGen/MachineMemOperand.h"
59#include "llvm/CodeGen/MachineModuleInfo.h"
60#include "llvm/CodeGen/MachineOperand.h"
61#include "llvm/CodeGen/MachineRegisterInfo.h"
62#include "llvm/CodeGen/StackMaps.h"
63#include "llvm/CodeGen/TargetInstrInfo.h"
64#include "llvm/CodeGen/TargetLowering.h"
65#include "llvm/CodeGen/TargetSubtargetInfo.h"
66#include "llvm/CodeGen/ValueTypes.h"
67#include "llvm/CodeGenTypes/MachineValueType.h"
68#include "llvm/IR/Argument.h"
69#include "llvm/IR/Attributes.h"
70#include "llvm/IR/BasicBlock.h"
71#include "llvm/IR/CallingConv.h"
72#include "llvm/IR/Constant.h"
73#include "llvm/IR/Constants.h"
74#include "llvm/IR/DataLayout.h"
75#include "llvm/IR/DebugLoc.h"
76#include "llvm/IR/DerivedTypes.h"
77#include "llvm/IR/DiagnosticInfo.h"
78#include "llvm/IR/Function.h"
79#include "llvm/IR/GetElementPtrTypeIterator.h"
80#include "llvm/IR/GlobalValue.h"
81#include "llvm/IR/InlineAsm.h"
82#include "llvm/IR/InstrTypes.h"
83#include "llvm/IR/Instruction.h"
84#include "llvm/IR/Instructions.h"
85#include "llvm/IR/IntrinsicInst.h"
86#include "llvm/IR/LLVMContext.h"
87#include "llvm/IR/Mangler.h"
88#include "llvm/IR/Metadata.h"
89#include "llvm/IR/Module.h"
90#include "llvm/IR/Operator.h"
91#include "llvm/IR/PatternMatch.h"
92#include "llvm/IR/Type.h"
93#include "llvm/IR/User.h"
94#include "llvm/IR/Value.h"
95#include "llvm/MC/MCContext.h"
96#include "llvm/MC/MCInstrDesc.h"
97#include "llvm/Support/Casting.h"
98#include "llvm/Support/Debug.h"
99#include "llvm/Support/ErrorHandling.h"
100#include "llvm/Support/MathExtras.h"
101#include "llvm/Support/raw_ostream.h"
102#include "llvm/Target/TargetMachine.h"
103#include "llvm/Target/TargetOptions.h"
104#include <cassert>
105#include <cstdint>
106#include <iterator>
107#include <optional>
108#include <utility>
109
110using namespace llvm;
111using namespace PatternMatch;
112
113#define DEBUG_TYPE "isel"
114
115STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
116 "target-independent selector");
117STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
118 "target-specific selector");
119STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
120
121/// Set the current block to which generated machine instructions will be
122/// appended.
123void FastISel::startNewBlock() {
124 assert(LocalValueMap.empty() &&
125 "local values should be cleared after finishing a BB");
126
127 // Instructions are appended to FuncInfo.MBB. If the basic block already
128 // contains labels or copies, use the last instruction as the last local
129 // value.
130 EmitStartPt = nullptr;
131 if (!FuncInfo.MBB->empty())
132 EmitStartPt = &FuncInfo.MBB->back();
133 LastLocalValue = EmitStartPt;
134}
135
136void FastISel::finishBasicBlock() { flushLocalValueMap(); }
137
138bool FastISel::lowerArguments() {
139 if (!FuncInfo.CanLowerReturn)
140 // Fallback to SDISel argument lowering code to deal with sret pointer
141 // parameter.
142 return false;
143
144 if (!fastLowerArguments())
145 return false;
146
147 // Enter arguments into ValueMap for uses in non-entry BBs.
148 for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(),
149 E = FuncInfo.Fn->arg_end();
150 I != E; ++I) {
151 DenseMap<const Value *, Register>::iterator VI = LocalValueMap.find(Val: &*I);
152 assert(VI != LocalValueMap.end() && "Missed an argument?");
153 FuncInfo.ValueMap[&*I] = VI->second;
154 }
155 return true;
156}
157
158/// Return the defined register if this instruction defines exactly one
159/// virtual register and uses no other virtual registers. Otherwise return
160/// Register();
161static Register findLocalRegDef(MachineInstr &MI) {
162 Register RegDef;
163 for (const MachineOperand &MO : MI.operands()) {
164 if (!MO.isReg())
165 continue;
166 if (MO.isDef()) {
167 if (RegDef)
168 return Register();
169 RegDef = MO.getReg();
170 } else if (MO.getReg().isVirtual()) {
171 // This is another use of a vreg. Don't delete it.
172 return Register();
173 }
174 }
175 return RegDef;
176}
177
178static bool isRegUsedByPhiNodes(Register DefReg,
179 FunctionLoweringInfo &FuncInfo) {
180 for (auto &P : FuncInfo.PHINodesToUpdate)
181 if (P.second == DefReg)
182 return true;
183 return false;
184}
185
186void FastISel::flushLocalValueMap() {
187 // If FastISel bails out, it could leave local value instructions behind
188 // that aren't used for anything. Detect and erase those.
189 if (LastLocalValue != EmitStartPt) {
190 // Save the first instruction after local values, for later.
191 MachineBasicBlock::iterator FirstNonValue(LastLocalValue);
192 ++FirstNonValue;
193
194 MachineBasicBlock::reverse_iterator RE =
195 EmitStartPt ? MachineBasicBlock::reverse_iterator(EmitStartPt)
196 : FuncInfo.MBB->rend();
197 MachineBasicBlock::reverse_iterator RI(LastLocalValue);
198 for (MachineInstr &LocalMI :
199 llvm::make_early_inc_range(Range: llvm::make_range(x: RI, y: RE))) {
200 Register DefReg = findLocalRegDef(MI&: LocalMI);
201 if (!DefReg)
202 continue;
203 if (FuncInfo.RegsWithFixups.count(V: DefReg))
204 continue;
205 bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
206 if (!UsedByPHI && MRI.use_nodbg_empty(RegNo: DefReg)) {
207 if (EmitStartPt == &LocalMI)
208 EmitStartPt = EmitStartPt->getPrevNode();
209 LLVM_DEBUG(dbgs() << "removing dead local value materialization"
210 << LocalMI);
211 LocalMI.eraseFromParent();
212 }
213 }
214
215 if (FirstNonValue != FuncInfo.MBB->end()) {
216 // See if there are any local value instructions left. If so, we want to
217 // make sure the first one has a debug location; if it doesn't, use the
218 // first non-value instruction's debug location.
219
220 // If EmitStartPt is non-null, this block had copies at the top before
221 // FastISel started doing anything; it points to the last one, so the
222 // first local value instruction is the one after EmitStartPt.
223 // If EmitStartPt is null, the first local value instruction is at the
224 // top of the block.
225 MachineBasicBlock::iterator FirstLocalValue =
226 EmitStartPt ? ++MachineBasicBlock::iterator(EmitStartPt)
227 : FuncInfo.MBB->begin();
228 if (FirstLocalValue != FirstNonValue && !FirstLocalValue->getDebugLoc())
229 FirstLocalValue->setDebugLoc(FirstNonValue->getDebugLoc());
230 }
231 }
232
233 LocalValueMap.clear();
234 LastLocalValue = EmitStartPt;
235 recomputeInsertPt();
236 SavedInsertPt = FuncInfo.InsertPt;
237}
238
239Register FastISel::getRegForValue(const Value *V) {
240 EVT RealVT = TLI.getValueType(DL, Ty: V->getType(), /*AllowUnknown=*/true);
241 // Don't handle non-simple values in FastISel.
242 if (!RealVT.isSimple())
243 return Register();
244
245 // Ignore illegal types. We must do this before looking up the value
246 // in ValueMap because Arguments are given virtual registers regardless
247 // of whether FastISel can handle them.
248 MVT VT = RealVT.getSimpleVT();
249 if (!TLI.isTypeLegal(VT)) {
250 // Handle integer promotions, though, because they're common and easy.
251 if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
252 VT = TLI.getTypeToTransformTo(Context&: V->getContext(), VT).getSimpleVT();
253 else
254 return Register();
255 }
256
257 // Look up the value to see if we already have a register for it.
258 Register Reg = lookUpRegForValue(V);
259 if (Reg)
260 return Reg;
261
262 // In bottom-up mode, just create the virtual register which will be used
263 // to hold the value. It will be materialized later.
264 if (isa<Instruction>(Val: V) &&
265 (!isa<AllocaInst>(Val: V) ||
266 !FuncInfo.StaticAllocaMap.count(Val: cast<AllocaInst>(Val: V))))
267 return FuncInfo.InitializeRegForValue(V);
268
269 SavePoint SaveInsertPt = enterLocalValueArea();
270
271 // Materialize the value in a register. Emit any instructions in the
272 // local value area.
273 Reg = materializeRegForValue(V, VT);
274
275 leaveLocalValueArea(Old: SaveInsertPt);
276
277 return Reg;
278}
279
280Register FastISel::materializeConstant(const Value *V, MVT VT) {
281 Register Reg;
282 if (const auto *CI = dyn_cast<ConstantInt>(Val: V)) {
283 if (CI->getValue().getActiveBits() <= 64)
284 Reg = fastEmit_i(VT, RetVT: VT, Opcode: ISD::Constant, Imm: CI->getZExtValue());
285 } else if (isa<AllocaInst>(Val: V))
286 Reg = fastMaterializeAlloca(C: cast<AllocaInst>(Val: V));
287 else if (isa<ConstantPointerNull>(Val: V))
288 // Translate this as an integer zero so that it can be
289 // local-CSE'd with actual integer zeros.
290 Reg =
291 getRegForValue(V: Constant::getNullValue(Ty: DL.getIntPtrType(V->getType())));
292 else if (const auto *CF = dyn_cast<ConstantFP>(Val: V)) {
293 if (CF->isNullValue())
294 Reg = fastMaterializeFloatZero(CF);
295 else
296 // Try to emit the constant directly.
297 Reg = fastEmit_f(VT, RetVT: VT, Opcode: ISD::ConstantFP, FPImm: CF);
298
299 if (!Reg) {
300 // Try to emit the constant by using an integer constant with a cast.
301 const APFloat &Flt = CF->getValueAPF();
302 EVT IntVT = TLI.getPointerTy(DL);
303 uint32_t IntBitWidth = IntVT.getSizeInBits();
304 APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
305 bool isExact;
306 (void)Flt.convertToInteger(Result&: SIntVal, RM: APFloat::rmTowardZero, IsExact: &isExact);
307 if (isExact) {
308 Register IntegerReg =
309 getRegForValue(V: ConstantInt::get(Context&: V->getContext(), V: SIntVal));
310 if (IntegerReg)
311 Reg = fastEmit_r(VT: IntVT.getSimpleVT(), RetVT: VT, Opcode: ISD::SINT_TO_FP,
312 Op0: IntegerReg);
313 }
314 }
315 } else if (const auto *Op = dyn_cast<Operator>(Val: V)) {
316 if (!selectOperator(I: Op, Opcode: Op->getOpcode()))
317 if (!isa<Instruction>(Val: Op) ||
318 !fastSelectInstruction(I: cast<Instruction>(Val: Op)))
319 return Register();
320 Reg = lookUpRegForValue(V: Op);
321 } else if (isa<UndefValue>(Val: V)) {
322 Reg = createResultReg(RC: TLI.getRegClassFor(VT));
323 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
324 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: Reg);
325 }
326 return Reg;
327}
328
329/// Helper for getRegForValue. This function is called when the value isn't
330/// already available in a register and must be materialized with new
331/// instructions.
332Register FastISel::materializeRegForValue(const Value *V, MVT VT) {
333 Register Reg;
334 // Give the target-specific code a try first.
335 if (isa<Constant>(Val: V))
336 Reg = fastMaterializeConstant(C: cast<Constant>(Val: V));
337
338 // If target-specific code couldn't or didn't want to handle the value, then
339 // give target-independent code a try.
340 if (!Reg)
341 Reg = materializeConstant(V, VT);
342
343 // Don't cache constant materializations in the general ValueMap.
344 // To do so would require tracking what uses they dominate.
345 if (Reg) {
346 LocalValueMap[V] = Reg;
347 LastLocalValue = MRI.getVRegDef(Reg);
348 }
349 return Reg;
350}
351
352Register FastISel::lookUpRegForValue(const Value *V) {
353 // Look up the value to see if we already have a register for it. We
354 // cache values defined by Instructions across blocks, and other values
355 // only locally. This is because Instructions already have the SSA
356 // def-dominates-use requirement enforced.
357 DenseMap<const Value *, Register>::iterator I = FuncInfo.ValueMap.find(Val: V);
358 if (I != FuncInfo.ValueMap.end())
359 return I->second;
360 return LocalValueMap[V];
361}
362
363void FastISel::updateValueMap(const Value *I, Register Reg, unsigned NumRegs) {
364 if (!isa<Instruction>(Val: I)) {
365 LocalValueMap[I] = Reg;
366 return;
367 }
368
369 Register &AssignedReg = FuncInfo.ValueMap[I];
370 if (!AssignedReg)
371 // Use the new register.
372 AssignedReg = Reg;
373 else if (Reg != AssignedReg) {
374 // Arrange for uses of AssignedReg to be replaced by uses of Reg.
375 for (unsigned i = 0; i < NumRegs; i++) {
376 FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
377 FuncInfo.RegsWithFixups.insert(V: Reg + i);
378 }
379
380 AssignedReg = Reg;
381 }
382}
383
384Register FastISel::getRegForGEPIndex(MVT PtrVT, const Value *Idx) {
385 Register IdxN = getRegForValue(V: Idx);
386 if (!IdxN)
387 // Unhandled operand. Halt "fast" selection and bail.
388 return Register();
389
390 // If the index is smaller or larger than intptr_t, truncate or extend it.
391 EVT IdxVT = EVT::getEVT(Ty: Idx->getType(), /*HandleUnknown=*/false);
392 if (IdxVT.bitsLT(VT: PtrVT)) {
393 IdxN = fastEmit_r(VT: IdxVT.getSimpleVT(), RetVT: PtrVT, Opcode: ISD::SIGN_EXTEND, Op0: IdxN);
394 } else if (IdxVT.bitsGT(VT: PtrVT)) {
395 IdxN =
396 fastEmit_r(VT: IdxVT.getSimpleVT(), RetVT: PtrVT, Opcode: ISD::TRUNCATE, Op0: IdxN);
397 }
398 return IdxN;
399}
400
401void FastISel::recomputeInsertPt() {
402 if (getLastLocalValue()) {
403 FuncInfo.InsertPt = getLastLocalValue();
404 FuncInfo.MBB = FuncInfo.InsertPt->getParent();
405 ++FuncInfo.InsertPt;
406 } else
407 FuncInfo.InsertPt = FuncInfo.MBB->getFirstNonPHI();
408}
409
410void FastISel::removeDeadCode(MachineBasicBlock::iterator I,
411 MachineBasicBlock::iterator E) {
412 assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
413 "Invalid iterator!");
414 while (I != E) {
415 if (SavedInsertPt == I)
416 SavedInsertPt = E;
417 if (EmitStartPt == I)
418 EmitStartPt = E.isValid() ? &*E : nullptr;
419 if (LastLocalValue == I)
420 LastLocalValue = E.isValid() ? &*E : nullptr;
421
422 MachineInstr *Dead = &*I;
423 ++I;
424 Dead->eraseFromParent();
425 ++NumFastIselDead;
426 }
427 recomputeInsertPt();
428}
429
430FastISel::SavePoint FastISel::enterLocalValueArea() {
431 SavePoint OldInsertPt = FuncInfo.InsertPt;
432 recomputeInsertPt();
433 return OldInsertPt;
434}
435
436void FastISel::leaveLocalValueArea(SavePoint OldInsertPt) {
437 if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
438 LastLocalValue = &*std::prev(x: FuncInfo.InsertPt);
439
440 // Restore the previous insert position.
441 FuncInfo.InsertPt = OldInsertPt;
442}
443
444bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
445 EVT VT = EVT::getEVT(Ty: I->getType(), /*HandleUnknown=*/true);
446 if (VT == MVT::Other || !VT.isSimple())
447 // Unhandled type. Halt "fast" selection and bail.
448 return false;
449
450 // We only handle legal types. For example, on x86-32 the instruction
451 // selector contains all of the 64-bit instructions from x86-64,
452 // under the assumption that i64 won't be used if the target doesn't
453 // support it.
454 if (!TLI.isTypeLegal(VT)) {
455 // MVT::i1 is special. Allow AND, OR, or XOR because they
456 // don't require additional zeroing, which makes them easy.
457 if (VT == MVT::i1 && ISD::isBitwiseLogicOp(Opcode: ISDOpcode))
458 VT = TLI.getTypeToTransformTo(Context&: I->getContext(), VT);
459 else
460 return false;
461 }
462
463 // Check if the first operand is a constant, and handle it as "ri". At -O0,
464 // we don't have anything that canonicalizes operand order.
465 if (const auto *CI = dyn_cast<ConstantInt>(Val: I->getOperand(i: 0)))
466 if (isa<Instruction>(Val: I) && cast<Instruction>(Val: I)->isCommutative()) {
467 Register Op1 = getRegForValue(V: I->getOperand(i: 1));
468 if (!Op1)
469 return false;
470
471 Register ResultReg =
472 fastEmit_ri_(VT: VT.getSimpleVT(), Opcode: ISDOpcode, Op0: Op1, Imm: CI->getZExtValue(),
473 ImmType: VT.getSimpleVT());
474 if (!ResultReg)
475 return false;
476
477 // We successfully emitted code for the given LLVM Instruction.
478 updateValueMap(I, Reg: ResultReg);
479 return true;
480 }
481
482 Register Op0 = getRegForValue(V: I->getOperand(i: 0));
483 if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
484 return false;
485
486 // Check if the second operand is a constant and handle it appropriately.
487 if (const auto *CI = dyn_cast<ConstantInt>(Val: I->getOperand(i: 1))) {
488 uint64_t Imm = CI->getSExtValue();
489
490 // Transform "sdiv exact X, 8" -> "sra X, 3".
491 if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(Val: I) &&
492 cast<BinaryOperator>(Val: I)->isExact() && isPowerOf2_64(Value: Imm)) {
493 Imm = Log2_64(Value: Imm);
494 ISDOpcode = ISD::SRA;
495 }
496
497 // Transform "urem x, pow2" -> "and x, pow2-1".
498 if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(Val: I) &&
499 isPowerOf2_64(Value: Imm)) {
500 --Imm;
501 ISDOpcode = ISD::AND;
502 }
503
504 Register ResultReg = fastEmit_ri_(VT: VT.getSimpleVT(), Opcode: ISDOpcode, Op0, Imm,
505 ImmType: VT.getSimpleVT());
506 if (!ResultReg)
507 return false;
508
509 // We successfully emitted code for the given LLVM Instruction.
510 updateValueMap(I, Reg: ResultReg);
511 return true;
512 }
513
514 Register Op1 = getRegForValue(V: I->getOperand(i: 1));
515 if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
516 return false;
517
518 // Now we have both operands in registers. Emit the instruction.
519 Register ResultReg = fastEmit_rr(VT: VT.getSimpleVT(), RetVT: VT.getSimpleVT(),
520 Opcode: ISDOpcode, Op0, Op1);
521 if (!ResultReg)
522 // Target-specific code wasn't able to find a machine opcode for
523 // the given ISD opcode and type. Halt "fast" selection and bail.
524 return false;
525
526 // We successfully emitted code for the given LLVM Instruction.
527 updateValueMap(I, Reg: ResultReg);
528 return true;
529}
530
531bool FastISel::selectGetElementPtr(const User *I) {
532 Register N = getRegForValue(V: I->getOperand(i: 0));
533 if (!N) // Unhandled operand. Halt "fast" selection and bail.
534 return false;
535
536 // FIXME: The code below does not handle vector GEPs. Halt "fast" selection
537 // and bail.
538 if (isa<VectorType>(Val: I->getType()))
539 return false;
540
541 // Keep a running tab of the total offset to coalesce multiple N = N + Offset
542 // into a single N = N + TotalOffset.
543 uint64_t TotalOffs = 0;
544 // FIXME: What's a good SWAG number for MaxOffs?
545 uint64_t MaxOffs = 2048;
546 MVT VT = TLI.getValueType(DL, Ty: I->getType()).getSimpleVT();
547
548 for (gep_type_iterator GTI = gep_type_begin(GEP: I), E = gep_type_end(GEP: I);
549 GTI != E; ++GTI) {
550 const Value *Idx = GTI.getOperand();
551 if (StructType *StTy = GTI.getStructTypeOrNull()) {
552 uint64_t Field = cast<ConstantInt>(Val: Idx)->getZExtValue();
553 if (Field) {
554 // N = N + Offset
555 TotalOffs += DL.getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
556 if (TotalOffs >= MaxOffs) {
557 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
558 if (!N) // Unhandled operand. Halt "fast" selection and bail.
559 return false;
560 TotalOffs = 0;
561 }
562 }
563 } else {
564 // If this is a constant subscript, handle it quickly.
565 if (const auto *CI = dyn_cast<ConstantInt>(Val: Idx)) {
566 if (CI->isZero())
567 continue;
568 // N = N + Offset
569 uint64_t IdxN = CI->getValue().sextOrTrunc(width: 64).getSExtValue();
570 TotalOffs += GTI.getSequentialElementStride(DL) * IdxN;
571 if (TotalOffs >= MaxOffs) {
572 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
573 if (!N) // Unhandled operand. Halt "fast" selection and bail.
574 return false;
575 TotalOffs = 0;
576 }
577 continue;
578 }
579 if (TotalOffs) {
580 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
581 if (!N) // Unhandled operand. Halt "fast" selection and bail.
582 return false;
583 TotalOffs = 0;
584 }
585
586 // N = N + Idx * ElementSize;
587 uint64_t ElementSize = GTI.getSequentialElementStride(DL);
588 Register IdxN = getRegForGEPIndex(PtrVT: VT, Idx);
589 if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
590 return false;
591
592 if (ElementSize != 1) {
593 IdxN = fastEmit_ri_(VT, Opcode: ISD::MUL, Op0: IdxN, Imm: ElementSize, ImmType: VT);
594 if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
595 return false;
596 }
597 N = fastEmit_rr(VT, RetVT: VT, Opcode: ISD::ADD, Op0: N, Op1: IdxN);
598 if (!N) // Unhandled operand. Halt "fast" selection and bail.
599 return false;
600 }
601 }
602 if (TotalOffs) {
603 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
604 if (!N) // Unhandled operand. Halt "fast" selection and bail.
605 return false;
606 }
607
608 // We successfully emitted code for the given LLVM Instruction.
609 updateValueMap(I, Reg: N);
610 return true;
611}
612
613bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
614 const CallInst *CI, unsigned StartIdx) {
615 for (unsigned i = StartIdx, e = CI->arg_size(); i != e; ++i) {
616 Value *Val = CI->getArgOperand(i);
617 // Check for constants and encode them with a StackMaps::ConstantOp prefix.
618 if (const auto *C = dyn_cast<ConstantInt>(Val)) {
619 Ops.push_back(Elt: MachineOperand::CreateImm(Val: StackMaps::ConstantOp));
620 Ops.push_back(Elt: MachineOperand::CreateImm(Val: C->getSExtValue()));
621 } else if (isa<ConstantPointerNull>(Val)) {
622 Ops.push_back(Elt: MachineOperand::CreateImm(Val: StackMaps::ConstantOp));
623 Ops.push_back(Elt: MachineOperand::CreateImm(Val: 0));
624 } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
625 // Values coming from a stack location also require a special encoding,
626 // but that is added later on by the target specific frame index
627 // elimination implementation.
628 auto SI = FuncInfo.StaticAllocaMap.find(Val: AI);
629 if (SI != FuncInfo.StaticAllocaMap.end())
630 Ops.push_back(Elt: MachineOperand::CreateFI(Idx: SI->second));
631 else
632 return false;
633 } else {
634 Register Reg = getRegForValue(V: Val);
635 if (!Reg)
636 return false;
637 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/false));
638 }
639 }
640 return true;
641}
642
643bool FastISel::selectStackmap(const CallInst *I) {
644 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
645 // [live variables...])
646 assert(I->getCalledFunction()->getReturnType()->isVoidTy() &&
647 "Stackmap cannot return a value.");
648
649 // The stackmap intrinsic only records the live variables (the arguments
650 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
651 // intrinsic, this won't be lowered to a function call. This means we don't
652 // have to worry about calling conventions and target-specific lowering code.
653 // Instead we perform the call lowering right here.
654 //
655 // CALLSEQ_START(0, 0...)
656 // STACKMAP(id, nbytes, ...)
657 // CALLSEQ_END(0, 0)
658 //
659 SmallVector<MachineOperand, 32> Ops;
660
661 // Add the <id> and <numBytes> constants.
662 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
663 "Expected a constant integer.");
664 const auto *ID = cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::IDPos));
665 Ops.push_back(Elt: MachineOperand::CreateImm(Val: ID->getZExtValue()));
666
667 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
668 "Expected a constant integer.");
669 const auto *NumBytes =
670 cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::NBytesPos));
671 Ops.push_back(Elt: MachineOperand::CreateImm(Val: NumBytes->getZExtValue()));
672
673 // Push live variables for the stack map (skipping the first two arguments
674 // <id> and <numBytes>).
675 if (!addStackMapLiveVars(Ops, CI: I, StartIdx: 2))
676 return false;
677
678 // We are not adding any register mask info here, because the stackmap doesn't
679 // clobber anything.
680
681 // Add scratch registers as implicit def and early clobber.
682 CallingConv::ID CC = I->getCallingConv();
683 const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
684 for (unsigned i = 0; ScratchRegs[i]; ++i)
685 Ops.push_back(Elt: MachineOperand::CreateReg(
686 Reg: ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
687 /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
688
689 // Issue CALLSEQ_START
690 unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
691 auto Builder =
692 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: AdjStackDown));
693 const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
694 for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
695 Builder.addImm(Val: 0);
696
697 // Issue STACKMAP.
698 MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
699 MCID: TII.get(Opcode: TargetOpcode::STACKMAP));
700 for (auto const &MO : Ops)
701 MIB.add(MO);
702
703 // Issue CALLSEQ_END
704 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
705 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: AdjStackUp))
706 .addImm(Val: 0)
707 .addImm(Val: 0);
708
709 // Inform the Frame Information that we have a stackmap in this function.
710 FuncInfo.MF->getFrameInfo().setHasStackMap();
711
712 return true;
713}
714
715/// Lower an argument list according to the target calling convention.
716///
717/// This is a helper for lowering intrinsics that follow a target calling
718/// convention or require stack pointer adjustment. Only a subset of the
719/// intrinsic's operands need to participate in the calling convention.
720bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
721 unsigned NumArgs, const Value *Callee,
722 bool ForceRetVoidTy, CallLoweringInfo &CLI) {
723 ArgListTy Args;
724 Args.reserve(n: NumArgs);
725
726 // Populate the argument list.
727 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
728 Value *V = CI->getOperand(i_nocapture: ArgI);
729
730 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
731
732 ArgListEntry Entry(V);
733 Entry.setAttributes(Call: CI, ArgIdx: ArgI);
734 Args.push_back(x: Entry);
735 }
736
737 Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(C&: CI->getType()->getContext())
738 : CI->getType();
739 CLI.setCallee(CC: CI->getCallingConv(), ResultTy: RetTy, Target: Callee, ArgsList: std::move(Args), FixedArgs: NumArgs);
740
741 return lowerCallTo(CLI);
742}
743
744FastISel::CallLoweringInfo &FastISel::CallLoweringInfo::setCallee(
745 const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
746 StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
747 SmallString<32> MangledName;
748 Mangler::getNameWithPrefix(OutName&: MangledName, GVName: Target, DL);
749 MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: MangledName);
750 return setCallee(CC, ResultTy, Target: Sym, ArgsList: std::move(ArgsList), FixedArgs);
751}
752
753bool FastISel::selectPatchpoint(const CallInst *I) {
754 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
755 // i32 <numBytes>,
756 // i8* <target>,
757 // i32 <numArgs>,
758 // [Args...],
759 // [live variables...])
760 CallingConv::ID CC = I->getCallingConv();
761 bool IsAnyRegCC = CC == CallingConv::AnyReg;
762 bool HasDef = !I->getType()->isVoidTy();
763 Value *Callee = I->getOperand(i_nocapture: PatchPointOpers::TargetPos)->stripPointerCasts();
764
765 // Check if we can lower the return type when using anyregcc.
766 MVT ValueType;
767 if (IsAnyRegCC && HasDef) {
768 ValueType = TLI.getSimpleValueType(DL, Ty: I->getType(), /*AllowUnknown=*/true);
769 if (ValueType == MVT::Other)
770 return false;
771 }
772
773 // Get the real number of arguments participating in the call <numArgs>
774 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
775 "Expected a constant integer.");
776 const auto *NumArgsVal =
777 cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::NArgPos));
778 unsigned NumArgs = NumArgsVal->getZExtValue();
779
780 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
781 // This includes all meta-operands up to but not including CC.
782 unsigned NumMetaOpers = PatchPointOpers::CCPos;
783 assert(I->arg_size() >= NumMetaOpers + NumArgs &&
784 "Not enough arguments provided to the patchpoint intrinsic");
785
786 // For AnyRegCC the arguments are lowered later on manually.
787 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
788 CallLoweringInfo CLI;
789 CLI.setIsPatchPoint();
790 if (!lowerCallOperands(CI: I, ArgIdx: NumMetaOpers, NumArgs: NumCallArgs, Callee, ForceRetVoidTy: IsAnyRegCC, CLI))
791 return false;
792
793 assert(CLI.Call && "No call instruction specified.");
794
795 SmallVector<MachineOperand, 32> Ops;
796
797 // Add an explicit result reg if we use the anyreg calling convention.
798 if (IsAnyRegCC && HasDef) {
799 assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
800 assert(ValueType.isValid());
801 CLI.ResultReg = createResultReg(RC: TLI.getRegClassFor(VT: ValueType));
802 CLI.NumResultRegs = 1;
803 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: CLI.ResultReg, /*isDef=*/true));
804 }
805
806 // Add the <id> and <numBytes> constants.
807 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
808 "Expected a constant integer.");
809 const auto *ID = cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::IDPos));
810 Ops.push_back(Elt: MachineOperand::CreateImm(Val: ID->getZExtValue()));
811
812 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
813 "Expected a constant integer.");
814 const auto *NumBytes =
815 cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::NBytesPos));
816 Ops.push_back(Elt: MachineOperand::CreateImm(Val: NumBytes->getZExtValue()));
817
818 // Add the call target.
819 if (const auto *C = dyn_cast<IntToPtrInst>(Val: Callee)) {
820 uint64_t CalleeConstAddr =
821 cast<ConstantInt>(Val: C->getOperand(i_nocapture: 0))->getZExtValue();
822 Ops.push_back(Elt: MachineOperand::CreateImm(Val: CalleeConstAddr));
823 } else if (const auto *C = dyn_cast<ConstantExpr>(Val: Callee)) {
824 if (C->getOpcode() == Instruction::IntToPtr) {
825 uint64_t CalleeConstAddr =
826 cast<ConstantInt>(Val: C->getOperand(i_nocapture: 0))->getZExtValue();
827 Ops.push_back(Elt: MachineOperand::CreateImm(Val: CalleeConstAddr));
828 } else
829 llvm_unreachable("Unsupported ConstantExpr.");
830 } else if (const auto *GV = dyn_cast<GlobalValue>(Val: Callee)) {
831 Ops.push_back(Elt: MachineOperand::CreateGA(GV, Offset: 0));
832 } else if (isa<ConstantPointerNull>(Val: Callee))
833 Ops.push_back(Elt: MachineOperand::CreateImm(Val: 0));
834 else
835 llvm_unreachable("Unsupported callee address.");
836
837 // Adjust <numArgs> to account for any arguments that have been passed on
838 // the stack instead.
839 unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
840 Ops.push_back(Elt: MachineOperand::CreateImm(Val: NumCallRegArgs));
841
842 // Add the calling convention
843 Ops.push_back(Elt: MachineOperand::CreateImm(Val: (unsigned)CC));
844
845 // Add the arguments we omitted previously. The register allocator should
846 // place these in any free register.
847 if (IsAnyRegCC) {
848 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
849 Register Reg = getRegForValue(V: I->getArgOperand(i));
850 if (!Reg)
851 return false;
852 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/false));
853 }
854 }
855
856 // Push the arguments from the call instruction.
857 for (auto Reg : CLI.OutRegs)
858 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/false));
859
860 // Push live variables for the stack map.
861 if (!addStackMapLiveVars(Ops, CI: I, StartIdx: NumMetaOpers + NumArgs))
862 return false;
863
864 // Push the register mask info.
865 Ops.push_back(Elt: MachineOperand::CreateRegMask(
866 Mask: TRI.getCallPreservedMask(MF: *FuncInfo.MF, CC)));
867
868 // Add scratch registers as implicit def and early clobber.
869 const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
870 for (unsigned i = 0; ScratchRegs[i]; ++i)
871 Ops.push_back(Elt: MachineOperand::CreateReg(
872 Reg: ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
873 /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
874
875 // Add implicit defs (return values).
876 for (auto Reg : CLI.InRegs)
877 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/true,
878 /*isImp=*/true));
879
880 // Insert the patchpoint instruction before the call generated by the target.
881 MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: CLI.Call, MIMD,
882 MCID: TII.get(Opcode: TargetOpcode::PATCHPOINT));
883
884 for (auto &MO : Ops)
885 MIB.add(MO);
886
887 MIB->setPhysRegsDeadExcept(UsedRegs: CLI.InRegs, TRI);
888
889 // Delete the original call instruction.
890 CLI.Call->eraseFromParent();
891
892 // Inform the Frame Information that we have a patchpoint in this function.
893 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
894
895 if (CLI.NumResultRegs)
896 updateValueMap(I, Reg: CLI.ResultReg, NumRegs: CLI.NumResultRegs);
897 return true;
898}
899
900bool FastISel::selectXRayCustomEvent(const CallInst *I) {
901 const auto &Triple = TM.getTargetTriple();
902 if (Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64)
903 return true; // don't do anything to this instruction.
904 SmallVector<MachineOperand, 8> Ops;
905 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 0)),
906 /*isDef=*/false));
907 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 1)),
908 /*isDef=*/false));
909 MachineInstrBuilder MIB =
910 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
911 MCID: TII.get(Opcode: TargetOpcode::PATCHABLE_EVENT_CALL));
912 for (auto &MO : Ops)
913 MIB.add(MO);
914
915 // Insert the Patchable Event Call instruction, that gets lowered properly.
916 return true;
917}
918
919bool FastISel::selectXRayTypedEvent(const CallInst *I) {
920 const auto &Triple = TM.getTargetTriple();
921 if (Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64)
922 return true; // don't do anything to this instruction.
923 SmallVector<MachineOperand, 8> Ops;
924 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 0)),
925 /*isDef=*/false));
926 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 1)),
927 /*isDef=*/false));
928 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 2)),
929 /*isDef=*/false));
930 MachineInstrBuilder MIB =
931 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
932 MCID: TII.get(Opcode: TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
933 for (auto &MO : Ops)
934 MIB.add(MO);
935
936 // Insert the Patchable Typed Event Call instruction, that gets lowered properly.
937 return true;
938}
939
940/// Returns an AttributeList representing the attributes applied to the return
941/// value of the given call.
942static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI) {
943 SmallVector<Attribute::AttrKind, 2> Attrs;
944 if (CLI.RetSExt)
945 Attrs.push_back(Elt: Attribute::SExt);
946 if (CLI.RetZExt)
947 Attrs.push_back(Elt: Attribute::ZExt);
948 if (CLI.IsInReg)
949 Attrs.push_back(Elt: Attribute::InReg);
950
951 return AttributeList::get(C&: CLI.RetTy->getContext(), Index: AttributeList::ReturnIndex,
952 Kinds: Attrs);
953}
954
955bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
956 unsigned NumArgs) {
957 MCContext &Ctx = MF->getContext();
958 SmallString<32> MangledName;
959 Mangler::getNameWithPrefix(OutName&: MangledName, GVName: SymName, DL);
960 MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: MangledName);
961 return lowerCallTo(CI, Symbol: Sym, NumArgs);
962}
963
964bool FastISel::lowerCallTo(const CallInst *CI, MCSymbol *Symbol,
965 unsigned NumArgs) {
966 FunctionType *FTy = CI->getFunctionType();
967 Type *RetTy = CI->getType();
968
969 ArgListTy Args;
970 Args.reserve(n: NumArgs);
971
972 // Populate the argument list.
973 // Attributes for args start at offset 1, after the return attribute.
974 for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
975 Value *V = CI->getOperand(i_nocapture: ArgI);
976
977 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
978
979 ArgListEntry Entry(V);
980 Entry.setAttributes(Call: CI, ArgIdx: ArgI);
981 Args.push_back(x: Entry);
982 }
983 TLI.markLibCallAttributes(MF, CC: CI->getCallingConv(), Args);
984
985 CallLoweringInfo CLI;
986 CLI.setCallee(ResultTy: RetTy, FuncTy: FTy, Target: Symbol, ArgsList: std::move(Args), Call: *CI, FixedArgs: NumArgs);
987
988 return lowerCallTo(CLI);
989}
990
991bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
992 // Handle the incoming return values from the call.
993 CLI.clearIns();
994 SmallVector<EVT, 4> RetTys;
995 ComputeValueVTs(TLI, DL, Ty: CLI.RetTy, ValueVTs&: RetTys);
996
997 SmallVector<ISD::OutputArg, 4> Outs;
998 GetReturnInfo(CC: CLI.CallConv, ReturnType: CLI.RetTy, attr: getReturnAttrs(CLI), Outs, TLI, DL);
999
1000 bool CanLowerReturn = TLI.CanLowerReturn(
1001 CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext(), RetTy: CLI.RetTy);
1002
1003 // FIXME: sret demotion isn't supported yet - bail out.
1004 if (!CanLowerReturn)
1005 return false;
1006
1007 for (EVT VT : RetTys) {
1008 MVT RegisterVT = TLI.getRegisterType(Context&: CLI.RetTy->getContext(), VT);
1009 unsigned NumRegs = TLI.getNumRegisters(Context&: CLI.RetTy->getContext(), VT);
1010 for (unsigned i = 0; i != NumRegs; ++i) {
1011 ISD::ArgFlagsTy Flags;
1012 if (CLI.RetSExt)
1013 Flags.setSExt();
1014 if (CLI.RetZExt)
1015 Flags.setZExt();
1016 if (CLI.IsInReg)
1017 Flags.setInReg();
1018 ISD::InputArg Ret(Flags, RegisterVT, VT, CLI.RetTy, CLI.IsReturnValueUsed,
1019 ISD::InputArg::NoArgIndex, 0);
1020 CLI.Ins.push_back(Elt: Ret);
1021 }
1022 }
1023
1024 // Handle all of the outgoing arguments.
1025 CLI.clearOuts();
1026 for (auto &Arg : CLI.getArgs()) {
1027 Type *FinalType = Arg.Ty;
1028 if (Arg.IsByVal)
1029 FinalType = Arg.IndirectType;
1030 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1031 Ty: FinalType, CallConv: CLI.CallConv, isVarArg: CLI.IsVarArg, DL);
1032
1033 ISD::ArgFlagsTy Flags;
1034 if (Arg.IsZExt)
1035 Flags.setZExt();
1036 if (Arg.IsSExt)
1037 Flags.setSExt();
1038 if (Arg.IsInReg)
1039 Flags.setInReg();
1040 if (Arg.IsSRet)
1041 Flags.setSRet();
1042 if (Arg.IsSwiftSelf)
1043 Flags.setSwiftSelf();
1044 if (Arg.IsSwiftAsync)
1045 Flags.setSwiftAsync();
1046 if (Arg.IsSwiftError)
1047 Flags.setSwiftError();
1048 if (Arg.IsCFGuardTarget)
1049 Flags.setCFGuardTarget();
1050 if (Arg.IsByVal)
1051 Flags.setByVal();
1052 if (Arg.IsInAlloca) {
1053 Flags.setInAlloca();
1054 // Set the byval flag for CCAssignFn callbacks that don't know about
1055 // inalloca. This way we can know how many bytes we should've allocated
1056 // and how many bytes a callee cleanup function will pop. If we port
1057 // inalloca to more targets, we'll have to add custom inalloca handling in
1058 // the various CC lowering callbacks.
1059 Flags.setByVal();
1060 }
1061 if (Arg.IsPreallocated) {
1062 Flags.setPreallocated();
1063 // Set the byval flag for CCAssignFn callbacks that don't know about
1064 // preallocated. This way we can know how many bytes we should've
1065 // allocated and how many bytes a callee cleanup function will pop. If we
1066 // port preallocated to more targets, we'll have to add custom
1067 // preallocated handling in the various CC lowering callbacks.
1068 Flags.setByVal();
1069 }
1070 MaybeAlign MemAlign = Arg.Alignment;
1071 if (Arg.IsByVal || Arg.IsInAlloca || Arg.IsPreallocated) {
1072 unsigned FrameSize = DL.getTypeAllocSize(Ty: Arg.IndirectType);
1073
1074 // For ByVal, alignment should come from FE. BE will guess if this info
1075 // is not there, but there are cases it cannot get right.
1076 if (!MemAlign)
1077 MemAlign = TLI.getByValTypeAlignment(Ty: Arg.IndirectType, DL);
1078 Flags.setByValSize(FrameSize);
1079 } else if (!MemAlign) {
1080 MemAlign = DL.getABITypeAlign(Ty: Arg.Ty);
1081 }
1082 Flags.setMemAlign(*MemAlign);
1083 if (Arg.IsNest)
1084 Flags.setNest();
1085 if (NeedsRegBlock)
1086 Flags.setInConsecutiveRegs();
1087 Flags.setOrigAlign(DL.getABITypeAlign(Ty: Arg.Ty));
1088 CLI.OutVals.push_back(Elt: Arg.Val);
1089 CLI.OutFlags.push_back(Elt: Flags);
1090 }
1091
1092 if (!fastLowerCall(CLI))
1093 return false;
1094
1095 // Set all unused physreg defs as dead.
1096 assert(CLI.Call && "No call instruction specified.");
1097 CLI.Call->setPhysRegsDeadExcept(UsedRegs: CLI.InRegs, TRI);
1098
1099 if (CLI.NumResultRegs && CLI.CB)
1100 updateValueMap(I: CLI.CB, Reg: CLI.ResultReg, NumRegs: CLI.NumResultRegs);
1101
1102 // Set labels for heapallocsite call.
1103 if (CLI.CB)
1104 if (MDNode *MD = CLI.CB->getMetadata(Kind: "heapallocsite"))
1105 CLI.Call->setHeapAllocMarker(MF&: *MF, MD);
1106
1107 return true;
1108}
1109
1110bool FastISel::lowerCall(const CallInst *CI) {
1111 FunctionType *FuncTy = CI->getFunctionType();
1112 Type *RetTy = CI->getType();
1113
1114 ArgListTy Args;
1115 Args.reserve(n: CI->arg_size());
1116
1117 for (auto i = CI->arg_begin(), e = CI->arg_end(); i != e; ++i) {
1118 Value *V = *i;
1119
1120 // Skip empty types
1121 if (V->getType()->isEmptyTy())
1122 continue;
1123
1124 ArgListEntry Entry(V);
1125 // Skip the first return-type Attribute to get to params.
1126 Entry.setAttributes(Call: CI, ArgIdx: i - CI->arg_begin());
1127 Args.push_back(x: Entry);
1128 }
1129
1130 // Check if target-independent constraints permit a tail call here.
1131 // Target-dependent constraints are checked within fastLowerCall.
1132 bool IsTailCall = CI->isTailCall();
1133 if (IsTailCall && !isInTailCallPosition(Call: *CI, TM))
1134 IsTailCall = false;
1135 if (IsTailCall && !CI->isMustTailCall() &&
1136 MF->getFunction().getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
1137 IsTailCall = false;
1138
1139 CallLoweringInfo CLI;
1140 CLI.setCallee(ResultTy: RetTy, FuncTy, Target: CI->getCalledOperand(), ArgsList: std::move(Args), Call: *CI)
1141 .setTailCall(IsTailCall);
1142
1143 if (lowerCallTo(CLI)) {
1144 diagnoseDontCall(CI: *CI);
1145 return true;
1146 }
1147
1148 return false;
1149}
1150
1151bool FastISel::selectCall(const User *I) {
1152 const CallInst *Call = cast<CallInst>(Val: I);
1153
1154 // Handle simple inline asms.
1155 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: Call->getCalledOperand())) {
1156 // Don't attempt to handle constraints.
1157 if (!IA->getConstraintString().empty())
1158 return false;
1159
1160 unsigned ExtraInfo = 0;
1161 if (IA->hasSideEffects())
1162 ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1163 if (IA->isAlignStack())
1164 ExtraInfo |= InlineAsm::Extra_IsAlignStack;
1165 if (IA->canThrow())
1166 ExtraInfo |= InlineAsm::Extra_MayUnwind;
1167 if (Call->isConvergent())
1168 ExtraInfo |= InlineAsm::Extra_IsConvergent;
1169 ExtraInfo |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
1170
1171 MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1172 MCID: TII.get(Opcode: TargetOpcode::INLINEASM));
1173 MIB.addExternalSymbol(FnName: IA->getAsmString().data());
1174 MIB.addImm(Val: ExtraInfo);
1175
1176 const MDNode *SrcLoc = Call->getMetadata(Kind: "srcloc");
1177 if (SrcLoc)
1178 MIB.addMetadata(MD: SrcLoc);
1179
1180 return true;
1181 }
1182
1183 // Handle intrinsic function calls.
1184 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Call))
1185 return selectIntrinsicCall(II);
1186
1187 return lowerCall(CI: Call);
1188}
1189
1190void FastISel::handleDbgInfo(const Instruction *II) {
1191 if (!II->hasDbgRecords())
1192 return;
1193
1194 // Clear any metadata.
1195 MIMD = MIMetadata();
1196
1197 // Reverse order of debug records, because fast-isel walks through backwards.
1198 for (DbgRecord &DR : llvm::reverse(C: II->getDbgRecordRange())) {
1199 flushLocalValueMap();
1200 recomputeInsertPt();
1201
1202 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
1203 assert(DLR->getLabel() && "Missing label");
1204 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DLR->getDebugLoc(),
1205 MCID: TII.get(Opcode: TargetOpcode::DBG_LABEL))
1206 .addMetadata(MD: DLR->getLabel());
1207 continue;
1208 }
1209
1210 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
1211
1212 Value *V = nullptr;
1213 if (!DVR.hasArgList())
1214 V = DVR.getVariableLocationOp(OpIdx: 0);
1215
1216 bool Res = false;
1217 if (DVR.getType() == DbgVariableRecord::LocationType::Value ||
1218 DVR.getType() == DbgVariableRecord::LocationType::Assign) {
1219 Res = lowerDbgValue(V, Expr: DVR.getExpression(), Var: DVR.getVariable(),
1220 DL: DVR.getDebugLoc());
1221 } else {
1222 assert(DVR.getType() == DbgVariableRecord::LocationType::Declare);
1223 if (FuncInfo.PreprocessedDVRDeclares.contains(Ptr: &DVR))
1224 continue;
1225 Res = lowerDbgDeclare(V, Expr: DVR.getExpression(), Var: DVR.getVariable(),
1226 DL: DVR.getDebugLoc());
1227 }
1228
1229 if (!Res)
1230 LLVM_DEBUG(dbgs() << "Dropping debug-info for " << DVR << "\n");
1231 }
1232}
1233
1234bool FastISel::lowerDbgValue(const Value *V, DIExpression *Expr,
1235 DILocalVariable *Var, const DebugLoc &DL) {
1236 // This form of DBG_VALUE is target-independent.
1237 const MCInstrDesc &II = TII.get(Opcode: TargetOpcode::DBG_VALUE);
1238 if (!V || isa<UndefValue>(Val: V)) {
1239 // DI is either undef or cannot produce a valid DBG_VALUE, so produce an
1240 // undef DBG_VALUE to terminate any prior location.
1241 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect: false, Reg: 0U, Variable: Var, Expr);
1242 return true;
1243 }
1244 if (const auto *CI = dyn_cast<ConstantInt>(Val: V)) {
1245 // See if there's an expression to constant-fold.
1246 if (Expr)
1247 std::tie(args&: Expr, args&: CI) = Expr->constantFold(CI);
1248 if (CI->getBitWidth() > 64)
1249 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1250 .addCImm(Val: CI)
1251 .addImm(Val: 0U)
1252 .addMetadata(MD: Var)
1253 .addMetadata(MD: Expr);
1254 else
1255 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1256 .addImm(Val: CI->getZExtValue())
1257 .addImm(Val: 0U)
1258 .addMetadata(MD: Var)
1259 .addMetadata(MD: Expr);
1260 return true;
1261 }
1262 if (const auto *CF = dyn_cast<ConstantFP>(Val: V)) {
1263 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1264 .addFPImm(Val: CF)
1265 .addImm(Val: 0U)
1266 .addMetadata(MD: Var)
1267 .addMetadata(MD: Expr);
1268 return true;
1269 }
1270 if (const auto *Arg = dyn_cast<Argument>(Val: V);
1271 Arg && Expr && Expr->isEntryValue()) {
1272 // As per the Verifier, this case is only valid for swift async Args.
1273 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
1274
1275 Register Reg = getRegForValue(V: Arg);
1276 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
1277 if (Reg == VirtReg || Reg == PhysReg) {
1278 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect: false /*IsIndirect*/,
1279 Reg: PhysReg, Variable: Var, Expr);
1280 return true;
1281 }
1282
1283 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
1284 "couldn't find a physical register\n");
1285 return false;
1286 }
1287 if (auto SI = FuncInfo.StaticAllocaMap.find(Val: dyn_cast<AllocaInst>(Val: V));
1288 SI != FuncInfo.StaticAllocaMap.end()) {
1289 MachineOperand FrameIndexOp = MachineOperand::CreateFI(Idx: SI->second);
1290 bool IsIndirect = false;
1291 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect, MOs: FrameIndexOp,
1292 Variable: Var, Expr);
1293 return true;
1294 }
1295 if (Register Reg = lookUpRegForValue(V)) {
1296 // FIXME: This does not handle register-indirect values at offset 0.
1297 if (!FuncInfo.MF->useDebugInstrRef()) {
1298 bool IsIndirect = false;
1299 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect, Reg, Variable: Var,
1300 Expr);
1301 return true;
1302 }
1303 // If using instruction referencing, produce this as a DBG_INSTR_REF,
1304 // to be later patched up by finalizeDebugInstrRefs.
1305 SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg(
1306 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
1307 /* isKill */ false, /* isDead */ false,
1308 /* isUndef */ false, /* isEarlyClobber */ false,
1309 /* SubReg */ 0, /* isDebug */ true)});
1310 SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0});
1311 auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops);
1312 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL,
1313 MCID: TII.get(Opcode: TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs,
1314 Variable: Var, Expr: NewExpr);
1315 return true;
1316 }
1317 return false;
1318}
1319
1320bool FastISel::lowerDbgDeclare(const Value *Address, DIExpression *Expr,
1321 DILocalVariable *Var, const DebugLoc &DL) {
1322 if (!Address || isa<UndefValue>(Val: Address)) {
1323 LLVM_DEBUG(dbgs() << "Dropping debug info (bad/undef address)\n");
1324 return false;
1325 }
1326
1327 std::optional<MachineOperand> Op;
1328 if (Register Reg = lookUpRegForValue(V: Address))
1329 Op = MachineOperand::CreateReg(Reg, isDef: false);
1330
1331 // If we have a VLA that has a "use" in a metadata node that's then used
1332 // here but it has no other uses, then we have a problem. E.g.,
1333 //
1334 // int foo (const int *x) {
1335 // char a[*x];
1336 // return 0;
1337 // }
1338 //
1339 // If we assign 'a' a vreg and fast isel later on has to use the selection
1340 // DAG isel, it will want to copy the value to the vreg. However, there are
1341 // no uses, which goes counter to what selection DAG isel expects.
1342 if (!Op && !Address->use_empty() && isa<Instruction>(Val: Address) &&
1343 (!isa<AllocaInst>(Val: Address) ||
1344 !FuncInfo.StaticAllocaMap.count(Val: cast<AllocaInst>(Val: Address))))
1345 Op = MachineOperand::CreateReg(Reg: FuncInfo.InitializeRegForValue(V: Address),
1346 isDef: false);
1347
1348 if (Op) {
1349 assert(Var->isValidLocationForIntrinsic(DL) &&
1350 "Expected inlined-at fields to agree");
1351 if (FuncInfo.MF->useDebugInstrRef() && Op->isReg()) {
1352 // If using instruction referencing, produce this as a DBG_INSTR_REF,
1353 // to be later patched up by finalizeDebugInstrRefs. Tack a deref onto
1354 // the expression, we don't have an "indirect" flag in DBG_INSTR_REF.
1355 SmallVector<uint64_t, 3> Ops(
1356 {dwarf::DW_OP_LLVM_arg, 0, dwarf::DW_OP_deref});
1357 auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops);
1358 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL,
1359 MCID: TII.get(Opcode: TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs: *Op,
1360 Variable: Var, Expr: NewExpr);
1361 return true;
1362 }
1363
1364 // A dbg.declare describes the address of a source variable, so lower it
1365 // into an indirect DBG_VALUE.
1366 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL,
1367 MCID: TII.get(Opcode: TargetOpcode::DBG_VALUE), /*IsIndirect*/ true, MOs: *Op, Variable: Var,
1368 Expr);
1369 return true;
1370 }
1371
1372 // We can't yet handle anything else here because it would require
1373 // generating code, thus altering codegen because of debug info.
1374 LLVM_DEBUG(
1375 dbgs() << "Dropping debug info (no materialized reg for address)\n");
1376 return false;
1377}
1378
1379bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
1380 switch (II->getIntrinsicID()) {
1381 default:
1382 break;
1383 // At -O0 we don't care about the lifetime intrinsics.
1384 case Intrinsic::lifetime_start:
1385 case Intrinsic::lifetime_end:
1386 // The donothing intrinsic does, well, nothing.
1387 case Intrinsic::donothing:
1388 // Neither does the sideeffect intrinsic.
1389 case Intrinsic::sideeffect:
1390 // Neither does the assume intrinsic; it's also OK not to codegen its operand.
1391 case Intrinsic::assume:
1392 // Neither does the llvm.experimental.noalias.scope.decl intrinsic
1393 case Intrinsic::experimental_noalias_scope_decl:
1394 return true;
1395 case Intrinsic::objectsize:
1396 llvm_unreachable("llvm.objectsize.* should have been lowered already");
1397
1398 case Intrinsic::is_constant:
1399 llvm_unreachable("llvm.is.constant.* should have been lowered already");
1400
1401 case Intrinsic::allow_runtime_check:
1402 case Intrinsic::allow_ubsan_check: {
1403 Register ResultReg = getRegForValue(V: ConstantInt::getTrue(Ty: II->getType()));
1404 if (!ResultReg)
1405 return false;
1406 updateValueMap(I: II, Reg: ResultReg);
1407 return true;
1408 }
1409
1410 case Intrinsic::launder_invariant_group:
1411 case Intrinsic::strip_invariant_group:
1412 case Intrinsic::expect:
1413 case Intrinsic::expect_with_probability: {
1414 Register ResultReg = getRegForValue(V: II->getArgOperand(i: 0));
1415 if (!ResultReg)
1416 return false;
1417 updateValueMap(I: II, Reg: ResultReg);
1418 return true;
1419 }
1420 case Intrinsic::fake_use:
1421 // At -O0, we don't need fake use, so just ignore it.
1422 return true;
1423 case Intrinsic::experimental_stackmap:
1424 return selectStackmap(I: II);
1425 case Intrinsic::experimental_patchpoint_void:
1426 case Intrinsic::experimental_patchpoint:
1427 return selectPatchpoint(I: II);
1428
1429 case Intrinsic::xray_customevent:
1430 return selectXRayCustomEvent(I: II);
1431 case Intrinsic::xray_typedevent:
1432 return selectXRayTypedEvent(I: II);
1433 }
1434
1435 return fastLowerIntrinsicCall(II);
1436}
1437
1438bool FastISel::selectCast(const User *I, unsigned Opcode) {
1439 EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1440 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
1441
1442 if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
1443 !DstVT.isSimple())
1444 // Unhandled type. Halt "fast" selection and bail.
1445 return false;
1446
1447 // Check if the destination type is legal.
1448 if (!TLI.isTypeLegal(VT: DstVT))
1449 return false;
1450
1451 // Check if the source operand is legal.
1452 if (!TLI.isTypeLegal(VT: SrcVT))
1453 return false;
1454
1455 Register InputReg = getRegForValue(V: I->getOperand(i: 0));
1456 if (!InputReg)
1457 // Unhandled operand. Halt "fast" selection and bail.
1458 return false;
1459
1460 Register ResultReg = fastEmit_r(VT: SrcVT.getSimpleVT(), RetVT: DstVT.getSimpleVT(),
1461 Opcode, Op0: InputReg);
1462 if (!ResultReg)
1463 return false;
1464
1465 updateValueMap(I, Reg: ResultReg);
1466 return true;
1467}
1468
1469bool FastISel::selectBitCast(const User *I) {
1470 EVT SrcEVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1471 EVT DstEVT = TLI.getValueType(DL, Ty: I->getType());
1472 if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
1473 !TLI.isTypeLegal(VT: SrcEVT) || !TLI.isTypeLegal(VT: DstEVT))
1474 // Unhandled type. Halt "fast" selection and bail.
1475 return false;
1476
1477 MVT SrcVT = SrcEVT.getSimpleVT();
1478 MVT DstVT = DstEVT.getSimpleVT();
1479 Register Op0 = getRegForValue(V: I->getOperand(i: 0));
1480 if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
1481 return false;
1482
1483 // If the bitcast doesn't change the type, just use the operand value.
1484 if (SrcVT == DstVT) {
1485 updateValueMap(I, Reg: Op0);
1486 return true;
1487 }
1488
1489 // Otherwise, select a BITCAST opcode.
1490 Register ResultReg = fastEmit_r(VT: SrcVT, RetVT: DstVT, Opcode: ISD::BITCAST, Op0);
1491 if (!ResultReg)
1492 return false;
1493
1494 updateValueMap(I, Reg: ResultReg);
1495 return true;
1496}
1497
1498bool FastISel::selectFreeze(const User *I) {
1499 Register Reg = getRegForValue(V: I->getOperand(i: 0));
1500 if (!Reg)
1501 // Unhandled operand.
1502 return false;
1503
1504 EVT ETy = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1505 if (ETy == MVT::Other || !TLI.isTypeLegal(VT: ETy))
1506 // Unhandled type, bail out.
1507 return false;
1508
1509 MVT Ty = ETy.getSimpleVT();
1510 const TargetRegisterClass *TyRegClass = TLI.getRegClassFor(VT: Ty);
1511 Register ResultReg = createResultReg(RC: TyRegClass);
1512 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1513 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ResultReg).addReg(RegNo: Reg);
1514
1515 updateValueMap(I, Reg: ResultReg);
1516 return true;
1517}
1518
1519// Remove local value instructions starting from the instruction after
1520// SavedLastLocalValue to the current function insert point.
1521void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
1522{
1523 MachineInstr *CurLastLocalValue = getLastLocalValue();
1524 if (CurLastLocalValue != SavedLastLocalValue) {
1525 // Find the first local value instruction to be deleted.
1526 // This is the instruction after SavedLastLocalValue if it is non-NULL.
1527 // Otherwise it's the first instruction in the block.
1528 MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
1529 if (SavedLastLocalValue)
1530 ++FirstDeadInst;
1531 else
1532 FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
1533 setLastLocalValue(SavedLastLocalValue);
1534 removeDeadCode(I: FirstDeadInst, E: FuncInfo.InsertPt);
1535 }
1536}
1537
1538bool FastISel::selectInstruction(const Instruction *I) {
1539 // Flush the local value map before starting each instruction.
1540 // This improves locality and debugging, and can reduce spills.
1541 // Reuse of values across IR instructions is relatively uncommon.
1542 flushLocalValueMap();
1543
1544 MachineInstr *SavedLastLocalValue = getLastLocalValue();
1545 // Just before the terminator instruction, insert instructions to
1546 // feed PHI nodes in successor blocks.
1547 if (I->isTerminator()) {
1548 if (!handlePHINodesInSuccessorBlocks(LLVMBB: I->getParent())) {
1549 // PHI node handling may have generated local value instructions,
1550 // even though it failed to handle all PHI nodes.
1551 // We remove these instructions because SelectionDAGISel will generate
1552 // them again.
1553 removeDeadLocalValueCode(SavedLastLocalValue);
1554 return false;
1555 }
1556 }
1557
1558 // FastISel does not handle any operand bundles except OB_funclet.
1559 if (auto *Call = dyn_cast<CallBase>(Val: I))
1560 for (unsigned i = 0, e = Call->getNumOperandBundles(); i != e; ++i)
1561 if (Call->getOperandBundleAt(Index: i).getTagID() != LLVMContext::OB_funclet)
1562 return false;
1563
1564 MIMD = MIMetadata(*I);
1565
1566 SavedInsertPt = FuncInfo.InsertPt;
1567
1568 if (const auto *Call = dyn_cast<CallInst>(Val: I)) {
1569 const Function *F = Call->getCalledFunction();
1570
1571 // Don't handle Intrinsic::trap if a trap function is specified.
1572 if (F && F->getIntrinsicID() == Intrinsic::trap &&
1573 Call->hasFnAttr(Kind: "trap-func-name"))
1574 return false;
1575 }
1576
1577 // First, try doing target-independent selection.
1578 if (!SkipTargetIndependentISel) {
1579 if (selectOperator(I, Opcode: I->getOpcode())) {
1580 ++NumFastIselSuccessIndependent;
1581 MIMD = {};
1582 return true;
1583 }
1584 // Remove dead code.
1585 recomputeInsertPt();
1586 if (SavedInsertPt != FuncInfo.InsertPt)
1587 removeDeadCode(I: FuncInfo.InsertPt, E: SavedInsertPt);
1588 SavedInsertPt = FuncInfo.InsertPt;
1589 }
1590 // Next, try calling the target to attempt to handle the instruction.
1591 if (fastSelectInstruction(I)) {
1592 ++NumFastIselSuccessTarget;
1593 MIMD = {};
1594 return true;
1595 }
1596 // Remove dead code.
1597 recomputeInsertPt();
1598 if (SavedInsertPt != FuncInfo.InsertPt)
1599 removeDeadCode(I: FuncInfo.InsertPt, E: SavedInsertPt);
1600
1601 MIMD = {};
1602 // Undo phi node updates, because they will be added again by SelectionDAG.
1603 if (I->isTerminator()) {
1604 // PHI node handling may have generated local value instructions.
1605 // We remove them because SelectionDAGISel will generate them again.
1606 removeDeadLocalValueCode(SavedLastLocalValue);
1607 FuncInfo.PHINodesToUpdate.resize(new_size: FuncInfo.OrigNumPHINodesToUpdate);
1608 }
1609 return false;
1610}
1611
1612/// Emit an unconditional branch to the given block, unless it is the immediate
1613/// (fall-through) successor, and update the CFG.
1614void FastISel::fastEmitBranch(MachineBasicBlock *MSucc,
1615 const DebugLoc &DbgLoc) {
1616 const BasicBlock *BB = FuncInfo.MBB->getBasicBlock();
1617 bool BlockHasMultipleInstrs = &BB->front() != &BB->back();
1618 if (BlockHasMultipleInstrs && FuncInfo.MBB->isLayoutSuccessor(MBB: MSucc)) {
1619 // For more accurate line information if this is the only non-debug
1620 // instruction in the block then emit it, otherwise we have the
1621 // unconditional fall-through case, which needs no instructions.
1622 } else {
1623 // The unconditional branch case.
1624 TII.insertBranch(MBB&: *FuncInfo.MBB, TBB: MSucc, FBB: nullptr,
1625 Cond: SmallVector<MachineOperand, 0>(), DL: DbgLoc);
1626 }
1627 if (FuncInfo.BPI) {
1628 auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
1629 Src: FuncInfo.MBB->getBasicBlock(), Dst: MSucc->getBasicBlock());
1630 FuncInfo.MBB->addSuccessor(Succ: MSucc, Prob: BranchProbability);
1631 } else
1632 FuncInfo.MBB->addSuccessorWithoutProb(Succ: MSucc);
1633}
1634
1635void FastISel::finishCondBranch(const BasicBlock *BranchBB,
1636 MachineBasicBlock *TrueMBB,
1637 MachineBasicBlock *FalseMBB) {
1638 // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
1639 // happen in degenerate IR and MachineIR forbids to have a block twice in the
1640 // successor/predecessor lists.
1641 if (TrueMBB != FalseMBB) {
1642 if (FuncInfo.BPI) {
1643 auto BranchProbability =
1644 FuncInfo.BPI->getEdgeProbability(Src: BranchBB, Dst: TrueMBB->getBasicBlock());
1645 FuncInfo.MBB->addSuccessor(Succ: TrueMBB, Prob: BranchProbability);
1646 } else
1647 FuncInfo.MBB->addSuccessorWithoutProb(Succ: TrueMBB);
1648 }
1649
1650 fastEmitBranch(MSucc: FalseMBB, DbgLoc: MIMD.getDL());
1651}
1652
1653/// Emit an FNeg operation.
1654bool FastISel::selectFNeg(const User *I, const Value *In) {
1655 Register OpReg = getRegForValue(V: In);
1656 if (!OpReg)
1657 return false;
1658
1659 // If the target has ISD::FNEG, use it.
1660 EVT VT = TLI.getValueType(DL, Ty: I->getType());
1661 Register ResultReg = fastEmit_r(VT: VT.getSimpleVT(), RetVT: VT.getSimpleVT(), Opcode: ISD::FNEG,
1662 Op0: OpReg);
1663 if (ResultReg) {
1664 updateValueMap(I, Reg: ResultReg);
1665 return true;
1666 }
1667
1668 // Bitcast the value to integer, twiddle the sign bit with xor,
1669 // and then bitcast it back to floating-point.
1670 if (VT.getSizeInBits() > 64)
1671 return false;
1672 EVT IntVT = EVT::getIntegerVT(Context&: I->getContext(), BitWidth: VT.getSizeInBits());
1673 if (!TLI.isTypeLegal(VT: IntVT))
1674 return false;
1675
1676 Register IntReg = fastEmit_r(VT: VT.getSimpleVT(), RetVT: IntVT.getSimpleVT(),
1677 Opcode: ISD::BITCAST, Op0: OpReg);
1678 if (!IntReg)
1679 return false;
1680
1681 Register IntResultReg = fastEmit_ri_(
1682 VT: IntVT.getSimpleVT(), Opcode: ISD::XOR, Op0: IntReg,
1683 UINT64_C(1) << (VT.getSizeInBits() - 1), ImmType: IntVT.getSimpleVT());
1684 if (!IntResultReg)
1685 return false;
1686
1687 ResultReg = fastEmit_r(VT: IntVT.getSimpleVT(), RetVT: VT.getSimpleVT(), Opcode: ISD::BITCAST,
1688 Op0: IntResultReg);
1689 if (!ResultReg)
1690 return false;
1691
1692 updateValueMap(I, Reg: ResultReg);
1693 return true;
1694}
1695
1696bool FastISel::selectExtractValue(const User *U) {
1697 const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: U);
1698 if (!EVI)
1699 return false;
1700
1701 // Make sure we only try to handle extracts with a legal result. But also
1702 // allow i1 because it's easy.
1703 EVT RealVT = TLI.getValueType(DL, Ty: EVI->getType(), /*AllowUnknown=*/true);
1704 if (!RealVT.isSimple())
1705 return false;
1706 MVT VT = RealVT.getSimpleVT();
1707 if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
1708 return false;
1709
1710 const Value *Op0 = EVI->getOperand(i_nocapture: 0);
1711 Type *AggTy = Op0->getType();
1712
1713 // Get the base result register.
1714 Register ResultReg;
1715 DenseMap<const Value *, Register>::iterator I = FuncInfo.ValueMap.find(Val: Op0);
1716 if (I != FuncInfo.ValueMap.end())
1717 ResultReg = I->second;
1718 else if (isa<Instruction>(Val: Op0))
1719 ResultReg = FuncInfo.InitializeRegForValue(V: Op0);
1720 else
1721 return false; // fast-isel can't handle aggregate constants at the moment
1722
1723 // Get the actual result register, which is an offset from the base register.
1724 unsigned VTIndex = ComputeLinearIndex(Ty: AggTy, Indices: EVI->getIndices());
1725
1726 SmallVector<EVT, 4> AggValueVTs;
1727 ComputeValueVTs(TLI, DL, Ty: AggTy, ValueVTs&: AggValueVTs);
1728
1729 for (unsigned i = 0; i < VTIndex; i++)
1730 ResultReg = ResultReg.id() +
1731 TLI.getNumRegisters(Context&: FuncInfo.Fn->getContext(), VT: AggValueVTs[i]);
1732
1733 updateValueMap(I: EVI, Reg: ResultReg);
1734 return true;
1735}
1736
1737bool FastISel::selectOperator(const User *I, unsigned Opcode) {
1738 switch (Opcode) {
1739 case Instruction::Add:
1740 return selectBinaryOp(I, ISDOpcode: ISD::ADD);
1741 case Instruction::FAdd:
1742 return selectBinaryOp(I, ISDOpcode: ISD::FADD);
1743 case Instruction::Sub:
1744 return selectBinaryOp(I, ISDOpcode: ISD::SUB);
1745 case Instruction::FSub:
1746 return selectBinaryOp(I, ISDOpcode: ISD::FSUB);
1747 case Instruction::Mul:
1748 return selectBinaryOp(I, ISDOpcode: ISD::MUL);
1749 case Instruction::FMul:
1750 return selectBinaryOp(I, ISDOpcode: ISD::FMUL);
1751 case Instruction::SDiv:
1752 return selectBinaryOp(I, ISDOpcode: ISD::SDIV);
1753 case Instruction::UDiv:
1754 return selectBinaryOp(I, ISDOpcode: ISD::UDIV);
1755 case Instruction::FDiv:
1756 return selectBinaryOp(I, ISDOpcode: ISD::FDIV);
1757 case Instruction::SRem:
1758 return selectBinaryOp(I, ISDOpcode: ISD::SREM);
1759 case Instruction::URem:
1760 return selectBinaryOp(I, ISDOpcode: ISD::UREM);
1761 case Instruction::FRem:
1762 return selectBinaryOp(I, ISDOpcode: ISD::FREM);
1763 case Instruction::Shl:
1764 return selectBinaryOp(I, ISDOpcode: ISD::SHL);
1765 case Instruction::LShr:
1766 return selectBinaryOp(I, ISDOpcode: ISD::SRL);
1767 case Instruction::AShr:
1768 return selectBinaryOp(I, ISDOpcode: ISD::SRA);
1769 case Instruction::And:
1770 return selectBinaryOp(I, ISDOpcode: ISD::AND);
1771 case Instruction::Or:
1772 return selectBinaryOp(I, ISDOpcode: ISD::OR);
1773 case Instruction::Xor:
1774 return selectBinaryOp(I, ISDOpcode: ISD::XOR);
1775
1776 case Instruction::FNeg:
1777 return selectFNeg(I, In: I->getOperand(i: 0));
1778
1779 case Instruction::GetElementPtr:
1780 return selectGetElementPtr(I);
1781
1782 case Instruction::Br: {
1783 const BranchInst *BI = cast<BranchInst>(Val: I);
1784
1785 if (BI->isUnconditional()) {
1786 const BasicBlock *LLVMSucc = BI->getSuccessor(i: 0);
1787 MachineBasicBlock *MSucc = FuncInfo.getMBB(BB: LLVMSucc);
1788 fastEmitBranch(MSucc, DbgLoc: BI->getDebugLoc());
1789 return true;
1790 }
1791
1792 // Conditional branches are not handed yet.
1793 // Halt "fast" selection and bail.
1794 return false;
1795 }
1796
1797 case Instruction::Unreachable: {
1798 auto UI = cast<UnreachableInst>(Val: I);
1799 if (!UI->shouldLowerToTrap(TrapUnreachable: TM.Options.TrapUnreachable,
1800 NoTrapAfterNoreturn: TM.Options.NoTrapAfterNoreturn))
1801 return true;
1802
1803 return fastEmit_(VT: MVT::Other, RetVT: MVT::Other, Opcode: ISD::TRAP) != 0;
1804 }
1805
1806 case Instruction::Alloca:
1807 // FunctionLowering has the static-sized case covered.
1808 if (FuncInfo.StaticAllocaMap.count(Val: cast<AllocaInst>(Val: I)))
1809 return true;
1810
1811 // Dynamic-sized alloca is not handled yet.
1812 return false;
1813
1814 case Instruction::Call:
1815 // On AIX, normal call lowering uses the DAG-ISEL path currently so that the
1816 // callee of the direct function call instruction will be mapped to the
1817 // symbol for the function's entry point, which is distinct from the
1818 // function descriptor symbol. The latter is the symbol whose XCOFF symbol
1819 // name is the C-linkage name of the source level function.
1820 // But fast isel still has the ability to do selection for intrinsics.
1821 if (TM.getTargetTriple().isOSAIX() && !isa<IntrinsicInst>(Val: I))
1822 return false;
1823 return selectCall(I);
1824
1825 case Instruction::BitCast:
1826 return selectBitCast(I);
1827
1828 case Instruction::FPToSI:
1829 return selectCast(I, Opcode: ISD::FP_TO_SINT);
1830 case Instruction::ZExt:
1831 return selectCast(I, Opcode: ISD::ZERO_EXTEND);
1832 case Instruction::SExt:
1833 return selectCast(I, Opcode: ISD::SIGN_EXTEND);
1834 case Instruction::Trunc:
1835 return selectCast(I, Opcode: ISD::TRUNCATE);
1836 case Instruction::SIToFP:
1837 return selectCast(I, Opcode: ISD::SINT_TO_FP);
1838
1839 case Instruction::IntToPtr: // Deliberate fall-through.
1840 case Instruction::PtrToInt:
1841 case Instruction::PtrToAddr: {
1842 EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1843 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
1844 if (DstVT.bitsGT(VT: SrcVT))
1845 return selectCast(I, Opcode: ISD::ZERO_EXTEND);
1846 if (DstVT.bitsLT(VT: SrcVT))
1847 return selectCast(I, Opcode: ISD::TRUNCATE);
1848 Register Reg = getRegForValue(V: I->getOperand(i: 0));
1849 if (!Reg)
1850 return false;
1851 updateValueMap(I, Reg);
1852 return true;
1853 }
1854
1855 case Instruction::ExtractValue:
1856 return selectExtractValue(U: I);
1857
1858 case Instruction::Freeze:
1859 return selectFreeze(I);
1860
1861 case Instruction::PHI:
1862 llvm_unreachable("FastISel shouldn't visit PHI nodes!");
1863
1864 default:
1865 // Unhandled instruction. Halt "fast" selection and bail.
1866 return false;
1867 }
1868}
1869
1870FastISel::FastISel(FunctionLoweringInfo &FuncInfo,
1871 const TargetLibraryInfo *LibInfo,
1872 const LibcallLoweringInfo *LibcallLowering,
1873 bool SkipTargetIndependentISel)
1874 : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
1875 MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
1876 TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
1877 TII(*MF->getSubtarget().getInstrInfo()),
1878 TLI(*MF->getSubtarget().getTargetLowering()),
1879 TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
1880 LibcallLowering(LibcallLowering),
1881 SkipTargetIndependentISel(SkipTargetIndependentISel) {}
1882
1883FastISel::~FastISel() = default;
1884
1885bool FastISel::fastLowerArguments() { return false; }
1886
1887bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
1888
1889bool FastISel::fastLowerIntrinsicCall(const IntrinsicInst * /*II*/) {
1890 return false;
1891}
1892
1893Register FastISel::fastEmit_(MVT, MVT, unsigned) { return Register(); }
1894
1895Register FastISel::fastEmit_r(MVT, MVT, unsigned, Register /*Op0*/) {
1896 return Register();
1897}
1898
1899Register FastISel::fastEmit_rr(MVT, MVT, unsigned, Register /*Op0*/,
1900 Register /*Op1*/) {
1901 return Register();
1902}
1903
1904Register FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
1905 return Register();
1906}
1907
1908Register FastISel::fastEmit_f(MVT, MVT, unsigned,
1909 const ConstantFP * /*FPImm*/) {
1910 return Register();
1911}
1912
1913Register FastISel::fastEmit_ri(MVT, MVT, unsigned, Register /*Op0*/,
1914 uint64_t /*Imm*/) {
1915 return Register();
1916}
1917
1918/// This method is a wrapper of fastEmit_ri. It first tries to emit an
1919/// instruction with an immediate operand using fastEmit_ri.
1920/// If that fails, it materializes the immediate into a register and try
1921/// fastEmit_rr instead.
1922Register FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, Register Op0,
1923 uint64_t Imm, MVT ImmType) {
1924 // If this is a multiply by a power of two, emit this as a shift left.
1925 if (Opcode == ISD::MUL && isPowerOf2_64(Value: Imm)) {
1926 Opcode = ISD::SHL;
1927 Imm = Log2_64(Value: Imm);
1928 } else if (Opcode == ISD::UDIV && isPowerOf2_64(Value: Imm)) {
1929 // div x, 8 -> srl x, 3
1930 Opcode = ISD::SRL;
1931 Imm = Log2_64(Value: Imm);
1932 }
1933
1934 // Horrible hack (to be removed), check to make sure shift amounts are
1935 // in-range.
1936 if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
1937 Imm >= VT.getSizeInBits())
1938 return Register();
1939
1940 // First check if immediate type is legal. If not, we can't use the ri form.
1941 Register ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Imm);
1942 if (ResultReg)
1943 return ResultReg;
1944 Register MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
1945 if (!MaterialReg) {
1946 // This is a bit ugly/slow, but failing here means falling out of
1947 // fast-isel, which would be very slow.
1948 IntegerType *ITy =
1949 IntegerType::get(C&: FuncInfo.Fn->getContext(), NumBits: VT.getSizeInBits());
1950 // TODO: Avoid implicit trunc?
1951 // See https://github.com/llvm/llvm-project/issues/112510.
1952 MaterialReg = getRegForValue(
1953 V: ConstantInt::get(Ty: ITy, V: Imm, /*IsSigned=*/false, /*ImplicitTrunc=*/true));
1954 if (!MaterialReg)
1955 return Register();
1956 }
1957 return fastEmit_rr(VT, VT, Opcode, Op0, MaterialReg);
1958}
1959
1960Register FastISel::createResultReg(const TargetRegisterClass *RC) {
1961 return MRI.createVirtualRegister(RegClass: RC);
1962}
1963
1964Register FastISel::constrainOperandRegClass(const MCInstrDesc &II, Register Op,
1965 unsigned OpNum) {
1966 if (Op.isVirtual()) {
1967 const TargetRegisterClass *RegClass = TII.getRegClass(MCID: II, OpNum);
1968 if (!MRI.constrainRegClass(Reg: Op, RC: RegClass)) {
1969 // If it's not legal to COPY between the register classes, something
1970 // has gone very wrong before we got here.
1971 Register NewOp = createResultReg(RC: RegClass);
1972 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1973 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: NewOp).addReg(RegNo: Op);
1974 return NewOp;
1975 }
1976 }
1977 return Op;
1978}
1979
1980Register FastISel::fastEmitInst_(unsigned MachineInstOpcode,
1981 const TargetRegisterClass *RC) {
1982 Register ResultReg = createResultReg(RC);
1983 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
1984
1985 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg);
1986 return ResultReg;
1987}
1988
1989Register FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
1990 const TargetRegisterClass *RC, Register Op0) {
1991 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
1992
1993 Register ResultReg = createResultReg(RC);
1994 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
1995
1996 if (II.getNumDefs() >= 1)
1997 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
1998 .addReg(RegNo: Op0);
1999 else {
2000 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2001 .addReg(RegNo: Op0);
2002 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2003 DestReg: ResultReg)
2004 .addReg(RegNo: II.implicit_defs()[0]);
2005 }
2006
2007 return ResultReg;
2008}
2009
2010Register FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
2011 const TargetRegisterClass *RC, Register Op0,
2012 Register Op1) {
2013 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2014
2015 Register ResultReg = createResultReg(RC);
2016 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2017 Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
2018
2019 if (II.getNumDefs() >= 1)
2020 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2021 .addReg(RegNo: Op0)
2022 .addReg(RegNo: Op1);
2023 else {
2024 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2025 .addReg(RegNo: Op0)
2026 .addReg(RegNo: Op1);
2027 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2028 DestReg: ResultReg)
2029 .addReg(RegNo: II.implicit_defs()[0]);
2030 }
2031 return ResultReg;
2032}
2033
2034Register FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
2035 const TargetRegisterClass *RC, Register Op0,
2036 Register Op1, Register Op2) {
2037 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2038
2039 Register ResultReg = createResultReg(RC);
2040 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2041 Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
2042 Op2 = constrainOperandRegClass(II, Op: Op2, OpNum: II.getNumDefs() + 2);
2043
2044 if (II.getNumDefs() >= 1)
2045 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2046 .addReg(RegNo: Op0)
2047 .addReg(RegNo: Op1)
2048 .addReg(RegNo: Op2);
2049 else {
2050 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2051 .addReg(RegNo: Op0)
2052 .addReg(RegNo: Op1)
2053 .addReg(RegNo: Op2);
2054 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2055 DestReg: ResultReg)
2056 .addReg(RegNo: II.implicit_defs()[0]);
2057 }
2058 return ResultReg;
2059}
2060
2061Register FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
2062 const TargetRegisterClass *RC, Register Op0,
2063 uint64_t Imm) {
2064 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2065
2066 Register ResultReg = createResultReg(RC);
2067 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2068
2069 if (II.getNumDefs() >= 1)
2070 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2071 .addReg(RegNo: Op0)
2072 .addImm(Val: Imm);
2073 else {
2074 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2075 .addReg(RegNo: Op0)
2076 .addImm(Val: Imm);
2077 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2078 DestReg: ResultReg)
2079 .addReg(RegNo: II.implicit_defs()[0]);
2080 }
2081 return ResultReg;
2082}
2083
2084Register FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
2085 const TargetRegisterClass *RC, Register Op0,
2086 uint64_t Imm1, uint64_t Imm2) {
2087 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2088
2089 Register ResultReg = createResultReg(RC);
2090 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2091
2092 if (II.getNumDefs() >= 1)
2093 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2094 .addReg(RegNo: Op0)
2095 .addImm(Val: Imm1)
2096 .addImm(Val: Imm2);
2097 else {
2098 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2099 .addReg(RegNo: Op0)
2100 .addImm(Val: Imm1)
2101 .addImm(Val: Imm2);
2102 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2103 DestReg: ResultReg)
2104 .addReg(RegNo: II.implicit_defs()[0]);
2105 }
2106 return ResultReg;
2107}
2108
2109Register FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
2110 const TargetRegisterClass *RC,
2111 const ConstantFP *FPImm) {
2112 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2113
2114 Register ResultReg = createResultReg(RC);
2115
2116 if (II.getNumDefs() >= 1)
2117 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2118 .addFPImm(Val: FPImm);
2119 else {
2120 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2121 .addFPImm(Val: FPImm);
2122 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2123 DestReg: ResultReg)
2124 .addReg(RegNo: II.implicit_defs()[0]);
2125 }
2126 return ResultReg;
2127}
2128
2129Register FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
2130 const TargetRegisterClass *RC, Register Op0,
2131 Register Op1, uint64_t Imm) {
2132 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2133
2134 Register ResultReg = createResultReg(RC);
2135 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2136 Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
2137
2138 if (II.getNumDefs() >= 1)
2139 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2140 .addReg(RegNo: Op0)
2141 .addReg(RegNo: Op1)
2142 .addImm(Val: Imm);
2143 else {
2144 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2145 .addReg(RegNo: Op0)
2146 .addReg(RegNo: Op1)
2147 .addImm(Val: Imm);
2148 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2149 DestReg: ResultReg)
2150 .addReg(RegNo: II.implicit_defs()[0]);
2151 }
2152 return ResultReg;
2153}
2154
2155Register FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
2156 const TargetRegisterClass *RC, uint64_t Imm) {
2157 Register ResultReg = createResultReg(RC);
2158 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2159
2160 if (II.getNumDefs() >= 1)
2161 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2162 .addImm(Val: Imm);
2163 else {
2164 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II).addImm(Val: Imm);
2165 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2166 DestReg: ResultReg)
2167 .addReg(RegNo: II.implicit_defs()[0]);
2168 }
2169 return ResultReg;
2170}
2171
2172Register FastISel::fastEmitInst_extractsubreg(MVT RetVT, Register Op0,
2173 uint32_t Idx) {
2174 Register ResultReg = createResultReg(RC: TLI.getRegClassFor(VT: RetVT));
2175 assert(Op0.isVirtual() && "Cannot yet extract from physregs");
2176 const TargetRegisterClass *RC = MRI.getRegClass(Reg: Op0);
2177 MRI.constrainRegClass(Reg: Op0, RC: TRI.getSubClassWithSubReg(RC, Idx));
2178 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2179 DestReg: ResultReg)
2180 .addReg(RegNo: Op0, Flags: {}, SubReg: Idx);
2181 return ResultReg;
2182}
2183
2184/// Emit MachineInstrs to compute the value of Op with all but the least
2185/// significant bit set to zero.
2186Register FastISel::fastEmitZExtFromI1(MVT VT, Register Op0) {
2187 return fastEmit_ri(VT, VT, ISD::AND, Op0, 1);
2188}
2189
2190/// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
2191/// Emit code to ensure constants are copied into registers when needed.
2192/// Remember the virtual registers that need to be added to the Machine PHI
2193/// nodes as input. We cannot just directly add them, because expansion
2194/// might result in multiple MBB's for one BB. As such, the start of the
2195/// BB might correspond to a different MBB than the end.
2196bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
2197 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
2198 FuncInfo.OrigNumPHINodesToUpdate = FuncInfo.PHINodesToUpdate.size();
2199
2200 // Check successor nodes' PHI nodes that expect a constant to be available
2201 // from this block.
2202 for (const BasicBlock *SuccBB : successors(BB: LLVMBB)) {
2203 if (!isa<PHINode>(Val: SuccBB->begin()))
2204 continue;
2205 MachineBasicBlock *SuccMBB = FuncInfo.getMBB(BB: SuccBB);
2206
2207 // If this terminator has multiple identical successors (common for
2208 // switches), only handle each succ once.
2209 if (!SuccsHandled.insert(Ptr: SuccMBB).second)
2210 continue;
2211
2212 MachineBasicBlock::iterator MBBI = SuccMBB->begin();
2213
2214 // At this point we know that there is a 1-1 correspondence between LLVM PHI
2215 // nodes and Machine PHI nodes, but the incoming operands have not been
2216 // emitted yet.
2217 for (const PHINode &PN : SuccBB->phis()) {
2218 // Ignore dead phi's.
2219 if (PN.use_empty())
2220 continue;
2221
2222 // Only handle legal types. Two interesting things to note here. First,
2223 // by bailing out early, we may leave behind some dead instructions,
2224 // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
2225 // own moves. Second, this check is necessary because FastISel doesn't
2226 // use CreateRegs to create registers, so it always creates
2227 // exactly one register for each non-void instruction.
2228 EVT VT = TLI.getValueType(DL, Ty: PN.getType(), /*AllowUnknown=*/true);
2229 if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
2230 // Handle integer promotions, though, because they're common and easy.
2231 if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
2232 FuncInfo.PHINodesToUpdate.resize(new_size: FuncInfo.OrigNumPHINodesToUpdate);
2233 return false;
2234 }
2235 }
2236
2237 const Value *PHIOp = PN.getIncomingValueForBlock(BB: LLVMBB);
2238
2239 // Set the DebugLoc for the copy. Use the location of the operand if
2240 // there is one; otherwise no location, flushLocalValueMap will fix it.
2241 MIMD = {};
2242 if (const auto *Inst = dyn_cast<Instruction>(Val: PHIOp))
2243 MIMD = MIMetadata(*Inst);
2244
2245 Register Reg = getRegForValue(V: PHIOp);
2246 if (!Reg) {
2247 FuncInfo.PHINodesToUpdate.resize(new_size: FuncInfo.OrigNumPHINodesToUpdate);
2248 return false;
2249 }
2250 FuncInfo.PHINodesToUpdate.emplace_back(args: &*MBBI++, args&: Reg);
2251 MIMD = {};
2252 }
2253 }
2254
2255 return true;
2256}
2257
2258bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
2259 assert(LI->hasOneUse() &&
2260 "tryToFoldLoad expected a LoadInst with a single use");
2261 // We know that the load has a single use, but don't know what it is. If it
2262 // isn't one of the folded instructions, then we can't succeed here. Handle
2263 // this by scanning the single-use users of the load until we get to FoldInst.
2264 unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
2265
2266 const Instruction *TheUser = LI->user_back();
2267 while (TheUser != FoldInst && // Scan up until we find FoldInst.
2268 // Stay in the right block.
2269 TheUser->getParent() == FoldInst->getParent() &&
2270 --MaxUsers) { // Don't scan too far.
2271 // If there are multiple or no uses of this instruction, then bail out.
2272 if (!TheUser->hasOneUse())
2273 return false;
2274
2275 TheUser = TheUser->user_back();
2276 }
2277
2278 // If we didn't find the fold instruction, then we failed to collapse the
2279 // sequence.
2280 if (TheUser != FoldInst)
2281 return false;
2282
2283 // Don't try to fold volatile loads. Target has to deal with alignment
2284 // constraints.
2285 if (LI->isVolatile())
2286 return false;
2287
2288 // Figure out which vreg this is going into. If there is no assigned vreg yet
2289 // then there actually was no reference to it. Perhaps the load is referenced
2290 // by a dead instruction.
2291 Register LoadReg = getRegForValue(V: LI);
2292 if (!LoadReg)
2293 return false;
2294
2295 // We can't fold if this vreg has no uses or more than one use. Multiple uses
2296 // may mean that the instruction got lowered to multiple MIs, or the use of
2297 // the loaded value ended up being multiple operands of the result.
2298 if (!MRI.hasOneUse(RegNo: LoadReg))
2299 return false;
2300
2301 // If the register has fixups, there may be additional uses through a
2302 // different alias of the register.
2303 if (FuncInfo.RegsWithFixups.contains(V: LoadReg))
2304 return false;
2305
2306 MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(RegNo: LoadReg);
2307 MachineInstr *User = RI->getParent();
2308
2309 // Set the insertion point properly. Folding the load can cause generation of
2310 // other random instructions (like sign extends) for addressing modes; make
2311 // sure they get inserted in a logical place before the new instruction.
2312 FuncInfo.InsertPt = User;
2313 FuncInfo.MBB = User->getParent();
2314
2315 // Ask the target to try folding the load.
2316 return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
2317}
2318
2319bool FastISel::canFoldAddIntoGEP(const User *GEP, const Value *Add) {
2320 // Must be an add.
2321 if (!isa<AddOperator>(Val: Add))
2322 return false;
2323 // Type size needs to match.
2324 if (DL.getTypeSizeInBits(Ty: GEP->getType()) !=
2325 DL.getTypeSizeInBits(Ty: Add->getType()))
2326 return false;
2327 // Must be in the same basic block.
2328 if (isa<Instruction>(Val: Add) &&
2329 FuncInfo.getMBB(BB: cast<Instruction>(Val: Add)->getParent()) != FuncInfo.MBB)
2330 return false;
2331 // Must have a constant operand.
2332 return isa<ConstantInt>(Val: cast<AddOperator>(Val: Add)->getOperand(i_nocapture: 1));
2333}
2334
2335MachineMemOperand *
2336FastISel::createMachineMemOperandFor(const Instruction *I) const {
2337 const Value *Ptr;
2338 Type *ValTy;
2339 MaybeAlign Alignment;
2340 MachineMemOperand::Flags Flags;
2341 bool IsVolatile;
2342
2343 if (const auto *LI = dyn_cast<LoadInst>(Val: I)) {
2344 Alignment = LI->getAlign();
2345 IsVolatile = LI->isVolatile();
2346 Flags = MachineMemOperand::MOLoad;
2347 Ptr = LI->getPointerOperand();
2348 ValTy = LI->getType();
2349 } else if (const auto *SI = dyn_cast<StoreInst>(Val: I)) {
2350 Alignment = SI->getAlign();
2351 IsVolatile = SI->isVolatile();
2352 Flags = MachineMemOperand::MOStore;
2353 Ptr = SI->getPointerOperand();
2354 ValTy = SI->getValueOperand()->getType();
2355 } else
2356 return nullptr;
2357
2358 bool IsNonTemporal = I->hasMetadata(KindID: LLVMContext::MD_nontemporal);
2359 bool IsInvariant = I->hasMetadata(KindID: LLVMContext::MD_invariant_load);
2360 bool IsDereferenceable = I->hasMetadata(KindID: LLVMContext::MD_dereferenceable);
2361 const MDNode *Ranges = I->getMetadata(KindID: LLVMContext::MD_range);
2362
2363 AAMDNodes AAInfo = I->getAAMetadata();
2364
2365 if (!Alignment) // Ensure that codegen never sees alignment 0.
2366 Alignment = DL.getABITypeAlign(Ty: ValTy);
2367
2368 unsigned Size = DL.getTypeStoreSize(Ty: ValTy);
2369
2370 if (IsVolatile)
2371 Flags |= MachineMemOperand::MOVolatile;
2372 if (IsNonTemporal)
2373 Flags |= MachineMemOperand::MONonTemporal;
2374 if (IsDereferenceable)
2375 Flags |= MachineMemOperand::MODereferenceable;
2376 if (IsInvariant)
2377 Flags |= MachineMemOperand::MOInvariant;
2378
2379 return FuncInfo.MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Ptr), F: Flags, Size,
2380 BaseAlignment: *Alignment, AAInfo, Ranges);
2381}
2382
2383CmpInst::Predicate FastISel::optimizeCmpPredicate(const CmpInst *CI) const {
2384 // If both operands are the same, then try to optimize or fold the cmp.
2385 CmpInst::Predicate Predicate = CI->getPredicate();
2386 if (CI->getOperand(i_nocapture: 0) != CI->getOperand(i_nocapture: 1))
2387 return Predicate;
2388
2389 switch (Predicate) {
2390 default: llvm_unreachable("Invalid predicate!");
2391 case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
2392 case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break;
2393 case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break;
2394 case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break;
2395 case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break;
2396 case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break;
2397 case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break;
2398 case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break;
2399 case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break;
2400 case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break;
2401 case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break;
2402 case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2403 case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break;
2404 case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2405 case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break;
2406 case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break;
2407
2408 case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break;
2409 case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break;
2410 case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break;
2411 case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2412 case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break;
2413 case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2414 case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break;
2415 case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break;
2416 case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break;
2417 case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break;
2418 }
2419
2420 return Predicate;
2421}
2422