1//===- FastISel.cpp - Implementation of the FastISel class ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the FastISel class.
10//
11// "Fast" instruction selection is designed to emit very poor code quickly.
12// Also, it is not designed to be able to do much lowering, so most illegal
13// types (e.g. i64 on 32-bit targets) and operations are not supported. It is
14// also not intended to be able to do much optimization, except in a few cases
15// where doing optimizations reduces overall compile time. For example, folding
16// constants into immediate fields is often done, because it's cheap and it
17// reduces the number of instructions later phases have to examine.
18//
19// "Fast" instruction selection is able to fail gracefully and transfer
20// control to the SelectionDAG selector for operations that it doesn't
21// support. In many cases, this allows us to avoid duplicating a lot of
22// the complicated lowering logic that SelectionDAG currently has.
23//
24// The intended use for "fast" instruction selection is "-O0" mode
25// compilation, where the quality of the generated code is irrelevant when
26// weighed against the speed at which the code can be generated. Also,
27// at -O0, the LLVM optimizers are not running, and this makes the
28// compile time of codegen a much higher portion of the overall compile
29// time. Despite its limitations, "fast" instruction selection is able to
30// handle enough code on its own to provide noticeable overall speedups
31// in -O0 compiles.
32//
33// Basic operations are supported in a target-independent way, by reading
34// the same instruction descriptions that the SelectionDAG selector reads,
35// and identifying simple arithmetic operations that can be directly selected
36// from simple operators. More complicated operations currently require
37// target-specific code.
38//
39//===----------------------------------------------------------------------===//
40
41#include "llvm/CodeGen/FastISel.h"
42#include "llvm/ADT/APFloat.h"
43#include "llvm/ADT/APSInt.h"
44#include "llvm/ADT/DenseMap.h"
45#include "llvm/ADT/SmallPtrSet.h"
46#include "llvm/ADT/SmallString.h"
47#include "llvm/ADT/SmallVector.h"
48#include "llvm/ADT/Statistic.h"
49#include "llvm/Analysis/BranchProbabilityInfo.h"
50#include "llvm/Analysis/TargetLibraryInfo.h"
51#include "llvm/CodeGen/Analysis.h"
52#include "llvm/CodeGen/FunctionLoweringInfo.h"
53#include "llvm/CodeGen/ISDOpcodes.h"
54#include "llvm/CodeGen/MachineBasicBlock.h"
55#include "llvm/CodeGen/MachineFrameInfo.h"
56#include "llvm/CodeGen/MachineInstr.h"
57#include "llvm/CodeGen/MachineInstrBuilder.h"
58#include "llvm/CodeGen/MachineMemOperand.h"
59#include "llvm/CodeGen/MachineModuleInfo.h"
60#include "llvm/CodeGen/MachineOperand.h"
61#include "llvm/CodeGen/MachineRegisterInfo.h"
62#include "llvm/CodeGen/StackMaps.h"
63#include "llvm/CodeGen/TargetInstrInfo.h"
64#include "llvm/CodeGen/TargetLowering.h"
65#include "llvm/CodeGen/TargetSubtargetInfo.h"
66#include "llvm/CodeGen/ValueTypes.h"
67#include "llvm/CodeGenTypes/MachineValueType.h"
68#include "llvm/IR/Argument.h"
69#include "llvm/IR/Attributes.h"
70#include "llvm/IR/BasicBlock.h"
71#include "llvm/IR/CallingConv.h"
72#include "llvm/IR/Constant.h"
73#include "llvm/IR/Constants.h"
74#include "llvm/IR/DataLayout.h"
75#include "llvm/IR/DebugLoc.h"
76#include "llvm/IR/DerivedTypes.h"
77#include "llvm/IR/DiagnosticInfo.h"
78#include "llvm/IR/Function.h"
79#include "llvm/IR/GetElementPtrTypeIterator.h"
80#include "llvm/IR/GlobalValue.h"
81#include "llvm/IR/InlineAsm.h"
82#include "llvm/IR/InstrTypes.h"
83#include "llvm/IR/Instruction.h"
84#include "llvm/IR/Instructions.h"
85#include "llvm/IR/IntrinsicInst.h"
86#include "llvm/IR/LLVMContext.h"
87#include "llvm/IR/Mangler.h"
88#include "llvm/IR/Metadata.h"
89#include "llvm/IR/Operator.h"
90#include "llvm/IR/PatternMatch.h"
91#include "llvm/IR/Type.h"
92#include "llvm/IR/User.h"
93#include "llvm/IR/Value.h"
94#include "llvm/MC/MCContext.h"
95#include "llvm/MC/MCInstrDesc.h"
96#include "llvm/Support/Casting.h"
97#include "llvm/Support/Debug.h"
98#include "llvm/Support/ErrorHandling.h"
99#include "llvm/Support/MathExtras.h"
100#include "llvm/Support/raw_ostream.h"
101#include "llvm/Target/TargetMachine.h"
102#include "llvm/Target/TargetOptions.h"
103#include <algorithm>
104#include <cassert>
105#include <cstdint>
106#include <iterator>
107#include <optional>
108#include <utility>
109
110using namespace llvm;
111using namespace PatternMatch;
112
113#define DEBUG_TYPE "isel"
114
115STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
116 "target-independent selector");
117STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
118 "target-specific selector");
119STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
120
121/// Set the current block to which generated machine instructions will be
122/// appended.
123void FastISel::startNewBlock() {
124 assert(LocalValueMap.empty() &&
125 "local values should be cleared after finishing a BB");
126
127 // Instructions are appended to FuncInfo.MBB. If the basic block already
128 // contains labels or copies, use the last instruction as the last local
129 // value.
130 EmitStartPt = nullptr;
131 if (!FuncInfo.MBB->empty())
132 EmitStartPt = &FuncInfo.MBB->back();
133 LastLocalValue = EmitStartPt;
134}
135
136void FastISel::finishBasicBlock() { flushLocalValueMap(); }
137
138bool FastISel::lowerArguments() {
139 if (!FuncInfo.CanLowerReturn)
140 // Fallback to SDISel argument lowering code to deal with sret pointer
141 // parameter.
142 return false;
143
144 if (!fastLowerArguments())
145 return false;
146
147 // Enter arguments into ValueMap for uses in non-entry BBs.
148 for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(),
149 E = FuncInfo.Fn->arg_end();
150 I != E; ++I) {
151 DenseMap<const Value *, Register>::iterator VI = LocalValueMap.find(Val: &*I);
152 assert(VI != LocalValueMap.end() && "Missed an argument?");
153 FuncInfo.ValueMap[&*I] = VI->second;
154 }
155 return true;
156}
157
158/// Return the defined register if this instruction defines exactly one
159/// virtual register and uses no other virtual registers. Otherwise return 0.
160static Register findLocalRegDef(MachineInstr &MI) {
161 Register RegDef;
162 for (const MachineOperand &MO : MI.operands()) {
163 if (!MO.isReg())
164 continue;
165 if (MO.isDef()) {
166 if (RegDef)
167 return Register();
168 RegDef = MO.getReg();
169 } else if (MO.getReg().isVirtual()) {
170 // This is another use of a vreg. Don't delete it.
171 return Register();
172 }
173 }
174 return RegDef;
175}
176
177static bool isRegUsedByPhiNodes(Register DefReg,
178 FunctionLoweringInfo &FuncInfo) {
179 for (auto &P : FuncInfo.PHINodesToUpdate)
180 if (P.second == DefReg)
181 return true;
182 return false;
183}
184
185void FastISel::flushLocalValueMap() {
186 // If FastISel bails out, it could leave local value instructions behind
187 // that aren't used for anything. Detect and erase those.
188 if (LastLocalValue != EmitStartPt) {
189 // Save the first instruction after local values, for later.
190 MachineBasicBlock::iterator FirstNonValue(LastLocalValue);
191 ++FirstNonValue;
192
193 MachineBasicBlock::reverse_iterator RE =
194 EmitStartPt ? MachineBasicBlock::reverse_iterator(EmitStartPt)
195 : FuncInfo.MBB->rend();
196 MachineBasicBlock::reverse_iterator RI(LastLocalValue);
197 for (MachineInstr &LocalMI :
198 llvm::make_early_inc_range(Range: llvm::make_range(x: RI, y: RE))) {
199 Register DefReg = findLocalRegDef(MI&: LocalMI);
200 if (!DefReg)
201 continue;
202 if (FuncInfo.RegsWithFixups.count(V: DefReg))
203 continue;
204 bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
205 if (!UsedByPHI && MRI.use_nodbg_empty(RegNo: DefReg)) {
206 if (EmitStartPt == &LocalMI)
207 EmitStartPt = EmitStartPt->getPrevNode();
208 LLVM_DEBUG(dbgs() << "removing dead local value materialization"
209 << LocalMI);
210 LocalMI.eraseFromParent();
211 }
212 }
213
214 if (FirstNonValue != FuncInfo.MBB->end()) {
215 // See if there are any local value instructions left. If so, we want to
216 // make sure the first one has a debug location; if it doesn't, use the
217 // first non-value instruction's debug location.
218
219 // If EmitStartPt is non-null, this block had copies at the top before
220 // FastISel started doing anything; it points to the last one, so the
221 // first local value instruction is the one after EmitStartPt.
222 // If EmitStartPt is null, the first local value instruction is at the
223 // top of the block.
224 MachineBasicBlock::iterator FirstLocalValue =
225 EmitStartPt ? ++MachineBasicBlock::iterator(EmitStartPt)
226 : FuncInfo.MBB->begin();
227 if (FirstLocalValue != FirstNonValue && !FirstLocalValue->getDebugLoc())
228 FirstLocalValue->setDebugLoc(FirstNonValue->getDebugLoc());
229 }
230 }
231
232 LocalValueMap.clear();
233 LastLocalValue = EmitStartPt;
234 recomputeInsertPt();
235 SavedInsertPt = FuncInfo.InsertPt;
236}
237
238Register FastISel::getRegForValue(const Value *V) {
239 EVT RealVT = TLI.getValueType(DL, Ty: V->getType(), /*AllowUnknown=*/true);
240 // Don't handle non-simple values in FastISel.
241 if (!RealVT.isSimple())
242 return Register();
243
244 // Ignore illegal types. We must do this before looking up the value
245 // in ValueMap because Arguments are given virtual registers regardless
246 // of whether FastISel can handle them.
247 MVT VT = RealVT.getSimpleVT();
248 if (!TLI.isTypeLegal(VT)) {
249 // Handle integer promotions, though, because they're common and easy.
250 if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
251 VT = TLI.getTypeToTransformTo(Context&: V->getContext(), VT).getSimpleVT();
252 else
253 return Register();
254 }
255
256 // Look up the value to see if we already have a register for it.
257 Register Reg = lookUpRegForValue(V);
258 if (Reg)
259 return Reg;
260
261 // In bottom-up mode, just create the virtual register which will be used
262 // to hold the value. It will be materialized later.
263 if (isa<Instruction>(Val: V) &&
264 (!isa<AllocaInst>(Val: V) ||
265 !FuncInfo.StaticAllocaMap.count(Val: cast<AllocaInst>(Val: V))))
266 return FuncInfo.InitializeRegForValue(V);
267
268 SavePoint SaveInsertPt = enterLocalValueArea();
269
270 // Materialize the value in a register. Emit any instructions in the
271 // local value area.
272 Reg = materializeRegForValue(V, VT);
273
274 leaveLocalValueArea(Old: SaveInsertPt);
275
276 return Reg;
277}
278
279Register FastISel::materializeConstant(const Value *V, MVT VT) {
280 Register Reg;
281 if (const auto *CI = dyn_cast<ConstantInt>(Val: V)) {
282 if (CI->getValue().getActiveBits() <= 64)
283 Reg = fastEmit_i(VT, RetVT: VT, Opcode: ISD::Constant, Imm: CI->getZExtValue());
284 } else if (isa<AllocaInst>(Val: V))
285 Reg = fastMaterializeAlloca(C: cast<AllocaInst>(Val: V));
286 else if (isa<ConstantPointerNull>(Val: V))
287 // Translate this as an integer zero so that it can be
288 // local-CSE'd with actual integer zeros.
289 Reg =
290 getRegForValue(V: Constant::getNullValue(Ty: DL.getIntPtrType(V->getType())));
291 else if (const auto *CF = dyn_cast<ConstantFP>(Val: V)) {
292 if (CF->isNullValue())
293 Reg = fastMaterializeFloatZero(CF);
294 else
295 // Try to emit the constant directly.
296 Reg = fastEmit_f(VT, RetVT: VT, Opcode: ISD::ConstantFP, FPImm: CF);
297
298 if (!Reg) {
299 // Try to emit the constant by using an integer constant with a cast.
300 const APFloat &Flt = CF->getValueAPF();
301 EVT IntVT = TLI.getPointerTy(DL);
302 uint32_t IntBitWidth = IntVT.getSizeInBits();
303 APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
304 bool isExact;
305 (void)Flt.convertToInteger(Result&: SIntVal, RM: APFloat::rmTowardZero, IsExact: &isExact);
306 if (isExact) {
307 Register IntegerReg =
308 getRegForValue(V: ConstantInt::get(Context&: V->getContext(), V: SIntVal));
309 if (IntegerReg)
310 Reg = fastEmit_r(VT: IntVT.getSimpleVT(), RetVT: VT, Opcode: ISD::SINT_TO_FP,
311 Op0: IntegerReg);
312 }
313 }
314 } else if (const auto *Op = dyn_cast<Operator>(Val: V)) {
315 if (!selectOperator(I: Op, Opcode: Op->getOpcode()))
316 if (!isa<Instruction>(Val: Op) ||
317 !fastSelectInstruction(I: cast<Instruction>(Val: Op)))
318 return 0;
319 Reg = lookUpRegForValue(V: Op);
320 } else if (isa<UndefValue>(Val: V)) {
321 Reg = createResultReg(RC: TLI.getRegClassFor(VT));
322 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
323 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: Reg);
324 }
325 return Reg;
326}
327
328/// Helper for getRegForValue. This function is called when the value isn't
329/// already available in a register and must be materialized with new
330/// instructions.
331Register FastISel::materializeRegForValue(const Value *V, MVT VT) {
332 Register Reg;
333 // Give the target-specific code a try first.
334 if (isa<Constant>(Val: V))
335 Reg = fastMaterializeConstant(C: cast<Constant>(Val: V));
336
337 // If target-specific code couldn't or didn't want to handle the value, then
338 // give target-independent code a try.
339 if (!Reg)
340 Reg = materializeConstant(V, VT);
341
342 // Don't cache constant materializations in the general ValueMap.
343 // To do so would require tracking what uses they dominate.
344 if (Reg) {
345 LocalValueMap[V] = Reg;
346 LastLocalValue = MRI.getVRegDef(Reg);
347 }
348 return Reg;
349}
350
351Register FastISel::lookUpRegForValue(const Value *V) {
352 // Look up the value to see if we already have a register for it. We
353 // cache values defined by Instructions across blocks, and other values
354 // only locally. This is because Instructions already have the SSA
355 // def-dominates-use requirement enforced.
356 DenseMap<const Value *, Register>::iterator I = FuncInfo.ValueMap.find(Val: V);
357 if (I != FuncInfo.ValueMap.end())
358 return I->second;
359 return LocalValueMap[V];
360}
361
362void FastISel::updateValueMap(const Value *I, Register Reg, unsigned NumRegs) {
363 if (!isa<Instruction>(Val: I)) {
364 LocalValueMap[I] = Reg;
365 return;
366 }
367
368 Register &AssignedReg = FuncInfo.ValueMap[I];
369 if (!AssignedReg)
370 // Use the new register.
371 AssignedReg = Reg;
372 else if (Reg != AssignedReg) {
373 // Arrange for uses of AssignedReg to be replaced by uses of Reg.
374 for (unsigned i = 0; i < NumRegs; i++) {
375 FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
376 FuncInfo.RegsWithFixups.insert(V: Reg + i);
377 }
378
379 AssignedReg = Reg;
380 }
381}
382
383Register FastISel::getRegForGEPIndex(const Value *Idx) {
384 Register IdxN = getRegForValue(V: Idx);
385 if (!IdxN)
386 // Unhandled operand. Halt "fast" selection and bail.
387 return Register();
388
389 // If the index is smaller or larger than intptr_t, truncate or extend it.
390 MVT PtrVT = TLI.getPointerTy(DL);
391 EVT IdxVT = EVT::getEVT(Ty: Idx->getType(), /*HandleUnknown=*/false);
392 if (IdxVT.bitsLT(VT: PtrVT)) {
393 IdxN = fastEmit_r(VT: IdxVT.getSimpleVT(), RetVT: PtrVT, Opcode: ISD::SIGN_EXTEND, Op0: IdxN);
394 } else if (IdxVT.bitsGT(VT: PtrVT)) {
395 IdxN =
396 fastEmit_r(VT: IdxVT.getSimpleVT(), RetVT: PtrVT, Opcode: ISD::TRUNCATE, Op0: IdxN);
397 }
398 return IdxN;
399}
400
401void FastISel::recomputeInsertPt() {
402 if (getLastLocalValue()) {
403 FuncInfo.InsertPt = getLastLocalValue();
404 FuncInfo.MBB = FuncInfo.InsertPt->getParent();
405 ++FuncInfo.InsertPt;
406 } else
407 FuncInfo.InsertPt = FuncInfo.MBB->getFirstNonPHI();
408}
409
410void FastISel::removeDeadCode(MachineBasicBlock::iterator I,
411 MachineBasicBlock::iterator E) {
412 assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
413 "Invalid iterator!");
414 while (I != E) {
415 if (SavedInsertPt == I)
416 SavedInsertPt = E;
417 if (EmitStartPt == I)
418 EmitStartPt = E.isValid() ? &*E : nullptr;
419 if (LastLocalValue == I)
420 LastLocalValue = E.isValid() ? &*E : nullptr;
421
422 MachineInstr *Dead = &*I;
423 ++I;
424 Dead->eraseFromParent();
425 ++NumFastIselDead;
426 }
427 recomputeInsertPt();
428}
429
430FastISel::SavePoint FastISel::enterLocalValueArea() {
431 SavePoint OldInsertPt = FuncInfo.InsertPt;
432 recomputeInsertPt();
433 return OldInsertPt;
434}
435
436void FastISel::leaveLocalValueArea(SavePoint OldInsertPt) {
437 if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
438 LastLocalValue = &*std::prev(x: FuncInfo.InsertPt);
439
440 // Restore the previous insert position.
441 FuncInfo.InsertPt = OldInsertPt;
442}
443
444bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
445 EVT VT = EVT::getEVT(Ty: I->getType(), /*HandleUnknown=*/true);
446 if (VT == MVT::Other || !VT.isSimple())
447 // Unhandled type. Halt "fast" selection and bail.
448 return false;
449
450 // We only handle legal types. For example, on x86-32 the instruction
451 // selector contains all of the 64-bit instructions from x86-64,
452 // under the assumption that i64 won't be used if the target doesn't
453 // support it.
454 if (!TLI.isTypeLegal(VT)) {
455 // MVT::i1 is special. Allow AND, OR, or XOR because they
456 // don't require additional zeroing, which makes them easy.
457 if (VT == MVT::i1 && ISD::isBitwiseLogicOp(Opcode: ISDOpcode))
458 VT = TLI.getTypeToTransformTo(Context&: I->getContext(), VT);
459 else
460 return false;
461 }
462
463 // Check if the first operand is a constant, and handle it as "ri". At -O0,
464 // we don't have anything that canonicalizes operand order.
465 if (const auto *CI = dyn_cast<ConstantInt>(Val: I->getOperand(i: 0)))
466 if (isa<Instruction>(Val: I) && cast<Instruction>(Val: I)->isCommutative()) {
467 Register Op1 = getRegForValue(V: I->getOperand(i: 1));
468 if (!Op1)
469 return false;
470
471 Register ResultReg =
472 fastEmit_ri_(VT: VT.getSimpleVT(), Opcode: ISDOpcode, Op0: Op1, Imm: CI->getZExtValue(),
473 ImmType: VT.getSimpleVT());
474 if (!ResultReg)
475 return false;
476
477 // We successfully emitted code for the given LLVM Instruction.
478 updateValueMap(I, Reg: ResultReg);
479 return true;
480 }
481
482 Register Op0 = getRegForValue(V: I->getOperand(i: 0));
483 if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
484 return false;
485
486 // Check if the second operand is a constant and handle it appropriately.
487 if (const auto *CI = dyn_cast<ConstantInt>(Val: I->getOperand(i: 1))) {
488 uint64_t Imm = CI->getSExtValue();
489
490 // Transform "sdiv exact X, 8" -> "sra X, 3".
491 if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(Val: I) &&
492 cast<BinaryOperator>(Val: I)->isExact() && isPowerOf2_64(Value: Imm)) {
493 Imm = Log2_64(Value: Imm);
494 ISDOpcode = ISD::SRA;
495 }
496
497 // Transform "urem x, pow2" -> "and x, pow2-1".
498 if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(Val: I) &&
499 isPowerOf2_64(Value: Imm)) {
500 --Imm;
501 ISDOpcode = ISD::AND;
502 }
503
504 Register ResultReg = fastEmit_ri_(VT: VT.getSimpleVT(), Opcode: ISDOpcode, Op0, Imm,
505 ImmType: VT.getSimpleVT());
506 if (!ResultReg)
507 return false;
508
509 // We successfully emitted code for the given LLVM Instruction.
510 updateValueMap(I, Reg: ResultReg);
511 return true;
512 }
513
514 Register Op1 = getRegForValue(V: I->getOperand(i: 1));
515 if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
516 return false;
517
518 // Now we have both operands in registers. Emit the instruction.
519 Register ResultReg = fastEmit_rr(VT: VT.getSimpleVT(), RetVT: VT.getSimpleVT(),
520 Opcode: ISDOpcode, Op0, Op1);
521 if (!ResultReg)
522 // Target-specific code wasn't able to find a machine opcode for
523 // the given ISD opcode and type. Halt "fast" selection and bail.
524 return false;
525
526 // We successfully emitted code for the given LLVM Instruction.
527 updateValueMap(I, Reg: ResultReg);
528 return true;
529}
530
531bool FastISel::selectGetElementPtr(const User *I) {
532 Register N = getRegForValue(V: I->getOperand(i: 0));
533 if (!N) // Unhandled operand. Halt "fast" selection and bail.
534 return false;
535
536 // FIXME: The code below does not handle vector GEPs. Halt "fast" selection
537 // and bail.
538 if (isa<VectorType>(Val: I->getType()))
539 return false;
540
541 // Keep a running tab of the total offset to coalesce multiple N = N + Offset
542 // into a single N = N + TotalOffset.
543 uint64_t TotalOffs = 0;
544 // FIXME: What's a good SWAG number for MaxOffs?
545 uint64_t MaxOffs = 2048;
546 MVT VT = TLI.getPointerTy(DL);
547 for (gep_type_iterator GTI = gep_type_begin(GEP: I), E = gep_type_end(GEP: I);
548 GTI != E; ++GTI) {
549 const Value *Idx = GTI.getOperand();
550 if (StructType *StTy = GTI.getStructTypeOrNull()) {
551 uint64_t Field = cast<ConstantInt>(Val: Idx)->getZExtValue();
552 if (Field) {
553 // N = N + Offset
554 TotalOffs += DL.getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
555 if (TotalOffs >= MaxOffs) {
556 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
557 if (!N) // Unhandled operand. Halt "fast" selection and bail.
558 return false;
559 TotalOffs = 0;
560 }
561 }
562 } else {
563 // If this is a constant subscript, handle it quickly.
564 if (const auto *CI = dyn_cast<ConstantInt>(Val: Idx)) {
565 if (CI->isZero())
566 continue;
567 // N = N + Offset
568 uint64_t IdxN = CI->getValue().sextOrTrunc(width: 64).getSExtValue();
569 TotalOffs += GTI.getSequentialElementStride(DL) * IdxN;
570 if (TotalOffs >= MaxOffs) {
571 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
572 if (!N) // Unhandled operand. Halt "fast" selection and bail.
573 return false;
574 TotalOffs = 0;
575 }
576 continue;
577 }
578 if (TotalOffs) {
579 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
580 if (!N) // Unhandled operand. Halt "fast" selection and bail.
581 return false;
582 TotalOffs = 0;
583 }
584
585 // N = N + Idx * ElementSize;
586 uint64_t ElementSize = GTI.getSequentialElementStride(DL);
587 Register IdxN = getRegForGEPIndex(Idx);
588 if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
589 return false;
590
591 if (ElementSize != 1) {
592 IdxN = fastEmit_ri_(VT, Opcode: ISD::MUL, Op0: IdxN, Imm: ElementSize, ImmType: VT);
593 if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
594 return false;
595 }
596 N = fastEmit_rr(VT, RetVT: VT, Opcode: ISD::ADD, Op0: N, Op1: IdxN);
597 if (!N) // Unhandled operand. Halt "fast" selection and bail.
598 return false;
599 }
600 }
601 if (TotalOffs) {
602 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
603 if (!N) // Unhandled operand. Halt "fast" selection and bail.
604 return false;
605 }
606
607 // We successfully emitted code for the given LLVM Instruction.
608 updateValueMap(I, Reg: N);
609 return true;
610}
611
612bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
613 const CallInst *CI, unsigned StartIdx) {
614 for (unsigned i = StartIdx, e = CI->arg_size(); i != e; ++i) {
615 Value *Val = CI->getArgOperand(i);
616 // Check for constants and encode them with a StackMaps::ConstantOp prefix.
617 if (const auto *C = dyn_cast<ConstantInt>(Val)) {
618 Ops.push_back(Elt: MachineOperand::CreateImm(Val: StackMaps::ConstantOp));
619 Ops.push_back(Elt: MachineOperand::CreateImm(Val: C->getSExtValue()));
620 } else if (isa<ConstantPointerNull>(Val)) {
621 Ops.push_back(Elt: MachineOperand::CreateImm(Val: StackMaps::ConstantOp));
622 Ops.push_back(Elt: MachineOperand::CreateImm(Val: 0));
623 } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
624 // Values coming from a stack location also require a special encoding,
625 // but that is added later on by the target specific frame index
626 // elimination implementation.
627 auto SI = FuncInfo.StaticAllocaMap.find(Val: AI);
628 if (SI != FuncInfo.StaticAllocaMap.end())
629 Ops.push_back(Elt: MachineOperand::CreateFI(Idx: SI->second));
630 else
631 return false;
632 } else {
633 Register Reg = getRegForValue(V: Val);
634 if (!Reg)
635 return false;
636 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/false));
637 }
638 }
639 return true;
640}
641
642bool FastISel::selectStackmap(const CallInst *I) {
643 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
644 // [live variables...])
645 assert(I->getCalledFunction()->getReturnType()->isVoidTy() &&
646 "Stackmap cannot return a value.");
647
648 // The stackmap intrinsic only records the live variables (the arguments
649 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
650 // intrinsic, this won't be lowered to a function call. This means we don't
651 // have to worry about calling conventions and target-specific lowering code.
652 // Instead we perform the call lowering right here.
653 //
654 // CALLSEQ_START(0, 0...)
655 // STACKMAP(id, nbytes, ...)
656 // CALLSEQ_END(0, 0)
657 //
658 SmallVector<MachineOperand, 32> Ops;
659
660 // Add the <id> and <numBytes> constants.
661 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
662 "Expected a constant integer.");
663 const auto *ID = cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::IDPos));
664 Ops.push_back(Elt: MachineOperand::CreateImm(Val: ID->getZExtValue()));
665
666 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
667 "Expected a constant integer.");
668 const auto *NumBytes =
669 cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::NBytesPos));
670 Ops.push_back(Elt: MachineOperand::CreateImm(Val: NumBytes->getZExtValue()));
671
672 // Push live variables for the stack map (skipping the first two arguments
673 // <id> and <numBytes>).
674 if (!addStackMapLiveVars(Ops, CI: I, StartIdx: 2))
675 return false;
676
677 // We are not adding any register mask info here, because the stackmap doesn't
678 // clobber anything.
679
680 // Add scratch registers as implicit def and early clobber.
681 CallingConv::ID CC = I->getCallingConv();
682 const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
683 for (unsigned i = 0; ScratchRegs[i]; ++i)
684 Ops.push_back(Elt: MachineOperand::CreateReg(
685 Reg: ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
686 /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
687
688 // Issue CALLSEQ_START
689 unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
690 auto Builder =
691 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: AdjStackDown));
692 const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
693 for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
694 Builder.addImm(Val: 0);
695
696 // Issue STACKMAP.
697 MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
698 MCID: TII.get(Opcode: TargetOpcode::STACKMAP));
699 for (auto const &MO : Ops)
700 MIB.add(MO);
701
702 // Issue CALLSEQ_END
703 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
704 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: AdjStackUp))
705 .addImm(Val: 0)
706 .addImm(Val: 0);
707
708 // Inform the Frame Information that we have a stackmap in this function.
709 FuncInfo.MF->getFrameInfo().setHasStackMap();
710
711 return true;
712}
713
714/// Lower an argument list according to the target calling convention.
715///
716/// This is a helper for lowering intrinsics that follow a target calling
717/// convention or require stack pointer adjustment. Only a subset of the
718/// intrinsic's operands need to participate in the calling convention.
719bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
720 unsigned NumArgs, const Value *Callee,
721 bool ForceRetVoidTy, CallLoweringInfo &CLI) {
722 ArgListTy Args;
723 Args.reserve(n: NumArgs);
724
725 // Populate the argument list.
726 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
727 Value *V = CI->getOperand(i_nocapture: ArgI);
728
729 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
730
731 ArgListEntry Entry;
732 Entry.Val = V;
733 Entry.Ty = V->getType();
734 Entry.setAttributes(Call: CI, ArgIdx: ArgI);
735 Args.push_back(x: Entry);
736 }
737
738 Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(C&: CI->getType()->getContext())
739 : CI->getType();
740 CLI.setCallee(CC: CI->getCallingConv(), ResultTy: RetTy, Target: Callee, ArgsList: std::move(Args), FixedArgs: NumArgs);
741
742 return lowerCallTo(CLI);
743}
744
745FastISel::CallLoweringInfo &FastISel::CallLoweringInfo::setCallee(
746 const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
747 StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
748 SmallString<32> MangledName;
749 Mangler::getNameWithPrefix(OutName&: MangledName, GVName: Target, DL);
750 MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: MangledName);
751 return setCallee(CC, ResultTy, Target: Sym, ArgsList: std::move(ArgsList), FixedArgs);
752}
753
754bool FastISel::selectPatchpoint(const CallInst *I) {
755 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
756 // i32 <numBytes>,
757 // i8* <target>,
758 // i32 <numArgs>,
759 // [Args...],
760 // [live variables...])
761 CallingConv::ID CC = I->getCallingConv();
762 bool IsAnyRegCC = CC == CallingConv::AnyReg;
763 bool HasDef = !I->getType()->isVoidTy();
764 Value *Callee = I->getOperand(i_nocapture: PatchPointOpers::TargetPos)->stripPointerCasts();
765
766 // Check if we can lower the return type when using anyregcc.
767 MVT ValueType;
768 if (IsAnyRegCC && HasDef) {
769 ValueType = TLI.getSimpleValueType(DL, Ty: I->getType(), /*AllowUnknown=*/true);
770 if (ValueType == MVT::Other)
771 return false;
772 }
773
774 // Get the real number of arguments participating in the call <numArgs>
775 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
776 "Expected a constant integer.");
777 const auto *NumArgsVal =
778 cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::NArgPos));
779 unsigned NumArgs = NumArgsVal->getZExtValue();
780
781 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
782 // This includes all meta-operands up to but not including CC.
783 unsigned NumMetaOpers = PatchPointOpers::CCPos;
784 assert(I->arg_size() >= NumMetaOpers + NumArgs &&
785 "Not enough arguments provided to the patchpoint intrinsic");
786
787 // For AnyRegCC the arguments are lowered later on manually.
788 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
789 CallLoweringInfo CLI;
790 CLI.setIsPatchPoint();
791 if (!lowerCallOperands(CI: I, ArgIdx: NumMetaOpers, NumArgs: NumCallArgs, Callee, ForceRetVoidTy: IsAnyRegCC, CLI))
792 return false;
793
794 assert(CLI.Call && "No call instruction specified.");
795
796 SmallVector<MachineOperand, 32> Ops;
797
798 // Add an explicit result reg if we use the anyreg calling convention.
799 if (IsAnyRegCC && HasDef) {
800 assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
801 assert(ValueType.isValid());
802 CLI.ResultReg = createResultReg(RC: TLI.getRegClassFor(VT: ValueType));
803 CLI.NumResultRegs = 1;
804 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: CLI.ResultReg, /*isDef=*/true));
805 }
806
807 // Add the <id> and <numBytes> constants.
808 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
809 "Expected a constant integer.");
810 const auto *ID = cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::IDPos));
811 Ops.push_back(Elt: MachineOperand::CreateImm(Val: ID->getZExtValue()));
812
813 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
814 "Expected a constant integer.");
815 const auto *NumBytes =
816 cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::NBytesPos));
817 Ops.push_back(Elt: MachineOperand::CreateImm(Val: NumBytes->getZExtValue()));
818
819 // Add the call target.
820 if (const auto *C = dyn_cast<IntToPtrInst>(Val: Callee)) {
821 uint64_t CalleeConstAddr =
822 cast<ConstantInt>(Val: C->getOperand(i_nocapture: 0))->getZExtValue();
823 Ops.push_back(Elt: MachineOperand::CreateImm(Val: CalleeConstAddr));
824 } else if (const auto *C = dyn_cast<ConstantExpr>(Val: Callee)) {
825 if (C->getOpcode() == Instruction::IntToPtr) {
826 uint64_t CalleeConstAddr =
827 cast<ConstantInt>(Val: C->getOperand(i_nocapture: 0))->getZExtValue();
828 Ops.push_back(Elt: MachineOperand::CreateImm(Val: CalleeConstAddr));
829 } else
830 llvm_unreachable("Unsupported ConstantExpr.");
831 } else if (const auto *GV = dyn_cast<GlobalValue>(Val: Callee)) {
832 Ops.push_back(Elt: MachineOperand::CreateGA(GV, Offset: 0));
833 } else if (isa<ConstantPointerNull>(Val: Callee))
834 Ops.push_back(Elt: MachineOperand::CreateImm(Val: 0));
835 else
836 llvm_unreachable("Unsupported callee address.");
837
838 // Adjust <numArgs> to account for any arguments that have been passed on
839 // the stack instead.
840 unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
841 Ops.push_back(Elt: MachineOperand::CreateImm(Val: NumCallRegArgs));
842
843 // Add the calling convention
844 Ops.push_back(Elt: MachineOperand::CreateImm(Val: (unsigned)CC));
845
846 // Add the arguments we omitted previously. The register allocator should
847 // place these in any free register.
848 if (IsAnyRegCC) {
849 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
850 Register Reg = getRegForValue(V: I->getArgOperand(i));
851 if (!Reg)
852 return false;
853 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/false));
854 }
855 }
856
857 // Push the arguments from the call instruction.
858 for (auto Reg : CLI.OutRegs)
859 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/false));
860
861 // Push live variables for the stack map.
862 if (!addStackMapLiveVars(Ops, CI: I, StartIdx: NumMetaOpers + NumArgs))
863 return false;
864
865 // Push the register mask info.
866 Ops.push_back(Elt: MachineOperand::CreateRegMask(
867 Mask: TRI.getCallPreservedMask(MF: *FuncInfo.MF, CC)));
868
869 // Add scratch registers as implicit def and early clobber.
870 const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
871 for (unsigned i = 0; ScratchRegs[i]; ++i)
872 Ops.push_back(Elt: MachineOperand::CreateReg(
873 Reg: ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
874 /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
875
876 // Add implicit defs (return values).
877 for (auto Reg : CLI.InRegs)
878 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/true,
879 /*isImp=*/true));
880
881 // Insert the patchpoint instruction before the call generated by the target.
882 MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: CLI.Call, MIMD,
883 MCID: TII.get(Opcode: TargetOpcode::PATCHPOINT));
884
885 for (auto &MO : Ops)
886 MIB.add(MO);
887
888 MIB->setPhysRegsDeadExcept(UsedRegs: CLI.InRegs, TRI);
889
890 // Delete the original call instruction.
891 CLI.Call->eraseFromParent();
892
893 // Inform the Frame Information that we have a patchpoint in this function.
894 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
895
896 if (CLI.NumResultRegs)
897 updateValueMap(I, Reg: CLI.ResultReg, NumRegs: CLI.NumResultRegs);
898 return true;
899}
900
901bool FastISel::selectXRayCustomEvent(const CallInst *I) {
902 const auto &Triple = TM.getTargetTriple();
903 if (Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64)
904 return true; // don't do anything to this instruction.
905 SmallVector<MachineOperand, 8> Ops;
906 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 0)),
907 /*isDef=*/false));
908 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 1)),
909 /*isDef=*/false));
910 MachineInstrBuilder MIB =
911 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
912 MCID: TII.get(Opcode: TargetOpcode::PATCHABLE_EVENT_CALL));
913 for (auto &MO : Ops)
914 MIB.add(MO);
915
916 // Insert the Patchable Event Call instruction, that gets lowered properly.
917 return true;
918}
919
920bool FastISel::selectXRayTypedEvent(const CallInst *I) {
921 const auto &Triple = TM.getTargetTriple();
922 if (Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64)
923 return true; // don't do anything to this instruction.
924 SmallVector<MachineOperand, 8> Ops;
925 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 0)),
926 /*isDef=*/false));
927 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 1)),
928 /*isDef=*/false));
929 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 2)),
930 /*isDef=*/false));
931 MachineInstrBuilder MIB =
932 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
933 MCID: TII.get(Opcode: TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
934 for (auto &MO : Ops)
935 MIB.add(MO);
936
937 // Insert the Patchable Typed Event Call instruction, that gets lowered properly.
938 return true;
939}
940
941/// Returns an AttributeList representing the attributes applied to the return
942/// value of the given call.
943static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI) {
944 SmallVector<Attribute::AttrKind, 2> Attrs;
945 if (CLI.RetSExt)
946 Attrs.push_back(Elt: Attribute::SExt);
947 if (CLI.RetZExt)
948 Attrs.push_back(Elt: Attribute::ZExt);
949 if (CLI.IsInReg)
950 Attrs.push_back(Elt: Attribute::InReg);
951
952 return AttributeList::get(C&: CLI.RetTy->getContext(), Index: AttributeList::ReturnIndex,
953 Kinds: Attrs);
954}
955
956bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
957 unsigned NumArgs) {
958 MCContext &Ctx = MF->getContext();
959 SmallString<32> MangledName;
960 Mangler::getNameWithPrefix(OutName&: MangledName, GVName: SymName, DL);
961 MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: MangledName);
962 return lowerCallTo(CI, Symbol: Sym, NumArgs);
963}
964
965bool FastISel::lowerCallTo(const CallInst *CI, MCSymbol *Symbol,
966 unsigned NumArgs) {
967 FunctionType *FTy = CI->getFunctionType();
968 Type *RetTy = CI->getType();
969
970 ArgListTy Args;
971 Args.reserve(n: NumArgs);
972
973 // Populate the argument list.
974 // Attributes for args start at offset 1, after the return attribute.
975 for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
976 Value *V = CI->getOperand(i_nocapture: ArgI);
977
978 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
979
980 ArgListEntry Entry;
981 Entry.Val = V;
982 Entry.Ty = V->getType();
983 Entry.setAttributes(Call: CI, ArgIdx: ArgI);
984 Args.push_back(x: Entry);
985 }
986 TLI.markLibCallAttributes(MF, CC: CI->getCallingConv(), Args);
987
988 CallLoweringInfo CLI;
989 CLI.setCallee(ResultTy: RetTy, FuncTy: FTy, Target: Symbol, ArgsList: std::move(Args), Call: *CI, FixedArgs: NumArgs);
990
991 return lowerCallTo(CLI);
992}
993
994bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
995 // Handle the incoming return values from the call.
996 CLI.clearIns();
997 SmallVector<EVT, 4> RetTys;
998 ComputeValueVTs(TLI, DL, Ty: CLI.RetTy, ValueVTs&: RetTys);
999
1000 SmallVector<ISD::OutputArg, 4> Outs;
1001 GetReturnInfo(CC: CLI.CallConv, ReturnType: CLI.RetTy, attr: getReturnAttrs(CLI), Outs, TLI, DL);
1002
1003 bool CanLowerReturn = TLI.CanLowerReturn(
1004 CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext());
1005
1006 // FIXME: sret demotion isn't supported yet - bail out.
1007 if (!CanLowerReturn)
1008 return false;
1009
1010 for (EVT VT : RetTys) {
1011 MVT RegisterVT = TLI.getRegisterType(Context&: CLI.RetTy->getContext(), VT);
1012 unsigned NumRegs = TLI.getNumRegisters(Context&: CLI.RetTy->getContext(), VT);
1013 for (unsigned i = 0; i != NumRegs; ++i) {
1014 ISD::InputArg MyFlags;
1015 MyFlags.VT = RegisterVT;
1016 MyFlags.ArgVT = VT;
1017 MyFlags.Used = CLI.IsReturnValueUsed;
1018 if (CLI.RetSExt)
1019 MyFlags.Flags.setSExt();
1020 if (CLI.RetZExt)
1021 MyFlags.Flags.setZExt();
1022 if (CLI.IsInReg)
1023 MyFlags.Flags.setInReg();
1024 CLI.Ins.push_back(Elt: MyFlags);
1025 }
1026 }
1027
1028 // Handle all of the outgoing arguments.
1029 CLI.clearOuts();
1030 for (auto &Arg : CLI.getArgs()) {
1031 Type *FinalType = Arg.Ty;
1032 if (Arg.IsByVal)
1033 FinalType = Arg.IndirectType;
1034 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1035 Ty: FinalType, CallConv: CLI.CallConv, isVarArg: CLI.IsVarArg, DL);
1036
1037 ISD::ArgFlagsTy Flags;
1038 if (Arg.IsZExt)
1039 Flags.setZExt();
1040 if (Arg.IsSExt)
1041 Flags.setSExt();
1042 if (Arg.IsInReg)
1043 Flags.setInReg();
1044 if (Arg.IsSRet)
1045 Flags.setSRet();
1046 if (Arg.IsSwiftSelf)
1047 Flags.setSwiftSelf();
1048 if (Arg.IsSwiftAsync)
1049 Flags.setSwiftAsync();
1050 if (Arg.IsSwiftError)
1051 Flags.setSwiftError();
1052 if (Arg.IsCFGuardTarget)
1053 Flags.setCFGuardTarget();
1054 if (Arg.IsByVal)
1055 Flags.setByVal();
1056 if (Arg.IsInAlloca) {
1057 Flags.setInAlloca();
1058 // Set the byval flag for CCAssignFn callbacks that don't know about
1059 // inalloca. This way we can know how many bytes we should've allocated
1060 // and how many bytes a callee cleanup function will pop. If we port
1061 // inalloca to more targets, we'll have to add custom inalloca handling in
1062 // the various CC lowering callbacks.
1063 Flags.setByVal();
1064 }
1065 if (Arg.IsPreallocated) {
1066 Flags.setPreallocated();
1067 // Set the byval flag for CCAssignFn callbacks that don't know about
1068 // preallocated. This way we can know how many bytes we should've
1069 // allocated and how many bytes a callee cleanup function will pop. If we
1070 // port preallocated to more targets, we'll have to add custom
1071 // preallocated handling in the various CC lowering callbacks.
1072 Flags.setByVal();
1073 }
1074 MaybeAlign MemAlign = Arg.Alignment;
1075 if (Arg.IsByVal || Arg.IsInAlloca || Arg.IsPreallocated) {
1076 unsigned FrameSize = DL.getTypeAllocSize(Ty: Arg.IndirectType);
1077
1078 // For ByVal, alignment should come from FE. BE will guess if this info
1079 // is not there, but there are cases it cannot get right.
1080 if (!MemAlign)
1081 MemAlign = Align(TLI.getByValTypeAlignment(Ty: Arg.IndirectType, DL));
1082 Flags.setByValSize(FrameSize);
1083 } else if (!MemAlign) {
1084 MemAlign = DL.getABITypeAlign(Ty: Arg.Ty);
1085 }
1086 Flags.setMemAlign(*MemAlign);
1087 if (Arg.IsNest)
1088 Flags.setNest();
1089 if (NeedsRegBlock)
1090 Flags.setInConsecutiveRegs();
1091 Flags.setOrigAlign(DL.getABITypeAlign(Ty: Arg.Ty));
1092 CLI.OutVals.push_back(Elt: Arg.Val);
1093 CLI.OutFlags.push_back(Elt: Flags);
1094 }
1095
1096 if (!fastLowerCall(CLI))
1097 return false;
1098
1099 // Set all unused physreg defs as dead.
1100 assert(CLI.Call && "No call instruction specified.");
1101 CLI.Call->setPhysRegsDeadExcept(UsedRegs: CLI.InRegs, TRI);
1102
1103 if (CLI.NumResultRegs && CLI.CB)
1104 updateValueMap(I: CLI.CB, Reg: CLI.ResultReg, NumRegs: CLI.NumResultRegs);
1105
1106 // Set labels for heapallocsite call.
1107 if (CLI.CB)
1108 if (MDNode *MD = CLI.CB->getMetadata(Kind: "heapallocsite"))
1109 CLI.Call->setHeapAllocMarker(MF&: *MF, MD);
1110
1111 return true;
1112}
1113
1114bool FastISel::lowerCall(const CallInst *CI) {
1115 FunctionType *FuncTy = CI->getFunctionType();
1116 Type *RetTy = CI->getType();
1117
1118 ArgListTy Args;
1119 ArgListEntry Entry;
1120 Args.reserve(n: CI->arg_size());
1121
1122 for (auto i = CI->arg_begin(), e = CI->arg_end(); i != e; ++i) {
1123 Value *V = *i;
1124
1125 // Skip empty types
1126 if (V->getType()->isEmptyTy())
1127 continue;
1128
1129 Entry.Val = V;
1130 Entry.Ty = V->getType();
1131
1132 // Skip the first return-type Attribute to get to params.
1133 Entry.setAttributes(Call: CI, ArgIdx: i - CI->arg_begin());
1134 Args.push_back(x: Entry);
1135 }
1136
1137 // Check if target-independent constraints permit a tail call here.
1138 // Target-dependent constraints are checked within fastLowerCall.
1139 bool IsTailCall = CI->isTailCall();
1140 if (IsTailCall && !isInTailCallPosition(Call: *CI, TM))
1141 IsTailCall = false;
1142 if (IsTailCall && !CI->isMustTailCall() &&
1143 MF->getFunction().getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
1144 IsTailCall = false;
1145
1146 CallLoweringInfo CLI;
1147 CLI.setCallee(ResultTy: RetTy, FuncTy, Target: CI->getCalledOperand(), ArgsList: std::move(Args), Call: *CI)
1148 .setTailCall(IsTailCall);
1149
1150 diagnoseDontCall(CI: *CI);
1151
1152 return lowerCallTo(CLI);
1153}
1154
1155bool FastISel::selectCall(const User *I) {
1156 const CallInst *Call = cast<CallInst>(Val: I);
1157
1158 // Handle simple inline asms.
1159 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: Call->getCalledOperand())) {
1160 // Don't attempt to handle constraints.
1161 if (!IA->getConstraintString().empty())
1162 return false;
1163
1164 unsigned ExtraInfo = 0;
1165 if (IA->hasSideEffects())
1166 ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1167 if (IA->isAlignStack())
1168 ExtraInfo |= InlineAsm::Extra_IsAlignStack;
1169 if (Call->isConvergent())
1170 ExtraInfo |= InlineAsm::Extra_IsConvergent;
1171 ExtraInfo |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
1172
1173 MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1174 MCID: TII.get(Opcode: TargetOpcode::INLINEASM));
1175 MIB.addExternalSymbol(FnName: IA->getAsmString().c_str());
1176 MIB.addImm(Val: ExtraInfo);
1177
1178 const MDNode *SrcLoc = Call->getMetadata(Kind: "srcloc");
1179 if (SrcLoc)
1180 MIB.addMetadata(MD: SrcLoc);
1181
1182 return true;
1183 }
1184
1185 // Handle intrinsic function calls.
1186 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Call))
1187 return selectIntrinsicCall(II);
1188
1189 return lowerCall(CI: Call);
1190}
1191
1192void FastISel::handleDbgInfo(const Instruction *II) {
1193 if (!II->hasDbgRecords())
1194 return;
1195
1196 // Clear any metadata.
1197 MIMD = MIMetadata();
1198
1199 // Reverse order of debug records, because fast-isel walks through backwards.
1200 for (DbgRecord &DR : llvm::reverse(C: II->getDbgRecordRange())) {
1201 flushLocalValueMap();
1202 recomputeInsertPt();
1203
1204 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
1205 assert(DLR->getLabel() && "Missing label");
1206 if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1207 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DLR << "\n");
1208 continue;
1209 }
1210
1211 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DLR->getDebugLoc(),
1212 MCID: TII.get(Opcode: TargetOpcode::DBG_LABEL))
1213 .addMetadata(MD: DLR->getLabel());
1214 continue;
1215 }
1216
1217 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
1218
1219 Value *V = nullptr;
1220 if (!DVR.hasArgList())
1221 V = DVR.getVariableLocationOp(OpIdx: 0);
1222
1223 bool Res = false;
1224 if (DVR.getType() == DbgVariableRecord::LocationType::Value ||
1225 DVR.getType() == DbgVariableRecord::LocationType::Assign) {
1226 Res = lowerDbgValue(V, Expr: DVR.getExpression(), Var: DVR.getVariable(),
1227 DL: DVR.getDebugLoc());
1228 } else {
1229 assert(DVR.getType() == DbgVariableRecord::LocationType::Declare);
1230 if (FuncInfo.PreprocessedDVRDeclares.contains(Ptr: &DVR))
1231 continue;
1232 Res = lowerDbgDeclare(V, Expr: DVR.getExpression(), Var: DVR.getVariable(),
1233 DL: DVR.getDebugLoc());
1234 }
1235
1236 if (!Res)
1237 LLVM_DEBUG(dbgs() << "Dropping debug-info for " << DVR << "\n";);
1238 }
1239}
1240
1241bool FastISel::lowerDbgValue(const Value *V, DIExpression *Expr,
1242 DILocalVariable *Var, const DebugLoc &DL) {
1243 // This form of DBG_VALUE is target-independent.
1244 const MCInstrDesc &II = TII.get(Opcode: TargetOpcode::DBG_VALUE);
1245 if (!V || isa<UndefValue>(Val: V)) {
1246 // DI is either undef or cannot produce a valid DBG_VALUE, so produce an
1247 // undef DBG_VALUE to terminate any prior location.
1248 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect: false, Reg: 0U, Variable: Var, Expr);
1249 return true;
1250 }
1251 if (const auto *CI = dyn_cast<ConstantInt>(Val: V)) {
1252 // See if there's an expression to constant-fold.
1253 if (Expr)
1254 std::tie(args&: Expr, args&: CI) = Expr->constantFold(CI);
1255 if (CI->getBitWidth() > 64)
1256 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1257 .addCImm(Val: CI)
1258 .addImm(Val: 0U)
1259 .addMetadata(MD: Var)
1260 .addMetadata(MD: Expr);
1261 else
1262 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1263 .addImm(Val: CI->getZExtValue())
1264 .addImm(Val: 0U)
1265 .addMetadata(MD: Var)
1266 .addMetadata(MD: Expr);
1267 return true;
1268 }
1269 if (const auto *CF = dyn_cast<ConstantFP>(Val: V)) {
1270 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1271 .addFPImm(Val: CF)
1272 .addImm(Val: 0U)
1273 .addMetadata(MD: Var)
1274 .addMetadata(MD: Expr);
1275 return true;
1276 }
1277 if (const auto *Arg = dyn_cast<Argument>(Val: V);
1278 Arg && Expr && Expr->isEntryValue()) {
1279 // As per the Verifier, this case is only valid for swift async Args.
1280 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
1281
1282 Register Reg = getRegForValue(V: Arg);
1283 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
1284 if (Reg == VirtReg || Reg == PhysReg) {
1285 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect: false /*IsIndirect*/,
1286 Reg: PhysReg, Variable: Var, Expr);
1287 return true;
1288 }
1289
1290 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
1291 "couldn't find a physical register\n");
1292 return false;
1293 }
1294 if (auto SI = FuncInfo.StaticAllocaMap.find(Val: dyn_cast<AllocaInst>(Val: V));
1295 SI != FuncInfo.StaticAllocaMap.end()) {
1296 MachineOperand FrameIndexOp = MachineOperand::CreateFI(Idx: SI->second);
1297 bool IsIndirect = false;
1298 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect, MOs: FrameIndexOp,
1299 Variable: Var, Expr);
1300 return true;
1301 }
1302 if (Register Reg = lookUpRegForValue(V)) {
1303 // FIXME: This does not handle register-indirect values at offset 0.
1304 if (!FuncInfo.MF->useDebugInstrRef()) {
1305 bool IsIndirect = false;
1306 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect, Reg, Variable: Var,
1307 Expr);
1308 return true;
1309 }
1310 // If using instruction referencing, produce this as a DBG_INSTR_REF,
1311 // to be later patched up by finalizeDebugInstrRefs.
1312 SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg(
1313 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
1314 /* isKill */ false, /* isDead */ false,
1315 /* isUndef */ false, /* isEarlyClobber */ false,
1316 /* SubReg */ 0, /* isDebug */ true)});
1317 SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0});
1318 auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops);
1319 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL,
1320 MCID: TII.get(Opcode: TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs,
1321 Variable: Var, Expr: NewExpr);
1322 return true;
1323 }
1324 return false;
1325}
1326
1327bool FastISel::lowerDbgDeclare(const Value *Address, DIExpression *Expr,
1328 DILocalVariable *Var, const DebugLoc &DL) {
1329 if (!Address || isa<UndefValue>(Val: Address)) {
1330 LLVM_DEBUG(dbgs() << "Dropping debug info (bad/undef address)\n");
1331 return false;
1332 }
1333
1334 std::optional<MachineOperand> Op;
1335 if (Register Reg = lookUpRegForValue(V: Address))
1336 Op = MachineOperand::CreateReg(Reg, isDef: false);
1337
1338 // If we have a VLA that has a "use" in a metadata node that's then used
1339 // here but it has no other uses, then we have a problem. E.g.,
1340 //
1341 // int foo (const int *x) {
1342 // char a[*x];
1343 // return 0;
1344 // }
1345 //
1346 // If we assign 'a' a vreg and fast isel later on has to use the selection
1347 // DAG isel, it will want to copy the value to the vreg. However, there are
1348 // no uses, which goes counter to what selection DAG isel expects.
1349 if (!Op && !Address->use_empty() && isa<Instruction>(Val: Address) &&
1350 (!isa<AllocaInst>(Val: Address) ||
1351 !FuncInfo.StaticAllocaMap.count(Val: cast<AllocaInst>(Val: Address))))
1352 Op = MachineOperand::CreateReg(Reg: FuncInfo.InitializeRegForValue(V: Address),
1353 isDef: false);
1354
1355 if (Op) {
1356 assert(Var->isValidLocationForIntrinsic(DL) &&
1357 "Expected inlined-at fields to agree");
1358 if (FuncInfo.MF->useDebugInstrRef() && Op->isReg()) {
1359 // If using instruction referencing, produce this as a DBG_INSTR_REF,
1360 // to be later patched up by finalizeDebugInstrRefs. Tack a deref onto
1361 // the expression, we don't have an "indirect" flag in DBG_INSTR_REF.
1362 SmallVector<uint64_t, 3> Ops(
1363 {dwarf::DW_OP_LLVM_arg, 0, dwarf::DW_OP_deref});
1364 auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops);
1365 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL,
1366 MCID: TII.get(Opcode: TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs: *Op,
1367 Variable: Var, Expr: NewExpr);
1368 return true;
1369 }
1370
1371 // A dbg.declare describes the address of a source variable, so lower it
1372 // into an indirect DBG_VALUE.
1373 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL,
1374 MCID: TII.get(Opcode: TargetOpcode::DBG_VALUE), /*IsIndirect*/ true, MOs: *Op, Variable: Var,
1375 Expr);
1376 return true;
1377 }
1378
1379 // We can't yet handle anything else here because it would require
1380 // generating code, thus altering codegen because of debug info.
1381 LLVM_DEBUG(
1382 dbgs() << "Dropping debug info (no materialized reg for address)\n");
1383 return false;
1384}
1385
1386bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
1387 switch (II->getIntrinsicID()) {
1388 default:
1389 break;
1390 // At -O0 we don't care about the lifetime intrinsics.
1391 case Intrinsic::lifetime_start:
1392 case Intrinsic::lifetime_end:
1393 // The donothing intrinsic does, well, nothing.
1394 case Intrinsic::donothing:
1395 // Neither does the sideeffect intrinsic.
1396 case Intrinsic::sideeffect:
1397 // Neither does the assume intrinsic; it's also OK not to codegen its operand.
1398 case Intrinsic::assume:
1399 // Neither does the llvm.experimental.noalias.scope.decl intrinsic
1400 case Intrinsic::experimental_noalias_scope_decl:
1401 return true;
1402 case Intrinsic::dbg_declare: {
1403 const DbgDeclareInst *DI = cast<DbgDeclareInst>(Val: II);
1404 assert(DI->getVariable() && "Missing variable");
1405 if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1406 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI
1407 << " (!hasDebugInfo)\n");
1408 return true;
1409 }
1410
1411 if (FuncInfo.PreprocessedDbgDeclares.contains(Ptr: DI))
1412 return true;
1413
1414 const Value *Address = DI->getAddress();
1415 if (!lowerDbgDeclare(Address, Expr: DI->getExpression(), Var: DI->getVariable(),
1416 DL: MIMD.getDL()))
1417 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI);
1418
1419 return true;
1420 }
1421 case Intrinsic::dbg_assign:
1422 // A dbg.assign is a dbg.value with more information, typically produced
1423 // during optimisation. If one reaches fastisel then something odd has
1424 // happened (such as an optimised function being always-inlined into an
1425 // optnone function). We will not be using the extra information in the
1426 // dbg.assign in that case, just use its dbg.value fields.
1427 [[fallthrough]];
1428 case Intrinsic::dbg_value: {
1429 // This form of DBG_VALUE is target-independent.
1430 const DbgValueInst *DI = cast<DbgValueInst>(Val: II);
1431 const Value *V = DI->getValue();
1432 DIExpression *Expr = DI->getExpression();
1433 DILocalVariable *Var = DI->getVariable();
1434 if (DI->hasArgList())
1435 // Signal that we don't have a location for this.
1436 V = nullptr;
1437
1438 assert(Var->isValidLocationForIntrinsic(MIMD.getDL()) &&
1439 "Expected inlined-at fields to agree");
1440
1441 if (!lowerDbgValue(V, Expr, Var, DL: MIMD.getDL()))
1442 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1443
1444 return true;
1445 }
1446 case Intrinsic::dbg_label: {
1447 const DbgLabelInst *DI = cast<DbgLabelInst>(Val: II);
1448 assert(DI->getLabel() && "Missing label");
1449 if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1450 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1451 return true;
1452 }
1453
1454 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1455 MCID: TII.get(Opcode: TargetOpcode::DBG_LABEL)).addMetadata(MD: DI->getLabel());
1456 return true;
1457 }
1458 case Intrinsic::objectsize:
1459 llvm_unreachable("llvm.objectsize.* should have been lowered already");
1460
1461 case Intrinsic::is_constant:
1462 llvm_unreachable("llvm.is.constant.* should have been lowered already");
1463
1464 case Intrinsic::allow_runtime_check:
1465 case Intrinsic::allow_ubsan_check: {
1466 Register ResultReg = getRegForValue(V: ConstantInt::getTrue(Ty: II->getType()));
1467 if (!ResultReg)
1468 return false;
1469 updateValueMap(I: II, Reg: ResultReg);
1470 return true;
1471 }
1472
1473 case Intrinsic::launder_invariant_group:
1474 case Intrinsic::strip_invariant_group:
1475 case Intrinsic::expect: {
1476 Register ResultReg = getRegForValue(V: II->getArgOperand(i: 0));
1477 if (!ResultReg)
1478 return false;
1479 updateValueMap(I: II, Reg: ResultReg);
1480 return true;
1481 }
1482 case Intrinsic::experimental_stackmap:
1483 return selectStackmap(I: II);
1484 case Intrinsic::experimental_patchpoint_void:
1485 case Intrinsic::experimental_patchpoint:
1486 return selectPatchpoint(I: II);
1487
1488 case Intrinsic::xray_customevent:
1489 return selectXRayCustomEvent(I: II);
1490 case Intrinsic::xray_typedevent:
1491 return selectXRayTypedEvent(I: II);
1492 }
1493
1494 return fastLowerIntrinsicCall(II);
1495}
1496
1497bool FastISel::selectCast(const User *I, unsigned Opcode) {
1498 EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1499 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
1500
1501 if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
1502 !DstVT.isSimple())
1503 // Unhandled type. Halt "fast" selection and bail.
1504 return false;
1505
1506 // Check if the destination type is legal.
1507 if (!TLI.isTypeLegal(VT: DstVT))
1508 return false;
1509
1510 // Check if the source operand is legal.
1511 if (!TLI.isTypeLegal(VT: SrcVT))
1512 return false;
1513
1514 Register InputReg = getRegForValue(V: I->getOperand(i: 0));
1515 if (!InputReg)
1516 // Unhandled operand. Halt "fast" selection and bail.
1517 return false;
1518
1519 Register ResultReg = fastEmit_r(VT: SrcVT.getSimpleVT(), RetVT: DstVT.getSimpleVT(),
1520 Opcode, Op0: InputReg);
1521 if (!ResultReg)
1522 return false;
1523
1524 updateValueMap(I, Reg: ResultReg);
1525 return true;
1526}
1527
1528bool FastISel::selectBitCast(const User *I) {
1529 EVT SrcEVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1530 EVT DstEVT = TLI.getValueType(DL, Ty: I->getType());
1531 if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
1532 !TLI.isTypeLegal(VT: SrcEVT) || !TLI.isTypeLegal(VT: DstEVT))
1533 // Unhandled type. Halt "fast" selection and bail.
1534 return false;
1535
1536 MVT SrcVT = SrcEVT.getSimpleVT();
1537 MVT DstVT = DstEVT.getSimpleVT();
1538 Register Op0 = getRegForValue(V: I->getOperand(i: 0));
1539 if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
1540 return false;
1541
1542 // If the bitcast doesn't change the type, just use the operand value.
1543 if (SrcVT == DstVT) {
1544 updateValueMap(I, Reg: Op0);
1545 return true;
1546 }
1547
1548 // Otherwise, select a BITCAST opcode.
1549 Register ResultReg = fastEmit_r(VT: SrcVT, RetVT: DstVT, Opcode: ISD::BITCAST, Op0);
1550 if (!ResultReg)
1551 return false;
1552
1553 updateValueMap(I, Reg: ResultReg);
1554 return true;
1555}
1556
1557bool FastISel::selectFreeze(const User *I) {
1558 Register Reg = getRegForValue(V: I->getOperand(i: 0));
1559 if (!Reg)
1560 // Unhandled operand.
1561 return false;
1562
1563 EVT ETy = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1564 if (ETy == MVT::Other || !TLI.isTypeLegal(VT: ETy))
1565 // Unhandled type, bail out.
1566 return false;
1567
1568 MVT Ty = ETy.getSimpleVT();
1569 const TargetRegisterClass *TyRegClass = TLI.getRegClassFor(VT: Ty);
1570 Register ResultReg = createResultReg(RC: TyRegClass);
1571 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1572 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ResultReg).addReg(RegNo: Reg);
1573
1574 updateValueMap(I, Reg: ResultReg);
1575 return true;
1576}
1577
1578// Remove local value instructions starting from the instruction after
1579// SavedLastLocalValue to the current function insert point.
1580void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
1581{
1582 MachineInstr *CurLastLocalValue = getLastLocalValue();
1583 if (CurLastLocalValue != SavedLastLocalValue) {
1584 // Find the first local value instruction to be deleted.
1585 // This is the instruction after SavedLastLocalValue if it is non-NULL.
1586 // Otherwise it's the first instruction in the block.
1587 MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
1588 if (SavedLastLocalValue)
1589 ++FirstDeadInst;
1590 else
1591 FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
1592 setLastLocalValue(SavedLastLocalValue);
1593 removeDeadCode(I: FirstDeadInst, E: FuncInfo.InsertPt);
1594 }
1595}
1596
1597bool FastISel::selectInstruction(const Instruction *I) {
1598 // Flush the local value map before starting each instruction.
1599 // This improves locality and debugging, and can reduce spills.
1600 // Reuse of values across IR instructions is relatively uncommon.
1601 flushLocalValueMap();
1602
1603 MachineInstr *SavedLastLocalValue = getLastLocalValue();
1604 // Just before the terminator instruction, insert instructions to
1605 // feed PHI nodes in successor blocks.
1606 if (I->isTerminator()) {
1607 if (!handlePHINodesInSuccessorBlocks(LLVMBB: I->getParent())) {
1608 // PHI node handling may have generated local value instructions,
1609 // even though it failed to handle all PHI nodes.
1610 // We remove these instructions because SelectionDAGISel will generate
1611 // them again.
1612 removeDeadLocalValueCode(SavedLastLocalValue);
1613 return false;
1614 }
1615 }
1616
1617 // FastISel does not handle any operand bundles except OB_funclet.
1618 if (auto *Call = dyn_cast<CallBase>(Val: I))
1619 for (unsigned i = 0, e = Call->getNumOperandBundles(); i != e; ++i)
1620 if (Call->getOperandBundleAt(Index: i).getTagID() != LLVMContext::OB_funclet)
1621 return false;
1622
1623 MIMD = MIMetadata(*I);
1624
1625 SavedInsertPt = FuncInfo.InsertPt;
1626
1627 if (const auto *Call = dyn_cast<CallInst>(Val: I)) {
1628 const Function *F = Call->getCalledFunction();
1629 LibFunc Func;
1630
1631 // As a special case, don't handle calls to builtin library functions that
1632 // may be translated directly to target instructions.
1633 if (F && !F->hasLocalLinkage() && F->hasName() &&
1634 LibInfo->getLibFunc(funcName: F->getName(), F&: Func) &&
1635 LibInfo->hasOptimizedCodeGen(F: Func))
1636 return false;
1637
1638 // Don't handle Intrinsic::trap if a trap function is specified.
1639 if (F && F->getIntrinsicID() == Intrinsic::trap &&
1640 Call->hasFnAttr(Kind: "trap-func-name"))
1641 return false;
1642 }
1643
1644 // First, try doing target-independent selection.
1645 if (!SkipTargetIndependentISel) {
1646 if (selectOperator(I, Opcode: I->getOpcode())) {
1647 ++NumFastIselSuccessIndependent;
1648 MIMD = {};
1649 return true;
1650 }
1651 // Remove dead code.
1652 recomputeInsertPt();
1653 if (SavedInsertPt != FuncInfo.InsertPt)
1654 removeDeadCode(I: FuncInfo.InsertPt, E: SavedInsertPt);
1655 SavedInsertPt = FuncInfo.InsertPt;
1656 }
1657 // Next, try calling the target to attempt to handle the instruction.
1658 if (fastSelectInstruction(I)) {
1659 ++NumFastIselSuccessTarget;
1660 MIMD = {};
1661 return true;
1662 }
1663 // Remove dead code.
1664 recomputeInsertPt();
1665 if (SavedInsertPt != FuncInfo.InsertPt)
1666 removeDeadCode(I: FuncInfo.InsertPt, E: SavedInsertPt);
1667
1668 MIMD = {};
1669 // Undo phi node updates, because they will be added again by SelectionDAG.
1670 if (I->isTerminator()) {
1671 // PHI node handling may have generated local value instructions.
1672 // We remove them because SelectionDAGISel will generate them again.
1673 removeDeadLocalValueCode(SavedLastLocalValue);
1674 FuncInfo.PHINodesToUpdate.resize(new_size: FuncInfo.OrigNumPHINodesToUpdate);
1675 }
1676 return false;
1677}
1678
1679/// Emit an unconditional branch to the given block, unless it is the immediate
1680/// (fall-through) successor, and update the CFG.
1681void FastISel::fastEmitBranch(MachineBasicBlock *MSucc,
1682 const DebugLoc &DbgLoc) {
1683 if (FuncInfo.MBB->getBasicBlock()->sizeWithoutDebug() > 1 &&
1684 FuncInfo.MBB->isLayoutSuccessor(MBB: MSucc)) {
1685 // For more accurate line information if this is the only non-debug
1686 // instruction in the block then emit it, otherwise we have the
1687 // unconditional fall-through case, which needs no instructions.
1688 } else {
1689 // The unconditional branch case.
1690 TII.insertBranch(MBB&: *FuncInfo.MBB, TBB: MSucc, FBB: nullptr,
1691 Cond: SmallVector<MachineOperand, 0>(), DL: DbgLoc);
1692 }
1693 if (FuncInfo.BPI) {
1694 auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
1695 Src: FuncInfo.MBB->getBasicBlock(), Dst: MSucc->getBasicBlock());
1696 FuncInfo.MBB->addSuccessor(Succ: MSucc, Prob: BranchProbability);
1697 } else
1698 FuncInfo.MBB->addSuccessorWithoutProb(Succ: MSucc);
1699}
1700
1701void FastISel::finishCondBranch(const BasicBlock *BranchBB,
1702 MachineBasicBlock *TrueMBB,
1703 MachineBasicBlock *FalseMBB) {
1704 // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
1705 // happen in degenerate IR and MachineIR forbids to have a block twice in the
1706 // successor/predecessor lists.
1707 if (TrueMBB != FalseMBB) {
1708 if (FuncInfo.BPI) {
1709 auto BranchProbability =
1710 FuncInfo.BPI->getEdgeProbability(Src: BranchBB, Dst: TrueMBB->getBasicBlock());
1711 FuncInfo.MBB->addSuccessor(Succ: TrueMBB, Prob: BranchProbability);
1712 } else
1713 FuncInfo.MBB->addSuccessorWithoutProb(Succ: TrueMBB);
1714 }
1715
1716 fastEmitBranch(MSucc: FalseMBB, DbgLoc: MIMD.getDL());
1717}
1718
1719/// Emit an FNeg operation.
1720bool FastISel::selectFNeg(const User *I, const Value *In) {
1721 Register OpReg = getRegForValue(V: In);
1722 if (!OpReg)
1723 return false;
1724
1725 // If the target has ISD::FNEG, use it.
1726 EVT VT = TLI.getValueType(DL, Ty: I->getType());
1727 Register ResultReg = fastEmit_r(VT: VT.getSimpleVT(), RetVT: VT.getSimpleVT(), Opcode: ISD::FNEG,
1728 Op0: OpReg);
1729 if (ResultReg) {
1730 updateValueMap(I, Reg: ResultReg);
1731 return true;
1732 }
1733
1734 // Bitcast the value to integer, twiddle the sign bit with xor,
1735 // and then bitcast it back to floating-point.
1736 if (VT.getSizeInBits() > 64)
1737 return false;
1738 EVT IntVT = EVT::getIntegerVT(Context&: I->getContext(), BitWidth: VT.getSizeInBits());
1739 if (!TLI.isTypeLegal(VT: IntVT))
1740 return false;
1741
1742 Register IntReg = fastEmit_r(VT: VT.getSimpleVT(), RetVT: IntVT.getSimpleVT(),
1743 Opcode: ISD::BITCAST, Op0: OpReg);
1744 if (!IntReg)
1745 return false;
1746
1747 Register IntResultReg = fastEmit_ri_(
1748 VT: IntVT.getSimpleVT(), Opcode: ISD::XOR, Op0: IntReg,
1749 UINT64_C(1) << (VT.getSizeInBits() - 1), ImmType: IntVT.getSimpleVT());
1750 if (!IntResultReg)
1751 return false;
1752
1753 ResultReg = fastEmit_r(VT: IntVT.getSimpleVT(), RetVT: VT.getSimpleVT(), Opcode: ISD::BITCAST,
1754 Op0: IntResultReg);
1755 if (!ResultReg)
1756 return false;
1757
1758 updateValueMap(I, Reg: ResultReg);
1759 return true;
1760}
1761
1762bool FastISel::selectExtractValue(const User *U) {
1763 const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: U);
1764 if (!EVI)
1765 return false;
1766
1767 // Make sure we only try to handle extracts with a legal result. But also
1768 // allow i1 because it's easy.
1769 EVT RealVT = TLI.getValueType(DL, Ty: EVI->getType(), /*AllowUnknown=*/true);
1770 if (!RealVT.isSimple())
1771 return false;
1772 MVT VT = RealVT.getSimpleVT();
1773 if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
1774 return false;
1775
1776 const Value *Op0 = EVI->getOperand(i_nocapture: 0);
1777 Type *AggTy = Op0->getType();
1778
1779 // Get the base result register.
1780 unsigned ResultReg;
1781 DenseMap<const Value *, Register>::iterator I = FuncInfo.ValueMap.find(Val: Op0);
1782 if (I != FuncInfo.ValueMap.end())
1783 ResultReg = I->second;
1784 else if (isa<Instruction>(Val: Op0))
1785 ResultReg = FuncInfo.InitializeRegForValue(V: Op0);
1786 else
1787 return false; // fast-isel can't handle aggregate constants at the moment
1788
1789 // Get the actual result register, which is an offset from the base register.
1790 unsigned VTIndex = ComputeLinearIndex(Ty: AggTy, Indices: EVI->getIndices());
1791
1792 SmallVector<EVT, 4> AggValueVTs;
1793 ComputeValueVTs(TLI, DL, Ty: AggTy, ValueVTs&: AggValueVTs);
1794
1795 for (unsigned i = 0; i < VTIndex; i++)
1796 ResultReg += TLI.getNumRegisters(Context&: FuncInfo.Fn->getContext(), VT: AggValueVTs[i]);
1797
1798 updateValueMap(I: EVI, Reg: ResultReg);
1799 return true;
1800}
1801
1802bool FastISel::selectOperator(const User *I, unsigned Opcode) {
1803 switch (Opcode) {
1804 case Instruction::Add:
1805 return selectBinaryOp(I, ISDOpcode: ISD::ADD);
1806 case Instruction::FAdd:
1807 return selectBinaryOp(I, ISDOpcode: ISD::FADD);
1808 case Instruction::Sub:
1809 return selectBinaryOp(I, ISDOpcode: ISD::SUB);
1810 case Instruction::FSub:
1811 return selectBinaryOp(I, ISDOpcode: ISD::FSUB);
1812 case Instruction::Mul:
1813 return selectBinaryOp(I, ISDOpcode: ISD::MUL);
1814 case Instruction::FMul:
1815 return selectBinaryOp(I, ISDOpcode: ISD::FMUL);
1816 case Instruction::SDiv:
1817 return selectBinaryOp(I, ISDOpcode: ISD::SDIV);
1818 case Instruction::UDiv:
1819 return selectBinaryOp(I, ISDOpcode: ISD::UDIV);
1820 case Instruction::FDiv:
1821 return selectBinaryOp(I, ISDOpcode: ISD::FDIV);
1822 case Instruction::SRem:
1823 return selectBinaryOp(I, ISDOpcode: ISD::SREM);
1824 case Instruction::URem:
1825 return selectBinaryOp(I, ISDOpcode: ISD::UREM);
1826 case Instruction::FRem:
1827 return selectBinaryOp(I, ISDOpcode: ISD::FREM);
1828 case Instruction::Shl:
1829 return selectBinaryOp(I, ISDOpcode: ISD::SHL);
1830 case Instruction::LShr:
1831 return selectBinaryOp(I, ISDOpcode: ISD::SRL);
1832 case Instruction::AShr:
1833 return selectBinaryOp(I, ISDOpcode: ISD::SRA);
1834 case Instruction::And:
1835 return selectBinaryOp(I, ISDOpcode: ISD::AND);
1836 case Instruction::Or:
1837 return selectBinaryOp(I, ISDOpcode: ISD::OR);
1838 case Instruction::Xor:
1839 return selectBinaryOp(I, ISDOpcode: ISD::XOR);
1840
1841 case Instruction::FNeg:
1842 return selectFNeg(I, In: I->getOperand(i: 0));
1843
1844 case Instruction::GetElementPtr:
1845 return selectGetElementPtr(I);
1846
1847 case Instruction::Br: {
1848 const BranchInst *BI = cast<BranchInst>(Val: I);
1849
1850 if (BI->isUnconditional()) {
1851 const BasicBlock *LLVMSucc = BI->getSuccessor(i: 0);
1852 MachineBasicBlock *MSucc = FuncInfo.MBBMap[LLVMSucc];
1853 fastEmitBranch(MSucc, DbgLoc: BI->getDebugLoc());
1854 return true;
1855 }
1856
1857 // Conditional branches are not handed yet.
1858 // Halt "fast" selection and bail.
1859 return false;
1860 }
1861
1862 case Instruction::Unreachable:
1863 if (TM.Options.TrapUnreachable)
1864 return fastEmit_(VT: MVT::Other, RetVT: MVT::Other, Opcode: ISD::TRAP) != 0;
1865 else
1866 return true;
1867
1868 case Instruction::Alloca:
1869 // FunctionLowering has the static-sized case covered.
1870 if (FuncInfo.StaticAllocaMap.count(Val: cast<AllocaInst>(Val: I)))
1871 return true;
1872
1873 // Dynamic-sized alloca is not handled yet.
1874 return false;
1875
1876 case Instruction::Call:
1877 // On AIX, normal call lowering uses the DAG-ISEL path currently so that the
1878 // callee of the direct function call instruction will be mapped to the
1879 // symbol for the function's entry point, which is distinct from the
1880 // function descriptor symbol. The latter is the symbol whose XCOFF symbol
1881 // name is the C-linkage name of the source level function.
1882 // But fast isel still has the ability to do selection for intrinsics.
1883 if (TM.getTargetTriple().isOSAIX() && !isa<IntrinsicInst>(Val: I))
1884 return false;
1885 return selectCall(I);
1886
1887 case Instruction::BitCast:
1888 return selectBitCast(I);
1889
1890 case Instruction::FPToSI:
1891 return selectCast(I, Opcode: ISD::FP_TO_SINT);
1892 case Instruction::ZExt:
1893 return selectCast(I, Opcode: ISD::ZERO_EXTEND);
1894 case Instruction::SExt:
1895 return selectCast(I, Opcode: ISD::SIGN_EXTEND);
1896 case Instruction::Trunc:
1897 return selectCast(I, Opcode: ISD::TRUNCATE);
1898 case Instruction::SIToFP:
1899 return selectCast(I, Opcode: ISD::SINT_TO_FP);
1900
1901 case Instruction::IntToPtr: // Deliberate fall-through.
1902 case Instruction::PtrToInt: {
1903 EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1904 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
1905 if (DstVT.bitsGT(VT: SrcVT))
1906 return selectCast(I, Opcode: ISD::ZERO_EXTEND);
1907 if (DstVT.bitsLT(VT: SrcVT))
1908 return selectCast(I, Opcode: ISD::TRUNCATE);
1909 Register Reg = getRegForValue(V: I->getOperand(i: 0));
1910 if (!Reg)
1911 return false;
1912 updateValueMap(I, Reg);
1913 return true;
1914 }
1915
1916 case Instruction::ExtractValue:
1917 return selectExtractValue(U: I);
1918
1919 case Instruction::Freeze:
1920 return selectFreeze(I);
1921
1922 case Instruction::PHI:
1923 llvm_unreachable("FastISel shouldn't visit PHI nodes!");
1924
1925 default:
1926 // Unhandled instruction. Halt "fast" selection and bail.
1927 return false;
1928 }
1929}
1930
1931FastISel::FastISel(FunctionLoweringInfo &FuncInfo,
1932 const TargetLibraryInfo *LibInfo,
1933 bool SkipTargetIndependentISel)
1934 : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
1935 MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
1936 TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
1937 TII(*MF->getSubtarget().getInstrInfo()),
1938 TLI(*MF->getSubtarget().getTargetLowering()),
1939 TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
1940 SkipTargetIndependentISel(SkipTargetIndependentISel) {}
1941
1942FastISel::~FastISel() = default;
1943
1944bool FastISel::fastLowerArguments() { return false; }
1945
1946bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
1947
1948bool FastISel::fastLowerIntrinsicCall(const IntrinsicInst * /*II*/) {
1949 return false;
1950}
1951
1952unsigned FastISel::fastEmit_(MVT, MVT, unsigned) { return 0; }
1953
1954unsigned FastISel::fastEmit_r(MVT, MVT, unsigned, unsigned /*Op0*/) {
1955 return 0;
1956}
1957
1958unsigned FastISel::fastEmit_rr(MVT, MVT, unsigned, unsigned /*Op0*/,
1959 unsigned /*Op1*/) {
1960 return 0;
1961}
1962
1963unsigned FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
1964 return 0;
1965}
1966
1967unsigned FastISel::fastEmit_f(MVT, MVT, unsigned,
1968 const ConstantFP * /*FPImm*/) {
1969 return 0;
1970}
1971
1972unsigned FastISel::fastEmit_ri(MVT, MVT, unsigned, unsigned /*Op0*/,
1973 uint64_t /*Imm*/) {
1974 return 0;
1975}
1976
1977/// This method is a wrapper of fastEmit_ri. It first tries to emit an
1978/// instruction with an immediate operand using fastEmit_ri.
1979/// If that fails, it materializes the immediate into a register and try
1980/// fastEmit_rr instead.
1981Register FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
1982 uint64_t Imm, MVT ImmType) {
1983 // If this is a multiply by a power of two, emit this as a shift left.
1984 if (Opcode == ISD::MUL && isPowerOf2_64(Value: Imm)) {
1985 Opcode = ISD::SHL;
1986 Imm = Log2_64(Value: Imm);
1987 } else if (Opcode == ISD::UDIV && isPowerOf2_64(Value: Imm)) {
1988 // div x, 8 -> srl x, 3
1989 Opcode = ISD::SRL;
1990 Imm = Log2_64(Value: Imm);
1991 }
1992
1993 // Horrible hack (to be removed), check to make sure shift amounts are
1994 // in-range.
1995 if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
1996 Imm >= VT.getSizeInBits())
1997 return 0;
1998
1999 // First check if immediate type is legal. If not, we can't use the ri form.
2000 Register ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Imm);
2001 if (ResultReg)
2002 return ResultReg;
2003 Register MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
2004 if (!MaterialReg) {
2005 // This is a bit ugly/slow, but failing here means falling out of
2006 // fast-isel, which would be very slow.
2007 IntegerType *ITy =
2008 IntegerType::get(C&: FuncInfo.Fn->getContext(), NumBits: VT.getSizeInBits());
2009 MaterialReg = getRegForValue(V: ConstantInt::get(Ty: ITy, V: Imm));
2010 if (!MaterialReg)
2011 return 0;
2012 }
2013 return fastEmit_rr(VT, VT, Opcode, Op0, MaterialReg);
2014}
2015
2016Register FastISel::createResultReg(const TargetRegisterClass *RC) {
2017 return MRI.createVirtualRegister(RegClass: RC);
2018}
2019
2020Register FastISel::constrainOperandRegClass(const MCInstrDesc &II, Register Op,
2021 unsigned OpNum) {
2022 if (Op.isVirtual()) {
2023 const TargetRegisterClass *RegClass =
2024 TII.getRegClass(MCID: II, OpNum, TRI: &TRI, MF: *FuncInfo.MF);
2025 if (!MRI.constrainRegClass(Reg: Op, RC: RegClass)) {
2026 // If it's not legal to COPY between the register classes, something
2027 // has gone very wrong before we got here.
2028 Register NewOp = createResultReg(RC: RegClass);
2029 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2030 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: NewOp).addReg(RegNo: Op);
2031 return NewOp;
2032 }
2033 }
2034 return Op;
2035}
2036
2037Register FastISel::fastEmitInst_(unsigned MachineInstOpcode,
2038 const TargetRegisterClass *RC) {
2039 Register ResultReg = createResultReg(RC);
2040 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2041
2042 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg);
2043 return ResultReg;
2044}
2045
2046Register FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
2047 const TargetRegisterClass *RC, unsigned Op0) {
2048 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2049
2050 Register ResultReg = createResultReg(RC);
2051 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2052
2053 if (II.getNumDefs() >= 1)
2054 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2055 .addReg(RegNo: Op0);
2056 else {
2057 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2058 .addReg(RegNo: Op0);
2059 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2060 DestReg: ResultReg)
2061 .addReg(RegNo: II.implicit_defs()[0]);
2062 }
2063
2064 return ResultReg;
2065}
2066
2067Register FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
2068 const TargetRegisterClass *RC, unsigned Op0,
2069 unsigned Op1) {
2070 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2071
2072 Register ResultReg = createResultReg(RC);
2073 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2074 Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
2075
2076 if (II.getNumDefs() >= 1)
2077 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2078 .addReg(RegNo: Op0)
2079 .addReg(RegNo: Op1);
2080 else {
2081 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2082 .addReg(RegNo: Op0)
2083 .addReg(RegNo: Op1);
2084 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2085 DestReg: ResultReg)
2086 .addReg(RegNo: II.implicit_defs()[0]);
2087 }
2088 return ResultReg;
2089}
2090
2091Register FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
2092 const TargetRegisterClass *RC, unsigned Op0,
2093 unsigned Op1, unsigned Op2) {
2094 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2095
2096 Register ResultReg = createResultReg(RC);
2097 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2098 Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
2099 Op2 = constrainOperandRegClass(II, Op: Op2, OpNum: II.getNumDefs() + 2);
2100
2101 if (II.getNumDefs() >= 1)
2102 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2103 .addReg(RegNo: Op0)
2104 .addReg(RegNo: Op1)
2105 .addReg(RegNo: Op2);
2106 else {
2107 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2108 .addReg(RegNo: Op0)
2109 .addReg(RegNo: Op1)
2110 .addReg(RegNo: Op2);
2111 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2112 DestReg: ResultReg)
2113 .addReg(RegNo: II.implicit_defs()[0]);
2114 }
2115 return ResultReg;
2116}
2117
2118Register FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
2119 const TargetRegisterClass *RC, unsigned Op0,
2120 uint64_t Imm) {
2121 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2122
2123 Register ResultReg = createResultReg(RC);
2124 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2125
2126 if (II.getNumDefs() >= 1)
2127 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2128 .addReg(RegNo: Op0)
2129 .addImm(Val: Imm);
2130 else {
2131 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2132 .addReg(RegNo: Op0)
2133 .addImm(Val: Imm);
2134 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2135 DestReg: ResultReg)
2136 .addReg(RegNo: II.implicit_defs()[0]);
2137 }
2138 return ResultReg;
2139}
2140
2141Register FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
2142 const TargetRegisterClass *RC, unsigned Op0,
2143 uint64_t Imm1, uint64_t Imm2) {
2144 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2145
2146 Register ResultReg = createResultReg(RC);
2147 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2148
2149 if (II.getNumDefs() >= 1)
2150 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2151 .addReg(RegNo: Op0)
2152 .addImm(Val: Imm1)
2153 .addImm(Val: Imm2);
2154 else {
2155 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2156 .addReg(RegNo: Op0)
2157 .addImm(Val: Imm1)
2158 .addImm(Val: Imm2);
2159 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2160 DestReg: ResultReg)
2161 .addReg(RegNo: II.implicit_defs()[0]);
2162 }
2163 return ResultReg;
2164}
2165
2166Register FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
2167 const TargetRegisterClass *RC,
2168 const ConstantFP *FPImm) {
2169 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2170
2171 Register ResultReg = createResultReg(RC);
2172
2173 if (II.getNumDefs() >= 1)
2174 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2175 .addFPImm(Val: FPImm);
2176 else {
2177 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2178 .addFPImm(Val: FPImm);
2179 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2180 DestReg: ResultReg)
2181 .addReg(RegNo: II.implicit_defs()[0]);
2182 }
2183 return ResultReg;
2184}
2185
2186Register FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
2187 const TargetRegisterClass *RC, unsigned Op0,
2188 unsigned Op1, uint64_t Imm) {
2189 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2190
2191 Register ResultReg = createResultReg(RC);
2192 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2193 Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
2194
2195 if (II.getNumDefs() >= 1)
2196 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2197 .addReg(RegNo: Op0)
2198 .addReg(RegNo: Op1)
2199 .addImm(Val: Imm);
2200 else {
2201 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2202 .addReg(RegNo: Op0)
2203 .addReg(RegNo: Op1)
2204 .addImm(Val: Imm);
2205 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2206 DestReg: ResultReg)
2207 .addReg(RegNo: II.implicit_defs()[0]);
2208 }
2209 return ResultReg;
2210}
2211
2212Register FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
2213 const TargetRegisterClass *RC, uint64_t Imm) {
2214 Register ResultReg = createResultReg(RC);
2215 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2216
2217 if (II.getNumDefs() >= 1)
2218 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2219 .addImm(Val: Imm);
2220 else {
2221 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II).addImm(Val: Imm);
2222 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2223 DestReg: ResultReg)
2224 .addReg(RegNo: II.implicit_defs()[0]);
2225 }
2226 return ResultReg;
2227}
2228
2229Register FastISel::fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0,
2230 uint32_t Idx) {
2231 Register ResultReg = createResultReg(RC: TLI.getRegClassFor(VT: RetVT));
2232 assert(Register::isVirtualRegister(Op0) &&
2233 "Cannot yet extract from physregs");
2234 const TargetRegisterClass *RC = MRI.getRegClass(Reg: Op0);
2235 MRI.constrainRegClass(Reg: Op0, RC: TRI.getSubClassWithSubReg(RC, Idx));
2236 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2237 DestReg: ResultReg).addReg(RegNo: Op0, flags: 0, SubReg: Idx);
2238 return ResultReg;
2239}
2240
2241/// Emit MachineInstrs to compute the value of Op with all but the least
2242/// significant bit set to zero.
2243Register FastISel::fastEmitZExtFromI1(MVT VT, unsigned Op0) {
2244 return fastEmit_ri(VT, VT, ISD::AND, Op0, 1);
2245}
2246
2247/// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
2248/// Emit code to ensure constants are copied into registers when needed.
2249/// Remember the virtual registers that need to be added to the Machine PHI
2250/// nodes as input. We cannot just directly add them, because expansion
2251/// might result in multiple MBB's for one BB. As such, the start of the
2252/// BB might correspond to a different MBB than the end.
2253bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
2254 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
2255 FuncInfo.OrigNumPHINodesToUpdate = FuncInfo.PHINodesToUpdate.size();
2256
2257 // Check successor nodes' PHI nodes that expect a constant to be available
2258 // from this block.
2259 for (const BasicBlock *SuccBB : successors(BB: LLVMBB)) {
2260 if (!isa<PHINode>(Val: SuccBB->begin()))
2261 continue;
2262 MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
2263
2264 // If this terminator has multiple identical successors (common for
2265 // switches), only handle each succ once.
2266 if (!SuccsHandled.insert(Ptr: SuccMBB).second)
2267 continue;
2268
2269 MachineBasicBlock::iterator MBBI = SuccMBB->begin();
2270
2271 // At this point we know that there is a 1-1 correspondence between LLVM PHI
2272 // nodes and Machine PHI nodes, but the incoming operands have not been
2273 // emitted yet.
2274 for (const PHINode &PN : SuccBB->phis()) {
2275 // Ignore dead phi's.
2276 if (PN.use_empty())
2277 continue;
2278
2279 // Only handle legal types. Two interesting things to note here. First,
2280 // by bailing out early, we may leave behind some dead instructions,
2281 // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
2282 // own moves. Second, this check is necessary because FastISel doesn't
2283 // use CreateRegs to create registers, so it always creates
2284 // exactly one register for each non-void instruction.
2285 EVT VT = TLI.getValueType(DL, Ty: PN.getType(), /*AllowUnknown=*/true);
2286 if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
2287 // Handle integer promotions, though, because they're common and easy.
2288 if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
2289 FuncInfo.PHINodesToUpdate.resize(new_size: FuncInfo.OrigNumPHINodesToUpdate);
2290 return false;
2291 }
2292 }
2293
2294 const Value *PHIOp = PN.getIncomingValueForBlock(BB: LLVMBB);
2295
2296 // Set the DebugLoc for the copy. Use the location of the operand if
2297 // there is one; otherwise no location, flushLocalValueMap will fix it.
2298 MIMD = {};
2299 if (const auto *Inst = dyn_cast<Instruction>(Val: PHIOp))
2300 MIMD = MIMetadata(*Inst);
2301
2302 Register Reg = getRegForValue(V: PHIOp);
2303 if (!Reg) {
2304 FuncInfo.PHINodesToUpdate.resize(new_size: FuncInfo.OrigNumPHINodesToUpdate);
2305 return false;
2306 }
2307 FuncInfo.PHINodesToUpdate.push_back(x: std::make_pair(x: &*MBBI++, y&: Reg));
2308 MIMD = {};
2309 }
2310 }
2311
2312 return true;
2313}
2314
2315bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
2316 assert(LI->hasOneUse() &&
2317 "tryToFoldLoad expected a LoadInst with a single use");
2318 // We know that the load has a single use, but don't know what it is. If it
2319 // isn't one of the folded instructions, then we can't succeed here. Handle
2320 // this by scanning the single-use users of the load until we get to FoldInst.
2321 unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
2322
2323 const Instruction *TheUser = LI->user_back();
2324 while (TheUser != FoldInst && // Scan up until we find FoldInst.
2325 // Stay in the right block.
2326 TheUser->getParent() == FoldInst->getParent() &&
2327 --MaxUsers) { // Don't scan too far.
2328 // If there are multiple or no uses of this instruction, then bail out.
2329 if (!TheUser->hasOneUse())
2330 return false;
2331
2332 TheUser = TheUser->user_back();
2333 }
2334
2335 // If we didn't find the fold instruction, then we failed to collapse the
2336 // sequence.
2337 if (TheUser != FoldInst)
2338 return false;
2339
2340 // Don't try to fold volatile loads. Target has to deal with alignment
2341 // constraints.
2342 if (LI->isVolatile())
2343 return false;
2344
2345 // Figure out which vreg this is going into. If there is no assigned vreg yet
2346 // then there actually was no reference to it. Perhaps the load is referenced
2347 // by a dead instruction.
2348 Register LoadReg = getRegForValue(V: LI);
2349 if (!LoadReg)
2350 return false;
2351
2352 // We can't fold if this vreg has no uses or more than one use. Multiple uses
2353 // may mean that the instruction got lowered to multiple MIs, or the use of
2354 // the loaded value ended up being multiple operands of the result.
2355 if (!MRI.hasOneUse(RegNo: LoadReg))
2356 return false;
2357
2358 // If the register has fixups, there may be additional uses through a
2359 // different alias of the register.
2360 if (FuncInfo.RegsWithFixups.contains(V: LoadReg))
2361 return false;
2362
2363 MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(RegNo: LoadReg);
2364 MachineInstr *User = RI->getParent();
2365
2366 // Set the insertion point properly. Folding the load can cause generation of
2367 // other random instructions (like sign extends) for addressing modes; make
2368 // sure they get inserted in a logical place before the new instruction.
2369 FuncInfo.InsertPt = User;
2370 FuncInfo.MBB = User->getParent();
2371
2372 // Ask the target to try folding the load.
2373 return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
2374}
2375
2376bool FastISel::canFoldAddIntoGEP(const User *GEP, const Value *Add) {
2377 // Must be an add.
2378 if (!isa<AddOperator>(Val: Add))
2379 return false;
2380 // Type size needs to match.
2381 if (DL.getTypeSizeInBits(Ty: GEP->getType()) !=
2382 DL.getTypeSizeInBits(Ty: Add->getType()))
2383 return false;
2384 // Must be in the same basic block.
2385 if (isa<Instruction>(Val: Add) &&
2386 FuncInfo.MBBMap[cast<Instruction>(Val: Add)->getParent()] != FuncInfo.MBB)
2387 return false;
2388 // Must have a constant operand.
2389 return isa<ConstantInt>(Val: cast<AddOperator>(Val: Add)->getOperand(i_nocapture: 1));
2390}
2391
2392MachineMemOperand *
2393FastISel::createMachineMemOperandFor(const Instruction *I) const {
2394 const Value *Ptr;
2395 Type *ValTy;
2396 MaybeAlign Alignment;
2397 MachineMemOperand::Flags Flags;
2398 bool IsVolatile;
2399
2400 if (const auto *LI = dyn_cast<LoadInst>(Val: I)) {
2401 Alignment = LI->getAlign();
2402 IsVolatile = LI->isVolatile();
2403 Flags = MachineMemOperand::MOLoad;
2404 Ptr = LI->getPointerOperand();
2405 ValTy = LI->getType();
2406 } else if (const auto *SI = dyn_cast<StoreInst>(Val: I)) {
2407 Alignment = SI->getAlign();
2408 IsVolatile = SI->isVolatile();
2409 Flags = MachineMemOperand::MOStore;
2410 Ptr = SI->getPointerOperand();
2411 ValTy = SI->getValueOperand()->getType();
2412 } else
2413 return nullptr;
2414
2415 bool IsNonTemporal = I->hasMetadata(KindID: LLVMContext::MD_nontemporal);
2416 bool IsInvariant = I->hasMetadata(KindID: LLVMContext::MD_invariant_load);
2417 bool IsDereferenceable = I->hasMetadata(KindID: LLVMContext::MD_dereferenceable);
2418 const MDNode *Ranges = I->getMetadata(KindID: LLVMContext::MD_range);
2419
2420 AAMDNodes AAInfo = I->getAAMetadata();
2421
2422 if (!Alignment) // Ensure that codegen never sees alignment 0.
2423 Alignment = DL.getABITypeAlign(Ty: ValTy);
2424
2425 unsigned Size = DL.getTypeStoreSize(Ty: ValTy);
2426
2427 if (IsVolatile)
2428 Flags |= MachineMemOperand::MOVolatile;
2429 if (IsNonTemporal)
2430 Flags |= MachineMemOperand::MONonTemporal;
2431 if (IsDereferenceable)
2432 Flags |= MachineMemOperand::MODereferenceable;
2433 if (IsInvariant)
2434 Flags |= MachineMemOperand::MOInvariant;
2435
2436 return FuncInfo.MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Ptr), F: Flags, Size,
2437 BaseAlignment: *Alignment, AAInfo, Ranges);
2438}
2439
2440CmpInst::Predicate FastISel::optimizeCmpPredicate(const CmpInst *CI) const {
2441 // If both operands are the same, then try to optimize or fold the cmp.
2442 CmpInst::Predicate Predicate = CI->getPredicate();
2443 if (CI->getOperand(i_nocapture: 0) != CI->getOperand(i_nocapture: 1))
2444 return Predicate;
2445
2446 switch (Predicate) {
2447 default: llvm_unreachable("Invalid predicate!");
2448 case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
2449 case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break;
2450 case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break;
2451 case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break;
2452 case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break;
2453 case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break;
2454 case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break;
2455 case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break;
2456 case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break;
2457 case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break;
2458 case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break;
2459 case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2460 case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break;
2461 case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2462 case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break;
2463 case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break;
2464
2465 case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break;
2466 case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break;
2467 case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break;
2468 case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2469 case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break;
2470 case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2471 case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break;
2472 case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break;
2473 case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break;
2474 case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break;
2475 }
2476
2477 return Predicate;
2478}
2479