1//===- FastISel.cpp - Implementation of the FastISel class ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the FastISel class.
10//
11// "Fast" instruction selection is designed to emit very poor code quickly.
12// Also, it is not designed to be able to do much lowering, so most illegal
13// types (e.g. i64 on 32-bit targets) and operations are not supported. It is
14// also not intended to be able to do much optimization, except in a few cases
15// where doing optimizations reduces overall compile time. For example, folding
16// constants into immediate fields is often done, because it's cheap and it
17// reduces the number of instructions later phases have to examine.
18//
19// "Fast" instruction selection is able to fail gracefully and transfer
20// control to the SelectionDAG selector for operations that it doesn't
21// support. In many cases, this allows us to avoid duplicating a lot of
22// the complicated lowering logic that SelectionDAG currently has.
23//
24// The intended use for "fast" instruction selection is "-O0" mode
25// compilation, where the quality of the generated code is irrelevant when
26// weighed against the speed at which the code can be generated. Also,
27// at -O0, the LLVM optimizers are not running, and this makes the
28// compile time of codegen a much higher portion of the overall compile
29// time. Despite its limitations, "fast" instruction selection is able to
30// handle enough code on its own to provide noticeable overall speedups
31// in -O0 compiles.
32//
33// Basic operations are supported in a target-independent way, by reading
34// the same instruction descriptions that the SelectionDAG selector reads,
35// and identifying simple arithmetic operations that can be directly selected
36// from simple operators. More complicated operations currently require
37// target-specific code.
38//
39//===----------------------------------------------------------------------===//
40
41#include "llvm/CodeGen/FastISel.h"
42#include "llvm/ADT/APFloat.h"
43#include "llvm/ADT/APSInt.h"
44#include "llvm/ADT/DenseMap.h"
45#include "llvm/ADT/SmallPtrSet.h"
46#include "llvm/ADT/SmallString.h"
47#include "llvm/ADT/SmallVector.h"
48#include "llvm/ADT/Statistic.h"
49#include "llvm/Analysis/BranchProbabilityInfo.h"
50#include "llvm/Analysis/TargetLibraryInfo.h"
51#include "llvm/CodeGen/Analysis.h"
52#include "llvm/CodeGen/FunctionLoweringInfo.h"
53#include "llvm/CodeGen/ISDOpcodes.h"
54#include "llvm/CodeGen/MachineBasicBlock.h"
55#include "llvm/CodeGen/MachineFrameInfo.h"
56#include "llvm/CodeGen/MachineInstr.h"
57#include "llvm/CodeGen/MachineInstrBuilder.h"
58#include "llvm/CodeGen/MachineMemOperand.h"
59#include "llvm/CodeGen/MachineModuleInfo.h"
60#include "llvm/CodeGen/MachineOperand.h"
61#include "llvm/CodeGen/MachineRegisterInfo.h"
62#include "llvm/CodeGen/StackMaps.h"
63#include "llvm/CodeGen/TargetInstrInfo.h"
64#include "llvm/CodeGen/TargetLowering.h"
65#include "llvm/CodeGen/TargetSubtargetInfo.h"
66#include "llvm/CodeGen/ValueTypes.h"
67#include "llvm/CodeGenTypes/MachineValueType.h"
68#include "llvm/IR/Argument.h"
69#include "llvm/IR/Attributes.h"
70#include "llvm/IR/BasicBlock.h"
71#include "llvm/IR/CallingConv.h"
72#include "llvm/IR/Constant.h"
73#include "llvm/IR/Constants.h"
74#include "llvm/IR/DataLayout.h"
75#include "llvm/IR/DebugLoc.h"
76#include "llvm/IR/DerivedTypes.h"
77#include "llvm/IR/DiagnosticInfo.h"
78#include "llvm/IR/Function.h"
79#include "llvm/IR/GetElementPtrTypeIterator.h"
80#include "llvm/IR/GlobalValue.h"
81#include "llvm/IR/InlineAsm.h"
82#include "llvm/IR/InstrTypes.h"
83#include "llvm/IR/Instruction.h"
84#include "llvm/IR/Instructions.h"
85#include "llvm/IR/IntrinsicInst.h"
86#include "llvm/IR/LLVMContext.h"
87#include "llvm/IR/Mangler.h"
88#include "llvm/IR/Metadata.h"
89#include "llvm/IR/Module.h"
90#include "llvm/IR/Operator.h"
91#include "llvm/IR/PatternMatch.h"
92#include "llvm/IR/Type.h"
93#include "llvm/IR/User.h"
94#include "llvm/IR/Value.h"
95#include "llvm/MC/MCContext.h"
96#include "llvm/MC/MCInstrDesc.h"
97#include "llvm/Support/Casting.h"
98#include "llvm/Support/Debug.h"
99#include "llvm/Support/ErrorHandling.h"
100#include "llvm/Support/MathExtras.h"
101#include "llvm/Support/raw_ostream.h"
102#include "llvm/Target/TargetMachine.h"
103#include "llvm/Target/TargetOptions.h"
104#include <cassert>
105#include <cstdint>
106#include <iterator>
107#include <optional>
108#include <utility>
109
110using namespace llvm;
111using namespace PatternMatch;
112
113#define DEBUG_TYPE "isel"
114
115STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
116 "target-independent selector");
117STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
118 "target-specific selector");
119STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
120
121/// Set the current block to which generated machine instructions will be
122/// appended.
123void FastISel::startNewBlock() {
124 assert(LocalValueMap.empty() &&
125 "local values should be cleared after finishing a BB");
126
127 // Instructions are appended to FuncInfo.MBB. If the basic block already
128 // contains labels or copies, use the last instruction as the last local
129 // value.
130 EmitStartPt = nullptr;
131 if (!FuncInfo.MBB->empty())
132 EmitStartPt = &FuncInfo.MBB->back();
133 LastLocalValue = EmitStartPt;
134}
135
136void FastISel::finishBasicBlock() { flushLocalValueMap(); }
137
138bool FastISel::lowerArguments() {
139 if (!FuncInfo.CanLowerReturn)
140 // Fallback to SDISel argument lowering code to deal with sret pointer
141 // parameter.
142 return false;
143
144 if (!fastLowerArguments())
145 return false;
146
147 // Enter arguments into ValueMap for uses in non-entry BBs.
148 for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(),
149 E = FuncInfo.Fn->arg_end();
150 I != E; ++I) {
151 auto VI = LocalValueMap.find(Val: &*I);
152 assert(VI != LocalValueMap.end() && "Missed an argument?");
153 FuncInfo.ValueMap[&*I] = VI->second;
154 }
155 return true;
156}
157
158/// Return the defined register if this instruction defines exactly one
159/// virtual register and uses no other virtual registers. Otherwise return
160/// Register();
161static Register findLocalRegDef(MachineInstr &MI) {
162 Register RegDef;
163 for (const MachineOperand &MO : MI.operands()) {
164 if (!MO.isReg())
165 continue;
166 if (MO.isDef()) {
167 if (RegDef)
168 return Register();
169 RegDef = MO.getReg();
170 } else if (MO.getReg().isVirtual()) {
171 // This is another use of a vreg. Don't delete it.
172 return Register();
173 }
174 }
175 return RegDef;
176}
177
178static bool isRegUsedByPhiNodes(Register DefReg,
179 FunctionLoweringInfo &FuncInfo) {
180 for (auto &P : FuncInfo.PHINodesToUpdate)
181 if (P.second == DefReg)
182 return true;
183 return false;
184}
185
186void FastISel::flushLocalValueMap() {
187 // If FastISel bails out, it could leave local value instructions behind
188 // that aren't used for anything. Detect and erase those.
189 if (LastLocalValue != EmitStartPt) {
190 // Save the first instruction after local values, for later.
191 MachineBasicBlock::iterator FirstNonValue(LastLocalValue);
192 ++FirstNonValue;
193
194 MachineBasicBlock::reverse_iterator RE =
195 EmitStartPt ? MachineBasicBlock::reverse_iterator(EmitStartPt)
196 : FuncInfo.MBB->rend();
197 MachineBasicBlock::reverse_iterator RI(LastLocalValue);
198 for (MachineInstr &LocalMI :
199 llvm::make_early_inc_range(Range: llvm::make_range(x: RI, y: RE))) {
200 Register DefReg = findLocalRegDef(MI&: LocalMI);
201 if (!DefReg)
202 continue;
203 if (FuncInfo.RegsWithFixups.count(V: DefReg))
204 continue;
205 bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
206 if (!UsedByPHI && MRI.use_nodbg_empty(RegNo: DefReg)) {
207 if (EmitStartPt == &LocalMI)
208 EmitStartPt = EmitStartPt->getPrevNode();
209 LLVM_DEBUG(dbgs() << "removing dead local value materialization"
210 << LocalMI);
211 LocalMI.eraseFromParent();
212 }
213 }
214
215 // See if there are any local value instructions left. If so, we want to
216 // make sure the first one has a debug location; if it doesn't, use the
217 // first non-value instruction's debug location.
218
219 // If EmitStartPt is non-null, this block had copies at the top before
220 // FastISel started doing anything; it points to the last one, so the
221 // first local value instruction is the one after EmitStartPt.
222 // If EmitStartPt is null, the first local value instruction is at the
223 // top of the block.
224 MachineBasicBlock::iterator FirstLocalValue =
225 EmitStartPt ? ++MachineBasicBlock::iterator(EmitStartPt)
226 : FuncInfo.MBB->begin();
227 if (FirstLocalValue != FirstNonValue && !FirstLocalValue->getDebugLoc()) {
228 if (FirstNonValue != FuncInfo.MBB->end()) {
229 FirstLocalValue->setDebugLoc(FirstNonValue->getDebugLoc());
230 } else if (const BasicBlock *BB = FuncInfo.MBB->getBasicBlock()) {
231 // Nothing follows them, e.g. a block only setting up a successor's PHI
232 // nodes before falling through. Use the terminator's location.
233 FirstLocalValue->setDebugLoc(BB->getTerminator()->getDebugLoc());
234 }
235 }
236 }
237
238 LocalValueMap.clear();
239 LastLocalValue = EmitStartPt;
240 recomputeInsertPt();
241 SavedInsertPt = FuncInfo.InsertPt;
242}
243
244Register FastISel::getRegForValue(const Value *V) {
245 EVT RealVT = TLI.getValueType(DL, Ty: V->getType(), /*AllowUnknown=*/true);
246 // Don't handle non-simple values in FastISel.
247 if (!RealVT.isSimple())
248 return Register();
249
250 // Ignore illegal types. We must do this before looking up the value
251 // in ValueMap because Arguments are given virtual registers regardless
252 // of whether FastISel can handle them.
253 MVT VT = RealVT.getSimpleVT();
254 if (!TLI.isTypeLegal(VT)) {
255 // Handle integer promotions, though, because they're common and easy.
256 if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
257 VT = TLI.getTypeToTransformTo(Context&: V->getContext(), VT).getSimpleVT();
258 else
259 return Register();
260 }
261
262 // Look up the value to see if we already have a register for it.
263 Register Reg = lookUpRegForValue(V);
264 if (Reg)
265 return Reg;
266
267 // In bottom-up mode, just create the virtual register which will be used
268 // to hold the value. It will be materialized later.
269 if (isa<Instruction>(Val: V) &&
270 (!isa<AllocaInst>(Val: V) ||
271 !FuncInfo.StaticAllocaMap.count(Val: cast<AllocaInst>(Val: V))))
272 return FuncInfo.InitializeRegForValue(V);
273
274 SavePoint SaveInsertPt = enterLocalValueArea();
275
276 // Materialize the value in a register. Emit any instructions in the
277 // local value area.
278 Reg = materializeRegForValue(V, VT);
279
280 leaveLocalValueArea(Old: SaveInsertPt);
281
282 return Reg;
283}
284
285Register FastISel::materializeConstant(const Value *V, MVT VT) {
286 Register Reg;
287 if (const auto *CI = dyn_cast<ConstantInt>(Val: V)) {
288 if (CI->getValue().getActiveBits() <= 64)
289 Reg = fastEmit_i(VT, RetVT: VT, Opcode: ISD::Constant, Imm: CI->getZExtValue());
290 } else if (isa<AllocaInst>(Val: V))
291 Reg = fastMaterializeAlloca(C: cast<AllocaInst>(Val: V));
292 else if (isa<ConstantPointerNull>(Val: V))
293 // Translate this as an integer zero so that it can be
294 // local-CSE'd with actual integer zeros.
295 Reg =
296 getRegForValue(V: Constant::getNullValue(Ty: DL.getIntPtrType(V->getType())));
297 else if (const auto *CF = dyn_cast<ConstantFP>(Val: V)) {
298 if (CF->isNullValue())
299 Reg = fastMaterializeFloatZero(CF);
300 else
301 // Try to emit the constant directly.
302 Reg = fastEmit_f(VT, RetVT: VT, Opcode: ISD::ConstantFP, FPImm: CF);
303
304 if (!Reg) {
305 // Try to emit the constant by using an integer constant with a cast.
306 const APFloat &Flt = CF->getValueAPF();
307 EVT IntVT = TLI.getPointerTy(DL);
308 uint32_t IntBitWidth = IntVT.getSizeInBits();
309 APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
310 bool isExact;
311 (void)Flt.convertToInteger(Result&: SIntVal, RM: APFloat::rmTowardZero, IsExact: &isExact);
312 if (isExact) {
313 Register IntegerReg =
314 getRegForValue(V: ConstantInt::get(Context&: V->getContext(), V: SIntVal));
315 if (IntegerReg)
316 Reg = fastEmit_r(VT: IntVT.getSimpleVT(), RetVT: VT, Opcode: ISD::SINT_TO_FP,
317 Op0: IntegerReg);
318 }
319 }
320 } else if (const auto *Op = dyn_cast<Operator>(Val: V)) {
321 if (!selectOperator(I: Op, Opcode: Op->getOpcode()))
322 if (!isa<Instruction>(Val: Op) ||
323 !fastSelectInstruction(I: cast<Instruction>(Val: Op)))
324 return Register();
325 Reg = lookUpRegForValue(V: Op);
326 } else if (isa<UndefValue>(Val: V)) {
327 Reg = createResultReg(RC: TLI.getRegClassFor(VT));
328 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
329 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: Reg);
330 }
331 return Reg;
332}
333
334/// Helper for getRegForValue. This function is called when the value isn't
335/// already available in a register and must be materialized with new
336/// instructions.
337Register FastISel::materializeRegForValue(const Value *V, MVT VT) {
338 Register Reg;
339 // Give the target-specific code a try first.
340 if (isa<Constant>(Val: V))
341 Reg = fastMaterializeConstant(C: cast<Constant>(Val: V));
342
343 // If target-specific code couldn't or didn't want to handle the value, then
344 // give target-independent code a try.
345 if (!Reg)
346 Reg = materializeConstant(V, VT);
347
348 // Don't cache constant materializations in the general ValueMap.
349 // To do so would require tracking what uses they dominate.
350 if (Reg) {
351 LocalValueMap[V] = Reg;
352 LastLocalValue = MRI.getVRegDef(Reg);
353 }
354 return Reg;
355}
356
357Register FastISel::lookUpRegForValue(const Value *V) {
358 // Look up the value to see if we already have a register for it. We
359 // cache values defined by Instructions across blocks, and other values
360 // only locally. This is because Instructions already have the SSA
361 // def-dominates-use requirement enforced.
362 auto I = FuncInfo.ValueMap.find(Val: V);
363 if (I != FuncInfo.ValueMap.end())
364 return I->second;
365 return LocalValueMap[V];
366}
367
368void FastISel::updateValueMap(const Value *I, Register Reg, unsigned NumRegs) {
369 if (!isa<Instruction>(Val: I)) {
370 LocalValueMap[I] = Reg;
371 return;
372 }
373
374 Register &AssignedReg = FuncInfo.ValueMap[I];
375 if (!AssignedReg)
376 // Use the new register.
377 AssignedReg = Reg;
378 else if (Reg != AssignedReg) {
379 // Arrange for uses of AssignedReg to be replaced by uses of Reg.
380 for (unsigned i = 0; i < NumRegs; i++) {
381 FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
382 FuncInfo.RegsWithFixups.insert(V: Reg + i);
383 }
384
385 AssignedReg = Reg;
386 }
387}
388
389Register FastISel::getRegForGEPIndex(MVT PtrVT, const Value *Idx) {
390 Register IdxN = getRegForValue(V: Idx);
391 if (!IdxN)
392 // Unhandled operand. Halt "fast" selection and bail.
393 return Register();
394
395 // If the index is smaller or larger than intptr_t, truncate or extend it.
396 EVT IdxVT = EVT::getEVT(Ty: Idx->getType(), /*HandleUnknown=*/false);
397 if (IdxVT.bitsLT(VT: PtrVT)) {
398 IdxN = fastEmit_r(VT: IdxVT.getSimpleVT(), RetVT: PtrVT, Opcode: ISD::SIGN_EXTEND, Op0: IdxN);
399 } else if (IdxVT.bitsGT(VT: PtrVT)) {
400 IdxN =
401 fastEmit_r(VT: IdxVT.getSimpleVT(), RetVT: PtrVT, Opcode: ISD::TRUNCATE, Op0: IdxN);
402 }
403 return IdxN;
404}
405
406void FastISel::recomputeInsertPt() {
407 if (getLastLocalValue()) {
408 FuncInfo.InsertPt = getLastLocalValue();
409 FuncInfo.MBB = FuncInfo.InsertPt->getParent();
410 ++FuncInfo.InsertPt;
411 } else
412 FuncInfo.InsertPt = FuncInfo.MBB->getFirstNonPHI();
413}
414
415void FastISel::removeDeadCode(MachineBasicBlock::iterator I,
416 MachineBasicBlock::iterator E) {
417 assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
418 "Invalid iterator!");
419 while (I != E) {
420 if (SavedInsertPt == I)
421 SavedInsertPt = E;
422 if (EmitStartPt == I)
423 EmitStartPt = E.isValid() ? &*E : nullptr;
424 if (LastLocalValue == I)
425 LastLocalValue = E.isValid() ? &*E : nullptr;
426
427 MachineInstr *Dead = &*I;
428 ++I;
429 Dead->eraseFromParent();
430 ++NumFastIselDead;
431 }
432 recomputeInsertPt();
433}
434
435FastISel::SavePoint FastISel::enterLocalValueArea() {
436 SavePoint OldInsertPt = FuncInfo.InsertPt;
437 recomputeInsertPt();
438 return OldInsertPt;
439}
440
441void FastISel::leaveLocalValueArea(SavePoint OldInsertPt) {
442 if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
443 LastLocalValue = &*std::prev(x: FuncInfo.InsertPt);
444
445 // Restore the previous insert position.
446 FuncInfo.InsertPt = OldInsertPt;
447}
448
449bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
450 EVT VT = EVT::getEVT(Ty: I->getType(), /*HandleUnknown=*/true);
451 if (VT == MVT::Other || !VT.isSimple())
452 // Unhandled type. Halt "fast" selection and bail.
453 return false;
454
455 // We only handle legal types. For example, on x86-32 the instruction
456 // selector contains all of the 64-bit instructions from x86-64,
457 // under the assumption that i64 won't be used if the target doesn't
458 // support it.
459 if (!TLI.isTypeLegal(VT)) {
460 // MVT::i1 is special. Allow AND, OR, or XOR because they
461 // don't require additional zeroing, which makes them easy.
462 if (VT == MVT::i1 && ISD::isBitwiseLogicOp(Opcode: ISDOpcode))
463 VT = TLI.getTypeToTransformTo(Context&: I->getContext(), VT);
464 else
465 return false;
466 }
467
468 // Check if the first operand is a constant, and handle it as "ri". At -O0,
469 // we don't have anything that canonicalizes operand order.
470 if (const auto *CI = dyn_cast<ConstantInt>(Val: I->getOperand(i: 0)))
471 if (isa<Instruction>(Val: I) && cast<Instruction>(Val: I)->isCommutative()) {
472 Register Op1 = getRegForValue(V: I->getOperand(i: 1));
473 if (!Op1)
474 return false;
475
476 Register ResultReg =
477 fastEmit_ri_(VT: VT.getSimpleVT(), Opcode: ISDOpcode, Op0: Op1, Imm: CI->getZExtValue(),
478 ImmType: VT.getSimpleVT());
479 if (!ResultReg)
480 return false;
481
482 // We successfully emitted code for the given LLVM Instruction.
483 updateValueMap(I, Reg: ResultReg);
484 return true;
485 }
486
487 Register Op0 = getRegForValue(V: I->getOperand(i: 0));
488 if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
489 return false;
490
491 // Check if the second operand is a constant and handle it appropriately.
492 if (const auto *CI = dyn_cast<ConstantInt>(Val: I->getOperand(i: 1))) {
493 uint64_t Imm = CI->getSExtValue();
494
495 // Transform "sdiv exact X, 8" -> "sra X, 3".
496 if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(Val: I) &&
497 cast<BinaryOperator>(Val: I)->isExact() && isPowerOf2_64(Value: Imm)) {
498 Imm = Log2_64(Value: Imm);
499 ISDOpcode = ISD::SRA;
500 }
501
502 // Transform "urem x, pow2" -> "and x, pow2-1".
503 if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(Val: I) &&
504 isPowerOf2_64(Value: Imm)) {
505 --Imm;
506 ISDOpcode = ISD::AND;
507 }
508
509 Register ResultReg = fastEmit_ri_(VT: VT.getSimpleVT(), Opcode: ISDOpcode, Op0, Imm,
510 ImmType: VT.getSimpleVT());
511 if (!ResultReg)
512 return false;
513
514 // We successfully emitted code for the given LLVM Instruction.
515 updateValueMap(I, Reg: ResultReg);
516 return true;
517 }
518
519 Register Op1 = getRegForValue(V: I->getOperand(i: 1));
520 if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
521 return false;
522
523 // Now we have both operands in registers. Emit the instruction.
524 Register ResultReg = fastEmit_rr(VT: VT.getSimpleVT(), RetVT: VT.getSimpleVT(),
525 Opcode: ISDOpcode, Op0, Op1);
526 if (!ResultReg)
527 // Target-specific code wasn't able to find a machine opcode for
528 // the given ISD opcode and type. Halt "fast" selection and bail.
529 return false;
530
531 // We successfully emitted code for the given LLVM Instruction.
532 updateValueMap(I, Reg: ResultReg);
533 return true;
534}
535
536bool FastISel::selectGetElementPtr(const User *I) {
537 Register N = getRegForValue(V: I->getOperand(i: 0));
538 if (!N) // Unhandled operand. Halt "fast" selection and bail.
539 return false;
540
541 // FIXME: The code below does not handle vector GEPs. Halt "fast" selection
542 // and bail.
543 if (isa<VectorType>(Val: I->getType()))
544 return false;
545
546 // Keep a running tab of the total offset to coalesce multiple N = N + Offset
547 // into a single N = N + TotalOffset.
548 uint64_t TotalOffs = 0;
549 // FIXME: What's a good SWAG number for MaxOffs?
550 uint64_t MaxOffs = 2048;
551 MVT VT = TLI.getValueType(DL, Ty: I->getType()).getSimpleVT();
552
553 for (gep_type_iterator GTI = gep_type_begin(GEP: I), E = gep_type_end(GEP: I);
554 GTI != E; ++GTI) {
555 const Value *Idx = GTI.getOperand();
556 if (StructType *StTy = GTI.getStructTypeOrNull()) {
557 uint64_t Field = cast<ConstantInt>(Val: Idx)->getZExtValue();
558 if (Field) {
559 // N = N + Offset
560 TotalOffs += DL.getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
561 if (TotalOffs >= MaxOffs) {
562 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
563 if (!N) // Unhandled operand. Halt "fast" selection and bail.
564 return false;
565 TotalOffs = 0;
566 }
567 }
568 } else {
569 // If this is a constant subscript, handle it quickly.
570 if (const auto *CI = dyn_cast<ConstantInt>(Val: Idx)) {
571 if (CI->isZero())
572 continue;
573 // N = N + Offset
574 uint64_t IdxN = CI->getValue().sextOrTrunc(width: 64).getSExtValue();
575 TotalOffs += GTI.getSequentialElementStride(DL) * IdxN;
576 if (TotalOffs >= MaxOffs) {
577 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
578 if (!N) // Unhandled operand. Halt "fast" selection and bail.
579 return false;
580 TotalOffs = 0;
581 }
582 continue;
583 }
584 if (TotalOffs) {
585 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
586 if (!N) // Unhandled operand. Halt "fast" selection and bail.
587 return false;
588 TotalOffs = 0;
589 }
590
591 // N = N + Idx * ElementSize;
592 uint64_t ElementSize = GTI.getSequentialElementStride(DL);
593 Register IdxN = getRegForGEPIndex(PtrVT: VT, Idx);
594 if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
595 return false;
596
597 if (ElementSize != 1) {
598 IdxN = fastEmit_ri_(VT, Opcode: ISD::MUL, Op0: IdxN, Imm: ElementSize, ImmType: VT);
599 if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
600 return false;
601 }
602 N = fastEmit_rr(VT, RetVT: VT, Opcode: ISD::ADD, Op0: N, Op1: IdxN);
603 if (!N) // Unhandled operand. Halt "fast" selection and bail.
604 return false;
605 }
606 }
607 if (TotalOffs) {
608 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
609 if (!N) // Unhandled operand. Halt "fast" selection and bail.
610 return false;
611 }
612
613 // We successfully emitted code for the given LLVM Instruction.
614 updateValueMap(I, Reg: N);
615 return true;
616}
617
618bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
619 const CallInst *CI, unsigned StartIdx) {
620 for (unsigned i = StartIdx, e = CI->arg_size(); i != e; ++i) {
621 Value *Val = CI->getArgOperand(i);
622 // Check for constants and encode them with a StackMaps::ConstantOp prefix.
623 if (const auto *C = dyn_cast<ConstantInt>(Val)) {
624 Ops.push_back(Elt: MachineOperand::CreateImm(Val: StackMaps::ConstantOp));
625 Ops.push_back(Elt: MachineOperand::CreateImm(Val: C->getSExtValue()));
626 } else if (isa<ConstantPointerNull>(Val)) {
627 Ops.push_back(Elt: MachineOperand::CreateImm(Val: StackMaps::ConstantOp));
628 Ops.push_back(Elt: MachineOperand::CreateImm(Val: 0));
629 } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
630 // Values coming from a stack location also require a special encoding,
631 // but that is added later on by the target specific frame index
632 // elimination implementation.
633 auto SI = FuncInfo.StaticAllocaMap.find(Val: AI);
634 if (SI != FuncInfo.StaticAllocaMap.end())
635 Ops.push_back(Elt: MachineOperand::CreateFI(Idx: SI->second));
636 else
637 return false;
638 } else {
639 Register Reg = getRegForValue(V: Val);
640 if (!Reg)
641 return false;
642 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/false));
643 }
644 }
645 return true;
646}
647
648bool FastISel::selectStackmap(const CallInst *I) {
649 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
650 // [live variables...])
651 assert(I->getCalledFunction()->getReturnType()->isVoidTy() &&
652 "Stackmap cannot return a value.");
653
654 // The stackmap intrinsic only records the live variables (the arguments
655 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
656 // intrinsic, this won't be lowered to a function call. This means we don't
657 // have to worry about calling conventions and target-specific lowering code.
658 // Instead we perform the call lowering right here.
659 //
660 // CALLSEQ_START(0, 0...)
661 // STACKMAP(id, nbytes, ...)
662 // CALLSEQ_END(0, 0)
663 //
664 SmallVector<MachineOperand, 32> Ops;
665
666 // Add the <id> and <numBytes> constants.
667 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
668 "Expected a constant integer.");
669 const auto *ID = cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::IDPos));
670 Ops.push_back(Elt: MachineOperand::CreateImm(Val: ID->getZExtValue()));
671
672 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
673 "Expected a constant integer.");
674 const auto *NumBytes =
675 cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::NBytesPos));
676 Ops.push_back(Elt: MachineOperand::CreateImm(Val: NumBytes->getZExtValue()));
677
678 // Push live variables for the stack map (skipping the first two arguments
679 // <id> and <numBytes>).
680 if (!addStackMapLiveVars(Ops, CI: I, StartIdx: 2))
681 return false;
682
683 // We are not adding any register mask info here, because the stackmap doesn't
684 // clobber anything.
685
686 // Add scratch registers as implicit def and early clobber.
687 CallingConv::ID CC = I->getCallingConv();
688 const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
689 for (unsigned i = 0; ScratchRegs[i]; ++i)
690 Ops.push_back(Elt: MachineOperand::CreateReg(
691 Reg: ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
692 /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
693
694 // Issue CALLSEQ_START
695 unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
696 auto Builder =
697 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: AdjStackDown));
698 const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
699 for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
700 Builder.addImm(Val: 0);
701
702 // Issue STACKMAP.
703 MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
704 MCID: TII.get(Opcode: TargetOpcode::STACKMAP));
705 for (auto const &MO : Ops)
706 MIB.add(MO);
707
708 // Issue CALLSEQ_END
709 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
710 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: AdjStackUp))
711 .addImm(Val: 0)
712 .addImm(Val: 0);
713
714 // Inform the Frame Information that we have a stackmap in this function.
715 FuncInfo.MF->getFrameInfo().setHasStackMap();
716
717 return true;
718}
719
720/// Lower an argument list according to the target calling convention.
721///
722/// This is a helper for lowering intrinsics that follow a target calling
723/// convention or require stack pointer adjustment. Only a subset of the
724/// intrinsic's operands need to participate in the calling convention.
725bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
726 unsigned NumArgs, const Value *Callee,
727 bool ForceRetVoidTy, CallLoweringInfo &CLI) {
728 ArgListTy Args;
729 Args.reserve(n: NumArgs);
730
731 // Populate the argument list.
732 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
733 Value *V = CI->getOperand(i_nocapture: ArgI);
734
735 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
736
737 ArgListEntry Entry(V);
738 Entry.setAttributes(Call: CI, ArgIdx: ArgI);
739 Args.push_back(x: Entry);
740 }
741
742 Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(C&: CI->getType()->getContext())
743 : CI->getType();
744 CLI.setCallee(CC: CI->getCallingConv(), ResultTy: RetTy, Target: Callee, ArgsList: std::move(Args), FixedArgs: NumArgs);
745
746 return lowerCallTo(CLI);
747}
748
749FastISel::CallLoweringInfo &FastISel::CallLoweringInfo::setCallee(
750 const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
751 StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
752 SmallString<32> MangledName;
753 Mangler::getNameWithPrefix(OutName&: MangledName, GVName: Target, DL);
754 MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: MangledName);
755 return setCallee(CC, ResultTy, Target: Sym, ArgsList: std::move(ArgsList), FixedArgs);
756}
757
758bool FastISel::selectPatchpoint(const CallInst *I) {
759 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
760 // i32 <numBytes>,
761 // i8* <target>,
762 // i32 <numArgs>,
763 // [Args...],
764 // [live variables...])
765 CallingConv::ID CC = I->getCallingConv();
766 bool IsAnyRegCC = CC == CallingConv::AnyReg;
767 bool HasDef = !I->getType()->isVoidTy();
768 Value *Callee = I->getOperand(i_nocapture: PatchPointOpers::TargetPos)->stripPointerCasts();
769
770 // Check if we can lower the return type when using anyregcc.
771 MVT ValueType;
772 if (IsAnyRegCC && HasDef) {
773 ValueType = TLI.getSimpleValueType(DL, Ty: I->getType(), /*AllowUnknown=*/true);
774 if (ValueType == MVT::Other)
775 return false;
776 }
777
778 // Get the real number of arguments participating in the call <numArgs>
779 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
780 "Expected a constant integer.");
781 const auto *NumArgsVal =
782 cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::NArgPos));
783 unsigned NumArgs = NumArgsVal->getZExtValue();
784
785 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
786 // This includes all meta-operands up to but not including CC.
787 unsigned NumMetaOpers = PatchPointOpers::CCPos;
788 assert(I->arg_size() >= NumMetaOpers + NumArgs &&
789 "Not enough arguments provided to the patchpoint intrinsic");
790
791 // For AnyRegCC the arguments are lowered later on manually.
792 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
793 CallLoweringInfo CLI;
794 CLI.setIsPatchPoint();
795 if (!lowerCallOperands(CI: I, ArgIdx: NumMetaOpers, NumArgs: NumCallArgs, Callee, ForceRetVoidTy: IsAnyRegCC, CLI))
796 return false;
797
798 assert(CLI.Call && "No call instruction specified.");
799
800 SmallVector<MachineOperand, 32> Ops;
801
802 // Add an explicit result reg if we use the anyreg calling convention.
803 if (IsAnyRegCC && HasDef) {
804 assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
805 assert(ValueType.isValid());
806 CLI.ResultReg = createResultReg(RC: TLI.getRegClassFor(VT: ValueType));
807 CLI.NumResultRegs = 1;
808 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: CLI.ResultReg, /*isDef=*/true));
809 }
810
811 // Add the <id> and <numBytes> constants.
812 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
813 "Expected a constant integer.");
814 const auto *ID = cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::IDPos));
815 Ops.push_back(Elt: MachineOperand::CreateImm(Val: ID->getZExtValue()));
816
817 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
818 "Expected a constant integer.");
819 const auto *NumBytes =
820 cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::NBytesPos));
821 Ops.push_back(Elt: MachineOperand::CreateImm(Val: NumBytes->getZExtValue()));
822
823 // Add the call target.
824 if (const auto *C = dyn_cast<IntToPtrInst>(Val: Callee)) {
825 uint64_t CalleeConstAddr =
826 cast<ConstantInt>(Val: C->getOperand(i_nocapture: 0))->getZExtValue();
827 Ops.push_back(Elt: MachineOperand::CreateImm(Val: CalleeConstAddr));
828 } else if (const auto *C = dyn_cast<ConstantExpr>(Val: Callee)) {
829 if (C->getOpcode() == Instruction::IntToPtr) {
830 uint64_t CalleeConstAddr =
831 cast<ConstantInt>(Val: C->getOperand(i_nocapture: 0))->getZExtValue();
832 Ops.push_back(Elt: MachineOperand::CreateImm(Val: CalleeConstAddr));
833 } else
834 llvm_unreachable("Unsupported ConstantExpr.");
835 } else if (const auto *GV = dyn_cast<GlobalValue>(Val: Callee)) {
836 Ops.push_back(Elt: MachineOperand::CreateGA(GV, Offset: 0));
837 } else if (isa<ConstantPointerNull>(Val: Callee))
838 Ops.push_back(Elt: MachineOperand::CreateImm(Val: 0));
839 else
840 llvm_unreachable("Unsupported callee address.");
841
842 // Adjust <numArgs> to account for any arguments that have been passed on
843 // the stack instead.
844 unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
845 Ops.push_back(Elt: MachineOperand::CreateImm(Val: NumCallRegArgs));
846
847 // Add the calling convention
848 Ops.push_back(Elt: MachineOperand::CreateImm(Val: (unsigned)CC));
849
850 // Add the arguments we omitted previously. The register allocator should
851 // place these in any free register.
852 if (IsAnyRegCC) {
853 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
854 Register Reg = getRegForValue(V: I->getArgOperand(i));
855 if (!Reg)
856 return false;
857 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/false));
858 }
859 }
860
861 // Push the arguments from the call instruction.
862 for (auto Reg : CLI.OutRegs)
863 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/false));
864
865 // Push live variables for the stack map.
866 if (!addStackMapLiveVars(Ops, CI: I, StartIdx: NumMetaOpers + NumArgs))
867 return false;
868
869 // Push the register mask info.
870 Ops.push_back(Elt: MachineOperand::CreateRegMask(
871 Mask: TRI.getCallPreservedMask(MF: *FuncInfo.MF, CC)));
872
873 // Add scratch registers as implicit def and early clobber.
874 const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
875 for (unsigned i = 0; ScratchRegs[i]; ++i)
876 Ops.push_back(Elt: MachineOperand::CreateReg(
877 Reg: ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
878 /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
879
880 // Add implicit defs (return values).
881 for (auto Reg : CLI.InRegs)
882 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/true,
883 /*isImp=*/true));
884
885 // Insert the patchpoint instruction before the call generated by the target.
886 MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: CLI.Call, MIMD,
887 MCID: TII.get(Opcode: TargetOpcode::PATCHPOINT));
888
889 for (auto &MO : Ops)
890 MIB.add(MO);
891
892 MIB->setPhysRegsDeadExcept(UsedRegs: CLI.InRegs, TRI);
893
894 // Delete the original call instruction.
895 CLI.Call->eraseFromParent();
896
897 // Inform the Frame Information that we have a patchpoint in this function.
898 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
899
900 if (CLI.NumResultRegs)
901 updateValueMap(I, Reg: CLI.ResultReg, NumRegs: CLI.NumResultRegs);
902 return true;
903}
904
905bool FastISel::selectXRayCustomEvent(const CallInst *I) {
906 const auto &Triple = TM.getTargetTriple();
907 if (Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64)
908 return true; // don't do anything to this instruction.
909 SmallVector<MachineOperand, 8> Ops;
910 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 0)),
911 /*isDef=*/false));
912 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 1)),
913 /*isDef=*/false));
914 MachineInstrBuilder MIB =
915 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
916 MCID: TII.get(Opcode: TargetOpcode::PATCHABLE_EVENT_CALL));
917 for (auto &MO : Ops)
918 MIB.add(MO);
919
920 // Insert the Patchable Event Call instruction, that gets lowered properly.
921 return true;
922}
923
924bool FastISel::selectXRayTypedEvent(const CallInst *I) {
925 const auto &Triple = TM.getTargetTriple();
926 if (Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64)
927 return true; // don't do anything to this instruction.
928 SmallVector<MachineOperand, 8> Ops;
929 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 0)),
930 /*isDef=*/false));
931 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 1)),
932 /*isDef=*/false));
933 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 2)),
934 /*isDef=*/false));
935 MachineInstrBuilder MIB =
936 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
937 MCID: TII.get(Opcode: TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
938 for (auto &MO : Ops)
939 MIB.add(MO);
940
941 // Insert the Patchable Typed Event Call instruction, that gets lowered properly.
942 return true;
943}
944
945/// Returns an AttributeList representing the attributes applied to the return
946/// value of the given call.
947static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI) {
948 SmallVector<Attribute::AttrKind, 2> Attrs;
949 if (CLI.RetSExt)
950 Attrs.push_back(Elt: Attribute::SExt);
951 if (CLI.RetZExt)
952 Attrs.push_back(Elt: Attribute::ZExt);
953 if (CLI.IsInReg)
954 Attrs.push_back(Elt: Attribute::InReg);
955
956 return AttributeList::get(C&: CLI.RetTy->getContext(), Index: AttributeList::ReturnIndex,
957 Kinds: Attrs);
958}
959
960bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
961 unsigned NumArgs) {
962 MCContext &Ctx = MF->getContext();
963 SmallString<32> MangledName;
964 Mangler::getNameWithPrefix(OutName&: MangledName, GVName: SymName, DL);
965 MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: MangledName);
966 return lowerCallTo(CI, Symbol: Sym, NumArgs);
967}
968
969bool FastISel::lowerCallTo(const CallInst *CI, MCSymbol *Symbol,
970 unsigned NumArgs) {
971 FunctionType *FTy = CI->getFunctionType();
972 Type *RetTy = CI->getType();
973
974 ArgListTy Args;
975 Args.reserve(n: NumArgs);
976
977 // Populate the argument list.
978 // Attributes for args start at offset 1, after the return attribute.
979 for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
980 Value *V = CI->getOperand(i_nocapture: ArgI);
981
982 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
983
984 ArgListEntry Entry(V);
985 Entry.setAttributes(Call: CI, ArgIdx: ArgI);
986 Args.push_back(x: Entry);
987 }
988 TLI.markLibCallAttributes(MF, CC: CI->getCallingConv(), Args);
989
990 CallLoweringInfo CLI;
991 CLI.setCallee(ResultTy: RetTy, FuncTy: FTy, Target: Symbol, ArgsList: std::move(Args), Call: *CI, FixedArgs: NumArgs);
992
993 return lowerCallTo(CLI);
994}
995
996bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
997 // Handle the incoming return values from the call.
998 CLI.clearIns();
999 SmallVector<EVT, 4> RetTys;
1000 ComputeValueVTs(TLI, DL, Ty: CLI.RetTy, ValueVTs&: RetTys);
1001
1002 SmallVector<ISD::OutputArg, 4> Outs;
1003 GetReturnInfo(CC: CLI.CallConv, ReturnType: CLI.RetTy, attr: getReturnAttrs(CLI), Outs, TLI, DL);
1004
1005 bool CanLowerReturn = TLI.CanLowerReturn(
1006 CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext(), RetTy: CLI.RetTy);
1007
1008 // FIXME: sret demotion isn't supported yet - bail out.
1009 if (!CanLowerReturn)
1010 return false;
1011
1012 for (EVT VT : RetTys) {
1013 MVT RegisterVT = TLI.getRegisterType(Context&: CLI.RetTy->getContext(), VT);
1014 unsigned NumRegs = TLI.getNumRegisters(Context&: CLI.RetTy->getContext(), VT);
1015 for (unsigned i = 0; i != NumRegs; ++i) {
1016 ISD::ArgFlagsTy Flags;
1017 if (CLI.RetSExt)
1018 Flags.setSExt();
1019 if (CLI.RetZExt)
1020 Flags.setZExt();
1021 if (CLI.IsInReg)
1022 Flags.setInReg();
1023 ISD::InputArg Ret(Flags, RegisterVT, VT, CLI.RetTy, CLI.IsReturnValueUsed,
1024 ISD::InputArg::NoArgIndex, 0);
1025 CLI.Ins.push_back(Elt: Ret);
1026 }
1027 }
1028
1029 // Handle all of the outgoing arguments.
1030 CLI.clearOuts();
1031 for (auto &Arg : CLI.getArgs()) {
1032 Type *FinalType = Arg.Ty;
1033 if (Arg.IsByVal)
1034 FinalType = Arg.IndirectType;
1035 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1036 Ty: FinalType, CallConv: CLI.CallConv, isVarArg: CLI.IsVarArg, DL);
1037
1038 ISD::ArgFlagsTy Flags;
1039 if (Arg.IsZExt)
1040 Flags.setZExt();
1041 if (Arg.IsSExt)
1042 Flags.setSExt();
1043 if (Arg.IsInReg)
1044 Flags.setInReg();
1045 if (Arg.IsSRet)
1046 Flags.setSRet();
1047 if (Arg.IsSwiftSelf)
1048 Flags.setSwiftSelf();
1049 if (Arg.IsSwiftAsync)
1050 Flags.setSwiftAsync();
1051 if (Arg.IsSwiftError)
1052 Flags.setSwiftError();
1053 if (Arg.IsCFGuardTarget)
1054 Flags.setCFGuardTarget();
1055 if (Arg.IsByVal)
1056 Flags.setByVal();
1057 if (Arg.IsInAlloca) {
1058 Flags.setInAlloca();
1059 // Set the byval flag for CCAssignFn callbacks that don't know about
1060 // inalloca. This way we can know how many bytes we should've allocated
1061 // and how many bytes a callee cleanup function will pop. If we port
1062 // inalloca to more targets, we'll have to add custom inalloca handling in
1063 // the various CC lowering callbacks.
1064 Flags.setByVal();
1065 }
1066 if (Arg.IsPreallocated) {
1067 Flags.setPreallocated();
1068 // Set the byval flag for CCAssignFn callbacks that don't know about
1069 // preallocated. This way we can know how many bytes we should've
1070 // allocated and how many bytes a callee cleanup function will pop. If we
1071 // port preallocated to more targets, we'll have to add custom
1072 // preallocated handling in the various CC lowering callbacks.
1073 Flags.setByVal();
1074 }
1075 MaybeAlign MemAlign = Arg.Alignment;
1076 if (Arg.IsByVal || Arg.IsInAlloca || Arg.IsPreallocated) {
1077 unsigned FrameSize = DL.getTypeAllocSize(Ty: Arg.IndirectType);
1078
1079 // For ByVal, alignment should come from FE. BE will guess if this info
1080 // is not there, but there are cases it cannot get right.
1081 if (!MemAlign)
1082 MemAlign = TLI.getByValTypeAlignment(Ty: Arg.IndirectType, DL);
1083 Flags.setByValSize(FrameSize);
1084 } else if (!MemAlign) {
1085 MemAlign = DL.getABITypeAlign(Ty: Arg.Ty);
1086 }
1087 Flags.setMemAlign(*MemAlign);
1088 if (Arg.IsNest)
1089 Flags.setNest();
1090 if (NeedsRegBlock)
1091 Flags.setInConsecutiveRegs();
1092 Flags.setOrigAlign(DL.getABITypeAlign(Ty: Arg.Ty));
1093 CLI.OutVals.push_back(Elt: Arg.Val);
1094 CLI.OutFlags.push_back(Elt: Flags);
1095 }
1096
1097 if (!fastLowerCall(CLI))
1098 return false;
1099
1100 // Set all unused physreg defs as dead.
1101 assert(CLI.Call && "No call instruction specified.");
1102 CLI.Call->setPhysRegsDeadExcept(UsedRegs: CLI.InRegs, TRI);
1103
1104 if (CLI.NumResultRegs && CLI.CB)
1105 updateValueMap(I: CLI.CB, Reg: CLI.ResultReg, NumRegs: CLI.NumResultRegs);
1106
1107 // Set labels for heapallocsite call.
1108 if (CLI.CB)
1109 if (MDNode *MD = CLI.CB->getMetadata(Kind: "heapallocsite"))
1110 CLI.Call->setHeapAllocMarker(MF&: *MF, MD);
1111
1112 return true;
1113}
1114
1115bool FastISel::lowerCall(const CallInst *CI) {
1116 FunctionType *FuncTy = CI->getFunctionType();
1117 Type *RetTy = CI->getType();
1118
1119 ArgListTy Args;
1120 Args.reserve(n: CI->arg_size());
1121
1122 for (auto i = CI->arg_begin(), e = CI->arg_end(); i != e; ++i) {
1123 Value *V = *i;
1124
1125 // Skip empty types
1126 if (V->getType()->isEmptyTy())
1127 continue;
1128
1129 ArgListEntry Entry(V);
1130 // Skip the first return-type Attribute to get to params.
1131 Entry.setAttributes(Call: CI, ArgIdx: i - CI->arg_begin());
1132 Args.push_back(x: Entry);
1133 }
1134
1135 // Check if target-independent constraints permit a tail call here.
1136 // Target-dependent constraints are checked within fastLowerCall.
1137 bool IsTailCall = CI->isTailCall();
1138 if (IsTailCall && !isInTailCallPosition(Call: *CI, TM))
1139 IsTailCall = false;
1140 if (IsTailCall && !CI->isMustTailCall() &&
1141 MF->getFunction().getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
1142 IsTailCall = false;
1143
1144 CallLoweringInfo CLI;
1145 CLI.setCallee(ResultTy: RetTy, FuncTy, Target: CI->getCalledOperand(), ArgsList: std::move(Args), Call: *CI)
1146 .setTailCall(IsTailCall);
1147
1148 if (lowerCallTo(CLI)) {
1149 diagnoseDontCall(CI: *CI);
1150 return true;
1151 }
1152
1153 return false;
1154}
1155
1156bool FastISel::selectCall(const User *I) {
1157 const CallInst *Call = cast<CallInst>(Val: I);
1158
1159 // Handle simple inline asms.
1160 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: Call->getCalledOperand())) {
1161 // Don't attempt to handle constraints.
1162 if (!IA->getConstraintString().empty())
1163 return false;
1164
1165 unsigned ExtraInfo = 0;
1166 if (IA->hasSideEffects())
1167 ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1168 if (IA->isAlignStack())
1169 ExtraInfo |= InlineAsm::Extra_IsAlignStack;
1170 if (IA->canThrow())
1171 ExtraInfo |= InlineAsm::Extra_MayUnwind;
1172 if (Call->isConvergent())
1173 ExtraInfo |= InlineAsm::Extra_IsConvergent;
1174 ExtraInfo |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
1175
1176 MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1177 MCID: TII.get(Opcode: TargetOpcode::INLINEASM));
1178 MIB.addExternalSymbol(FnName: IA->getAsmString().data());
1179 MIB.addImm(Val: ExtraInfo);
1180
1181 const MDNode *SrcLoc = Call->getMetadata(Kind: "srcloc");
1182 if (SrcLoc)
1183 MIB.addMetadata(MD: SrcLoc);
1184
1185 return true;
1186 }
1187
1188 // Handle intrinsic function calls.
1189 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Call))
1190 return selectIntrinsicCall(II);
1191
1192 return lowerCall(CI: Call);
1193}
1194
1195void FastISel::handleDbgInfo(const Instruction *II) {
1196 if (!II->hasDbgRecords())
1197 return;
1198
1199 // Clear any metadata.
1200 MIMD = MIMetadata();
1201
1202 // Reverse order of debug records, because fast-isel walks through backwards.
1203 for (DbgRecord &DR : llvm::reverse(C: II->getDbgRecordRange())) {
1204 flushLocalValueMap();
1205 recomputeInsertPt();
1206
1207 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
1208 assert(DLR->getLabel() && "Missing label");
1209 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DLR->getDebugLoc(),
1210 MCID: TII.get(Opcode: TargetOpcode::DBG_LABEL))
1211 .addMetadata(MD: DLR->getLabel());
1212 continue;
1213 }
1214
1215 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
1216
1217 Value *V = nullptr;
1218 if (!DVR.hasArgList())
1219 V = DVR.getVariableLocationOp(OpIdx: 0);
1220
1221 bool Res = false;
1222 if (DVR.getType() == DbgVariableRecord::LocationType::Value ||
1223 DVR.getType() == DbgVariableRecord::LocationType::Assign) {
1224 Res = lowerDbgValue(V, Expr: DVR.getExpression(), Var: DVR.getVariable(),
1225 DL: DVR.getDebugLoc());
1226 } else {
1227 assert(DVR.getType() == DbgVariableRecord::LocationType::Declare);
1228 if (FuncInfo.PreprocessedDVRDeclares.contains(Ptr: &DVR))
1229 continue;
1230 Res = lowerDbgDeclare(V, Expr: DVR.getExpression(), Var: DVR.getVariable(),
1231 DL: DVR.getDebugLoc());
1232 }
1233
1234 if (!Res)
1235 LLVM_DEBUG(dbgs() << "Dropping debug-info for " << DVR << "\n");
1236 }
1237}
1238
1239bool FastISel::lowerDbgValue(const Value *V, DIExpression *Expr,
1240 DILocalVariable *Var, const DebugLoc &DL) {
1241 // This form of DBG_VALUE is target-independent.
1242 const MCInstrDesc &II = TII.get(Opcode: TargetOpcode::DBG_VALUE);
1243 if (!V || isa<UndefValue>(Val: V)) {
1244 // DI is either undef or cannot produce a valid DBG_VALUE, so produce an
1245 // undef DBG_VALUE to terminate any prior location.
1246 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect: false, Reg: 0U, Variable: Var, Expr);
1247 return true;
1248 }
1249 if (auto *CE = dyn_cast<ConstantExpr>(Val: V);
1250 CE && CE->getOpcode() == Instruction::IntToPtr)
1251 if (auto *Int = dyn_cast<ConstantInt>(Val: CE->getOperand(i_nocapture: 0))) {
1252 unsigned PtrSize =
1253 FuncInfo.MF->getDataLayout().getPointerTypeSizeInBits(CE->getType());
1254 V = ConstantInt::get(Context&: CE->getContext(),
1255 V: Int->getValue().zextOrTrunc(width: PtrSize));
1256 }
1257 if (const auto *CI = dyn_cast<ConstantInt>(Val: V)) {
1258 // See if there's an expression to constant-fold.
1259 if (Expr)
1260 std::tie(args&: Expr, args&: CI) = Expr->constantFold(CI);
1261 if (CI->getBitWidth() > 64)
1262 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1263 .addCImm(Val: CI)
1264 .addImm(Val: 0U)
1265 .addMetadata(MD: Var)
1266 .addMetadata(MD: Expr);
1267 else
1268 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1269 .addImm(Val: CI->getZExtValue())
1270 .addImm(Val: 0U)
1271 .addMetadata(MD: Var)
1272 .addMetadata(MD: Expr);
1273 return true;
1274 }
1275 if (isa<ConstantPointerNull>(Val: V)) {
1276 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1277 .addImm(Val: 0)
1278 .addImm(Val: 0U)
1279 .addMetadata(MD: Var)
1280 .addMetadata(MD: Expr);
1281 return true;
1282 }
1283 if (const auto *CF = dyn_cast<ConstantFP>(Val: V)) {
1284 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1285 .addFPImm(Val: CF)
1286 .addImm(Val: 0U)
1287 .addMetadata(MD: Var)
1288 .addMetadata(MD: Expr);
1289 return true;
1290 }
1291 if (const auto *Arg = dyn_cast<Argument>(Val: V);
1292 Arg && Expr && Expr->isEntryValue()) {
1293 // As per the Verifier, this case is only valid for swift async Args.
1294 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
1295
1296 Register Reg = getRegForValue(V: Arg);
1297 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
1298 if (Reg == VirtReg || Reg == PhysReg) {
1299 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect: false /*IsIndirect*/,
1300 Reg: PhysReg, Variable: Var, Expr);
1301 return true;
1302 }
1303
1304 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
1305 "couldn't find a physical register\n");
1306 return false;
1307 }
1308 if (auto SI = FuncInfo.StaticAllocaMap.find(Val: dyn_cast<AllocaInst>(Val: V));
1309 SI != FuncInfo.StaticAllocaMap.end()) {
1310 MachineOperand FrameIndexOp = MachineOperand::CreateFI(Idx: SI->second);
1311 bool IsIndirect = false;
1312 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect, MOs: FrameIndexOp,
1313 Variable: Var, Expr);
1314 return true;
1315 }
1316 if (Register Reg = lookUpRegForValue(V)) {
1317 // FIXME: This does not handle register-indirect values at offset 0.
1318 if (!FuncInfo.MF->useDebugInstrRef()) {
1319 bool IsIndirect = false;
1320 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect, Reg, Variable: Var,
1321 Expr);
1322 return true;
1323 }
1324 // If using instruction referencing, produce this as a DBG_INSTR_REF,
1325 // to be later patched up by finalizeDebugInstrRefs.
1326 SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg(
1327 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
1328 /* isKill */ false, /* isDead */ false,
1329 /* isUndef */ false, /* isEarlyClobber */ false,
1330 /* SubReg */ 0, /* isDebug */ true)});
1331 SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0});
1332 auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops);
1333 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL,
1334 MCID: TII.get(Opcode: TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs,
1335 Variable: Var, Expr: NewExpr);
1336 return true;
1337 }
1338 return false;
1339}
1340
1341bool FastISel::lowerDbgDeclare(const Value *Address, DIExpression *Expr,
1342 DILocalVariable *Var, const DebugLoc &DL) {
1343 if (!Address || isa<UndefValue>(Val: Address)) {
1344 LLVM_DEBUG(dbgs() << "Dropping debug info (bad/undef address)\n");
1345 return false;
1346 }
1347
1348 std::optional<MachineOperand> Op;
1349 if (Register Reg = lookUpRegForValue(V: Address))
1350 Op = MachineOperand::CreateReg(Reg, isDef: false);
1351
1352 // If we have a VLA that has a "use" in a metadata node that's then used
1353 // here but it has no other uses, then we have a problem. E.g.,
1354 //
1355 // int foo (const int *x) {
1356 // char a[*x];
1357 // return 0;
1358 // }
1359 //
1360 // If we assign 'a' a vreg and fast isel later on has to use the selection
1361 // DAG isel, it will want to copy the value to the vreg. However, there are
1362 // no uses, which goes counter to what selection DAG isel expects.
1363 if (!Op && !Address->use_empty() && isa<Instruction>(Val: Address) &&
1364 (!isa<AllocaInst>(Val: Address) ||
1365 !FuncInfo.StaticAllocaMap.count(Val: cast<AllocaInst>(Val: Address))))
1366 Op = MachineOperand::CreateReg(Reg: FuncInfo.InitializeRegForValue(V: Address),
1367 isDef: false);
1368
1369 if (Op) {
1370 assert(Var->isValidLocationForIntrinsic(DL) &&
1371 "Expected inlined-at fields to agree");
1372 if (FuncInfo.MF->useDebugInstrRef() && Op->isReg()) {
1373 // If using instruction referencing, produce this as a DBG_INSTR_REF,
1374 // to be later patched up by finalizeDebugInstrRefs. Tack a deref onto
1375 // the expression, we don't have an "indirect" flag in DBG_INSTR_REF.
1376 SmallVector<uint64_t, 3> Ops(
1377 {dwarf::DW_OP_LLVM_arg, 0, dwarf::DW_OP_deref});
1378 auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops);
1379 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL,
1380 MCID: TII.get(Opcode: TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs: *Op,
1381 Variable: Var, Expr: NewExpr);
1382 return true;
1383 }
1384
1385 // A dbg.declare describes the address of a source variable, so lower it
1386 // into an indirect DBG_VALUE.
1387 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL,
1388 MCID: TII.get(Opcode: TargetOpcode::DBG_VALUE), /*IsIndirect*/ true, MOs: *Op, Variable: Var,
1389 Expr);
1390 return true;
1391 }
1392
1393 // We can't yet handle anything else here because it would require
1394 // generating code, thus altering codegen because of debug info.
1395 LLVM_DEBUG(
1396 dbgs() << "Dropping debug info (no materialized reg for address)\n");
1397 return false;
1398}
1399
1400bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
1401 switch (II->getIntrinsicID()) {
1402 default:
1403 break;
1404 // At -O0 we don't care about the lifetime intrinsics.
1405 case Intrinsic::lifetime_start:
1406 case Intrinsic::lifetime_end:
1407 // The donothing intrinsic does, well, nothing.
1408 case Intrinsic::donothing:
1409 // Neither does the sideeffect intrinsic.
1410 case Intrinsic::sideeffect:
1411 // Neither does the assume intrinsic; it's also OK not to codegen its operand.
1412 case Intrinsic::assume:
1413 // Neither does the llvm.experimental.noalias.scope.decl intrinsic
1414 case Intrinsic::experimental_noalias_scope_decl:
1415 return true;
1416 case Intrinsic::objectsize:
1417 llvm_unreachable("llvm.objectsize.* should have been lowered already");
1418
1419 case Intrinsic::is_constant:
1420 llvm_unreachable("llvm.is.constant.* should have been lowered already");
1421
1422 case Intrinsic::allow_runtime_check:
1423 case Intrinsic::allow_ubsan_check: {
1424 Register ResultReg = getRegForValue(V: ConstantInt::getTrue(Ty: II->getType()));
1425 if (!ResultReg)
1426 return false;
1427 updateValueMap(I: II, Reg: ResultReg);
1428 return true;
1429 }
1430
1431 case Intrinsic::launder_invariant_group:
1432 case Intrinsic::expect:
1433 case Intrinsic::expect_with_probability: {
1434 Register ResultReg = getRegForValue(V: II->getArgOperand(i: 0));
1435 if (!ResultReg)
1436 return false;
1437 updateValueMap(I: II, Reg: ResultReg);
1438 return true;
1439 }
1440 case Intrinsic::fake_use: {
1441 const Value *V = II->getArgOperand(i: 0);
1442 if (Register Reg = getRegForValue(V))
1443 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1444 MCID: TII.get(Opcode: TargetOpcode::FAKE_USE))
1445 .addReg(RegNo: Reg);
1446 return true;
1447 }
1448 case Intrinsic::experimental_stackmap:
1449 return selectStackmap(I: II);
1450 case Intrinsic::experimental_patchpoint_void:
1451 case Intrinsic::experimental_patchpoint:
1452 return selectPatchpoint(I: II);
1453
1454 case Intrinsic::xray_customevent:
1455 return selectXRayCustomEvent(I: II);
1456 case Intrinsic::xray_typedevent:
1457 return selectXRayTypedEvent(I: II);
1458 }
1459
1460 return fastLowerIntrinsicCall(II);
1461}
1462
1463bool FastISel::selectCast(const User *I, unsigned Opcode) {
1464 EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1465 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
1466
1467 if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
1468 !DstVT.isSimple())
1469 // Unhandled type. Halt "fast" selection and bail.
1470 return false;
1471
1472 // Check if the destination type is legal.
1473 if (!TLI.isTypeLegal(VT: DstVT))
1474 return false;
1475
1476 // Check if the source operand is legal.
1477 if (!TLI.isTypeLegal(VT: SrcVT))
1478 return false;
1479
1480 Register InputReg = getRegForValue(V: I->getOperand(i: 0));
1481 if (!InputReg)
1482 // Unhandled operand. Halt "fast" selection and bail.
1483 return false;
1484
1485 Register ResultReg = fastEmit_r(VT: SrcVT.getSimpleVT(), RetVT: DstVT.getSimpleVT(),
1486 Opcode, Op0: InputReg);
1487 if (!ResultReg)
1488 return false;
1489
1490 updateValueMap(I, Reg: ResultReg);
1491 return true;
1492}
1493
1494bool FastISel::selectBitCast(const User *I) {
1495 EVT SrcEVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1496 EVT DstEVT = TLI.getValueType(DL, Ty: I->getType());
1497 if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
1498 !TLI.isTypeLegal(VT: SrcEVT) || !TLI.isTypeLegal(VT: DstEVT))
1499 // Unhandled type. Halt "fast" selection and bail.
1500 return false;
1501
1502 MVT SrcVT = SrcEVT.getSimpleVT();
1503 MVT DstVT = DstEVT.getSimpleVT();
1504 Register Op0 = getRegForValue(V: I->getOperand(i: 0));
1505 if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
1506 return false;
1507
1508 // If the bitcast doesn't change the type, just use the operand value.
1509 if (SrcVT == DstVT) {
1510 updateValueMap(I, Reg: Op0);
1511 return true;
1512 }
1513
1514 // Otherwise, select a BITCAST opcode.
1515 Register ResultReg = fastEmit_r(VT: SrcVT, RetVT: DstVT, Opcode: ISD::BITCAST, Op0);
1516 if (!ResultReg)
1517 return false;
1518
1519 updateValueMap(I, Reg: ResultReg);
1520 return true;
1521}
1522
1523bool FastISel::selectFreeze(const User *I) {
1524 Register Reg = getRegForValue(V: I->getOperand(i: 0));
1525 if (!Reg)
1526 // Unhandled operand.
1527 return false;
1528
1529 EVT ETy = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1530 if (ETy == MVT::Other || !TLI.isTypeLegal(VT: ETy))
1531 // Unhandled type, bail out.
1532 return false;
1533
1534 MVT Ty = ETy.getSimpleVT();
1535 const TargetRegisterClass *TyRegClass = TLI.getRegClassFor(VT: Ty);
1536 Register ResultReg = createResultReg(RC: TyRegClass);
1537 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1538 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ResultReg).addReg(RegNo: Reg);
1539
1540 updateValueMap(I, Reg: ResultReg);
1541 return true;
1542}
1543
1544// Remove local value instructions starting from the instruction after
1545// SavedLastLocalValue to the current function insert point.
1546void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
1547{
1548 MachineInstr *CurLastLocalValue = getLastLocalValue();
1549 if (CurLastLocalValue != SavedLastLocalValue) {
1550 // Find the first local value instruction to be deleted.
1551 // This is the instruction after SavedLastLocalValue if it is non-NULL.
1552 // Otherwise it's the first instruction in the block.
1553 MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
1554 if (SavedLastLocalValue)
1555 ++FirstDeadInst;
1556 else
1557 FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
1558 setLastLocalValue(SavedLastLocalValue);
1559 removeDeadCode(I: FirstDeadInst, E: FuncInfo.InsertPt);
1560 }
1561}
1562
1563bool FastISel::selectInstruction(const Instruction *I) {
1564 // Flush the local value map before starting each instruction.
1565 // This improves locality and debugging, and can reduce spills.
1566 // Reuse of values across IR instructions is relatively uncommon.
1567 flushLocalValueMap();
1568
1569 MachineInstr *SavedLastLocalValue = getLastLocalValue();
1570 // Just before the terminator instruction, insert instructions to
1571 // feed PHI nodes in successor blocks.
1572 if (I->isTerminator()) {
1573 if (!handlePHINodesInSuccessorBlocks(LLVMBB: I->getParent())) {
1574 // PHI node handling may have generated local value instructions,
1575 // even though it failed to handle all PHI nodes.
1576 // We remove these instructions because SelectionDAGISel will generate
1577 // them again.
1578 removeDeadLocalValueCode(SavedLastLocalValue);
1579 return false;
1580 }
1581 }
1582
1583 // FastISel does not handle any operand bundles except OB_funclet.
1584 if (auto *Call = dyn_cast<CallBase>(Val: I))
1585 for (unsigned i = 0, e = Call->getNumOperandBundles(); i != e; ++i)
1586 if (Call->getOperandBundleAt(Index: i).getTagID() != LLVMContext::OB_funclet)
1587 return false;
1588
1589 MIMD = MIMetadata(*I);
1590
1591 SavedInsertPt = FuncInfo.InsertPt;
1592
1593 if (const auto *Call = dyn_cast<CallInst>(Val: I)) {
1594 const Function *F = Call->getCalledFunction();
1595
1596 // Don't handle Intrinsic::trap if a trap function is specified.
1597 if (F && F->getIntrinsicID() == Intrinsic::trap &&
1598 Call->hasFnAttr(Kind: "trap-func-name"))
1599 return false;
1600 }
1601
1602 // First, try doing target-independent selection.
1603 if (!SkipTargetIndependentISel) {
1604 if (selectOperator(I, Opcode: I->getOpcode())) {
1605 ++NumFastIselSuccessIndependent;
1606 MIMD = {};
1607 return true;
1608 }
1609 // Remove dead code.
1610 recomputeInsertPt();
1611 if (SavedInsertPt != FuncInfo.InsertPt)
1612 removeDeadCode(I: FuncInfo.InsertPt, E: SavedInsertPt);
1613 SavedInsertPt = FuncInfo.InsertPt;
1614 }
1615 // Next, try calling the target to attempt to handle the instruction.
1616 if (fastSelectInstruction(I)) {
1617 ++NumFastIselSuccessTarget;
1618 MIMD = {};
1619 return true;
1620 }
1621 // Remove dead code.
1622 recomputeInsertPt();
1623 if (SavedInsertPt != FuncInfo.InsertPt)
1624 removeDeadCode(I: FuncInfo.InsertPt, E: SavedInsertPt);
1625
1626 MIMD = {};
1627 // Undo phi node updates, because they will be added again by SelectionDAG.
1628 if (I->isTerminator()) {
1629 // PHI node handling may have generated local value instructions.
1630 // We remove them because SelectionDAGISel will generate them again.
1631 removeDeadLocalValueCode(SavedLastLocalValue);
1632 FuncInfo.PHINodesToUpdate.resize(new_size: FuncInfo.OrigNumPHINodesToUpdate);
1633 }
1634 return false;
1635}
1636
1637/// Emit an unconditional branch to the given block, unless it is the immediate
1638/// (fall-through) successor, and update the CFG.
1639void FastISel::fastEmitBranch(MachineBasicBlock *MSucc,
1640 const DebugLoc &DbgLoc) {
1641 const BasicBlock *BB = FuncInfo.MBB->getBasicBlock();
1642 bool BlockHasMultipleInstrs = &BB->front() != &BB->back();
1643 if (BlockHasMultipleInstrs && FuncInfo.MBB->isLayoutSuccessor(MBB: MSucc)) {
1644 // For more accurate line information if this is the only non-debug
1645 // instruction in the block then emit it, otherwise we have the
1646 // unconditional fall-through case, which needs no instructions.
1647 } else {
1648 // The unconditional branch case.
1649 TII.insertBranch(MBB&: *FuncInfo.MBB, TBB: MSucc, FBB: nullptr,
1650 Cond: SmallVector<MachineOperand, 0>(), DL: DbgLoc);
1651 }
1652 if (FuncInfo.BPI) {
1653 auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
1654 Src: FuncInfo.MBB->getBasicBlock(), Dst: MSucc->getBasicBlock());
1655 FuncInfo.MBB->addSuccessor(Succ: MSucc, Prob: BranchProbability);
1656 } else
1657 FuncInfo.MBB->addSuccessorWithoutProb(Succ: MSucc);
1658}
1659
1660void FastISel::finishCondBranch(const BasicBlock *BranchBB,
1661 MachineBasicBlock *TrueMBB,
1662 MachineBasicBlock *FalseMBB) {
1663 // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
1664 // happen in degenerate IR and MachineIR forbids to have a block twice in the
1665 // successor/predecessor lists.
1666 if (TrueMBB != FalseMBB) {
1667 if (FuncInfo.BPI) {
1668 auto BranchProbability =
1669 FuncInfo.BPI->getEdgeProbability(Src: BranchBB, Dst: TrueMBB->getBasicBlock());
1670 FuncInfo.MBB->addSuccessor(Succ: TrueMBB, Prob: BranchProbability);
1671 } else
1672 FuncInfo.MBB->addSuccessorWithoutProb(Succ: TrueMBB);
1673 }
1674
1675 fastEmitBranch(MSucc: FalseMBB, DbgLoc: MIMD.getDL());
1676}
1677
1678/// Emit an FNeg operation.
1679bool FastISel::selectFNeg(const User *I, const Value *In) {
1680 Register OpReg = getRegForValue(V: In);
1681 if (!OpReg)
1682 return false;
1683
1684 // If the target has ISD::FNEG, use it.
1685 EVT VT = TLI.getValueType(DL, Ty: I->getType());
1686 Register ResultReg = fastEmit_r(VT: VT.getSimpleVT(), RetVT: VT.getSimpleVT(), Opcode: ISD::FNEG,
1687 Op0: OpReg);
1688 if (ResultReg) {
1689 updateValueMap(I, Reg: ResultReg);
1690 return true;
1691 }
1692
1693 // Bitcast the value to integer, twiddle the sign bit with xor,
1694 // and then bitcast it back to floating-point.
1695 if (VT.getSizeInBits() > 64)
1696 return false;
1697 EVT IntVT = EVT::getIntegerVT(Context&: I->getContext(), BitWidth: VT.getSizeInBits());
1698 if (!TLI.isTypeLegal(VT: IntVT))
1699 return false;
1700
1701 Register IntReg = fastEmit_r(VT: VT.getSimpleVT(), RetVT: IntVT.getSimpleVT(),
1702 Opcode: ISD::BITCAST, Op0: OpReg);
1703 if (!IntReg)
1704 return false;
1705
1706 Register IntResultReg = fastEmit_ri_(
1707 VT: IntVT.getSimpleVT(), Opcode: ISD::XOR, Op0: IntReg,
1708 UINT64_C(1) << (VT.getSizeInBits() - 1), ImmType: IntVT.getSimpleVT());
1709 if (!IntResultReg)
1710 return false;
1711
1712 ResultReg = fastEmit_r(VT: IntVT.getSimpleVT(), RetVT: VT.getSimpleVT(), Opcode: ISD::BITCAST,
1713 Op0: IntResultReg);
1714 if (!ResultReg)
1715 return false;
1716
1717 updateValueMap(I, Reg: ResultReg);
1718 return true;
1719}
1720
1721bool FastISel::selectExtractValue(const User *U) {
1722 const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: U);
1723 if (!EVI)
1724 return false;
1725
1726 // Make sure we only try to handle extracts with a legal result. But also
1727 // allow i1 because it's easy.
1728 EVT RealVT = TLI.getValueType(DL, Ty: EVI->getType(), /*AllowUnknown=*/true);
1729 if (!RealVT.isSimple())
1730 return false;
1731 MVT VT = RealVT.getSimpleVT();
1732 if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
1733 return false;
1734
1735 const Value *Op0 = EVI->getOperand(i_nocapture: 0);
1736 Type *AggTy = Op0->getType();
1737
1738 // Get the base result register.
1739 Register ResultReg;
1740 auto I = FuncInfo.ValueMap.find(Val: Op0);
1741 if (I != FuncInfo.ValueMap.end())
1742 ResultReg = I->second;
1743 else if (isa<Instruction>(Val: Op0))
1744 ResultReg = FuncInfo.InitializeRegForValue(V: Op0);
1745 else
1746 return false; // fast-isel can't handle aggregate constants at the moment
1747
1748 // Get the actual result register, which is an offset from the base register.
1749 unsigned VTIndex = ComputeLinearIndex(Ty: AggTy, Indices: EVI->getIndices());
1750
1751 SmallVector<EVT, 4> AggValueVTs;
1752 ComputeValueVTs(TLI, DL, Ty: AggTy, ValueVTs&: AggValueVTs);
1753
1754 for (unsigned i = 0; i < VTIndex; i++)
1755 ResultReg = ResultReg.id() +
1756 TLI.getNumRegisters(Context&: FuncInfo.Fn->getContext(), VT: AggValueVTs[i]);
1757
1758 updateValueMap(I: EVI, Reg: ResultReg);
1759 return true;
1760}
1761
1762bool FastISel::selectOperator(const User *I, unsigned Opcode) {
1763 switch (Opcode) {
1764 case Instruction::Add:
1765 return selectBinaryOp(I, ISDOpcode: ISD::ADD);
1766 case Instruction::FAdd:
1767 return selectBinaryOp(I, ISDOpcode: ISD::FADD);
1768 case Instruction::Sub:
1769 return selectBinaryOp(I, ISDOpcode: ISD::SUB);
1770 case Instruction::FSub:
1771 return selectBinaryOp(I, ISDOpcode: ISD::FSUB);
1772 case Instruction::Mul:
1773 return selectBinaryOp(I, ISDOpcode: ISD::MUL);
1774 case Instruction::FMul:
1775 return selectBinaryOp(I, ISDOpcode: ISD::FMUL);
1776 case Instruction::SDiv:
1777 return selectBinaryOp(I, ISDOpcode: ISD::SDIV);
1778 case Instruction::UDiv:
1779 return selectBinaryOp(I, ISDOpcode: ISD::UDIV);
1780 case Instruction::FDiv:
1781 return selectBinaryOp(I, ISDOpcode: ISD::FDIV);
1782 case Instruction::SRem:
1783 return selectBinaryOp(I, ISDOpcode: ISD::SREM);
1784 case Instruction::URem:
1785 return selectBinaryOp(I, ISDOpcode: ISD::UREM);
1786 case Instruction::FRem:
1787 return selectBinaryOp(I, ISDOpcode: ISD::FREM);
1788 case Instruction::Shl:
1789 return selectBinaryOp(I, ISDOpcode: ISD::SHL);
1790 case Instruction::LShr:
1791 return selectBinaryOp(I, ISDOpcode: ISD::SRL);
1792 case Instruction::AShr:
1793 return selectBinaryOp(I, ISDOpcode: ISD::SRA);
1794 case Instruction::And:
1795 return selectBinaryOp(I, ISDOpcode: ISD::AND);
1796 case Instruction::Or:
1797 return selectBinaryOp(I, ISDOpcode: ISD::OR);
1798 case Instruction::Xor:
1799 return selectBinaryOp(I, ISDOpcode: ISD::XOR);
1800
1801 case Instruction::FNeg:
1802 return selectFNeg(I, In: I->getOperand(i: 0));
1803
1804 case Instruction::GetElementPtr:
1805 return selectGetElementPtr(I);
1806
1807 case Instruction::UncondBr: {
1808 const UncondBrInst *BI = cast<UncondBrInst>(Val: I);
1809 const BasicBlock *LLVMSucc = BI->getSuccessor(i: 0);
1810 MachineBasicBlock *MSucc = FuncInfo.getMBB(BB: LLVMSucc);
1811 fastEmitBranch(MSucc, DbgLoc: BI->getDebugLoc());
1812 return true;
1813 }
1814
1815 case Instruction::Unreachable: {
1816 auto UI = cast<UnreachableInst>(Val: I);
1817 if (!UI->shouldLowerToTrap(TrapUnreachable: TM.Options.TrapUnreachable,
1818 NoTrapAfterNoreturn: TM.Options.NoTrapAfterNoreturn))
1819 return true;
1820
1821 return fastEmit_(VT: MVT::Other, RetVT: MVT::Other, Opcode: ISD::TRAP) != 0;
1822 }
1823
1824 case Instruction::Alloca:
1825 // FunctionLowering has the static-sized case covered.
1826 if (FuncInfo.StaticAllocaMap.count(Val: cast<AllocaInst>(Val: I)))
1827 return true;
1828
1829 // Dynamic-sized alloca is not handled yet.
1830 return false;
1831
1832 case Instruction::Call:
1833 // On AIX, normal call lowering uses the DAG-ISEL path currently so that the
1834 // callee of the direct function call instruction will be mapped to the
1835 // symbol for the function's entry point, which is distinct from the
1836 // function descriptor symbol. The latter is the symbol whose XCOFF symbol
1837 // name is the C-linkage name of the source level function.
1838 // But fast isel still has the ability to do selection for intrinsics.
1839 if (TM.getTargetTriple().isOSAIX() && !isa<IntrinsicInst>(Val: I))
1840 return false;
1841 return selectCall(I);
1842
1843 case Instruction::BitCast:
1844 return selectBitCast(I);
1845
1846 case Instruction::FPToSI:
1847 return selectCast(I, Opcode: ISD::FP_TO_SINT);
1848 case Instruction::ZExt:
1849 return selectCast(I, Opcode: ISD::ZERO_EXTEND);
1850 case Instruction::SExt:
1851 return selectCast(I, Opcode: ISD::SIGN_EXTEND);
1852 case Instruction::Trunc:
1853 return selectCast(I, Opcode: ISD::TRUNCATE);
1854 case Instruction::SIToFP:
1855 return selectCast(I, Opcode: ISD::SINT_TO_FP);
1856
1857 case Instruction::IntToPtr: // Deliberate fall-through.
1858 case Instruction::PtrToInt:
1859 case Instruction::PtrToAddr: {
1860 EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1861 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
1862 if (DstVT.bitsGT(VT: SrcVT))
1863 return selectCast(I, Opcode: ISD::ZERO_EXTEND);
1864 if (DstVT.bitsLT(VT: SrcVT))
1865 return selectCast(I, Opcode: ISD::TRUNCATE);
1866 Register Reg = getRegForValue(V: I->getOperand(i: 0));
1867 if (!Reg)
1868 return false;
1869 updateValueMap(I, Reg);
1870 return true;
1871 }
1872
1873 case Instruction::ExtractValue:
1874 return selectExtractValue(U: I);
1875
1876 case Instruction::Freeze:
1877 return selectFreeze(I);
1878
1879 case Instruction::PHI:
1880 llvm_unreachable("FastISel shouldn't visit PHI nodes!");
1881
1882 default:
1883 // Unhandled instruction. Halt "fast" selection and bail.
1884 return false;
1885 }
1886}
1887
1888FastISel::FastISel(FunctionLoweringInfo &FuncInfo,
1889 const TargetLibraryInfo *LibInfo,
1890 const LibcallLoweringInfo *LibcallLowering,
1891 bool SkipTargetIndependentISel)
1892 : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
1893 MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
1894 TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
1895 TII(*MF->getSubtarget().getInstrInfo()),
1896 TLI(*MF->getSubtarget().getTargetLowering()),
1897 TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
1898 LibcallLowering(LibcallLowering),
1899 SkipTargetIndependentISel(SkipTargetIndependentISel) {}
1900
1901FastISel::~FastISel() = default;
1902
1903bool FastISel::fastLowerArguments() { return false; }
1904
1905bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
1906
1907bool FastISel::fastLowerIntrinsicCall(const IntrinsicInst * /*II*/) {
1908 return false;
1909}
1910
1911Register FastISel::fastEmit_(MVT, MVT, unsigned) { return Register(); }
1912
1913Register FastISel::fastEmit_r(MVT, MVT, unsigned, Register /*Op0*/) {
1914 return Register();
1915}
1916
1917Register FastISel::fastEmit_rr(MVT, MVT, unsigned, Register /*Op0*/,
1918 Register /*Op1*/) {
1919 return Register();
1920}
1921
1922Register FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
1923 return Register();
1924}
1925
1926Register FastISel::fastEmit_f(MVT, MVT, unsigned,
1927 const ConstantFP * /*FPImm*/) {
1928 return Register();
1929}
1930
1931Register FastISel::fastEmit_ri(MVT, MVT, unsigned, Register /*Op0*/,
1932 uint64_t /*Imm*/) {
1933 return Register();
1934}
1935
1936/// This method is a wrapper of fastEmit_ri. It first tries to emit an
1937/// instruction with an immediate operand using fastEmit_ri.
1938/// If that fails, it materializes the immediate into a register and try
1939/// fastEmit_rr instead.
1940Register FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, Register Op0,
1941 uint64_t Imm, MVT ImmType) {
1942 // If this is a multiply by a power of two, emit this as a shift left.
1943 if (Opcode == ISD::MUL && isPowerOf2_64(Value: Imm)) {
1944 Opcode = ISD::SHL;
1945 Imm = Log2_64(Value: Imm);
1946 } else if (Opcode == ISD::UDIV && isPowerOf2_64(Value: Imm)) {
1947 // div x, 8 -> srl x, 3
1948 Opcode = ISD::SRL;
1949 Imm = Log2_64(Value: Imm);
1950 }
1951
1952 // Horrible hack (to be removed), check to make sure shift amounts are
1953 // in-range.
1954 if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
1955 Imm >= VT.getSizeInBits())
1956 return Register();
1957
1958 // First check if immediate type is legal. If not, we can't use the ri form.
1959 Register ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Imm);
1960 if (ResultReg)
1961 return ResultReg;
1962 Register MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
1963 if (!MaterialReg) {
1964 // This is a bit ugly/slow, but failing here means falling out of
1965 // fast-isel, which would be very slow.
1966 IntegerType *ITy =
1967 IntegerType::get(C&: FuncInfo.Fn->getContext(), NumBits: VT.getSizeInBits());
1968 // TODO: Avoid implicit trunc?
1969 // See https://github.com/llvm/llvm-project/issues/112510.
1970 MaterialReg = getRegForValue(
1971 V: ConstantInt::get(Ty: ITy, V: Imm, /*IsSigned=*/false, /*ImplicitTrunc=*/true));
1972 if (!MaterialReg)
1973 return Register();
1974 }
1975 return fastEmit_rr(VT, VT, Opcode, Op0, MaterialReg);
1976}
1977
1978Register FastISel::createResultReg(const TargetRegisterClass *RC) {
1979 return MRI.createVirtualRegister(RegClass: RC);
1980}
1981
1982Register FastISel::constrainOperandRegClass(const MCInstrDesc &II, Register Op,
1983 unsigned OpNum) {
1984 if (Op.isVirtual()) {
1985 const TargetRegisterClass *RegClass = TII.getRegClass(MCID: II, OpNum);
1986 if (!MRI.constrainRegClass(Reg: Op, RC: RegClass)) {
1987 // If it's not legal to COPY between the register classes, something
1988 // has gone very wrong before we got here.
1989 Register NewOp = createResultReg(RC: RegClass);
1990 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1991 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: NewOp).addReg(RegNo: Op);
1992 return NewOp;
1993 }
1994 }
1995 return Op;
1996}
1997
1998Register FastISel::fastEmitInst_(unsigned MachineInstOpcode,
1999 const TargetRegisterClass *RC) {
2000 Register ResultReg = createResultReg(RC);
2001 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2002
2003 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2004 ->setImplicitPhysRegDefsDead();
2005 return ResultReg;
2006}
2007
2008Register FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
2009 const TargetRegisterClass *RC, Register Op0) {
2010 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2011
2012 Register ResultReg = createResultReg(RC);
2013 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2014
2015 assert(II.getNumDefs() >= 1 && "instruction must define the result");
2016 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2017 .addReg(RegNo: Op0)
2018 ->setImplicitPhysRegDefsDead();
2019
2020 return ResultReg;
2021}
2022
2023Register FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
2024 const TargetRegisterClass *RC, Register Op0,
2025 Register Op1) {
2026 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2027
2028 Register ResultReg = createResultReg(RC);
2029 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2030 Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
2031
2032 assert(II.getNumDefs() >= 1 && "instruction must define the result");
2033 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2034 .addReg(RegNo: Op0)
2035 .addReg(RegNo: Op1)
2036 ->setImplicitPhysRegDefsDead();
2037 return ResultReg;
2038}
2039
2040Register FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
2041 const TargetRegisterClass *RC, Register Op0,
2042 Register Op1, Register Op2) {
2043 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2044
2045 Register ResultReg = createResultReg(RC);
2046 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2047 Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
2048 Op2 = constrainOperandRegClass(II, Op: Op2, OpNum: II.getNumDefs() + 2);
2049
2050 assert(II.getNumDefs() >= 1 && "instruction must define the result");
2051 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2052 .addReg(RegNo: Op0)
2053 .addReg(RegNo: Op1)
2054 .addReg(RegNo: Op2)
2055 ->setImplicitPhysRegDefsDead();
2056 return ResultReg;
2057}
2058
2059Register FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
2060 const TargetRegisterClass *RC, Register Op0,
2061 uint64_t Imm) {
2062 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2063
2064 Register ResultReg = createResultReg(RC);
2065 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2066
2067 assert(II.getNumDefs() >= 1 && "instruction must define the result");
2068 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2069 .addReg(RegNo: Op0)
2070 .addImm(Val: Imm)
2071 ->setImplicitPhysRegDefsDead();
2072 return ResultReg;
2073}
2074
2075Register FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
2076 const TargetRegisterClass *RC, Register Op0,
2077 uint64_t Imm1, uint64_t Imm2) {
2078 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2079
2080 Register ResultReg = createResultReg(RC);
2081 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2082
2083 assert(II.getNumDefs() >= 1 && "instruction must define the result");
2084 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2085 .addReg(RegNo: Op0)
2086 .addImm(Val: Imm1)
2087 .addImm(Val: Imm2)
2088 ->setImplicitPhysRegDefsDead();
2089 return ResultReg;
2090}
2091
2092Register FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
2093 const TargetRegisterClass *RC,
2094 const ConstantFP *FPImm) {
2095 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2096
2097 Register ResultReg = createResultReg(RC);
2098
2099 assert(II.getNumDefs() >= 1 && "instruction must define the result");
2100 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2101 .addFPImm(Val: FPImm)
2102 ->setImplicitPhysRegDefsDead();
2103 return ResultReg;
2104}
2105
2106Register FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
2107 const TargetRegisterClass *RC, Register Op0,
2108 Register Op1, uint64_t Imm) {
2109 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2110
2111 Register ResultReg = createResultReg(RC);
2112 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2113 Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
2114
2115 assert(II.getNumDefs() >= 1 && "instruction must define the result");
2116 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2117 .addReg(RegNo: Op0)
2118 .addReg(RegNo: Op1)
2119 .addImm(Val: Imm)
2120 ->setImplicitPhysRegDefsDead();
2121 return ResultReg;
2122}
2123
2124Register FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
2125 const TargetRegisterClass *RC, uint64_t Imm) {
2126 Register ResultReg = createResultReg(RC);
2127 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2128
2129 assert(II.getNumDefs() >= 1 && "instruction must define the result");
2130 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2131 .addImm(Val: Imm)
2132 ->setImplicitPhysRegDefsDead();
2133 return ResultReg;
2134}
2135
2136Register FastISel::fastEmitInst_extractsubreg(MVT RetVT, Register Op0,
2137 uint32_t Idx) {
2138 Register ResultReg = createResultReg(RC: TLI.getRegClassFor(VT: RetVT));
2139 assert(Op0.isVirtual() && "Cannot yet extract from physregs");
2140 const TargetRegisterClass *RC = MRI.getRegClass(Reg: Op0);
2141 MRI.constrainRegClass(Reg: Op0, RC: TRI.getSubClassWithSubReg(RC, Idx));
2142 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2143 DestReg: ResultReg)
2144 .addReg(RegNo: Op0, Flags: {}, SubReg: Idx);
2145 return ResultReg;
2146}
2147
2148/// Emit MachineInstrs to compute the value of Op with all but the least
2149/// significant bit set to zero.
2150Register FastISel::fastEmitZExtFromI1(MVT VT, Register Op0) {
2151 return fastEmit_ri(VT, VT, ISD::AND, Op0, 1);
2152}
2153
2154/// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
2155/// Emit code to ensure constants are copied into registers when needed.
2156/// Remember the virtual registers that need to be added to the Machine PHI
2157/// nodes as input. We cannot just directly add them, because expansion
2158/// might result in multiple MBB's for one BB. As such, the start of the
2159/// BB might correspond to a different MBB than the end.
2160bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
2161 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
2162 FuncInfo.OrigNumPHINodesToUpdate = FuncInfo.PHINodesToUpdate.size();
2163
2164 // Check successor nodes' PHI nodes that expect a constant to be available
2165 // from this block.
2166 for (const BasicBlock *SuccBB : successors(BB: LLVMBB)) {
2167 if (!isa<PHINode>(Val: SuccBB->begin()))
2168 continue;
2169 MachineBasicBlock *SuccMBB = FuncInfo.getMBB(BB: SuccBB);
2170
2171 // If this terminator has multiple identical successors (common for
2172 // switches), only handle each succ once.
2173 if (!SuccsHandled.insert(Ptr: SuccMBB).second)
2174 continue;
2175
2176 MachineBasicBlock::iterator MBBI = SuccMBB->begin();
2177
2178 // At this point we know that there is a 1-1 correspondence between LLVM PHI
2179 // nodes and Machine PHI nodes, but the incoming operands have not been
2180 // emitted yet.
2181 for (const PHINode &PN : SuccBB->phis()) {
2182 // Ignore dead phi's.
2183 if (PN.use_empty())
2184 continue;
2185
2186 // Only handle legal types. Two interesting things to note here. First,
2187 // by bailing out early, we may leave behind some dead instructions,
2188 // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
2189 // own moves. Second, this check is necessary because FastISel doesn't
2190 // use CreateRegs to create registers, so it always creates
2191 // exactly one register for each non-void instruction.
2192 EVT VT = TLI.getValueType(DL, Ty: PN.getType(), /*AllowUnknown=*/true);
2193 if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
2194 // Handle integer promotions, though, because they're common and easy.
2195 if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
2196 FuncInfo.PHINodesToUpdate.resize(new_size: FuncInfo.OrigNumPHINodesToUpdate);
2197 return false;
2198 }
2199 }
2200
2201 const Value *PHIOp = PN.getIncomingValueForBlock(BB: LLVMBB);
2202
2203 // Set the DebugLoc for the copy. Use the location of the operand if
2204 // there is one; otherwise no location, flushLocalValueMap will fix it.
2205 MIMD = {};
2206 if (const auto *Inst = dyn_cast<Instruction>(Val: PHIOp))
2207 MIMD = MIMetadata(*Inst);
2208
2209 Register Reg = getRegForValue(V: PHIOp);
2210 if (!Reg) {
2211 FuncInfo.PHINodesToUpdate.resize(new_size: FuncInfo.OrigNumPHINodesToUpdate);
2212 return false;
2213 }
2214 FuncInfo.PHINodesToUpdate.emplace_back(args: &*MBBI++, args&: Reg);
2215 MIMD = {};
2216 }
2217 }
2218
2219 return true;
2220}
2221
2222bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
2223 assert(LI->hasOneUse() &&
2224 "tryToFoldLoad expected a LoadInst with a single use");
2225 // We know that the load has a single use, but don't know what it is. If it
2226 // isn't one of the folded instructions, then we can't succeed here. Handle
2227 // this by scanning the single-use users of the load until we get to FoldInst.
2228 unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
2229
2230 const Instruction *TheUser = LI->user_back();
2231 while (TheUser != FoldInst && // Scan up until we find FoldInst.
2232 // Stay in the right block.
2233 TheUser->getParent() == FoldInst->getParent() &&
2234 --MaxUsers) { // Don't scan too far.
2235 // If there are multiple or no uses of this instruction, then bail out.
2236 if (!TheUser->hasOneUse())
2237 return false;
2238
2239 TheUser = TheUser->user_back();
2240 }
2241
2242 // If we didn't find the fold instruction, then we failed to collapse the
2243 // sequence.
2244 if (TheUser != FoldInst)
2245 return false;
2246
2247 // Don't try to fold ordered loads. Target has to deal with alignment
2248 // constraints and synchronization.
2249 if (!LI->isUnordered())
2250 return false;
2251
2252 // Where the target supports swifterror, the value lives in a register, so a
2253 // load from it can't become a memory operand.
2254 if (TLI.supportSwiftError() && LI->getPointerOperand()->isSwiftError())
2255 return false;
2256
2257 // Figure out which vreg this is going into. If there is no assigned vreg yet
2258 // then there actually was no reference to it. Perhaps the load is referenced
2259 // by a dead instruction.
2260 Register LoadReg = getRegForValue(V: LI);
2261 if (!LoadReg)
2262 return false;
2263
2264 // We can't fold if this vreg has no uses or more than one non-debug use.
2265 // Multiple uses may mean that the instruction got lowered to multiple MIs, or
2266 // the use of the loaded value ended up being multiple operands of the result.
2267 if (!MRI.hasOneNonDBGUse(RegNo: LoadReg))
2268 return false;
2269
2270 // If the register has fixups, there may be additional uses through a
2271 // different alias of the register.
2272 if (FuncInfo.RegsWithFixups.contains(V: LoadReg))
2273 return false;
2274
2275 MachineRegisterInfo::use_nodbg_iterator RI = MRI.use_nodbg_begin(RegNo: LoadReg);
2276 MachineInstr *User = RI->getParent();
2277
2278 // Set the insertion point properly. Folding the load can cause generation of
2279 // other random instructions (like sign extends) for addressing modes; make
2280 // sure they get inserted in a logical place before the new instruction.
2281 FuncInfo.InsertPt = User;
2282 FuncInfo.MBB = User->getParent();
2283
2284 // Ask the target to try folding the load.
2285 if (!tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI))
2286 return false;
2287
2288 // The loaded value no longer lives in LoadReg.
2289 MRI.markUsesInDebugValueAsUndef(Reg: LoadReg);
2290 return true;
2291}
2292
2293bool FastISel::canFoldAddIntoGEP(const User *GEP, const Value *Add) {
2294 // Must be an add.
2295 if (!isa<AddOperator>(Val: Add))
2296 return false;
2297 // Type size needs to match.
2298 if (DL.getTypeSizeInBits(Ty: GEP->getType()) !=
2299 DL.getTypeSizeInBits(Ty: Add->getType()))
2300 return false;
2301 // Must be in the same basic block.
2302 if (isa<Instruction>(Val: Add) &&
2303 FuncInfo.getMBB(BB: cast<Instruction>(Val: Add)->getParent()) != FuncInfo.MBB)
2304 return false;
2305 // Must have a constant operand.
2306 return isa<ConstantInt>(Val: cast<AddOperator>(Val: Add)->getOperand(i_nocapture: 1));
2307}
2308
2309MachineMemOperand *
2310FastISel::createMachineMemOperandFor(const Instruction *I) const {
2311 const Value *Ptr;
2312 Type *ValTy;
2313 MaybeAlign Alignment;
2314 MachineMemOperand::Flags Flags;
2315 bool IsVolatile;
2316
2317 if (const auto *LI = dyn_cast<LoadInst>(Val: I)) {
2318 Alignment = LI->getAlign();
2319 IsVolatile = LI->isVolatile();
2320 Flags = MachineMemOperand::MOLoad;
2321 Ptr = LI->getPointerOperand();
2322 ValTy = LI->getType();
2323 } else if (const auto *SI = dyn_cast<StoreInst>(Val: I)) {
2324 Alignment = SI->getAlign();
2325 IsVolatile = SI->isVolatile();
2326 Flags = MachineMemOperand::MOStore;
2327 Ptr = SI->getPointerOperand();
2328 ValTy = SI->getValueOperand()->getType();
2329 } else
2330 return nullptr;
2331
2332 bool IsNonTemporal = I->hasMetadata(KindID: LLVMContext::MD_nontemporal);
2333 bool IsInvariant = I->hasMetadata(KindID: LLVMContext::MD_invariant_load);
2334 const MDNode *Ranges = I->getMetadata(KindID: LLVMContext::MD_range);
2335
2336 AAMDNodes AAInfo = I->getAAMetadata();
2337
2338 if (!Alignment) // Ensure that codegen never sees alignment 0.
2339 Alignment = DL.getABITypeAlign(Ty: ValTy);
2340
2341 unsigned Size = DL.getTypeStoreSize(Ty: ValTy);
2342
2343 if (IsVolatile)
2344 Flags |= MachineMemOperand::MOVolatile;
2345 if (IsNonTemporal)
2346 Flags |= MachineMemOperand::MONonTemporal;
2347 if (IsInvariant)
2348 Flags |= MachineMemOperand::MOInvariant;
2349
2350 return FuncInfo.MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Ptr), F: Flags, Size,
2351 BaseAlignment: *Alignment,
2352 Metadata: MMOMetadata(AAInfo, Ranges));
2353}
2354
2355CmpInst::Predicate FastISel::optimizeCmpPredicate(const CmpInst *CI) const {
2356 // If both operands are the same, then try to optimize or fold the cmp.
2357 CmpInst::Predicate Predicate = CI->getPredicate();
2358 if (CI->getOperand(i_nocapture: 0) != CI->getOperand(i_nocapture: 1))
2359 return Predicate;
2360
2361 switch (Predicate) {
2362 default: llvm_unreachable("Invalid predicate!");
2363 case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
2364 case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break;
2365 case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break;
2366 case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break;
2367 case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break;
2368 case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break;
2369 case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break;
2370 case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break;
2371 case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break;
2372 case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break;
2373 case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break;
2374 case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2375 case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break;
2376 case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2377 case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break;
2378 case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break;
2379
2380 case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break;
2381 case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break;
2382 case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break;
2383 case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2384 case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break;
2385 case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2386 case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break;
2387 case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break;
2388 case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break;
2389 case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break;
2390 }
2391
2392 return Predicate;
2393}
2394