1//===- FastISel.cpp - Implementation of the FastISel class ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the FastISel class.
10//
11// "Fast" instruction selection is designed to emit very poor code quickly.
12// Also, it is not designed to be able to do much lowering, so most illegal
13// types (e.g. i64 on 32-bit targets) and operations are not supported. It is
14// also not intended to be able to do much optimization, except in a few cases
15// where doing optimizations reduces overall compile time. For example, folding
16// constants into immediate fields is often done, because it's cheap and it
17// reduces the number of instructions later phases have to examine.
18//
19// "Fast" instruction selection is able to fail gracefully and transfer
20// control to the SelectionDAG selector for operations that it doesn't
21// support. In many cases, this allows us to avoid duplicating a lot of
22// the complicated lowering logic that SelectionDAG currently has.
23//
24// The intended use for "fast" instruction selection is "-O0" mode
25// compilation, where the quality of the generated code is irrelevant when
26// weighed against the speed at which the code can be generated. Also,
27// at -O0, the LLVM optimizers are not running, and this makes the
28// compile time of codegen a much higher portion of the overall compile
29// time. Despite its limitations, "fast" instruction selection is able to
30// handle enough code on its own to provide noticeable overall speedups
31// in -O0 compiles.
32//
33// Basic operations are supported in a target-independent way, by reading
34// the same instruction descriptions that the SelectionDAG selector reads,
35// and identifying simple arithmetic operations that can be directly selected
36// from simple operators. More complicated operations currently require
37// target-specific code.
38//
39//===----------------------------------------------------------------------===//
40
41#include "llvm/CodeGen/FastISel.h"
42#include "llvm/ADT/APFloat.h"
43#include "llvm/ADT/APSInt.h"
44#include "llvm/ADT/DenseMap.h"
45#include "llvm/ADT/SmallPtrSet.h"
46#include "llvm/ADT/SmallString.h"
47#include "llvm/ADT/SmallVector.h"
48#include "llvm/ADT/Statistic.h"
49#include "llvm/Analysis/BranchProbabilityInfo.h"
50#include "llvm/Analysis/TargetLibraryInfo.h"
51#include "llvm/CodeGen/Analysis.h"
52#include "llvm/CodeGen/FunctionLoweringInfo.h"
53#include "llvm/CodeGen/ISDOpcodes.h"
54#include "llvm/CodeGen/MachineBasicBlock.h"
55#include "llvm/CodeGen/MachineFrameInfo.h"
56#include "llvm/CodeGen/MachineInstr.h"
57#include "llvm/CodeGen/MachineInstrBuilder.h"
58#include "llvm/CodeGen/MachineMemOperand.h"
59#include "llvm/CodeGen/MachineModuleInfo.h"
60#include "llvm/CodeGen/MachineOperand.h"
61#include "llvm/CodeGen/MachineRegisterInfo.h"
62#include "llvm/CodeGen/StackMaps.h"
63#include "llvm/CodeGen/TargetInstrInfo.h"
64#include "llvm/CodeGen/TargetLowering.h"
65#include "llvm/CodeGen/TargetSubtargetInfo.h"
66#include "llvm/CodeGen/ValueTypes.h"
67#include "llvm/CodeGenTypes/MachineValueType.h"
68#include "llvm/IR/Argument.h"
69#include "llvm/IR/Attributes.h"
70#include "llvm/IR/BasicBlock.h"
71#include "llvm/IR/CallingConv.h"
72#include "llvm/IR/Constant.h"
73#include "llvm/IR/Constants.h"
74#include "llvm/IR/DataLayout.h"
75#include "llvm/IR/DebugLoc.h"
76#include "llvm/IR/DerivedTypes.h"
77#include "llvm/IR/DiagnosticInfo.h"
78#include "llvm/IR/Function.h"
79#include "llvm/IR/GetElementPtrTypeIterator.h"
80#include "llvm/IR/GlobalValue.h"
81#include "llvm/IR/InlineAsm.h"
82#include "llvm/IR/InstrTypes.h"
83#include "llvm/IR/Instruction.h"
84#include "llvm/IR/Instructions.h"
85#include "llvm/IR/IntrinsicInst.h"
86#include "llvm/IR/LLVMContext.h"
87#include "llvm/IR/Mangler.h"
88#include "llvm/IR/Metadata.h"
89#include "llvm/IR/Module.h"
90#include "llvm/IR/Operator.h"
91#include "llvm/IR/PatternMatch.h"
92#include "llvm/IR/Type.h"
93#include "llvm/IR/User.h"
94#include "llvm/IR/Value.h"
95#include "llvm/MC/MCContext.h"
96#include "llvm/MC/MCInstrDesc.h"
97#include "llvm/Support/Casting.h"
98#include "llvm/Support/Debug.h"
99#include "llvm/Support/ErrorHandling.h"
100#include "llvm/Support/MathExtras.h"
101#include "llvm/Support/raw_ostream.h"
102#include "llvm/Target/TargetMachine.h"
103#include "llvm/Target/TargetOptions.h"
104#include <cassert>
105#include <cstdint>
106#include <iterator>
107#include <optional>
108#include <utility>
109
110using namespace llvm;
111using namespace PatternMatch;
112
113#define DEBUG_TYPE "isel"
114
115STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
116 "target-independent selector");
117STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
118 "target-specific selector");
119STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
120
121/// Set the current block to which generated machine instructions will be
122/// appended.
123void FastISel::startNewBlock() {
124 assert(LocalValueMap.empty() &&
125 "local values should be cleared after finishing a BB");
126
127 // Instructions are appended to FuncInfo.MBB. If the basic block already
128 // contains labels or copies, use the last instruction as the last local
129 // value.
130 EmitStartPt = nullptr;
131 if (!FuncInfo.MBB->empty())
132 EmitStartPt = &FuncInfo.MBB->back();
133 LastLocalValue = EmitStartPt;
134}
135
136void FastISel::finishBasicBlock() { flushLocalValueMap(); }
137
138bool FastISel::lowerArguments() {
139 if (!FuncInfo.CanLowerReturn)
140 // Fallback to SDISel argument lowering code to deal with sret pointer
141 // parameter.
142 return false;
143
144 if (!fastLowerArguments())
145 return false;
146
147 // Enter arguments into ValueMap for uses in non-entry BBs.
148 for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(),
149 E = FuncInfo.Fn->arg_end();
150 I != E; ++I) {
151 auto VI = LocalValueMap.find(Val: &*I);
152 assert(VI != LocalValueMap.end() && "Missed an argument?");
153 FuncInfo.ValueMap[&*I] = VI->second;
154 }
155 return true;
156}
157
158/// Return the defined register if this instruction defines exactly one
159/// virtual register and uses no other virtual registers. Otherwise return
160/// Register();
161static Register findLocalRegDef(MachineInstr &MI) {
162 Register RegDef;
163 for (const MachineOperand &MO : MI.operands()) {
164 if (!MO.isReg())
165 continue;
166 if (MO.isDef()) {
167 if (RegDef)
168 return Register();
169 RegDef = MO.getReg();
170 } else if (MO.getReg().isVirtual()) {
171 // This is another use of a vreg. Don't delete it.
172 return Register();
173 }
174 }
175 return RegDef;
176}
177
178static bool isRegUsedByPhiNodes(Register DefReg,
179 FunctionLoweringInfo &FuncInfo) {
180 for (auto &P : FuncInfo.PHINodesToUpdate)
181 if (P.second == DefReg)
182 return true;
183 return false;
184}
185
186void FastISel::flushLocalValueMap() {
187 // If FastISel bails out, it could leave local value instructions behind
188 // that aren't used for anything. Detect and erase those.
189 if (LastLocalValue != EmitStartPt) {
190 // Save the first instruction after local values, for later.
191 MachineBasicBlock::iterator FirstNonValue(LastLocalValue);
192 ++FirstNonValue;
193
194 MachineBasicBlock::reverse_iterator RE =
195 EmitStartPt ? MachineBasicBlock::reverse_iterator(EmitStartPt)
196 : FuncInfo.MBB->rend();
197 MachineBasicBlock::reverse_iterator RI(LastLocalValue);
198 for (MachineInstr &LocalMI :
199 llvm::make_early_inc_range(Range: llvm::make_range(x: RI, y: RE))) {
200 Register DefReg = findLocalRegDef(MI&: LocalMI);
201 if (!DefReg)
202 continue;
203 if (FuncInfo.RegsWithFixups.count(V: DefReg))
204 continue;
205 bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
206 if (!UsedByPHI && MRI.use_nodbg_empty(RegNo: DefReg)) {
207 if (EmitStartPt == &LocalMI)
208 EmitStartPt = EmitStartPt->getPrevNode();
209 LLVM_DEBUG(dbgs() << "removing dead local value materialization"
210 << LocalMI);
211 LocalMI.eraseFromParent();
212 }
213 }
214
215 // See if there are any local value instructions left. If so, we want to
216 // make sure the first one has a debug location; if it doesn't, use the
217 // first non-value instruction's debug location.
218
219 // If EmitStartPt is non-null, this block had copies at the top before
220 // FastISel started doing anything; it points to the last one, so the
221 // first local value instruction is the one after EmitStartPt.
222 // If EmitStartPt is null, the first local value instruction is at the
223 // top of the block.
224 MachineBasicBlock::iterator FirstLocalValue =
225 EmitStartPt ? ++MachineBasicBlock::iterator(EmitStartPt)
226 : FuncInfo.MBB->begin();
227 if (FirstLocalValue != FirstNonValue && !FirstLocalValue->getDebugLoc()) {
228 if (FirstNonValue != FuncInfo.MBB->end()) {
229 FirstLocalValue->setDebugLoc(FirstNonValue->getDebugLoc());
230 } else if (const BasicBlock *BB = FuncInfo.MBB->getBasicBlock()) {
231 // Nothing follows them, e.g. a block only setting up a successor's PHI
232 // nodes before falling through. Use the terminator's location.
233 FirstLocalValue->setDebugLoc(BB->getTerminator()->getDebugLoc());
234 }
235 }
236 }
237
238 LocalValueMap.clear();
239 LastLocalValue = EmitStartPt;
240 recomputeInsertPt();
241 SavedInsertPt = FuncInfo.InsertPt;
242}
243
244Register FastISel::getRegForValue(const Value *V) {
245 EVT RealVT = TLI.getValueType(DL, Ty: V->getType(), /*AllowUnknown=*/true);
246 // Don't handle non-simple values in FastISel.
247 if (!RealVT.isSimple())
248 return Register();
249
250 // Ignore illegal types. We must do this before looking up the value
251 // in ValueMap because Arguments are given virtual registers regardless
252 // of whether FastISel can handle them.
253 MVT VT = RealVT.getSimpleVT();
254 if (!TLI.isTypeLegal(VT)) {
255 // Handle integer promotions, though, because they're common and easy.
256 if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
257 VT = TLI.getTypeToTransformTo(Context&: V->getContext(), VT).getSimpleVT();
258 else
259 return Register();
260 }
261
262 // Look up the value to see if we already have a register for it.
263 Register Reg = lookUpRegForValue(V);
264 if (Reg)
265 return Reg;
266
267 // In bottom-up mode, just create the virtual register which will be used
268 // to hold the value. It will be materialized later.
269 if (isa<Instruction>(Val: V) &&
270 (!isa<AllocaInst>(Val: V) ||
271 !FuncInfo.StaticAllocaMap.count(Val: cast<AllocaInst>(Val: V))))
272 return FuncInfo.InitializeRegForValue(V);
273
274 SavePoint SaveInsertPt = enterLocalValueArea();
275
276 // Materialize the value in a register. Emit any instructions in the
277 // local value area.
278 Reg = materializeRegForValue(V, VT);
279
280 leaveLocalValueArea(Old: SaveInsertPt);
281
282 return Reg;
283}
284
285Register FastISel::materializeConstant(const Value *V, MVT VT) {
286 Register Reg;
287 if (const auto *CI = dyn_cast<ConstantInt>(Val: V)) {
288 if (CI->getValue().getActiveBits() <= 64)
289 Reg = fastEmit_i(VT, RetVT: VT, Opcode: ISD::Constant, Imm: CI->getZExtValue());
290 } else if (isa<AllocaInst>(Val: V))
291 Reg = fastMaterializeAlloca(C: cast<AllocaInst>(Val: V));
292 else if (isa<ConstantPointerNull>(Val: V))
293 // Translate this as an integer zero so that it can be
294 // local-CSE'd with actual integer zeros.
295 Reg =
296 getRegForValue(V: Constant::getNullValue(Ty: DL.getIntPtrType(V->getType())));
297 else if (const auto *CF = dyn_cast<ConstantFP>(Val: V)) {
298 if (CF->isNullValue())
299 Reg = fastMaterializeFloatZero(CF);
300 else
301 // Try to emit the constant directly.
302 Reg = fastEmit_f(VT, RetVT: VT, Opcode: ISD::ConstantFP, FPImm: CF);
303
304 if (!Reg) {
305 // Try to emit the constant by using an integer constant with a cast.
306 const APFloat &Flt = CF->getValueAPF();
307 EVT IntVT = TLI.getPointerTy(DL);
308 uint32_t IntBitWidth = IntVT.getSizeInBits();
309 APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
310 bool isExact;
311 (void)Flt.convertToInteger(Result&: SIntVal, RM: APFloat::rmTowardZero, IsExact: &isExact);
312 if (isExact) {
313 Register IntegerReg =
314 getRegForValue(V: ConstantInt::get(Context&: V->getContext(), V: SIntVal));
315 if (IntegerReg)
316 Reg = fastEmit_r(VT: IntVT.getSimpleVT(), RetVT: VT, Opcode: ISD::SINT_TO_FP,
317 Op0: IntegerReg);
318 }
319 }
320 } else if (const auto *Op = dyn_cast<Operator>(Val: V)) {
321 if (!selectOperator(I: Op, Opcode: Op->getOpcode()))
322 if (!isa<Instruction>(Val: Op) ||
323 !fastSelectInstruction(I: cast<Instruction>(Val: Op)))
324 return Register();
325 Reg = lookUpRegForValue(V: Op);
326 } else if (isa<UndefValue>(Val: V)) {
327 Reg = createResultReg(RC: TLI.getRegClassFor(VT));
328 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
329 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: Reg);
330 }
331 return Reg;
332}
333
334/// Helper for getRegForValue. This function is called when the value isn't
335/// already available in a register and must be materialized with new
336/// instructions.
337Register FastISel::materializeRegForValue(const Value *V, MVT VT) {
338 Register Reg;
339 // Give the target-specific code a try first.
340 if (isa<Constant>(Val: V))
341 Reg = fastMaterializeConstant(C: cast<Constant>(Val: V));
342
343 // If target-specific code couldn't or didn't want to handle the value, then
344 // give target-independent code a try.
345 if (!Reg)
346 Reg = materializeConstant(V, VT);
347
348 // Don't cache constant materializations in the general ValueMap.
349 // To do so would require tracking what uses they dominate.
350 if (Reg) {
351 LocalValueMap[V] = Reg;
352 LastLocalValue = MRI.getVRegDef(Reg);
353 }
354 return Reg;
355}
356
357Register FastISel::lookUpRegForValue(const Value *V) {
358 // Look up the value to see if we already have a register for it. We
359 // cache values defined by Instructions across blocks, and other values
360 // only locally. This is because Instructions already have the SSA
361 // def-dominates-use requirement enforced.
362 auto I = FuncInfo.ValueMap.find(Val: V);
363 if (I != FuncInfo.ValueMap.end())
364 return I->second;
365 return LocalValueMap[V];
366}
367
368void FastISel::updateValueMap(const Value *I, Register Reg, unsigned NumRegs) {
369 if (!isa<Instruction>(Val: I)) {
370 LocalValueMap[I] = Reg;
371 return;
372 }
373
374 Register &AssignedReg = FuncInfo.ValueMap[I];
375 if (!AssignedReg)
376 // Use the new register.
377 AssignedReg = Reg;
378 else if (Reg != AssignedReg) {
379 // Arrange for uses of AssignedReg to be replaced by uses of Reg.
380 for (unsigned i = 0; i < NumRegs; i++) {
381 FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
382 FuncInfo.RegsWithFixups.insert(V: Reg + i);
383 }
384
385 AssignedReg = Reg;
386 }
387}
388
389Register FastISel::getRegForGEPIndex(MVT PtrVT, const Value *Idx) {
390 Register IdxN = getRegForValue(V: Idx);
391 if (!IdxN)
392 // Unhandled operand. Halt "fast" selection and bail.
393 return Register();
394
395 // If the index is smaller or larger than intptr_t, truncate or extend it.
396 EVT IdxVT = EVT::getEVT(Ty: Idx->getType(), /*HandleUnknown=*/false);
397 if (IdxVT.bitsLT(VT: PtrVT)) {
398 IdxN = fastEmit_r(VT: IdxVT.getSimpleVT(), RetVT: PtrVT, Opcode: ISD::SIGN_EXTEND, Op0: IdxN);
399 } else if (IdxVT.bitsGT(VT: PtrVT)) {
400 IdxN =
401 fastEmit_r(VT: IdxVT.getSimpleVT(), RetVT: PtrVT, Opcode: ISD::TRUNCATE, Op0: IdxN);
402 }
403 return IdxN;
404}
405
406void FastISel::recomputeInsertPt() {
407 if (getLastLocalValue()) {
408 FuncInfo.InsertPt = getLastLocalValue();
409 FuncInfo.MBB = FuncInfo.InsertPt->getParent();
410 ++FuncInfo.InsertPt;
411 } else
412 FuncInfo.InsertPt = FuncInfo.MBB->getFirstNonPHI();
413}
414
415void FastISel::removeDeadCode(MachineBasicBlock::iterator I,
416 MachineBasicBlock::iterator E) {
417 assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
418 "Invalid iterator!");
419 while (I != E) {
420 if (SavedInsertPt == I)
421 SavedInsertPt = E;
422 if (EmitStartPt == I)
423 EmitStartPt = E.isValid() ? &*E : nullptr;
424 if (LastLocalValue == I)
425 LastLocalValue = E.isValid() ? &*E : nullptr;
426
427 MachineInstr *Dead = &*I;
428 ++I;
429 Dead->eraseFromParent();
430 ++NumFastIselDead;
431 }
432 recomputeInsertPt();
433}
434
435FastISel::SavePoint FastISel::enterLocalValueArea() {
436 SavePoint OldInsertPt = FuncInfo.InsertPt;
437 recomputeInsertPt();
438 return OldInsertPt;
439}
440
441void FastISel::leaveLocalValueArea(SavePoint OldInsertPt) {
442 if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
443 LastLocalValue = &*std::prev(x: FuncInfo.InsertPt);
444
445 // Restore the previous insert position.
446 FuncInfo.InsertPt = OldInsertPt;
447}
448
449bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
450 EVT VT = EVT::getEVT(Ty: I->getType(), /*HandleUnknown=*/true);
451 if (VT == MVT::Other || !VT.isSimple())
452 // Unhandled type. Halt "fast" selection and bail.
453 return false;
454
455 // We only handle legal types. For example, on x86-32 the instruction
456 // selector contains all of the 64-bit instructions from x86-64,
457 // under the assumption that i64 won't be used if the target doesn't
458 // support it.
459 if (!TLI.isTypeLegal(VT)) {
460 // MVT::i1 is special. Allow AND, OR, or XOR because they
461 // don't require additional zeroing, which makes them easy.
462 if (VT == MVT::i1 && ISD::isBitwiseLogicOp(Opcode: ISDOpcode))
463 VT = TLI.getTypeToTransformTo(Context&: I->getContext(), VT);
464 else
465 return false;
466 }
467
468 // Check if the first operand is a constant, and handle it as "ri". At -O0,
469 // we don't have anything that canonicalizes operand order.
470 if (const auto *CI = dyn_cast<ConstantInt>(Val: I->getOperand(i: 0)))
471 if (isa<Instruction>(Val: I) && cast<Instruction>(Val: I)->isCommutative()) {
472 Register Op1 = getRegForValue(V: I->getOperand(i: 1));
473 if (!Op1)
474 return false;
475
476 Register ResultReg =
477 fastEmit_ri_(VT: VT.getSimpleVT(), Opcode: ISDOpcode, Op0: Op1, Imm: CI->getZExtValue(),
478 ImmType: VT.getSimpleVT());
479 if (!ResultReg)
480 return false;
481
482 // We successfully emitted code for the given LLVM Instruction.
483 updateValueMap(I, Reg: ResultReg);
484 return true;
485 }
486
487 Register Op0 = getRegForValue(V: I->getOperand(i: 0));
488 if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
489 return false;
490
491 // Check if the second operand is a constant and handle it appropriately.
492 if (const auto *CI = dyn_cast<ConstantInt>(Val: I->getOperand(i: 1))) {
493 uint64_t Imm = CI->getSExtValue();
494
495 // Transform "sdiv exact X, 8" -> "sra X, 3".
496 if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(Val: I) &&
497 cast<BinaryOperator>(Val: I)->isExact() && isPowerOf2_64(Value: Imm)) {
498 Imm = Log2_64(Value: Imm);
499 ISDOpcode = ISD::SRA;
500 }
501
502 // Transform "urem x, pow2" -> "and x, pow2-1".
503 if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(Val: I) &&
504 isPowerOf2_64(Value: Imm)) {
505 --Imm;
506 ISDOpcode = ISD::AND;
507 }
508
509 Register ResultReg = fastEmit_ri_(VT: VT.getSimpleVT(), Opcode: ISDOpcode, Op0, Imm,
510 ImmType: VT.getSimpleVT());
511 if (!ResultReg)
512 return false;
513
514 // We successfully emitted code for the given LLVM Instruction.
515 updateValueMap(I, Reg: ResultReg);
516 return true;
517 }
518
519 Register Op1 = getRegForValue(V: I->getOperand(i: 1));
520 if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
521 return false;
522
523 // Now we have both operands in registers. Emit the instruction.
524 Register ResultReg = fastEmit_rr(VT: VT.getSimpleVT(), RetVT: VT.getSimpleVT(),
525 Opcode: ISDOpcode, Op0, Op1);
526 if (!ResultReg)
527 // Target-specific code wasn't able to find a machine opcode for
528 // the given ISD opcode and type. Halt "fast" selection and bail.
529 return false;
530
531 // We successfully emitted code for the given LLVM Instruction.
532 updateValueMap(I, Reg: ResultReg);
533 return true;
534}
535
536bool FastISel::selectGetElementPtr(const User *I) {
537 Register N = getRegForValue(V: I->getOperand(i: 0));
538 if (!N) // Unhandled operand. Halt "fast" selection and bail.
539 return false;
540
541 // FIXME: The code below does not handle vector GEPs. Halt "fast" selection
542 // and bail.
543 if (isa<VectorType>(Val: I->getType()))
544 return false;
545
546 // Keep a running tab of the total offset to coalesce multiple N = N + Offset
547 // into a single N = N + TotalOffset.
548 uint64_t TotalOffs = 0;
549 // FIXME: What's a good SWAG number for MaxOffs?
550 uint64_t MaxOffs = 2048;
551 MVT VT = TLI.getValueType(DL, Ty: I->getType()).getSimpleVT();
552
553 for (gep_type_iterator GTI = gep_type_begin(GEP: I), E = gep_type_end(GEP: I);
554 GTI != E; ++GTI) {
555 const Value *Idx = GTI.getOperand();
556 if (StructType *StTy = GTI.getStructTypeOrNull()) {
557 uint64_t Field = cast<ConstantInt>(Val: Idx)->getZExtValue();
558 if (Field) {
559 // N = N + Offset
560 TotalOffs += DL.getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
561 if (TotalOffs >= MaxOffs) {
562 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
563 if (!N) // Unhandled operand. Halt "fast" selection and bail.
564 return false;
565 TotalOffs = 0;
566 }
567 }
568 } else {
569 // If this is a constant subscript, handle it quickly.
570 if (const auto *CI = dyn_cast<ConstantInt>(Val: Idx)) {
571 if (CI->isZero())
572 continue;
573 // N = N + Offset
574 uint64_t IdxN = CI->getValue().sextOrTrunc(width: 64).getSExtValue();
575 TotalOffs += GTI.getSequentialElementStride(DL) * IdxN;
576 if (TotalOffs >= MaxOffs) {
577 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
578 if (!N) // Unhandled operand. Halt "fast" selection and bail.
579 return false;
580 TotalOffs = 0;
581 }
582 continue;
583 }
584 if (TotalOffs) {
585 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
586 if (!N) // Unhandled operand. Halt "fast" selection and bail.
587 return false;
588 TotalOffs = 0;
589 }
590
591 // N = N + Idx * ElementSize;
592 uint64_t ElementSize = GTI.getSequentialElementStride(DL);
593 Register IdxN = getRegForGEPIndex(PtrVT: VT, Idx);
594 if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
595 return false;
596
597 if (ElementSize != 1) {
598 IdxN = fastEmit_ri_(VT, Opcode: ISD::MUL, Op0: IdxN, Imm: ElementSize, ImmType: VT);
599 if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
600 return false;
601 }
602 N = fastEmit_rr(VT, RetVT: VT, Opcode: ISD::ADD, Op0: N, Op1: IdxN);
603 if (!N) // Unhandled operand. Halt "fast" selection and bail.
604 return false;
605 }
606 }
607 if (TotalOffs) {
608 N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
609 if (!N) // Unhandled operand. Halt "fast" selection and bail.
610 return false;
611 }
612
613 // We successfully emitted code for the given LLVM Instruction.
614 updateValueMap(I, Reg: N);
615 return true;
616}
617
618bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
619 const CallInst *CI, unsigned StartIdx) {
620 for (unsigned i = StartIdx, e = CI->arg_size(); i != e; ++i) {
621 Value *Val = CI->getArgOperand(i);
622 // Check for constants and encode them with a StackMaps::ConstantOp prefix.
623 if (const auto *C = dyn_cast<ConstantInt>(Val)) {
624 Ops.push_back(Elt: MachineOperand::CreateImm(Val: StackMaps::ConstantOp));
625 Ops.push_back(Elt: MachineOperand::CreateImm(Val: C->getSExtValue()));
626 } else if (isa<ConstantPointerNull>(Val)) {
627 Ops.push_back(Elt: MachineOperand::CreateImm(Val: StackMaps::ConstantOp));
628 Ops.push_back(Elt: MachineOperand::CreateImm(Val: 0));
629 } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
630 // Values coming from a stack location also require a special encoding,
631 // but that is added later on by the target specific frame index
632 // elimination implementation.
633 auto SI = FuncInfo.StaticAllocaMap.find(Val: AI);
634 if (SI != FuncInfo.StaticAllocaMap.end())
635 Ops.push_back(Elt: MachineOperand::CreateFI(Idx: SI->second));
636 else
637 return false;
638 } else {
639 Register Reg = getRegForValue(V: Val);
640 if (!Reg)
641 return false;
642 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/false));
643 }
644 }
645 return true;
646}
647
648bool FastISel::selectStackmap(const CallInst *I) {
649 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
650 // [live variables...])
651 assert(I->getCalledFunction()->getReturnType()->isVoidTy() &&
652 "Stackmap cannot return a value.");
653
654 // The stackmap intrinsic only records the live variables (the arguments
655 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
656 // intrinsic, this won't be lowered to a function call. This means we don't
657 // have to worry about calling conventions and target-specific lowering code.
658 // Instead we perform the call lowering right here.
659 //
660 // CALLSEQ_START(0, 0...)
661 // STACKMAP(id, nbytes, ...)
662 // CALLSEQ_END(0, 0)
663 //
664 SmallVector<MachineOperand, 32> Ops;
665
666 // Add the <id> and <numBytes> constants.
667 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
668 "Expected a constant integer.");
669 const auto *ID = cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::IDPos));
670 Ops.push_back(Elt: MachineOperand::CreateImm(Val: ID->getZExtValue()));
671
672 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
673 "Expected a constant integer.");
674 const auto *NumBytes =
675 cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::NBytesPos));
676 Ops.push_back(Elt: MachineOperand::CreateImm(Val: NumBytes->getZExtValue()));
677
678 // Push live variables for the stack map (skipping the first two arguments
679 // <id> and <numBytes>).
680 if (!addStackMapLiveVars(Ops, CI: I, StartIdx: 2))
681 return false;
682
683 // We are not adding any register mask info here, because the stackmap doesn't
684 // clobber anything.
685
686 // Add scratch registers as implicit def and early clobber.
687 CallingConv::ID CC = I->getCallingConv();
688 const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
689 for (unsigned i = 0; ScratchRegs[i]; ++i)
690 Ops.push_back(Elt: MachineOperand::CreateReg(
691 Reg: ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
692 /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
693
694 // Issue CALLSEQ_START
695 unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
696 auto Builder =
697 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: AdjStackDown));
698 const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
699 for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
700 Builder.addImm(Val: 0);
701
702 // Issue STACKMAP.
703 MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
704 MCID: TII.get(Opcode: TargetOpcode::STACKMAP));
705 for (auto const &MO : Ops)
706 MIB.add(MO);
707
708 // Issue CALLSEQ_END
709 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
710 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: AdjStackUp))
711 .addImm(Val: 0)
712 .addImm(Val: 0);
713
714 // Inform the Frame Information that we have a stackmap in this function.
715 FuncInfo.MF->getFrameInfo().setHasStackMap();
716
717 return true;
718}
719
720/// Lower an argument list according to the target calling convention.
721///
722/// This is a helper for lowering intrinsics that follow a target calling
723/// convention or require stack pointer adjustment. Only a subset of the
724/// intrinsic's operands need to participate in the calling convention.
725bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
726 unsigned NumArgs, const Value *Callee,
727 bool ForceRetVoidTy, CallLoweringInfo &CLI) {
728 ArgListTy Args;
729 Args.reserve(n: NumArgs);
730
731 // Populate the argument list.
732 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
733 Value *V = CI->getOperand(i_nocapture: ArgI);
734
735 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
736
737 ArgListEntry Entry(V);
738 Entry.setAttributes(Call: CI, ArgIdx: ArgI);
739 Args.push_back(x: Entry);
740 }
741
742 Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(C&: CI->getType()->getContext())
743 : CI->getType();
744 CLI.setCallee(CC: CI->getCallingConv(), ResultTy: RetTy, Target: Callee, ArgsList: std::move(Args), FixedArgs: NumArgs);
745
746 return lowerCallTo(CLI);
747}
748
749FastISel::CallLoweringInfo &FastISel::CallLoweringInfo::setCallee(
750 const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
751 StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
752 SmallString<32> MangledName;
753 Mangler::getNameWithPrefix(OutName&: MangledName, GVName: Target, DL);
754 MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: MangledName);
755 return setCallee(CC, ResultTy, Target: Sym, ArgsList: std::move(ArgsList), FixedArgs);
756}
757
758bool FastISel::selectPatchpoint(const CallInst *I) {
759 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
760 // i32 <numBytes>,
761 // i8* <target>,
762 // i32 <numArgs>,
763 // [Args...],
764 // [live variables...])
765 CallingConv::ID CC = I->getCallingConv();
766 bool IsAnyRegCC = CC == CallingConv::AnyReg;
767 bool HasDef = !I->getType()->isVoidTy();
768 Value *Callee = I->getOperand(i_nocapture: PatchPointOpers::TargetPos)->stripPointerCasts();
769
770 // Check if we can lower the return type when using anyregcc.
771 MVT ValueType;
772 if (IsAnyRegCC && HasDef) {
773 ValueType = TLI.getSimpleValueType(DL, Ty: I->getType(), /*AllowUnknown=*/true);
774 if (ValueType == MVT::Other)
775 return false;
776 }
777
778 // Get the real number of arguments participating in the call <numArgs>
779 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
780 "Expected a constant integer.");
781 const auto *NumArgsVal =
782 cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::NArgPos));
783 unsigned NumArgs = NumArgsVal->getZExtValue();
784
785 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
786 // This includes all meta-operands up to but not including CC.
787 unsigned NumMetaOpers = PatchPointOpers::CCPos;
788 assert(I->arg_size() >= NumMetaOpers + NumArgs &&
789 "Not enough arguments provided to the patchpoint intrinsic");
790
791 // For AnyRegCC the arguments are lowered later on manually.
792 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
793 CallLoweringInfo CLI;
794 CLI.setIsPatchPoint();
795 if (!lowerCallOperands(CI: I, ArgIdx: NumMetaOpers, NumArgs: NumCallArgs, Callee, ForceRetVoidTy: IsAnyRegCC, CLI))
796 return false;
797
798 assert(CLI.Call && "No call instruction specified.");
799
800 SmallVector<MachineOperand, 32> Ops;
801
802 // Add an explicit result reg if we use the anyreg calling convention.
803 if (IsAnyRegCC && HasDef) {
804 assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
805 assert(ValueType.isValid());
806 CLI.ResultReg = createResultReg(RC: TLI.getRegClassFor(VT: ValueType));
807 CLI.NumResultRegs = 1;
808 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: CLI.ResultReg, /*isDef=*/true));
809 }
810
811 // Add the <id> and <numBytes> constants.
812 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
813 "Expected a constant integer.");
814 const auto *ID = cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::IDPos));
815 Ops.push_back(Elt: MachineOperand::CreateImm(Val: ID->getZExtValue()));
816
817 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
818 "Expected a constant integer.");
819 const auto *NumBytes =
820 cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::NBytesPos));
821 Ops.push_back(Elt: MachineOperand::CreateImm(Val: NumBytes->getZExtValue()));
822
823 // Add the call target.
824 if (const auto *C = dyn_cast<IntToPtrInst>(Val: Callee)) {
825 uint64_t CalleeConstAddr =
826 cast<ConstantInt>(Val: C->getOperand(i_nocapture: 0))->getZExtValue();
827 Ops.push_back(Elt: MachineOperand::CreateImm(Val: CalleeConstAddr));
828 } else if (const auto *C = dyn_cast<ConstantExpr>(Val: Callee)) {
829 if (C->getOpcode() == Instruction::IntToPtr) {
830 uint64_t CalleeConstAddr =
831 cast<ConstantInt>(Val: C->getOperand(i_nocapture: 0))->getZExtValue();
832 Ops.push_back(Elt: MachineOperand::CreateImm(Val: CalleeConstAddr));
833 } else
834 llvm_unreachable("Unsupported ConstantExpr.");
835 } else if (const auto *GV = dyn_cast<GlobalValue>(Val: Callee)) {
836 Ops.push_back(Elt: MachineOperand::CreateGA(GV, Offset: 0));
837 } else if (isa<ConstantPointerNull>(Val: Callee))
838 Ops.push_back(Elt: MachineOperand::CreateImm(Val: 0));
839 else
840 llvm_unreachable("Unsupported callee address.");
841
842 // Adjust <numArgs> to account for any arguments that have been passed on
843 // the stack instead.
844 unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
845 Ops.push_back(Elt: MachineOperand::CreateImm(Val: NumCallRegArgs));
846
847 // Add the calling convention
848 Ops.push_back(Elt: MachineOperand::CreateImm(Val: (unsigned)CC));
849
850 // Add the arguments we omitted previously. The register allocator should
851 // place these in any free register.
852 if (IsAnyRegCC) {
853 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
854 Register Reg = getRegForValue(V: I->getArgOperand(i));
855 if (!Reg)
856 return false;
857 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/false));
858 }
859 }
860
861 // Push the arguments from the call instruction.
862 for (auto Reg : CLI.OutRegs)
863 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/false));
864
865 // Push live variables for the stack map.
866 if (!addStackMapLiveVars(Ops, CI: I, StartIdx: NumMetaOpers + NumArgs))
867 return false;
868
869 // Push the register mask info.
870 Ops.push_back(Elt: MachineOperand::CreateRegMask(
871 Mask: TRI.getCallPreservedMask(MF: *FuncInfo.MF, CC)));
872
873 // Add scratch registers as implicit def and early clobber.
874 const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
875 for (unsigned i = 0; ScratchRegs[i]; ++i)
876 Ops.push_back(Elt: MachineOperand::CreateReg(
877 Reg: ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
878 /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
879
880 // Add implicit defs (return values).
881 for (auto Reg : CLI.InRegs)
882 Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/true,
883 /*isImp=*/true));
884
885 // Insert the patchpoint instruction before the call generated by the target.
886 MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: CLI.Call, MIMD,
887 MCID: TII.get(Opcode: TargetOpcode::PATCHPOINT));
888
889 for (auto &MO : Ops)
890 MIB.add(MO);
891
892 MIB->setPhysRegsDeadExcept(UsedRegs: CLI.InRegs, TRI);
893
894 // Delete the original call instruction.
895 CLI.Call->eraseFromParent();
896
897 // Inform the Frame Information that we have a patchpoint in this function.
898 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
899
900 if (CLI.NumResultRegs)
901 updateValueMap(I, Reg: CLI.ResultReg, NumRegs: CLI.NumResultRegs);
902 return true;
903}
904
905bool FastISel::selectXRayCustomEvent(const CallInst *I) {
906 const auto &Triple = TM.getTargetTriple();
907 if (Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64)
908 return true; // don't do anything to this instruction.
909 SmallVector<MachineOperand, 8> Ops;
910 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 0)),
911 /*isDef=*/false));
912 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 1)),
913 /*isDef=*/false));
914 MachineInstrBuilder MIB =
915 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
916 MCID: TII.get(Opcode: TargetOpcode::PATCHABLE_EVENT_CALL));
917 for (auto &MO : Ops)
918 MIB.add(MO);
919
920 // Insert the Patchable Event Call instruction, that gets lowered properly.
921 return true;
922}
923
924bool FastISel::selectXRayTypedEvent(const CallInst *I) {
925 const auto &Triple = TM.getTargetTriple();
926 if (Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64)
927 return true; // don't do anything to this instruction.
928 SmallVector<MachineOperand, 8> Ops;
929 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 0)),
930 /*isDef=*/false));
931 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 1)),
932 /*isDef=*/false));
933 Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 2)),
934 /*isDef=*/false));
935 MachineInstrBuilder MIB =
936 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
937 MCID: TII.get(Opcode: TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
938 for (auto &MO : Ops)
939 MIB.add(MO);
940
941 // Insert the Patchable Typed Event Call instruction, that gets lowered properly.
942 return true;
943}
944
945/// Returns an AttributeList representing the attributes applied to the return
946/// value of the given call.
947static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI) {
948 SmallVector<Attribute::AttrKind, 2> Attrs;
949 if (CLI.RetSExt)
950 Attrs.push_back(Elt: Attribute::SExt);
951 if (CLI.RetZExt)
952 Attrs.push_back(Elt: Attribute::ZExt);
953 if (CLI.IsInReg)
954 Attrs.push_back(Elt: Attribute::InReg);
955
956 return AttributeList::get(C&: CLI.RetTy->getContext(), Index: AttributeList::ReturnIndex,
957 Kinds: Attrs);
958}
959
960bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
961 unsigned NumArgs) {
962 MCContext &Ctx = MF->getContext();
963 SmallString<32> MangledName;
964 Mangler::getNameWithPrefix(OutName&: MangledName, GVName: SymName, DL);
965 MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: MangledName);
966 return lowerCallTo(CI, Symbol: Sym, NumArgs);
967}
968
969bool FastISel::lowerCallTo(const CallInst *CI, MCSymbol *Symbol,
970 unsigned NumArgs) {
971 FunctionType *FTy = CI->getFunctionType();
972 Type *RetTy = CI->getType();
973
974 ArgListTy Args;
975 Args.reserve(n: NumArgs);
976
977 // Populate the argument list.
978 // Attributes for args start at offset 1, after the return attribute.
979 for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
980 Value *V = CI->getOperand(i_nocapture: ArgI);
981
982 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
983
984 ArgListEntry Entry(V);
985 Entry.setAttributes(Call: CI, ArgIdx: ArgI);
986 Args.push_back(x: Entry);
987 }
988 TLI.markLibCallAttributes(MF, CC: CI->getCallingConv(), Args);
989
990 CallLoweringInfo CLI;
991 CLI.setCallee(ResultTy: RetTy, FuncTy: FTy, Target: Symbol, ArgsList: std::move(Args), Call: *CI, FixedArgs: NumArgs);
992
993 return lowerCallTo(CLI);
994}
995
996bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
997 // Handle the incoming return values from the call.
998 CLI.clearIns();
999 SmallVector<EVT, 4> RetTys;
1000 ComputeValueVTs(TLI, DL, Ty: CLI.RetTy, ValueVTs&: RetTys);
1001
1002 SmallVector<ISD::OutputArg, 4> Outs;
1003 GetReturnInfo(CC: CLI.CallConv, ReturnType: CLI.RetTy, attr: getReturnAttrs(CLI), Outs, TLI, DL);
1004
1005 bool CanLowerReturn = TLI.CanLowerReturn(
1006 CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext(), RetTy: CLI.RetTy);
1007
1008 // FIXME: sret demotion isn't supported yet - bail out.
1009 if (!CanLowerReturn)
1010 return false;
1011
1012 for (EVT VT : RetTys) {
1013 MVT RegisterVT = TLI.getRegisterType(Context&: CLI.RetTy->getContext(), VT);
1014 unsigned NumRegs = TLI.getNumRegisters(Context&: CLI.RetTy->getContext(), VT);
1015 for (unsigned i = 0; i != NumRegs; ++i) {
1016 ISD::ArgFlagsTy Flags;
1017 if (CLI.RetSExt)
1018 Flags.setSExt();
1019 if (CLI.RetZExt)
1020 Flags.setZExt();
1021 if (CLI.IsInReg)
1022 Flags.setInReg();
1023 ISD::InputArg Ret(Flags, RegisterVT, VT, CLI.RetTy, CLI.IsReturnValueUsed,
1024 ISD::InputArg::NoArgIndex, 0);
1025 CLI.Ins.push_back(Elt: Ret);
1026 }
1027 }
1028
1029 // Handle all of the outgoing arguments.
1030 CLI.clearOuts();
1031 for (auto &Arg : CLI.getArgs()) {
1032 Type *FinalType = Arg.Ty;
1033 if (Arg.IsByVal)
1034 FinalType = Arg.IndirectType;
1035 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1036 Ty: FinalType, CallConv: CLI.CallConv, isVarArg: CLI.IsVarArg, DL);
1037
1038 ISD::ArgFlagsTy Flags;
1039 if (Arg.IsZExt)
1040 Flags.setZExt();
1041 if (Arg.IsSExt)
1042 Flags.setSExt();
1043 if (Arg.IsInReg)
1044 Flags.setInReg();
1045 if (Arg.IsSRet)
1046 Flags.setSRet();
1047 if (Arg.IsSwiftSelf)
1048 Flags.setSwiftSelf();
1049 if (Arg.IsSwiftAsync)
1050 Flags.setSwiftAsync();
1051 if (Arg.IsSwiftError)
1052 Flags.setSwiftError();
1053 if (Arg.IsCFGuardTarget)
1054 Flags.setCFGuardTarget();
1055 if (Arg.IsByVal)
1056 Flags.setByVal();
1057 if (Arg.IsInAlloca) {
1058 Flags.setInAlloca();
1059 // Set the byval flag for CCAssignFn callbacks that don't know about
1060 // inalloca. This way we can know how many bytes we should've allocated
1061 // and how many bytes a callee cleanup function will pop. If we port
1062 // inalloca to more targets, we'll have to add custom inalloca handling in
1063 // the various CC lowering callbacks.
1064 Flags.setByVal();
1065 }
1066 if (Arg.IsPreallocated) {
1067 Flags.setPreallocated();
1068 // Set the byval flag for CCAssignFn callbacks that don't know about
1069 // preallocated. This way we can know how many bytes we should've
1070 // allocated and how many bytes a callee cleanup function will pop. If we
1071 // port preallocated to more targets, we'll have to add custom
1072 // preallocated handling in the various CC lowering callbacks.
1073 Flags.setByVal();
1074 }
1075 MaybeAlign MemAlign = Arg.Alignment;
1076 if (Arg.IsByVal || Arg.IsInAlloca || Arg.IsPreallocated) {
1077 unsigned FrameSize = DL.getTypeAllocSize(Ty: Arg.IndirectType);
1078
1079 // For ByVal, alignment should come from FE. BE will guess if this info
1080 // is not there, but there are cases it cannot get right.
1081 if (!MemAlign)
1082 MemAlign = TLI.getByValTypeAlignment(Ty: Arg.IndirectType, DL);
1083 Flags.setByValSize(FrameSize);
1084 } else if (!MemAlign) {
1085 MemAlign = DL.getABITypeAlign(Ty: Arg.Ty);
1086 }
1087 Flags.setMemAlign(*MemAlign);
1088 if (Arg.IsNest)
1089 Flags.setNest();
1090 if (NeedsRegBlock)
1091 Flags.setInConsecutiveRegs();
1092 Flags.setOrigAlign(DL.getABITypeAlign(Ty: Arg.Ty));
1093 CLI.OutVals.push_back(Elt: Arg.Val);
1094 CLI.OutFlags.push_back(Elt: Flags);
1095 }
1096
1097 if (!fastLowerCall(CLI))
1098 return false;
1099
1100 // Set all unused physreg defs as dead.
1101 assert(CLI.Call && "No call instruction specified.");
1102 CLI.Call->setPhysRegsDeadExcept(UsedRegs: CLI.InRegs, TRI);
1103
1104 if (CLI.NumResultRegs && CLI.CB)
1105 updateValueMap(I: CLI.CB, Reg: CLI.ResultReg, NumRegs: CLI.NumResultRegs);
1106
1107 // Set labels for heapallocsite call.
1108 if (CLI.CB)
1109 if (MDNode *MD = CLI.CB->getMetadata(Kind: "heapallocsite"))
1110 CLI.Call->setHeapAllocMarker(MF&: *MF, MD);
1111
1112 return true;
1113}
1114
1115bool FastISel::lowerCall(const CallInst *CI) {
1116 FunctionType *FuncTy = CI->getFunctionType();
1117 Type *RetTy = CI->getType();
1118
1119 ArgListTy Args;
1120 Args.reserve(n: CI->arg_size());
1121
1122 for (auto i = CI->arg_begin(), e = CI->arg_end(); i != e; ++i) {
1123 Value *V = *i;
1124
1125 // Skip empty types
1126 if (V->getType()->isEmptyTy())
1127 continue;
1128
1129 ArgListEntry Entry(V);
1130 // Skip the first return-type Attribute to get to params.
1131 Entry.setAttributes(Call: CI, ArgIdx: i - CI->arg_begin());
1132 Args.push_back(x: Entry);
1133 }
1134
1135 // Check if target-independent constraints permit a tail call here.
1136 // Target-dependent constraints are checked within fastLowerCall.
1137 bool IsTailCall = CI->isTailCall();
1138 if (IsTailCall && !isInTailCallPosition(Call: *CI, TM))
1139 IsTailCall = false;
1140 if (IsTailCall && !CI->isMustTailCall() &&
1141 MF->getFunction().getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
1142 IsTailCall = false;
1143
1144 CallLoweringInfo CLI;
1145 CLI.setCallee(ResultTy: RetTy, FuncTy, Target: CI->getCalledOperand(), ArgsList: std::move(Args), Call: *CI)
1146 .setTailCall(IsTailCall);
1147
1148 if (lowerCallTo(CLI)) {
1149 diagnoseDontCall(CI: *CI);
1150 return true;
1151 }
1152
1153 return false;
1154}
1155
1156bool FastISel::selectCall(const User *I) {
1157 const CallInst *Call = cast<CallInst>(Val: I);
1158
1159 // Handle simple inline asms.
1160 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: Call->getCalledOperand())) {
1161 // Don't attempt to handle constraints.
1162 if (!IA->getConstraintString().empty())
1163 return false;
1164
1165 unsigned ExtraInfo = 0;
1166 if (IA->hasSideEffects())
1167 ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1168 if (IA->isAlignStack())
1169 ExtraInfo |= InlineAsm::Extra_IsAlignStack;
1170 if (IA->canThrow())
1171 ExtraInfo |= InlineAsm::Extra_MayUnwind;
1172 if (Call->isConvergent())
1173 ExtraInfo |= InlineAsm::Extra_IsConvergent;
1174 ExtraInfo |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
1175
1176 MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1177 MCID: TII.get(Opcode: TargetOpcode::INLINEASM));
1178 MIB.addExternalSymbol(FnName: IA->getAsmString().data());
1179 MIB.addImm(Val: ExtraInfo);
1180
1181 const MDNode *SrcLoc = Call->getMetadata(Kind: "srcloc");
1182 if (SrcLoc)
1183 MIB.addMetadata(MD: SrcLoc);
1184
1185 return true;
1186 }
1187
1188 // Handle intrinsic function calls.
1189 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Call))
1190 return selectIntrinsicCall(II);
1191
1192 return lowerCall(CI: Call);
1193}
1194
1195void FastISel::handleDbgInfo(const Instruction *II) {
1196 if (!II->hasDbgRecords())
1197 return;
1198
1199 // Clear any metadata.
1200 MIMD = MIMetadata();
1201
1202 // Reverse order of debug records, because fast-isel walks through backwards.
1203 for (DbgRecord &DR : llvm::reverse(C: II->getDbgRecordRange())) {
1204 flushLocalValueMap();
1205 recomputeInsertPt();
1206
1207 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
1208 assert(DLR->getLabel() && "Missing label");
1209 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DLR->getDebugLoc(),
1210 MCID: TII.get(Opcode: TargetOpcode::DBG_LABEL))
1211 .addMetadata(MD: DLR->getLabel());
1212 continue;
1213 }
1214
1215 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
1216
1217 Value *V = nullptr;
1218 if (!DVR.hasArgList())
1219 V = DVR.getVariableLocationOp(OpIdx: 0);
1220
1221 bool Res = false;
1222 if (DVR.getType() == DbgVariableRecord::LocationType::Value ||
1223 DVR.getType() == DbgVariableRecord::LocationType::Assign) {
1224 Res = lowerDbgValue(V, Expr: DVR.getExpression(), Var: DVR.getVariable(),
1225 DL: DVR.getDebugLoc());
1226 } else {
1227 assert(DVR.getType() == DbgVariableRecord::LocationType::Declare);
1228 if (FuncInfo.PreprocessedDVRDeclares.contains(Ptr: &DVR))
1229 continue;
1230 Res = lowerDbgDeclare(V, Expr: DVR.getExpression(), Var: DVR.getVariable(),
1231 DL: DVR.getDebugLoc());
1232 }
1233
1234 if (!Res)
1235 LLVM_DEBUG(dbgs() << "Dropping debug-info for " << DVR << "\n");
1236 }
1237}
1238
1239bool FastISel::lowerDbgValue(const Value *V, DIExpression *Expr,
1240 DILocalVariable *Var, const DebugLoc &DL) {
1241 // This form of DBG_VALUE is target-independent.
1242 const MCInstrDesc &II = TII.get(Opcode: TargetOpcode::DBG_VALUE);
1243 if (!V || isa<UndefValue>(Val: V)) {
1244 // DI is either undef or cannot produce a valid DBG_VALUE, so produce an
1245 // undef DBG_VALUE to terminate any prior location.
1246 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect: false, Reg: 0U, Variable: Var, Expr);
1247 return true;
1248 }
1249 if (const auto *CI = dyn_cast<ConstantInt>(Val: V)) {
1250 // See if there's an expression to constant-fold.
1251 if (Expr)
1252 std::tie(args&: Expr, args&: CI) = Expr->constantFold(CI);
1253 if (CI->getBitWidth() > 64)
1254 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1255 .addCImm(Val: CI)
1256 .addImm(Val: 0U)
1257 .addMetadata(MD: Var)
1258 .addMetadata(MD: Expr);
1259 else
1260 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1261 .addImm(Val: CI->getZExtValue())
1262 .addImm(Val: 0U)
1263 .addMetadata(MD: Var)
1264 .addMetadata(MD: Expr);
1265 return true;
1266 }
1267 if (const auto *CF = dyn_cast<ConstantFP>(Val: V)) {
1268 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1269 .addFPImm(Val: CF)
1270 .addImm(Val: 0U)
1271 .addMetadata(MD: Var)
1272 .addMetadata(MD: Expr);
1273 return true;
1274 }
1275 if (const auto *Arg = dyn_cast<Argument>(Val: V);
1276 Arg && Expr && Expr->isEntryValue()) {
1277 // As per the Verifier, this case is only valid for swift async Args.
1278 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
1279
1280 Register Reg = getRegForValue(V: Arg);
1281 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
1282 if (Reg == VirtReg || Reg == PhysReg) {
1283 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect: false /*IsIndirect*/,
1284 Reg: PhysReg, Variable: Var, Expr);
1285 return true;
1286 }
1287
1288 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
1289 "couldn't find a physical register\n");
1290 return false;
1291 }
1292 if (auto SI = FuncInfo.StaticAllocaMap.find(Val: dyn_cast<AllocaInst>(Val: V));
1293 SI != FuncInfo.StaticAllocaMap.end()) {
1294 MachineOperand FrameIndexOp = MachineOperand::CreateFI(Idx: SI->second);
1295 bool IsIndirect = false;
1296 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect, MOs: FrameIndexOp,
1297 Variable: Var, Expr);
1298 return true;
1299 }
1300 if (Register Reg = lookUpRegForValue(V)) {
1301 // FIXME: This does not handle register-indirect values at offset 0.
1302 if (!FuncInfo.MF->useDebugInstrRef()) {
1303 bool IsIndirect = false;
1304 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect, Reg, Variable: Var,
1305 Expr);
1306 return true;
1307 }
1308 // If using instruction referencing, produce this as a DBG_INSTR_REF,
1309 // to be later patched up by finalizeDebugInstrRefs.
1310 SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg(
1311 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
1312 /* isKill */ false, /* isDead */ false,
1313 /* isUndef */ false, /* isEarlyClobber */ false,
1314 /* SubReg */ 0, /* isDebug */ true)});
1315 SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0});
1316 auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops);
1317 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL,
1318 MCID: TII.get(Opcode: TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs,
1319 Variable: Var, Expr: NewExpr);
1320 return true;
1321 }
1322 return false;
1323}
1324
1325bool FastISel::lowerDbgDeclare(const Value *Address, DIExpression *Expr,
1326 DILocalVariable *Var, const DebugLoc &DL) {
1327 if (!Address || isa<UndefValue>(Val: Address)) {
1328 LLVM_DEBUG(dbgs() << "Dropping debug info (bad/undef address)\n");
1329 return false;
1330 }
1331
1332 std::optional<MachineOperand> Op;
1333 if (Register Reg = lookUpRegForValue(V: Address))
1334 Op = MachineOperand::CreateReg(Reg, isDef: false);
1335
1336 // If we have a VLA that has a "use" in a metadata node that's then used
1337 // here but it has no other uses, then we have a problem. E.g.,
1338 //
1339 // int foo (const int *x) {
1340 // char a[*x];
1341 // return 0;
1342 // }
1343 //
1344 // If we assign 'a' a vreg and fast isel later on has to use the selection
1345 // DAG isel, it will want to copy the value to the vreg. However, there are
1346 // no uses, which goes counter to what selection DAG isel expects.
1347 if (!Op && !Address->use_empty() && isa<Instruction>(Val: Address) &&
1348 (!isa<AllocaInst>(Val: Address) ||
1349 !FuncInfo.StaticAllocaMap.count(Val: cast<AllocaInst>(Val: Address))))
1350 Op = MachineOperand::CreateReg(Reg: FuncInfo.InitializeRegForValue(V: Address),
1351 isDef: false);
1352
1353 if (Op) {
1354 assert(Var->isValidLocationForIntrinsic(DL) &&
1355 "Expected inlined-at fields to agree");
1356 if (FuncInfo.MF->useDebugInstrRef() && Op->isReg()) {
1357 // If using instruction referencing, produce this as a DBG_INSTR_REF,
1358 // to be later patched up by finalizeDebugInstrRefs. Tack a deref onto
1359 // the expression, we don't have an "indirect" flag in DBG_INSTR_REF.
1360 SmallVector<uint64_t, 3> Ops(
1361 {dwarf::DW_OP_LLVM_arg, 0, dwarf::DW_OP_deref});
1362 auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops);
1363 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL,
1364 MCID: TII.get(Opcode: TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs: *Op,
1365 Variable: Var, Expr: NewExpr);
1366 return true;
1367 }
1368
1369 // A dbg.declare describes the address of a source variable, so lower it
1370 // into an indirect DBG_VALUE.
1371 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL,
1372 MCID: TII.get(Opcode: TargetOpcode::DBG_VALUE), /*IsIndirect*/ true, MOs: *Op, Variable: Var,
1373 Expr);
1374 return true;
1375 }
1376
1377 // We can't yet handle anything else here because it would require
1378 // generating code, thus altering codegen because of debug info.
1379 LLVM_DEBUG(
1380 dbgs() << "Dropping debug info (no materialized reg for address)\n");
1381 return false;
1382}
1383
1384bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
1385 switch (II->getIntrinsicID()) {
1386 default:
1387 break;
1388 // At -O0 we don't care about the lifetime intrinsics.
1389 case Intrinsic::lifetime_start:
1390 case Intrinsic::lifetime_end:
1391 // The donothing intrinsic does, well, nothing.
1392 case Intrinsic::donothing:
1393 // Neither does the sideeffect intrinsic.
1394 case Intrinsic::sideeffect:
1395 // Neither does the assume intrinsic; it's also OK not to codegen its operand.
1396 case Intrinsic::assume:
1397 // Neither does the llvm.experimental.noalias.scope.decl intrinsic
1398 case Intrinsic::experimental_noalias_scope_decl:
1399 return true;
1400 case Intrinsic::objectsize:
1401 llvm_unreachable("llvm.objectsize.* should have been lowered already");
1402
1403 case Intrinsic::is_constant:
1404 llvm_unreachable("llvm.is.constant.* should have been lowered already");
1405
1406 case Intrinsic::allow_runtime_check:
1407 case Intrinsic::allow_ubsan_check: {
1408 Register ResultReg = getRegForValue(V: ConstantInt::getTrue(Ty: II->getType()));
1409 if (!ResultReg)
1410 return false;
1411 updateValueMap(I: II, Reg: ResultReg);
1412 return true;
1413 }
1414
1415 case Intrinsic::launder_invariant_group:
1416 case Intrinsic::strip_invariant_group:
1417 case Intrinsic::expect:
1418 case Intrinsic::expect_with_probability: {
1419 Register ResultReg = getRegForValue(V: II->getArgOperand(i: 0));
1420 if (!ResultReg)
1421 return false;
1422 updateValueMap(I: II, Reg: ResultReg);
1423 return true;
1424 }
1425 case Intrinsic::fake_use: {
1426 const Value *V = II->getArgOperand(i: 0);
1427 if (Register Reg = getRegForValue(V))
1428 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1429 MCID: TII.get(Opcode: TargetOpcode::FAKE_USE))
1430 .addReg(RegNo: Reg);
1431 return true;
1432 }
1433 case Intrinsic::experimental_stackmap:
1434 return selectStackmap(I: II);
1435 case Intrinsic::experimental_patchpoint_void:
1436 case Intrinsic::experimental_patchpoint:
1437 return selectPatchpoint(I: II);
1438
1439 case Intrinsic::xray_customevent:
1440 return selectXRayCustomEvent(I: II);
1441 case Intrinsic::xray_typedevent:
1442 return selectXRayTypedEvent(I: II);
1443 }
1444
1445 return fastLowerIntrinsicCall(II);
1446}
1447
1448bool FastISel::selectCast(const User *I, unsigned Opcode) {
1449 EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1450 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
1451
1452 if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
1453 !DstVT.isSimple())
1454 // Unhandled type. Halt "fast" selection and bail.
1455 return false;
1456
1457 // Check if the destination type is legal.
1458 if (!TLI.isTypeLegal(VT: DstVT))
1459 return false;
1460
1461 // Check if the source operand is legal.
1462 if (!TLI.isTypeLegal(VT: SrcVT))
1463 return false;
1464
1465 Register InputReg = getRegForValue(V: I->getOperand(i: 0));
1466 if (!InputReg)
1467 // Unhandled operand. Halt "fast" selection and bail.
1468 return false;
1469
1470 Register ResultReg = fastEmit_r(VT: SrcVT.getSimpleVT(), RetVT: DstVT.getSimpleVT(),
1471 Opcode, Op0: InputReg);
1472 if (!ResultReg)
1473 return false;
1474
1475 updateValueMap(I, Reg: ResultReg);
1476 return true;
1477}
1478
1479bool FastISel::selectBitCast(const User *I) {
1480 EVT SrcEVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1481 EVT DstEVT = TLI.getValueType(DL, Ty: I->getType());
1482 if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
1483 !TLI.isTypeLegal(VT: SrcEVT) || !TLI.isTypeLegal(VT: DstEVT))
1484 // Unhandled type. Halt "fast" selection and bail.
1485 return false;
1486
1487 MVT SrcVT = SrcEVT.getSimpleVT();
1488 MVT DstVT = DstEVT.getSimpleVT();
1489 Register Op0 = getRegForValue(V: I->getOperand(i: 0));
1490 if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
1491 return false;
1492
1493 // If the bitcast doesn't change the type, just use the operand value.
1494 if (SrcVT == DstVT) {
1495 updateValueMap(I, Reg: Op0);
1496 return true;
1497 }
1498
1499 // Otherwise, select a BITCAST opcode.
1500 Register ResultReg = fastEmit_r(VT: SrcVT, RetVT: DstVT, Opcode: ISD::BITCAST, Op0);
1501 if (!ResultReg)
1502 return false;
1503
1504 updateValueMap(I, Reg: ResultReg);
1505 return true;
1506}
1507
1508bool FastISel::selectFreeze(const User *I) {
1509 Register Reg = getRegForValue(V: I->getOperand(i: 0));
1510 if (!Reg)
1511 // Unhandled operand.
1512 return false;
1513
1514 EVT ETy = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1515 if (ETy == MVT::Other || !TLI.isTypeLegal(VT: ETy))
1516 // Unhandled type, bail out.
1517 return false;
1518
1519 MVT Ty = ETy.getSimpleVT();
1520 const TargetRegisterClass *TyRegClass = TLI.getRegClassFor(VT: Ty);
1521 Register ResultReg = createResultReg(RC: TyRegClass);
1522 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1523 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ResultReg).addReg(RegNo: Reg);
1524
1525 updateValueMap(I, Reg: ResultReg);
1526 return true;
1527}
1528
1529// Remove local value instructions starting from the instruction after
1530// SavedLastLocalValue to the current function insert point.
1531void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
1532{
1533 MachineInstr *CurLastLocalValue = getLastLocalValue();
1534 if (CurLastLocalValue != SavedLastLocalValue) {
1535 // Find the first local value instruction to be deleted.
1536 // This is the instruction after SavedLastLocalValue if it is non-NULL.
1537 // Otherwise it's the first instruction in the block.
1538 MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
1539 if (SavedLastLocalValue)
1540 ++FirstDeadInst;
1541 else
1542 FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
1543 setLastLocalValue(SavedLastLocalValue);
1544 removeDeadCode(I: FirstDeadInst, E: FuncInfo.InsertPt);
1545 }
1546}
1547
1548bool FastISel::selectInstruction(const Instruction *I) {
1549 // Flush the local value map before starting each instruction.
1550 // This improves locality and debugging, and can reduce spills.
1551 // Reuse of values across IR instructions is relatively uncommon.
1552 flushLocalValueMap();
1553
1554 MachineInstr *SavedLastLocalValue = getLastLocalValue();
1555 // Just before the terminator instruction, insert instructions to
1556 // feed PHI nodes in successor blocks.
1557 if (I->isTerminator()) {
1558 if (!handlePHINodesInSuccessorBlocks(LLVMBB: I->getParent())) {
1559 // PHI node handling may have generated local value instructions,
1560 // even though it failed to handle all PHI nodes.
1561 // We remove these instructions because SelectionDAGISel will generate
1562 // them again.
1563 removeDeadLocalValueCode(SavedLastLocalValue);
1564 return false;
1565 }
1566 }
1567
1568 // FastISel does not handle any operand bundles except OB_funclet.
1569 if (auto *Call = dyn_cast<CallBase>(Val: I))
1570 for (unsigned i = 0, e = Call->getNumOperandBundles(); i != e; ++i)
1571 if (Call->getOperandBundleAt(Index: i).getTagID() != LLVMContext::OB_funclet)
1572 return false;
1573
1574 MIMD = MIMetadata(*I);
1575
1576 SavedInsertPt = FuncInfo.InsertPt;
1577
1578 if (const auto *Call = dyn_cast<CallInst>(Val: I)) {
1579 const Function *F = Call->getCalledFunction();
1580
1581 // Don't handle Intrinsic::trap if a trap function is specified.
1582 if (F && F->getIntrinsicID() == Intrinsic::trap &&
1583 Call->hasFnAttr(Kind: "trap-func-name"))
1584 return false;
1585 }
1586
1587 // First, try doing target-independent selection.
1588 if (!SkipTargetIndependentISel) {
1589 if (selectOperator(I, Opcode: I->getOpcode())) {
1590 ++NumFastIselSuccessIndependent;
1591 MIMD = {};
1592 return true;
1593 }
1594 // Remove dead code.
1595 recomputeInsertPt();
1596 if (SavedInsertPt != FuncInfo.InsertPt)
1597 removeDeadCode(I: FuncInfo.InsertPt, E: SavedInsertPt);
1598 SavedInsertPt = FuncInfo.InsertPt;
1599 }
1600 // Next, try calling the target to attempt to handle the instruction.
1601 if (fastSelectInstruction(I)) {
1602 ++NumFastIselSuccessTarget;
1603 MIMD = {};
1604 return true;
1605 }
1606 // Remove dead code.
1607 recomputeInsertPt();
1608 if (SavedInsertPt != FuncInfo.InsertPt)
1609 removeDeadCode(I: FuncInfo.InsertPt, E: SavedInsertPt);
1610
1611 MIMD = {};
1612 // Undo phi node updates, because they will be added again by SelectionDAG.
1613 if (I->isTerminator()) {
1614 // PHI node handling may have generated local value instructions.
1615 // We remove them because SelectionDAGISel will generate them again.
1616 removeDeadLocalValueCode(SavedLastLocalValue);
1617 FuncInfo.PHINodesToUpdate.resize(new_size: FuncInfo.OrigNumPHINodesToUpdate);
1618 }
1619 return false;
1620}
1621
1622/// Emit an unconditional branch to the given block, unless it is the immediate
1623/// (fall-through) successor, and update the CFG.
1624void FastISel::fastEmitBranch(MachineBasicBlock *MSucc,
1625 const DebugLoc &DbgLoc) {
1626 const BasicBlock *BB = FuncInfo.MBB->getBasicBlock();
1627 bool BlockHasMultipleInstrs = &BB->front() != &BB->back();
1628 if (BlockHasMultipleInstrs && FuncInfo.MBB->isLayoutSuccessor(MBB: MSucc)) {
1629 // For more accurate line information if this is the only non-debug
1630 // instruction in the block then emit it, otherwise we have the
1631 // unconditional fall-through case, which needs no instructions.
1632 } else {
1633 // The unconditional branch case.
1634 TII.insertBranch(MBB&: *FuncInfo.MBB, TBB: MSucc, FBB: nullptr,
1635 Cond: SmallVector<MachineOperand, 0>(), DL: DbgLoc);
1636 }
1637 if (FuncInfo.BPI) {
1638 auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
1639 Src: FuncInfo.MBB->getBasicBlock(), Dst: MSucc->getBasicBlock());
1640 FuncInfo.MBB->addSuccessor(Succ: MSucc, Prob: BranchProbability);
1641 } else
1642 FuncInfo.MBB->addSuccessorWithoutProb(Succ: MSucc);
1643}
1644
1645void FastISel::finishCondBranch(const BasicBlock *BranchBB,
1646 MachineBasicBlock *TrueMBB,
1647 MachineBasicBlock *FalseMBB) {
1648 // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
1649 // happen in degenerate IR and MachineIR forbids to have a block twice in the
1650 // successor/predecessor lists.
1651 if (TrueMBB != FalseMBB) {
1652 if (FuncInfo.BPI) {
1653 auto BranchProbability =
1654 FuncInfo.BPI->getEdgeProbability(Src: BranchBB, Dst: TrueMBB->getBasicBlock());
1655 FuncInfo.MBB->addSuccessor(Succ: TrueMBB, Prob: BranchProbability);
1656 } else
1657 FuncInfo.MBB->addSuccessorWithoutProb(Succ: TrueMBB);
1658 }
1659
1660 fastEmitBranch(MSucc: FalseMBB, DbgLoc: MIMD.getDL());
1661}
1662
1663/// Emit an FNeg operation.
1664bool FastISel::selectFNeg(const User *I, const Value *In) {
1665 Register OpReg = getRegForValue(V: In);
1666 if (!OpReg)
1667 return false;
1668
1669 // If the target has ISD::FNEG, use it.
1670 EVT VT = TLI.getValueType(DL, Ty: I->getType());
1671 Register ResultReg = fastEmit_r(VT: VT.getSimpleVT(), RetVT: VT.getSimpleVT(), Opcode: ISD::FNEG,
1672 Op0: OpReg);
1673 if (ResultReg) {
1674 updateValueMap(I, Reg: ResultReg);
1675 return true;
1676 }
1677
1678 // Bitcast the value to integer, twiddle the sign bit with xor,
1679 // and then bitcast it back to floating-point.
1680 if (VT.getSizeInBits() > 64)
1681 return false;
1682 EVT IntVT = EVT::getIntegerVT(Context&: I->getContext(), BitWidth: VT.getSizeInBits());
1683 if (!TLI.isTypeLegal(VT: IntVT))
1684 return false;
1685
1686 Register IntReg = fastEmit_r(VT: VT.getSimpleVT(), RetVT: IntVT.getSimpleVT(),
1687 Opcode: ISD::BITCAST, Op0: OpReg);
1688 if (!IntReg)
1689 return false;
1690
1691 Register IntResultReg = fastEmit_ri_(
1692 VT: IntVT.getSimpleVT(), Opcode: ISD::XOR, Op0: IntReg,
1693 UINT64_C(1) << (VT.getSizeInBits() - 1), ImmType: IntVT.getSimpleVT());
1694 if (!IntResultReg)
1695 return false;
1696
1697 ResultReg = fastEmit_r(VT: IntVT.getSimpleVT(), RetVT: VT.getSimpleVT(), Opcode: ISD::BITCAST,
1698 Op0: IntResultReg);
1699 if (!ResultReg)
1700 return false;
1701
1702 updateValueMap(I, Reg: ResultReg);
1703 return true;
1704}
1705
1706bool FastISel::selectExtractValue(const User *U) {
1707 const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: U);
1708 if (!EVI)
1709 return false;
1710
1711 // Make sure we only try to handle extracts with a legal result. But also
1712 // allow i1 because it's easy.
1713 EVT RealVT = TLI.getValueType(DL, Ty: EVI->getType(), /*AllowUnknown=*/true);
1714 if (!RealVT.isSimple())
1715 return false;
1716 MVT VT = RealVT.getSimpleVT();
1717 if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
1718 return false;
1719
1720 const Value *Op0 = EVI->getOperand(i_nocapture: 0);
1721 Type *AggTy = Op0->getType();
1722
1723 // Get the base result register.
1724 Register ResultReg;
1725 auto I = FuncInfo.ValueMap.find(Val: Op0);
1726 if (I != FuncInfo.ValueMap.end())
1727 ResultReg = I->second;
1728 else if (isa<Instruction>(Val: Op0))
1729 ResultReg = FuncInfo.InitializeRegForValue(V: Op0);
1730 else
1731 return false; // fast-isel can't handle aggregate constants at the moment
1732
1733 // Get the actual result register, which is an offset from the base register.
1734 unsigned VTIndex = ComputeLinearIndex(Ty: AggTy, Indices: EVI->getIndices());
1735
1736 SmallVector<EVT, 4> AggValueVTs;
1737 ComputeValueVTs(TLI, DL, Ty: AggTy, ValueVTs&: AggValueVTs);
1738
1739 for (unsigned i = 0; i < VTIndex; i++)
1740 ResultReg = ResultReg.id() +
1741 TLI.getNumRegisters(Context&: FuncInfo.Fn->getContext(), VT: AggValueVTs[i]);
1742
1743 updateValueMap(I: EVI, Reg: ResultReg);
1744 return true;
1745}
1746
1747bool FastISel::selectOperator(const User *I, unsigned Opcode) {
1748 switch (Opcode) {
1749 case Instruction::Add:
1750 return selectBinaryOp(I, ISDOpcode: ISD::ADD);
1751 case Instruction::FAdd:
1752 return selectBinaryOp(I, ISDOpcode: ISD::FADD);
1753 case Instruction::Sub:
1754 return selectBinaryOp(I, ISDOpcode: ISD::SUB);
1755 case Instruction::FSub:
1756 return selectBinaryOp(I, ISDOpcode: ISD::FSUB);
1757 case Instruction::Mul:
1758 return selectBinaryOp(I, ISDOpcode: ISD::MUL);
1759 case Instruction::FMul:
1760 return selectBinaryOp(I, ISDOpcode: ISD::FMUL);
1761 case Instruction::SDiv:
1762 return selectBinaryOp(I, ISDOpcode: ISD::SDIV);
1763 case Instruction::UDiv:
1764 return selectBinaryOp(I, ISDOpcode: ISD::UDIV);
1765 case Instruction::FDiv:
1766 return selectBinaryOp(I, ISDOpcode: ISD::FDIV);
1767 case Instruction::SRem:
1768 return selectBinaryOp(I, ISDOpcode: ISD::SREM);
1769 case Instruction::URem:
1770 return selectBinaryOp(I, ISDOpcode: ISD::UREM);
1771 case Instruction::FRem:
1772 return selectBinaryOp(I, ISDOpcode: ISD::FREM);
1773 case Instruction::Shl:
1774 return selectBinaryOp(I, ISDOpcode: ISD::SHL);
1775 case Instruction::LShr:
1776 return selectBinaryOp(I, ISDOpcode: ISD::SRL);
1777 case Instruction::AShr:
1778 return selectBinaryOp(I, ISDOpcode: ISD::SRA);
1779 case Instruction::And:
1780 return selectBinaryOp(I, ISDOpcode: ISD::AND);
1781 case Instruction::Or:
1782 return selectBinaryOp(I, ISDOpcode: ISD::OR);
1783 case Instruction::Xor:
1784 return selectBinaryOp(I, ISDOpcode: ISD::XOR);
1785
1786 case Instruction::FNeg:
1787 return selectFNeg(I, In: I->getOperand(i: 0));
1788
1789 case Instruction::GetElementPtr:
1790 return selectGetElementPtr(I);
1791
1792 case Instruction::UncondBr: {
1793 const UncondBrInst *BI = cast<UncondBrInst>(Val: I);
1794 const BasicBlock *LLVMSucc = BI->getSuccessor(i: 0);
1795 MachineBasicBlock *MSucc = FuncInfo.getMBB(BB: LLVMSucc);
1796 fastEmitBranch(MSucc, DbgLoc: BI->getDebugLoc());
1797 return true;
1798 }
1799
1800 case Instruction::Unreachable: {
1801 auto UI = cast<UnreachableInst>(Val: I);
1802 if (!UI->shouldLowerToTrap(TrapUnreachable: TM.Options.TrapUnreachable,
1803 NoTrapAfterNoreturn: TM.Options.NoTrapAfterNoreturn))
1804 return true;
1805
1806 return fastEmit_(VT: MVT::Other, RetVT: MVT::Other, Opcode: ISD::TRAP) != 0;
1807 }
1808
1809 case Instruction::Alloca:
1810 // FunctionLowering has the static-sized case covered.
1811 if (FuncInfo.StaticAllocaMap.count(Val: cast<AllocaInst>(Val: I)))
1812 return true;
1813
1814 // Dynamic-sized alloca is not handled yet.
1815 return false;
1816
1817 case Instruction::Call:
1818 // On AIX, normal call lowering uses the DAG-ISEL path currently so that the
1819 // callee of the direct function call instruction will be mapped to the
1820 // symbol for the function's entry point, which is distinct from the
1821 // function descriptor symbol. The latter is the symbol whose XCOFF symbol
1822 // name is the C-linkage name of the source level function.
1823 // But fast isel still has the ability to do selection for intrinsics.
1824 if (TM.getTargetTriple().isOSAIX() && !isa<IntrinsicInst>(Val: I))
1825 return false;
1826 return selectCall(I);
1827
1828 case Instruction::BitCast:
1829 return selectBitCast(I);
1830
1831 case Instruction::FPToSI:
1832 return selectCast(I, Opcode: ISD::FP_TO_SINT);
1833 case Instruction::ZExt:
1834 return selectCast(I, Opcode: ISD::ZERO_EXTEND);
1835 case Instruction::SExt:
1836 return selectCast(I, Opcode: ISD::SIGN_EXTEND);
1837 case Instruction::Trunc:
1838 return selectCast(I, Opcode: ISD::TRUNCATE);
1839 case Instruction::SIToFP:
1840 return selectCast(I, Opcode: ISD::SINT_TO_FP);
1841
1842 case Instruction::IntToPtr: // Deliberate fall-through.
1843 case Instruction::PtrToInt:
1844 case Instruction::PtrToAddr: {
1845 EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1846 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
1847 if (DstVT.bitsGT(VT: SrcVT))
1848 return selectCast(I, Opcode: ISD::ZERO_EXTEND);
1849 if (DstVT.bitsLT(VT: SrcVT))
1850 return selectCast(I, Opcode: ISD::TRUNCATE);
1851 Register Reg = getRegForValue(V: I->getOperand(i: 0));
1852 if (!Reg)
1853 return false;
1854 updateValueMap(I, Reg);
1855 return true;
1856 }
1857
1858 case Instruction::ExtractValue:
1859 return selectExtractValue(U: I);
1860
1861 case Instruction::Freeze:
1862 return selectFreeze(I);
1863
1864 case Instruction::PHI:
1865 llvm_unreachable("FastISel shouldn't visit PHI nodes!");
1866
1867 default:
1868 // Unhandled instruction. Halt "fast" selection and bail.
1869 return false;
1870 }
1871}
1872
1873FastISel::FastISel(FunctionLoweringInfo &FuncInfo,
1874 const TargetLibraryInfo *LibInfo,
1875 const LibcallLoweringInfo *LibcallLowering,
1876 bool SkipTargetIndependentISel)
1877 : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
1878 MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
1879 TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
1880 TII(*MF->getSubtarget().getInstrInfo()),
1881 TLI(*MF->getSubtarget().getTargetLowering()),
1882 TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
1883 LibcallLowering(LibcallLowering),
1884 SkipTargetIndependentISel(SkipTargetIndependentISel) {}
1885
1886FastISel::~FastISel() = default;
1887
1888bool FastISel::fastLowerArguments() { return false; }
1889
1890bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
1891
1892bool FastISel::fastLowerIntrinsicCall(const IntrinsicInst * /*II*/) {
1893 return false;
1894}
1895
1896Register FastISel::fastEmit_(MVT, MVT, unsigned) { return Register(); }
1897
1898Register FastISel::fastEmit_r(MVT, MVT, unsigned, Register /*Op0*/) {
1899 return Register();
1900}
1901
1902Register FastISel::fastEmit_rr(MVT, MVT, unsigned, Register /*Op0*/,
1903 Register /*Op1*/) {
1904 return Register();
1905}
1906
1907Register FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
1908 return Register();
1909}
1910
1911Register FastISel::fastEmit_f(MVT, MVT, unsigned,
1912 const ConstantFP * /*FPImm*/) {
1913 return Register();
1914}
1915
1916Register FastISel::fastEmit_ri(MVT, MVT, unsigned, Register /*Op0*/,
1917 uint64_t /*Imm*/) {
1918 return Register();
1919}
1920
1921/// This method is a wrapper of fastEmit_ri. It first tries to emit an
1922/// instruction with an immediate operand using fastEmit_ri.
1923/// If that fails, it materializes the immediate into a register and try
1924/// fastEmit_rr instead.
1925Register FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, Register Op0,
1926 uint64_t Imm, MVT ImmType) {
1927 // If this is a multiply by a power of two, emit this as a shift left.
1928 if (Opcode == ISD::MUL && isPowerOf2_64(Value: Imm)) {
1929 Opcode = ISD::SHL;
1930 Imm = Log2_64(Value: Imm);
1931 } else if (Opcode == ISD::UDIV && isPowerOf2_64(Value: Imm)) {
1932 // div x, 8 -> srl x, 3
1933 Opcode = ISD::SRL;
1934 Imm = Log2_64(Value: Imm);
1935 }
1936
1937 // Horrible hack (to be removed), check to make sure shift amounts are
1938 // in-range.
1939 if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
1940 Imm >= VT.getSizeInBits())
1941 return Register();
1942
1943 // First check if immediate type is legal. If not, we can't use the ri form.
1944 Register ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Imm);
1945 if (ResultReg)
1946 return ResultReg;
1947 Register MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
1948 if (!MaterialReg) {
1949 // This is a bit ugly/slow, but failing here means falling out of
1950 // fast-isel, which would be very slow.
1951 IntegerType *ITy =
1952 IntegerType::get(C&: FuncInfo.Fn->getContext(), NumBits: VT.getSizeInBits());
1953 // TODO: Avoid implicit trunc?
1954 // See https://github.com/llvm/llvm-project/issues/112510.
1955 MaterialReg = getRegForValue(
1956 V: ConstantInt::get(Ty: ITy, V: Imm, /*IsSigned=*/false, /*ImplicitTrunc=*/true));
1957 if (!MaterialReg)
1958 return Register();
1959 }
1960 return fastEmit_rr(VT, VT, Opcode, Op0, MaterialReg);
1961}
1962
1963Register FastISel::createResultReg(const TargetRegisterClass *RC) {
1964 return MRI.createVirtualRegister(RegClass: RC);
1965}
1966
1967Register FastISel::constrainOperandRegClass(const MCInstrDesc &II, Register Op,
1968 unsigned OpNum) {
1969 if (Op.isVirtual()) {
1970 const TargetRegisterClass *RegClass = TII.getRegClass(MCID: II, OpNum);
1971 if (!MRI.constrainRegClass(Reg: Op, RC: RegClass)) {
1972 // If it's not legal to COPY between the register classes, something
1973 // has gone very wrong before we got here.
1974 Register NewOp = createResultReg(RC: RegClass);
1975 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1976 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: NewOp).addReg(RegNo: Op);
1977 return NewOp;
1978 }
1979 }
1980 return Op;
1981}
1982
1983Register FastISel::fastEmitInst_(unsigned MachineInstOpcode,
1984 const TargetRegisterClass *RC) {
1985 Register ResultReg = createResultReg(RC);
1986 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
1987
1988 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg);
1989 return ResultReg;
1990}
1991
1992Register FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
1993 const TargetRegisterClass *RC, Register Op0) {
1994 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
1995
1996 Register ResultReg = createResultReg(RC);
1997 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
1998
1999 if (II.getNumDefs() >= 1)
2000 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2001 .addReg(RegNo: Op0);
2002 else {
2003 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2004 .addReg(RegNo: Op0);
2005 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2006 DestReg: ResultReg)
2007 .addReg(RegNo: II.implicit_defs()[0]);
2008 }
2009
2010 return ResultReg;
2011}
2012
2013Register FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
2014 const TargetRegisterClass *RC, Register Op0,
2015 Register Op1) {
2016 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2017
2018 Register ResultReg = createResultReg(RC);
2019 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2020 Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
2021
2022 if (II.getNumDefs() >= 1)
2023 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2024 .addReg(RegNo: Op0)
2025 .addReg(RegNo: Op1);
2026 else {
2027 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2028 .addReg(RegNo: Op0)
2029 .addReg(RegNo: Op1);
2030 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2031 DestReg: ResultReg)
2032 .addReg(RegNo: II.implicit_defs()[0]);
2033 }
2034 return ResultReg;
2035}
2036
2037Register FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
2038 const TargetRegisterClass *RC, Register Op0,
2039 Register Op1, Register Op2) {
2040 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2041
2042 Register ResultReg = createResultReg(RC);
2043 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2044 Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
2045 Op2 = constrainOperandRegClass(II, Op: Op2, OpNum: II.getNumDefs() + 2);
2046
2047 if (II.getNumDefs() >= 1)
2048 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2049 .addReg(RegNo: Op0)
2050 .addReg(RegNo: Op1)
2051 .addReg(RegNo: Op2);
2052 else {
2053 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2054 .addReg(RegNo: Op0)
2055 .addReg(RegNo: Op1)
2056 .addReg(RegNo: Op2);
2057 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2058 DestReg: ResultReg)
2059 .addReg(RegNo: II.implicit_defs()[0]);
2060 }
2061 return ResultReg;
2062}
2063
2064Register FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
2065 const TargetRegisterClass *RC, Register Op0,
2066 uint64_t Imm) {
2067 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2068
2069 Register ResultReg = createResultReg(RC);
2070 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2071
2072 if (II.getNumDefs() >= 1)
2073 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2074 .addReg(RegNo: Op0)
2075 .addImm(Val: Imm);
2076 else {
2077 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2078 .addReg(RegNo: Op0)
2079 .addImm(Val: Imm);
2080 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2081 DestReg: ResultReg)
2082 .addReg(RegNo: II.implicit_defs()[0]);
2083 }
2084 return ResultReg;
2085}
2086
2087Register FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
2088 const TargetRegisterClass *RC, Register Op0,
2089 uint64_t Imm1, uint64_t Imm2) {
2090 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2091
2092 Register ResultReg = createResultReg(RC);
2093 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2094
2095 if (II.getNumDefs() >= 1)
2096 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2097 .addReg(RegNo: Op0)
2098 .addImm(Val: Imm1)
2099 .addImm(Val: Imm2);
2100 else {
2101 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2102 .addReg(RegNo: Op0)
2103 .addImm(Val: Imm1)
2104 .addImm(Val: Imm2);
2105 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2106 DestReg: ResultReg)
2107 .addReg(RegNo: II.implicit_defs()[0]);
2108 }
2109 return ResultReg;
2110}
2111
2112Register FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
2113 const TargetRegisterClass *RC,
2114 const ConstantFP *FPImm) {
2115 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2116
2117 Register ResultReg = createResultReg(RC);
2118
2119 if (II.getNumDefs() >= 1)
2120 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2121 .addFPImm(Val: FPImm);
2122 else {
2123 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2124 .addFPImm(Val: FPImm);
2125 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2126 DestReg: ResultReg)
2127 .addReg(RegNo: II.implicit_defs()[0]);
2128 }
2129 return ResultReg;
2130}
2131
2132Register FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
2133 const TargetRegisterClass *RC, Register Op0,
2134 Register Op1, uint64_t Imm) {
2135 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2136
2137 Register ResultReg = createResultReg(RC);
2138 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2139 Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
2140
2141 if (II.getNumDefs() >= 1)
2142 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2143 .addReg(RegNo: Op0)
2144 .addReg(RegNo: Op1)
2145 .addImm(Val: Imm);
2146 else {
2147 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2148 .addReg(RegNo: Op0)
2149 .addReg(RegNo: Op1)
2150 .addImm(Val: Imm);
2151 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2152 DestReg: ResultReg)
2153 .addReg(RegNo: II.implicit_defs()[0]);
2154 }
2155 return ResultReg;
2156}
2157
2158Register FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
2159 const TargetRegisterClass *RC, uint64_t Imm) {
2160 Register ResultReg = createResultReg(RC);
2161 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2162
2163 if (II.getNumDefs() >= 1)
2164 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2165 .addImm(Val: Imm);
2166 else {
2167 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II).addImm(Val: Imm);
2168 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2169 DestReg: ResultReg)
2170 .addReg(RegNo: II.implicit_defs()[0]);
2171 }
2172 return ResultReg;
2173}
2174
2175Register FastISel::fastEmitInst_extractsubreg(MVT RetVT, Register Op0,
2176 uint32_t Idx) {
2177 Register ResultReg = createResultReg(RC: TLI.getRegClassFor(VT: RetVT));
2178 assert(Op0.isVirtual() && "Cannot yet extract from physregs");
2179 const TargetRegisterClass *RC = MRI.getRegClass(Reg: Op0);
2180 MRI.constrainRegClass(Reg: Op0, RC: TRI.getSubClassWithSubReg(RC, Idx));
2181 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2182 DestReg: ResultReg)
2183 .addReg(RegNo: Op0, Flags: {}, SubReg: Idx);
2184 return ResultReg;
2185}
2186
2187/// Emit MachineInstrs to compute the value of Op with all but the least
2188/// significant bit set to zero.
2189Register FastISel::fastEmitZExtFromI1(MVT VT, Register Op0) {
2190 return fastEmit_ri(VT, VT, ISD::AND, Op0, 1);
2191}
2192
2193/// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
2194/// Emit code to ensure constants are copied into registers when needed.
2195/// Remember the virtual registers that need to be added to the Machine PHI
2196/// nodes as input. We cannot just directly add them, because expansion
2197/// might result in multiple MBB's for one BB. As such, the start of the
2198/// BB might correspond to a different MBB than the end.
2199bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
2200 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
2201 FuncInfo.OrigNumPHINodesToUpdate = FuncInfo.PHINodesToUpdate.size();
2202
2203 // Check successor nodes' PHI nodes that expect a constant to be available
2204 // from this block.
2205 for (const BasicBlock *SuccBB : successors(BB: LLVMBB)) {
2206 if (!isa<PHINode>(Val: SuccBB->begin()))
2207 continue;
2208 MachineBasicBlock *SuccMBB = FuncInfo.getMBB(BB: SuccBB);
2209
2210 // If this terminator has multiple identical successors (common for
2211 // switches), only handle each succ once.
2212 if (!SuccsHandled.insert(Ptr: SuccMBB).second)
2213 continue;
2214
2215 MachineBasicBlock::iterator MBBI = SuccMBB->begin();
2216
2217 // At this point we know that there is a 1-1 correspondence between LLVM PHI
2218 // nodes and Machine PHI nodes, but the incoming operands have not been
2219 // emitted yet.
2220 for (const PHINode &PN : SuccBB->phis()) {
2221 // Ignore dead phi's.
2222 if (PN.use_empty())
2223 continue;
2224
2225 // Only handle legal types. Two interesting things to note here. First,
2226 // by bailing out early, we may leave behind some dead instructions,
2227 // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
2228 // own moves. Second, this check is necessary because FastISel doesn't
2229 // use CreateRegs to create registers, so it always creates
2230 // exactly one register for each non-void instruction.
2231 EVT VT = TLI.getValueType(DL, Ty: PN.getType(), /*AllowUnknown=*/true);
2232 if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
2233 // Handle integer promotions, though, because they're common and easy.
2234 if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
2235 FuncInfo.PHINodesToUpdate.resize(new_size: FuncInfo.OrigNumPHINodesToUpdate);
2236 return false;
2237 }
2238 }
2239
2240 const Value *PHIOp = PN.getIncomingValueForBlock(BB: LLVMBB);
2241
2242 // Set the DebugLoc for the copy. Use the location of the operand if
2243 // there is one; otherwise no location, flushLocalValueMap will fix it.
2244 MIMD = {};
2245 if (const auto *Inst = dyn_cast<Instruction>(Val: PHIOp))
2246 MIMD = MIMetadata(*Inst);
2247
2248 Register Reg = getRegForValue(V: PHIOp);
2249 if (!Reg) {
2250 FuncInfo.PHINodesToUpdate.resize(new_size: FuncInfo.OrigNumPHINodesToUpdate);
2251 return false;
2252 }
2253 FuncInfo.PHINodesToUpdate.emplace_back(args: &*MBBI++, args&: Reg);
2254 MIMD = {};
2255 }
2256 }
2257
2258 return true;
2259}
2260
2261bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
2262 assert(LI->hasOneUse() &&
2263 "tryToFoldLoad expected a LoadInst with a single use");
2264 // We know that the load has a single use, but don't know what it is. If it
2265 // isn't one of the folded instructions, then we can't succeed here. Handle
2266 // this by scanning the single-use users of the load until we get to FoldInst.
2267 unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
2268
2269 const Instruction *TheUser = LI->user_back();
2270 while (TheUser != FoldInst && // Scan up until we find FoldInst.
2271 // Stay in the right block.
2272 TheUser->getParent() == FoldInst->getParent() &&
2273 --MaxUsers) { // Don't scan too far.
2274 // If there are multiple or no uses of this instruction, then bail out.
2275 if (!TheUser->hasOneUse())
2276 return false;
2277
2278 TheUser = TheUser->user_back();
2279 }
2280
2281 // If we didn't find the fold instruction, then we failed to collapse the
2282 // sequence.
2283 if (TheUser != FoldInst)
2284 return false;
2285
2286 // Don't try to fold ordered loads. Target has to deal with alignment
2287 // constraints and synchronization.
2288 if (!LI->isUnordered())
2289 return false;
2290
2291 // Figure out which vreg this is going into. If there is no assigned vreg yet
2292 // then there actually was no reference to it. Perhaps the load is referenced
2293 // by a dead instruction.
2294 Register LoadReg = getRegForValue(V: LI);
2295 if (!LoadReg)
2296 return false;
2297
2298 // We can't fold if this vreg has no uses or more than one use. Multiple uses
2299 // may mean that the instruction got lowered to multiple MIs, or the use of
2300 // the loaded value ended up being multiple operands of the result.
2301 if (!MRI.hasOneUse(RegNo: LoadReg))
2302 return false;
2303
2304 // If the register has fixups, there may be additional uses through a
2305 // different alias of the register.
2306 if (FuncInfo.RegsWithFixups.contains(V: LoadReg))
2307 return false;
2308
2309 MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(RegNo: LoadReg);
2310 MachineInstr *User = RI->getParent();
2311
2312 // Set the insertion point properly. Folding the load can cause generation of
2313 // other random instructions (like sign extends) for addressing modes; make
2314 // sure they get inserted in a logical place before the new instruction.
2315 FuncInfo.InsertPt = User;
2316 FuncInfo.MBB = User->getParent();
2317
2318 // Ask the target to try folding the load.
2319 return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
2320}
2321
2322bool FastISel::canFoldAddIntoGEP(const User *GEP, const Value *Add) {
2323 // Must be an add.
2324 if (!isa<AddOperator>(Val: Add))
2325 return false;
2326 // Type size needs to match.
2327 if (DL.getTypeSizeInBits(Ty: GEP->getType()) !=
2328 DL.getTypeSizeInBits(Ty: Add->getType()))
2329 return false;
2330 // Must be in the same basic block.
2331 if (isa<Instruction>(Val: Add) &&
2332 FuncInfo.getMBB(BB: cast<Instruction>(Val: Add)->getParent()) != FuncInfo.MBB)
2333 return false;
2334 // Must have a constant operand.
2335 return isa<ConstantInt>(Val: cast<AddOperator>(Val: Add)->getOperand(i_nocapture: 1));
2336}
2337
2338MachineMemOperand *
2339FastISel::createMachineMemOperandFor(const Instruction *I) const {
2340 const Value *Ptr;
2341 Type *ValTy;
2342 MaybeAlign Alignment;
2343 MachineMemOperand::Flags Flags;
2344 bool IsVolatile;
2345
2346 if (const auto *LI = dyn_cast<LoadInst>(Val: I)) {
2347 Alignment = LI->getAlign();
2348 IsVolatile = LI->isVolatile();
2349 Flags = MachineMemOperand::MOLoad;
2350 Ptr = LI->getPointerOperand();
2351 ValTy = LI->getType();
2352 } else if (const auto *SI = dyn_cast<StoreInst>(Val: I)) {
2353 Alignment = SI->getAlign();
2354 IsVolatile = SI->isVolatile();
2355 Flags = MachineMemOperand::MOStore;
2356 Ptr = SI->getPointerOperand();
2357 ValTy = SI->getValueOperand()->getType();
2358 } else
2359 return nullptr;
2360
2361 bool IsNonTemporal = I->hasMetadata(KindID: LLVMContext::MD_nontemporal);
2362 bool IsInvariant = I->hasMetadata(KindID: LLVMContext::MD_invariant_load);
2363 const MDNode *Ranges = I->getMetadata(KindID: LLVMContext::MD_range);
2364
2365 AAMDNodes AAInfo = I->getAAMetadata();
2366
2367 if (!Alignment) // Ensure that codegen never sees alignment 0.
2368 Alignment = DL.getABITypeAlign(Ty: ValTy);
2369
2370 unsigned Size = DL.getTypeStoreSize(Ty: ValTy);
2371
2372 if (IsVolatile)
2373 Flags |= MachineMemOperand::MOVolatile;
2374 if (IsNonTemporal)
2375 Flags |= MachineMemOperand::MONonTemporal;
2376 if (IsInvariant)
2377 Flags |= MachineMemOperand::MOInvariant;
2378
2379 return FuncInfo.MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Ptr), F: Flags, Size,
2380 BaseAlignment: *Alignment,
2381 Metadata: MMOMetadata(AAInfo, Ranges));
2382}
2383
2384CmpInst::Predicate FastISel::optimizeCmpPredicate(const CmpInst *CI) const {
2385 // If both operands are the same, then try to optimize or fold the cmp.
2386 CmpInst::Predicate Predicate = CI->getPredicate();
2387 if (CI->getOperand(i_nocapture: 0) != CI->getOperand(i_nocapture: 1))
2388 return Predicate;
2389
2390 switch (Predicate) {
2391 default: llvm_unreachable("Invalid predicate!");
2392 case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
2393 case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break;
2394 case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break;
2395 case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break;
2396 case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break;
2397 case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break;
2398 case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break;
2399 case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break;
2400 case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break;
2401 case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break;
2402 case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break;
2403 case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2404 case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break;
2405 case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2406 case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break;
2407 case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break;
2408
2409 case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break;
2410 case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break;
2411 case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break;
2412 case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2413 case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break;
2414 case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2415 case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break;
2416 case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break;
2417 case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break;
2418 case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break;
2419 }
2420
2421 return Predicate;
2422}
2423