1//===- llvm/CodeGen/GlobalISel/Utils.cpp -------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file This file implements the utility functions used by the GlobalISel
9/// pipeline.
10//===----------------------------------------------------------------------===//
11
12#include "llvm/CodeGen/GlobalISel/Utils.h"
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/Analysis/ConstantFolding.h"
16#include "llvm/Analysis/ValueTracking.h"
17#include "llvm/CodeGen/CodeGenCommonISel.h"
18#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
19#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
20#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/LowLevelTypeUtils.h"
25#include "llvm/CodeGen/MachineInstr.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
28#include "llvm/CodeGen/MachineRegisterInfo.h"
29#include "llvm/CodeGen/MachineSizeOpts.h"
30#include "llvm/CodeGen/RegisterBankInfo.h"
31#include "llvm/CodeGen/StackProtector.h"
32#include "llvm/CodeGen/TargetInstrInfo.h"
33#include "llvm/CodeGen/TargetLowering.h"
34#include "llvm/CodeGen/TargetOpcodes.h"
35#include "llvm/CodeGen/TargetPassConfig.h"
36#include "llvm/CodeGen/TargetRegisterInfo.h"
37#include "llvm/IR/Constants.h"
38#include "llvm/IR/Function.h"
39#include "llvm/IR/Type.h"
40#include "llvm/Support/UndefPoison.h"
41#include "llvm/Target/TargetMachine.h"
42#include "llvm/Transforms/Utils/SizeOpts.h"
43#include <cmath>
44#include <limits>
45#include <numeric>
46#include <optional>
47#include <tuple>
48
49#define DEBUG_TYPE "globalisel-utils"
50
51using namespace llvm;
52using namespace MIPatternMatch;
53
54Register llvm::constrainRegToClass(MachineRegisterInfo &MRI,
55 const TargetInstrInfo &TII,
56 const RegisterBankInfo &RBI, Register Reg,
57 const TargetRegisterClass &RegClass) {
58 if (!RBI.constrainGenericRegister(Reg, RC: RegClass, MRI))
59 return MRI.createVirtualRegister(RegClass: &RegClass);
60
61 return Reg;
62}
63
64Register llvm::constrainOperandRegClass(
65 const MachineFunction &MF, const TargetRegisterInfo &TRI,
66 MachineRegisterInfo &MRI, const TargetInstrInfo &TII,
67 const RegisterBankInfo &RBI, MachineInstr &InsertPt,
68 const TargetRegisterClass &RegClass, MachineOperand &RegMO) {
69 Register Reg = RegMO.getReg();
70 // Assume physical registers are properly constrained.
71 assert(Reg.isVirtual() && "PhysReg not implemented");
72
73 // Save the old register class to check whether
74 // the change notifications will be required.
75 // TODO: A better approach would be to pass
76 // the observers to constrainRegToClass().
77 auto *OldRegClass = MRI.getRegClassOrNull(Reg);
78 Register ConstrainedReg = constrainRegToClass(MRI, TII, RBI, Reg, RegClass);
79 // If we created a new virtual register because the class is not compatible
80 // then create a copy between the new and the old register.
81 if (ConstrainedReg != Reg) {
82 MachineBasicBlock::iterator InsertIt(&InsertPt);
83 MachineBasicBlock &MBB = *InsertPt.getParent();
84 // FIXME: The copy needs to have the classes constrained for its operands.
85 // Use operand's regbank to get the class for old register (Reg).
86 if (RegMO.isUse()) {
87 BuildMI(BB&: MBB, I: InsertIt, MIMD: InsertPt.getDebugLoc(),
88 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ConstrainedReg)
89 .addReg(RegNo: Reg);
90 } else {
91 assert(RegMO.isDef() && "Must be a definition");
92 BuildMI(BB&: MBB, I: std::next(x: InsertIt), MIMD: InsertPt.getDebugLoc(),
93 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Reg)
94 .addReg(RegNo: ConstrainedReg);
95 }
96 if (GISelChangeObserver *Observer = MF.getObserver()) {
97 Observer->changingInstr(MI&: *RegMO.getParent());
98 }
99 RegMO.setReg(ConstrainedReg);
100 if (GISelChangeObserver *Observer = MF.getObserver()) {
101 Observer->changedInstr(MI&: *RegMO.getParent());
102 }
103 } else if (OldRegClass != MRI.getRegClassOrNull(Reg)) {
104 if (GISelChangeObserver *Observer = MF.getObserver()) {
105 if (!RegMO.isDef()) {
106 MachineInstr *RegDef = MRI.getVRegDef(Reg);
107 Observer->changedInstr(MI&: *RegDef);
108 }
109 Observer->changingAllUsesOfReg(MRI, Reg);
110 Observer->finishedChangingAllUsesOfReg();
111 }
112 }
113 return ConstrainedReg;
114}
115
116Register llvm::constrainOperandRegClass(
117 const MachineFunction &MF, const TargetRegisterInfo &TRI,
118 MachineRegisterInfo &MRI, const TargetInstrInfo &TII,
119 const RegisterBankInfo &RBI, MachineInstr &InsertPt, const MCInstrDesc &II,
120 MachineOperand &RegMO, unsigned OpIdx) {
121 Register Reg = RegMO.getReg();
122 // Assume physical registers are properly constrained.
123 assert(Reg.isVirtual() && "PhysReg not implemented");
124
125 const TargetRegisterClass *OpRC = TII.getRegClass(MCID: II, OpNum: OpIdx);
126 // Some of the target independent instructions, like COPY, may not impose any
127 // register class constraints on some of their operands: If it's a use, we can
128 // skip constraining as the instruction defining the register would constrain
129 // it.
130
131 if (OpRC) {
132 // Obtain the RC from incoming regbank if it is a proper sub-class. Operands
133 // can have multiple regbanks for a superclass that combine different
134 // register types (E.g., AMDGPU's VGPR and AGPR). The regbank ambiguity
135 // resolved by targets during regbankselect should not be overridden.
136 if (const auto *SubRC = TRI.getCommonSubClass(
137 A: OpRC, B: TRI.getConstrainedRegClassForReg(Reg, MRI)))
138 OpRC = SubRC;
139
140 OpRC = TRI.getAllocatableClass(RC: OpRC);
141 }
142
143 if (!OpRC) {
144 assert((!isTargetSpecificOpcode(II.getOpcode()) || RegMO.isUse()) &&
145 "Register class constraint is required unless either the "
146 "instruction is target independent or the operand is a use");
147 // FIXME: Just bailing out like this here could be not enough, unless we
148 // expect the users of this function to do the right thing for PHIs and
149 // COPY:
150 // v1 = COPY v0
151 // v2 = COPY v1
152 // v1 here may end up not being constrained at all. Please notice that to
153 // reproduce the issue we likely need a destination pattern of a selection
154 // rule producing such extra copies, not just an input GMIR with them as
155 // every existing target using selectImpl handles copies before calling it
156 // and they never reach this function.
157 return Reg;
158 }
159 return constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt, RegClass: *OpRC,
160 RegMO);
161}
162
163void llvm::constrainSelectedInstRegOperands(MachineInstr &I,
164 const TargetInstrInfo &TII,
165 const TargetRegisterInfo &TRI,
166 const RegisterBankInfo &RBI) {
167 assert(!isPreISelGenericOpcode(I.getOpcode()) &&
168 "A selected instruction is expected");
169 MachineBasicBlock &MBB = *I.getParent();
170 MachineFunction &MF = *MBB.getParent();
171 MachineRegisterInfo &MRI = MF.getRegInfo();
172
173 for (unsigned OpI = 0, OpE = I.getNumExplicitOperands(); OpI != OpE; ++OpI) {
174 MachineOperand &MO = I.getOperand(i: OpI);
175
176 // There's nothing to be done on non-register operands.
177 if (!MO.isReg())
178 continue;
179
180 LLVM_DEBUG(dbgs() << "Converting operand: " << MO << '\n');
181
182 Register Reg = MO.getReg();
183 // Physical registers don't need to be constrained.
184 if (Reg.isPhysical())
185 continue;
186
187 // Register operands with a value of 0 (e.g. predicate operands) don't need
188 // to be constrained.
189 if (Reg == 0)
190 continue;
191
192 // If the operand is a vreg, we should constrain its regclass, and only
193 // insert COPYs if that's impossible.
194 // constrainOperandRegClass does that for us.
195 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt&: I, II: I.getDesc(), RegMO&: MO, OpIdx: OpI);
196
197 // Tie uses to defs as indicated in MCInstrDesc if this hasn't already been
198 // done.
199 if (MO.isUse()) {
200 int DefIdx = I.getDesc().getOperandConstraint(OpNum: OpI, Constraint: MCOI::TIED_TO);
201 if (DefIdx != -1 && !I.isRegTiedToUseOperand(DefOpIdx: DefIdx))
202 I.tieOperands(DefIdx, UseIdx: OpI);
203 }
204 }
205}
206
207bool llvm::canReplaceReg(Register DstReg, Register SrcReg,
208 MachineRegisterInfo &MRI) {
209 // Give up if either DstReg or SrcReg is a physical register.
210 if (DstReg.isPhysical() || SrcReg.isPhysical())
211 return false;
212 // Give up if the types don't match.
213 if (MRI.getType(Reg: DstReg) != MRI.getType(Reg: SrcReg))
214 return false;
215 // Replace if either DstReg has no constraints or the register
216 // constraints match.
217 const auto &DstRBC = MRI.getRegClassOrRegBank(Reg: DstReg);
218 if (!DstRBC || DstRBC == MRI.getRegClassOrRegBank(Reg: SrcReg))
219 return true;
220
221 // Otherwise match if the Src is already a regclass that is covered by the Dst
222 // RegBank.
223 return isa<const RegisterBank *>(Val: DstRBC) && MRI.getRegClassOrNull(Reg: SrcReg) &&
224 cast<const RegisterBank *>(Val: DstRBC)->covers(
225 RC: *MRI.getRegClassOrNull(Reg: SrcReg));
226}
227
228bool llvm::isTriviallyDead(const MachineInstr &MI,
229 const MachineRegisterInfo &MRI) {
230 // Instructions without side-effects are dead iff they only define dead regs.
231 // This function is hot and this loop returns early in the common case,
232 // so only perform additional checks before this if absolutely necessary.
233 for (const auto &MO : MI.all_defs()) {
234 Register Reg = MO.getReg();
235 if (Reg.isPhysical() || !MRI.use_nodbg_empty(RegNo: Reg))
236 return false;
237 }
238 return MI.wouldBeTriviallyDead();
239}
240
241static void reportGISelDiagnostic(DiagnosticSeverity Severity,
242 MachineFunction &MF,
243 MachineOptimizationRemarkEmitter &MORE,
244 MachineOptimizationRemarkMissed &R) {
245 bool IsGlobalISelAbortEnabled =
246 MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
247 bool IsFatal = Severity == DS_Error && IsGlobalISelAbortEnabled;
248 // Print the function name explicitly if we don't have a debug location (which
249 // makes the diagnostic less useful) or if we're going to emit a raw error.
250 if (!R.getLocation().isValid() || IsFatal)
251 R << (" (in function: " + MF.getName() + ")").str();
252
253 if (IsFatal)
254 reportFatalUsageError(reason: Twine(R.getMsg()));
255 else
256 MORE.emit(OptDiag&: R);
257}
258
259void llvm::reportGISelWarning(MachineFunction &MF,
260 MachineOptimizationRemarkEmitter &MORE,
261 MachineOptimizationRemarkMissed &R) {
262 reportGISelDiagnostic(Severity: DS_Warning, MF, MORE, R);
263}
264
265void llvm::reportGISelFailure(MachineFunction &MF,
266 MachineOptimizationRemarkEmitter &MORE,
267 MachineOptimizationRemarkMissed &R) {
268 MF.getProperties().setFailedISel();
269 reportGISelDiagnostic(Severity: DS_Error, MF, MORE, R);
270}
271
272void llvm::reportGISelFailure(MachineFunction &MF,
273 MachineOptimizationRemarkEmitter &MORE,
274 const char *PassName, StringRef Msg,
275 const MachineInstr &MI) {
276 MachineOptimizationRemarkMissed R(PassName, "GISelFailure: ",
277 MI.getDebugLoc(), MI.getParent());
278 R << Msg;
279 // Printing MI is expensive; only do it if expensive remarks are enabled.
280 if (MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable ||
281 MORE.allowExtraAnalysis(PassName))
282 R << ": " << ore::MNV("Inst", MI);
283 reportGISelFailure(MF, MORE, R);
284}
285
286unsigned llvm::getInverseGMinMaxOpcode(unsigned MinMaxOpc) {
287 switch (MinMaxOpc) {
288 case TargetOpcode::G_SMIN:
289 return TargetOpcode::G_SMAX;
290 case TargetOpcode::G_SMAX:
291 return TargetOpcode::G_SMIN;
292 case TargetOpcode::G_UMIN:
293 return TargetOpcode::G_UMAX;
294 case TargetOpcode::G_UMAX:
295 return TargetOpcode::G_UMIN;
296 default:
297 llvm_unreachable("unrecognized opcode");
298 }
299}
300
301std::optional<APInt> llvm::getIConstantVRegVal(Register VReg,
302 const MachineRegisterInfo &MRI) {
303 std::optional<ValueAndVReg> ValAndVReg = getIConstantVRegValWithLookThrough(
304 VReg, MRI, /*LookThroughInstrs*/ false);
305 assert((!ValAndVReg || ValAndVReg->VReg == VReg) &&
306 "Value found while looking through instrs");
307 if (!ValAndVReg)
308 return std::nullopt;
309 return ValAndVReg->Value;
310}
311
312const APInt &llvm::getIConstantFromReg(Register Reg,
313 const MachineRegisterInfo &MRI) {
314 MachineInstr *Const = MRI.getVRegDef(Reg);
315 assert((Const && Const->getOpcode() == TargetOpcode::G_CONSTANT) &&
316 "expected a G_CONSTANT on Reg");
317 return Const->getOperand(i: 1).getCImm()->getValue();
318}
319
320std::optional<int64_t>
321llvm::getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI) {
322 std::optional<APInt> Val = getIConstantVRegVal(VReg, MRI);
323 if (Val && Val->getBitWidth() <= 64)
324 return Val->getSExtValue();
325 return std::nullopt;
326}
327
328namespace {
329
330// This function is used in many places, and as such, it has some
331// micro-optimizations to try and make it as fast as it can be.
332//
333// - We use template arguments to avoid an indirect call caused by passing a
334// function_ref/std::function
335// - GetAPCstValue does not return std::optional<APInt> as that's expensive.
336// Instead it returns true/false and places the result in a pre-constructed
337// APInt.
338//
339// Please change this function carefully and benchmark your changes.
340template <bool (*IsConstantOpcode)(const MachineInstr *),
341 bool (*GetAPCstValue)(const MachineInstr *MI, APInt &)>
342std::optional<ValueAndVReg>
343getConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI,
344 bool LookThroughInstrs = true,
345 bool LookThroughAnyExt = false) {
346 SmallVector<std::pair<unsigned, unsigned>, 4> SeenOpcodes;
347 MachineInstr *MI;
348
349 while ((MI = MRI.getVRegDef(Reg: VReg)) && !IsConstantOpcode(MI) &&
350 LookThroughInstrs) {
351 switch (MI->getOpcode()) {
352 case TargetOpcode::G_ANYEXT:
353 if (!LookThroughAnyExt)
354 return std::nullopt;
355 [[fallthrough]];
356 case TargetOpcode::G_TRUNC:
357 case TargetOpcode::G_SEXT:
358 case TargetOpcode::G_ZEXT:
359 SeenOpcodes.push_back(Elt: std::make_pair(
360 x: MI->getOpcode(),
361 y: MRI.getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits()));
362 VReg = MI->getOperand(i: 1).getReg();
363 break;
364 case TargetOpcode::COPY:
365 VReg = MI->getOperand(i: 1).getReg();
366 if (VReg.isPhysical())
367 return std::nullopt;
368 break;
369 case TargetOpcode::G_INTTOPTR:
370 VReg = MI->getOperand(i: 1).getReg();
371 break;
372 default:
373 return std::nullopt;
374 }
375 }
376 if (!MI || !IsConstantOpcode(MI))
377 return std::nullopt;
378
379 APInt Val;
380 if (!GetAPCstValue(MI, Val))
381 return std::nullopt;
382 for (auto &Pair : reverse(C&: SeenOpcodes)) {
383 switch (Pair.first) {
384 case TargetOpcode::G_TRUNC:
385 Val = Val.trunc(width: Pair.second);
386 break;
387 case TargetOpcode::G_ANYEXT:
388 case TargetOpcode::G_SEXT:
389 Val = Val.sext(width: Pair.second);
390 break;
391 case TargetOpcode::G_ZEXT:
392 Val = Val.zext(width: Pair.second);
393 break;
394 }
395 }
396
397 return ValueAndVReg{.Value: std::move(Val), .VReg: VReg};
398}
399
400bool isIConstant(const MachineInstr *MI) {
401 if (!MI)
402 return false;
403 return MI->getOpcode() == TargetOpcode::G_CONSTANT;
404}
405
406bool isFConstant(const MachineInstr *MI) {
407 if (!MI)
408 return false;
409 return MI->getOpcode() == TargetOpcode::G_FCONSTANT;
410}
411
412bool isAnyConstant(const MachineInstr *MI) {
413 if (!MI)
414 return false;
415 unsigned Opc = MI->getOpcode();
416 return Opc == TargetOpcode::G_CONSTANT || Opc == TargetOpcode::G_FCONSTANT;
417}
418
419bool getCImmAsAPInt(const MachineInstr *MI, APInt &Result) {
420 const MachineOperand &CstVal = MI->getOperand(i: 1);
421 if (!CstVal.isCImm())
422 return false;
423 Result = CstVal.getCImm()->getValue();
424 return true;
425}
426
427bool getCImmOrFPImmAsAPInt(const MachineInstr *MI, APInt &Result) {
428 const MachineOperand &CstVal = MI->getOperand(i: 1);
429 if (CstVal.isCImm())
430 Result = CstVal.getCImm()->getValue();
431 else if (CstVal.isFPImm())
432 Result = CstVal.getFPImm()->getValueAPF().bitcastToAPInt();
433 else
434 return false;
435 return true;
436}
437
438} // end anonymous namespace
439
440std::optional<ValueAndVReg> llvm::getIConstantVRegValWithLookThrough(
441 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs) {
442 return getConstantVRegValWithLookThrough<isIConstant, getCImmAsAPInt>(
443 VReg, MRI, LookThroughInstrs);
444}
445
446std::optional<ValueAndVReg> llvm::getAnyConstantVRegValWithLookThrough(
447 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs,
448 bool LookThroughAnyExt) {
449 return getConstantVRegValWithLookThrough<isAnyConstant,
450 getCImmOrFPImmAsAPInt>(
451 VReg, MRI, LookThroughInstrs, LookThroughAnyExt);
452}
453
454std::optional<FPValueAndVReg> llvm::getFConstantVRegValWithLookThrough(
455 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs) {
456 auto Reg =
457 getConstantVRegValWithLookThrough<isFConstant, getCImmOrFPImmAsAPInt>(
458 VReg, MRI, LookThroughInstrs);
459 if (!Reg)
460 return std::nullopt;
461
462 unsigned Width = Reg->Value.getBitWidth();
463 LLT Ty = MRI.getType(Reg: VReg);
464 if (Ty.isValid())
465 Ty = Ty.getScalarType();
466 // An FP-typed VReg is the format its users read, which a looked-through
467 // G_ZEXT/G_TRUNC round trip can make differ from the G_FCONSTANT's.
468 if (Ty.isFloat() && Ty.getSizeInBits() == Width)
469 return FPValueAndVReg{.Value: APFloat(getFltSemanticForLLT(Ty), Reg->Value),
470 .VReg: Reg->VReg};
471
472 // Width alone can't tell half from bfloat, or identify x87/PPC formats.
473 const fltSemantics &CstSem =
474 getConstantFPVRegVal(VReg: Reg->VReg, MRI)->getValueAPF().getSemantics();
475 if (APFloat::getSizeInBits(Sem: CstSem) == Width)
476 return FPValueAndVReg{.Value: APFloat(CstSem, Reg->Value), .VReg: Reg->VReg};
477
478 // Resized by a looked-through G_TRUNC/G_SEXT/G_ZEXT: only the width is left,
479 // and it only identifies IEEE formats.
480 if (Width != 16 && Width != 32 && Width != 64 && Width != 128)
481 return std::nullopt;
482 return FPValueAndVReg{
483 .Value: APFloat(getFltSemanticForLLT(Ty: LLT::scalar(SizeInBits: Width)), Reg->Value), .VReg: Reg->VReg};
484}
485
486const ConstantFP *
487llvm::getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI) {
488 MachineInstr *MI = MRI.getVRegDef(Reg: VReg);
489 if (TargetOpcode::G_FCONSTANT != MI->getOpcode())
490 return nullptr;
491 return MI->getOperand(i: 1).getFPImm();
492}
493
494std::optional<DefinitionAndSourceRegister>
495llvm::getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI) {
496 Register DefSrcReg = Reg;
497 // This assumes that the code is in SSA form, so there should only be one
498 // definition.
499 auto DefIt = MRI.def_begin(RegNo: Reg);
500 if (DefIt == MRI.def_end())
501 return {};
502 MachineOperand &DefOpnd = *DefIt;
503 MachineInstr *DefMI = DefOpnd.getParent();
504 auto DstTy = MRI.getType(Reg: DefOpnd.getReg());
505 if (!DstTy.isValid())
506 return std::nullopt;
507 unsigned Opc = DefMI->getOpcode();
508 while (Opc == TargetOpcode::COPY || isPreISelGenericOptimizationHint(Opcode: Opc)) {
509 Register SrcReg = DefMI->getOperand(i: 1).getReg();
510 auto SrcTy = MRI.getType(Reg: SrcReg);
511 if (!SrcTy.isValid())
512 break;
513 DefMI = MRI.getVRegDef(Reg: SrcReg);
514 DefSrcReg = SrcReg;
515 Opc = DefMI->getOpcode();
516 }
517 return DefinitionAndSourceRegister{.MI: DefMI, .Reg: DefSrcReg};
518}
519
520MachineInstr *llvm::getDefIgnoringCopies(Register Reg,
521 const MachineRegisterInfo &MRI) {
522 std::optional<DefinitionAndSourceRegister> DefSrcReg =
523 getDefSrcRegIgnoringCopies(Reg, MRI);
524 return DefSrcReg ? DefSrcReg->MI : nullptr;
525}
526
527Register llvm::getSrcRegIgnoringCopies(Register Reg,
528 const MachineRegisterInfo &MRI) {
529 std::optional<DefinitionAndSourceRegister> DefSrcReg =
530 getDefSrcRegIgnoringCopies(Reg, MRI);
531 return DefSrcReg ? DefSrcReg->Reg : Register();
532}
533
534void llvm::extractParts(Register Reg, LLT Ty, int NumParts,
535 SmallVectorImpl<Register> &VRegs,
536 MachineIRBuilder &MIRBuilder,
537 MachineRegisterInfo &MRI) {
538 for (int i = 0; i < NumParts; ++i)
539 VRegs.push_back(Elt: MRI.createGenericVirtualRegister(Ty));
540 MIRBuilder.buildUnmerge(Res: VRegs, Op: Reg);
541}
542
543bool llvm::extractParts(Register Reg, LLT RegTy, LLT MainTy, LLT &LeftoverTy,
544 SmallVectorImpl<Register> &VRegs,
545 SmallVectorImpl<Register> &LeftoverRegs,
546 MachineIRBuilder &MIRBuilder,
547 MachineRegisterInfo &MRI) {
548 assert(!LeftoverTy.isValid() && "this is an out argument");
549
550 unsigned RegSize = RegTy.getSizeInBits();
551 unsigned MainSize = MainTy.getSizeInBits();
552 unsigned NumParts = RegSize / MainSize;
553 unsigned LeftoverSize = RegSize - NumParts * MainSize;
554
555 // Use an unmerge when possible.
556 if (LeftoverSize == 0) {
557 for (unsigned I = 0; I < NumParts; ++I)
558 VRegs.push_back(Elt: MRI.createGenericVirtualRegister(Ty: MainTy));
559 MIRBuilder.buildUnmerge(Res: VRegs, Op: Reg);
560 return true;
561 }
562
563 // Try to use unmerge for irregular vector split where possible
564 // For example when splitting a <6 x i32> into <4 x i32> with <2 x i32>
565 // leftover, it becomes:
566 // <2 x i32> %2, <2 x i32>%3, <2 x i32> %4 = G_UNMERGE_VALUE <6 x i32> %1
567 // <4 x i32> %5 = G_CONCAT_VECTOR <2 x i32> %2, <2 x i32> %3
568 if (RegTy.isVector() && MainTy.isVector()) {
569 unsigned RegNumElts = RegTy.getNumElements();
570 unsigned MainNumElts = MainTy.getNumElements();
571 unsigned LeftoverNumElts = RegNumElts % MainNumElts;
572 // If can unmerge to LeftoverTy, do it
573 if (MainNumElts % LeftoverNumElts == 0 &&
574 RegNumElts % LeftoverNumElts == 0 &&
575 RegTy.getScalarSizeInBits() == MainTy.getScalarSizeInBits() &&
576 LeftoverNumElts > 1) {
577 LeftoverTy = LLT::fixed_vector(NumElements: LeftoverNumElts, ScalarTy: RegTy.getElementType());
578
579 // Unmerge the SrcReg to LeftoverTy vectors
580 SmallVector<Register, 4> UnmergeValues;
581 extractParts(Reg, Ty: LeftoverTy, NumParts: RegNumElts / LeftoverNumElts, VRegs&: UnmergeValues,
582 MIRBuilder, MRI);
583
584 // Find how many LeftoverTy makes one MainTy
585 unsigned LeftoverPerMain = MainNumElts / LeftoverNumElts;
586 unsigned NumOfLeftoverVal =
587 ((RegNumElts % MainNumElts) / LeftoverNumElts);
588
589 // Create as many MainTy as possible using unmerged value
590 SmallVector<Register, 4> MergeValues;
591 for (unsigned I = 0; I < UnmergeValues.size() - NumOfLeftoverVal; I++) {
592 MergeValues.push_back(Elt: UnmergeValues[I]);
593 if (MergeValues.size() == LeftoverPerMain) {
594 VRegs.push_back(
595 Elt: MIRBuilder.buildMergeLikeInstr(Res: MainTy, Ops: MergeValues).getReg(Idx: 0));
596 MergeValues.clear();
597 }
598 }
599 // Populate LeftoverRegs with the leftovers
600 for (unsigned I = UnmergeValues.size() - NumOfLeftoverVal;
601 I < UnmergeValues.size(); I++) {
602 LeftoverRegs.push_back(Elt: UnmergeValues[I]);
603 }
604 return true;
605 }
606 }
607 // Perform irregular split. Leftover is last element of RegPieces.
608 if (MainTy.isVector()) {
609 SmallVector<Register, 8> RegPieces;
610 extractVectorParts(Reg, NumElts: MainTy.getNumElements(), VRegs&: RegPieces, MIRBuilder,
611 MRI);
612 for (unsigned i = 0; i < RegPieces.size() - 1; ++i)
613 VRegs.push_back(Elt: RegPieces[i]);
614 LeftoverRegs.push_back(Elt: RegPieces[RegPieces.size() - 1]);
615 LeftoverTy = MRI.getType(Reg: LeftoverRegs[0]);
616 return true;
617 }
618
619 LeftoverTy = LLT::integer(SizeInBits: LeftoverSize);
620 // For irregular sizes, extract the individual parts.
621 for (unsigned I = 0; I != NumParts; ++I) {
622 Register NewReg = MRI.createGenericVirtualRegister(Ty: MainTy);
623 VRegs.push_back(Elt: NewReg);
624 MIRBuilder.buildExtract(Res: NewReg, Src: Reg, Index: MainSize * I);
625 }
626
627 for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
628 Offset += LeftoverSize) {
629 Register NewReg = MRI.createGenericVirtualRegister(Ty: LeftoverTy);
630 LeftoverRegs.push_back(Elt: NewReg);
631 MIRBuilder.buildExtract(Res: NewReg, Src: Reg, Index: Offset);
632 }
633
634 return true;
635}
636
637void llvm::extractVectorParts(Register Reg, unsigned NumElts,
638 SmallVectorImpl<Register> &VRegs,
639 MachineIRBuilder &MIRBuilder,
640 MachineRegisterInfo &MRI) {
641 LLT RegTy = MRI.getType(Reg);
642 assert(RegTy.isVector() && "Expected a vector type");
643
644 LLT EltTy = RegTy.getElementType();
645 LLT NarrowTy = (NumElts == 1) ? EltTy : LLT::fixed_vector(NumElements: NumElts, ScalarTy: EltTy);
646 unsigned RegNumElts = RegTy.getNumElements();
647 unsigned LeftoverNumElts = RegNumElts % NumElts;
648 unsigned NumNarrowTyPieces = RegNumElts / NumElts;
649
650 // Perfect split without leftover
651 if (LeftoverNumElts == 0)
652 return extractParts(Reg, Ty: NarrowTy, NumParts: NumNarrowTyPieces, VRegs, MIRBuilder,
653 MRI);
654
655 // Irregular split. Provide direct access to all elements for artifact
656 // combiner using unmerge to elements. Then build vectors with NumElts
657 // elements. Remaining element(s) will be (used to build vector) Leftover.
658 SmallVector<Register, 8> Elts;
659 extractParts(Reg, Ty: EltTy, NumParts: RegNumElts, VRegs&: Elts, MIRBuilder, MRI);
660
661 unsigned Offset = 0;
662 // Requested sub-vectors of NarrowTy.
663 for (unsigned i = 0; i < NumNarrowTyPieces; ++i, Offset += NumElts) {
664 ArrayRef<Register> Pieces(&Elts[Offset], NumElts);
665 VRegs.push_back(Elt: MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: Pieces).getReg(Idx: 0));
666 }
667
668 // Leftover element(s).
669 if (LeftoverNumElts == 1) {
670 VRegs.push_back(Elt: Elts[Offset]);
671 } else {
672 LLT LeftoverTy = LLT::fixed_vector(NumElements: LeftoverNumElts, ScalarTy: EltTy);
673 ArrayRef<Register> Pieces(&Elts[Offset], LeftoverNumElts);
674 VRegs.push_back(
675 Elt: MIRBuilder.buildMergeLikeInstr(Res: LeftoverTy, Ops: Pieces).getReg(Idx: 0));
676 }
677}
678
679MachineInstr *llvm::getOpcodeDef(unsigned Opcode, Register Reg,
680 const MachineRegisterInfo &MRI) {
681 MachineInstr *DefMI = getDefIgnoringCopies(Reg, MRI);
682 return DefMI && DefMI->getOpcode() == Opcode ? DefMI : nullptr;
683}
684
685std::optional<APInt> llvm::ConstantFoldBinOp(unsigned Opcode,
686 const Register Op1,
687 const Register Op2,
688 const MachineRegisterInfo &MRI) {
689 auto MaybeOp2Cst = getAnyConstantVRegValWithLookThrough(VReg: Op2, MRI, LookThroughInstrs: false);
690 if (!MaybeOp2Cst)
691 return std::nullopt;
692
693 auto MaybeOp1Cst = getAnyConstantVRegValWithLookThrough(VReg: Op1, MRI, LookThroughInstrs: false);
694 if (!MaybeOp1Cst)
695 return std::nullopt;
696
697 const APInt &C1 = MaybeOp1Cst->Value;
698 const APInt &C2 = MaybeOp2Cst->Value;
699 switch (Opcode) {
700 default:
701 break;
702 case TargetOpcode::G_ADD:
703 return C1 + C2;
704 case TargetOpcode::G_PTR_ADD:
705 // Types can be of different width here.
706 // Result needs to be the same width as C1, so trunc or sext C2.
707 return C1 + C2.sextOrTrunc(width: C1.getBitWidth());
708 case TargetOpcode::G_AND:
709 return C1 & C2;
710 case TargetOpcode::G_ASHR:
711 return C1.ashr(ShiftAmt: C2);
712 case TargetOpcode::G_LSHR:
713 return C1.lshr(ShiftAmt: C2);
714 case TargetOpcode::G_MUL:
715 return C1 * C2;
716 case TargetOpcode::G_OR:
717 return C1 | C2;
718 case TargetOpcode::G_SHL:
719 return C1 << C2;
720 case TargetOpcode::G_SUB:
721 return C1 - C2;
722 case TargetOpcode::G_XOR:
723 return C1 ^ C2;
724 case TargetOpcode::G_UDIV:
725 if (!C2.getBoolValue())
726 break;
727 return C1.udiv(RHS: C2);
728 case TargetOpcode::G_SDIV:
729 if (!C2.getBoolValue())
730 break;
731 return C1.sdiv(RHS: C2);
732 case TargetOpcode::G_UREM:
733 if (!C2.getBoolValue())
734 break;
735 return C1.urem(RHS: C2);
736 case TargetOpcode::G_SREM:
737 if (!C2.getBoolValue())
738 break;
739 return C1.srem(RHS: C2);
740 case TargetOpcode::G_SMIN:
741 return APIntOps::smin(A: C1, B: C2);
742 case TargetOpcode::G_SMAX:
743 return APIntOps::smax(A: C1, B: C2);
744 case TargetOpcode::G_UMIN:
745 return APIntOps::umin(A: C1, B: C2);
746 case TargetOpcode::G_UMAX:
747 return APIntOps::umax(A: C1, B: C2);
748 }
749
750 return std::nullopt;
751}
752
753std::optional<APFloat>
754llvm::ConstantFoldFPBinOp(unsigned Opcode, const Register Op1,
755 const Register Op2, const MachineRegisterInfo &MRI) {
756 const ConstantFP *Op2Cst = getConstantFPVRegVal(VReg: Op2, MRI);
757 if (!Op2Cst)
758 return std::nullopt;
759
760 const ConstantFP *Op1Cst = getConstantFPVRegVal(VReg: Op1, MRI);
761 if (!Op1Cst)
762 return std::nullopt;
763
764 APFloat C1 = Op1Cst->getValueAPF();
765 const APFloat &C2 = Op2Cst->getValueAPF();
766 switch (Opcode) {
767 case TargetOpcode::G_FADD:
768 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
769 return C1;
770 case TargetOpcode::G_FSUB:
771 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
772 return C1;
773 case TargetOpcode::G_FMUL:
774 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
775 return C1;
776 case TargetOpcode::G_FDIV:
777 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
778 return C1;
779 case TargetOpcode::G_FREM:
780 C1.mod(RHS: C2);
781 return C1;
782 case TargetOpcode::G_FCOPYSIGN:
783 C1.copySign(RHS: C2);
784 return C1;
785 case TargetOpcode::G_FMINNUM:
786 return minnum(A: C1, B: C2);
787 case TargetOpcode::G_FMAXNUM:
788 return maxnum(A: C1, B: C2);
789 case TargetOpcode::G_FMINIMUM:
790 return minimum(A: C1, B: C2);
791 case TargetOpcode::G_FMAXIMUM:
792 return maximum(A: C1, B: C2);
793 case TargetOpcode::G_FMINIMUMNUM:
794 return minimumnum(A: C1, B: C2);
795 case TargetOpcode::G_FMAXIMUMNUM:
796 return maximumnum(A: C1, B: C2);
797 case TargetOpcode::G_FMINNUM_IEEE:
798 case TargetOpcode::G_FMAXNUM_IEEE:
799 // FIXME: These operations were unfortunately named. fminnum/fmaxnum do not
800 // follow the IEEE behavior for signaling nans and follow libm's fmin/fmax,
801 // and currently there isn't a nice wrapper in APFloat for the version with
802 // correct snan handling.
803 break;
804 default:
805 break;
806 }
807
808 return std::nullopt;
809}
810
811static GBuildVector *getBuildVectorLikeDef(Register Reg,
812 const MachineRegisterInfo &MRI) {
813 if (auto *BV = getOpcodeDef<GBuildVector>(Reg, MRI))
814 return BV;
815
816 auto *Bitcast = getOpcodeDef(Opcode: TargetOpcode::G_BITCAST, Reg, MRI);
817 if (!Bitcast)
818 return nullptr;
819
820 auto [Dst, DstTy, Src, SrcTy] = Bitcast->getFirst2RegLLTs();
821 if (!SrcTy.isVector() || !DstTy.isVector())
822 return nullptr;
823 if (SrcTy.getElementCount() != DstTy.getElementCount())
824 return nullptr;
825 if (SrcTy.getScalarSizeInBits() != DstTy.getScalarSizeInBits())
826 return nullptr;
827
828 return getOpcodeDef<GBuildVector>(Reg: Src, MRI);
829}
830
831SmallVector<APInt>
832llvm::ConstantFoldVectorBinop(unsigned Opcode, const Register Op1,
833 const Register Op2,
834 const MachineRegisterInfo &MRI) {
835 auto *SrcVec2 = getBuildVectorLikeDef(Reg: Op2, MRI);
836 if (!SrcVec2)
837 return SmallVector<APInt>();
838
839 auto *SrcVec1 = getBuildVectorLikeDef(Reg: Op1, MRI);
840 if (!SrcVec1)
841 return SmallVector<APInt>();
842
843 SmallVector<APInt> FoldedElements;
844 for (unsigned Idx = 0, E = SrcVec1->getNumSources(); Idx < E; ++Idx) {
845 auto MaybeCst = ConstantFoldBinOp(Opcode, Op1: SrcVec1->getSourceReg(I: Idx),
846 Op2: SrcVec2->getSourceReg(I: Idx), MRI);
847 if (!MaybeCst)
848 return SmallVector<APInt>();
849 FoldedElements.push_back(Elt: *MaybeCst);
850 }
851 return FoldedElements;
852}
853
854SmallVector<APFloat>
855llvm::ConstantFoldVectorFPBinop(unsigned Opcode, const Register Op1,
856 const Register Op2,
857 const MachineRegisterInfo &MRI) {
858 // Unlike the integer fold, don't look through G_BITCAST: it may reinterpret
859 // the element bits in a different FP semantics format (e.g. bf16 to f16).
860 auto *SrcVec2 = getOpcodeDef<GBuildVector>(Reg: Op2, MRI);
861 if (!SrcVec2)
862 return {};
863
864 auto *SrcVec1 = getOpcodeDef<GBuildVector>(Reg: Op1, MRI);
865 if (!SrcVec1)
866 return {};
867
868 SmallVector<APFloat> FoldedElements;
869 for (unsigned Idx = 0, E = SrcVec1->getNumSources(); Idx < E; ++Idx) {
870 auto MaybeCst = ConstantFoldFPBinOp(Opcode, Op1: SrcVec1->getSourceReg(I: Idx),
871 Op2: SrcVec2->getSourceReg(I: Idx), MRI);
872 if (!MaybeCst)
873 return {};
874 FoldedElements.push_back(Elt: *MaybeCst);
875 }
876 return FoldedElements;
877}
878
879Align llvm::inferAlignFromPtrInfo(MachineFunction &MF,
880 const MachinePointerInfo &MPO) {
881 auto PSV = dyn_cast_if_present<const PseudoSourceValue *>(Val: MPO.V);
882 if (auto FSPV = dyn_cast_or_null<FixedStackPseudoSourceValue>(Val: PSV)) {
883 MachineFrameInfo &MFI = MF.getFrameInfo();
884 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FSPV->getFrameIndex()),
885 Offset: MPO.Offset);
886 }
887
888 if (const Value *V = dyn_cast_if_present<const Value *>(Val: MPO.V)) {
889 const Module *M = MF.getFunction().getParent();
890 return V->getPointerAlignment(DL: M->getDataLayout());
891 }
892
893 return Align(1);
894}
895
896Register llvm::getFunctionLiveInPhysReg(MachineFunction &MF,
897 const TargetInstrInfo &TII,
898 MCRegister PhysReg,
899 const TargetRegisterClass &RC,
900 const DebugLoc &DL, LLT RegTy) {
901 MachineBasicBlock &EntryMBB = MF.front();
902 MachineRegisterInfo &MRI = MF.getRegInfo();
903 Register LiveIn = MRI.getLiveInVirtReg(PReg: PhysReg);
904 if (LiveIn) {
905 MachineInstr *Def = MRI.getVRegDef(Reg: LiveIn);
906 if (Def) {
907 // FIXME: Should the verifier check this is in the entry block?
908 assert(Def->getParent() == &EntryMBB && "live-in copy not in entry block");
909 return LiveIn;
910 }
911
912 // It's possible the incoming argument register and copy was added during
913 // lowering, but later deleted due to being/becoming dead. If this happens,
914 // re-insert the copy.
915 } else {
916 // The live in register was not present, so add it.
917 LiveIn = MF.addLiveIn(PReg: PhysReg, RC: &RC);
918 if (RegTy.isValid())
919 MRI.setType(VReg: LiveIn, Ty: RegTy);
920 }
921
922 BuildMI(BB&: EntryMBB, I: EntryMBB.begin(), MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: LiveIn)
923 .addReg(RegNo: PhysReg);
924 if (!EntryMBB.isLiveIn(Reg: PhysReg))
925 EntryMBB.addLiveIn(PhysReg);
926 return LiveIn;
927}
928
929std::optional<APInt> llvm::ConstantFoldExtOp(unsigned Opcode,
930 const Register Op1, uint64_t Imm,
931 const MachineRegisterInfo &MRI) {
932 auto MaybeOp1Cst = getIConstantVRegVal(VReg: Op1, MRI);
933 if (MaybeOp1Cst) {
934 switch (Opcode) {
935 default:
936 break;
937 case TargetOpcode::G_SEXT_INREG: {
938 LLT Ty = MRI.getType(Reg: Op1);
939 return MaybeOp1Cst->trunc(width: Imm).sext(width: Ty.getScalarSizeInBits());
940 }
941 }
942 }
943 return std::nullopt;
944}
945
946std::optional<APInt> llvm::ConstantFoldCastOp(unsigned Opcode, LLT DstTy,
947 const Register Op0,
948 const MachineRegisterInfo &MRI) {
949 std::optional<APInt> Val = getIConstantVRegVal(VReg: Op0, MRI);
950 if (!Val)
951 return Val;
952
953 const unsigned DstSize = DstTy.getScalarSizeInBits();
954
955 switch (Opcode) {
956 case TargetOpcode::G_SEXT:
957 return Val->sext(width: DstSize);
958 case TargetOpcode::G_ZEXT:
959 case TargetOpcode::G_ANYEXT:
960 // TODO: DAG considers target preference when constant folding any_extend.
961 return Val->zext(width: DstSize);
962 default:
963 break;
964 }
965
966 llvm_unreachable("unexpected cast opcode to constant fold");
967}
968
969std::optional<APFloat>
970llvm::ConstantFoldIntToFloat(unsigned Opcode, LLT DstTy, Register Src,
971 const MachineRegisterInfo &MRI) {
972 assert(Opcode == TargetOpcode::G_SITOFP || Opcode == TargetOpcode::G_UITOFP);
973 if (auto MaybeSrcVal = getIConstantVRegVal(VReg: Src, MRI)) {
974 APFloat DstVal(getFltSemanticForLLT(Ty: DstTy));
975 DstVal.convertFromAPInt(Input: *MaybeSrcVal, IsSigned: Opcode == TargetOpcode::G_SITOFP,
976 RM: APFloat::rmNearestTiesToEven);
977 return DstVal;
978 }
979 return std::nullopt;
980}
981
982SmallVector<APInt>
983llvm::ConstantFoldUnaryIntOp(unsigned Opcode, LLT DstTy, Register Src,
984 const MachineRegisterInfo &MRI) {
985 unsigned EltBits = DstTy.getScalarSizeInBits();
986 auto Fold = [Opcode, EltBits](const APInt &V) -> APInt {
987 switch (Opcode) {
988 case TargetOpcode::G_CTLZ:
989 case TargetOpcode::G_CTLZ_ZERO_POISON:
990 return APInt(EltBits, V.countl_zero());
991 case TargetOpcode::G_CTTZ:
992 case TargetOpcode::G_CTTZ_ZERO_POISON:
993 return APInt(EltBits, V.countr_zero());
994 case TargetOpcode::G_CTPOP:
995 return APInt(EltBits, V.popcount());
996 case TargetOpcode::G_ABS:
997 return V.abs();
998 case TargetOpcode::G_BSWAP:
999 return V.byteSwap();
1000 case TargetOpcode::G_BITREVERSE:
1001 return V.reverseBits();
1002 }
1003 llvm_unreachable("unexpected opcode in ConstantFoldUnaryIntOp");
1004 };
1005
1006 auto tryFoldScalar = [&](Register R) -> std::optional<APInt> {
1007 if (auto MaybeCst = getIConstantVRegVal(VReg: R, MRI))
1008 return Fold(*MaybeCst);
1009 return std::nullopt;
1010 };
1011 if (MRI.getType(Reg: Src).isVector()) {
1012 auto *BV = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
1013 if (!BV)
1014 return {};
1015 SmallVector<APInt> Folded;
1016 for (unsigned SrcIdx = 0; SrcIdx < BV->getNumSources(); ++SrcIdx) {
1017 if (auto MaybeFold = tryFoldScalar(BV->getSourceReg(I: SrcIdx))) {
1018 Folded.emplace_back(Args: std::move(*MaybeFold));
1019 continue;
1020 }
1021 return {};
1022 }
1023 return Folded;
1024 }
1025 if (auto MaybeCst = tryFoldScalar(Src))
1026 return {std::move(*MaybeCst)};
1027 return {};
1028}
1029
1030SmallVector<APFloat>
1031llvm::ConstantFoldUnaryFPOp(unsigned Opcode, LLT DstTy, Register Src,
1032 const MachineRegisterInfo &MRI) {
1033 LLT DstEltTy = DstTy.getScalarType();
1034 auto Fold = [Opcode, DstEltTy,
1035 &MRI](const APFloat &V) -> std::optional<APFloat> {
1036 APFloat Result(V);
1037 bool Unused;
1038 switch (Opcode) {
1039 case TargetOpcode::G_FNEG:
1040 Result.changeSign();
1041 return Result;
1042 case TargetOpcode::G_FABS:
1043 Result.clearSign();
1044 return Result;
1045 case TargetOpcode::G_FCEIL:
1046 Result.roundToIntegral(RM: APFloat::rmTowardPositive);
1047 return Result;
1048 case TargetOpcode::G_FFLOOR:
1049 Result.roundToIntegral(RM: APFloat::rmTowardNegative);
1050 return Result;
1051 case TargetOpcode::G_INTRINSIC_TRUNC:
1052 Result.roundToIntegral(RM: APFloat::rmTowardZero);
1053 return Result;
1054 case TargetOpcode::G_INTRINSIC_ROUND:
1055 Result.roundToIntegral(RM: APFloat::rmNearestTiesToAway);
1056 return Result;
1057 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
1058 case TargetOpcode::G_FRINT:
1059 case TargetOpcode::G_FNEARBYINT:
1060 Result.roundToIntegral(RM: APFloat::rmNearestTiesToEven);
1061 return Result;
1062 case TargetOpcode::G_FPEXT:
1063 case TargetOpcode::G_FPTRUNC:
1064 Result.convert(ToSemantics: getFltSemanticForLLT(Ty: DstEltTy),
1065 RM: APFloat::rmNearestTiesToEven, losesInfo: &Unused);
1066 return Result;
1067 case TargetOpcode::G_FSQRT:
1068 case TargetOpcode::G_FLOG2: {
1069 // ConstantFoldFP only accepts IEEE-like formats up to double. fp128,
1070 // x86_fp80, and ppc_fp128 are not supported.
1071 const fltSemantics &FltSem = getFltSemanticForLLT(Ty: DstEltTy);
1072 if (!APFloat::isIEEELikeFP(FltSem) || APFloat::getSizeInBits(Sem: FltSem) > 64)
1073 return std::nullopt;
1074 Type *Ty = Type::getFloatingPointTy(
1075 C&: MRI.getMF().getFunction().getContext(), S: FltSem);
1076 Constant *C = Opcode == TargetOpcode::G_FSQRT
1077 ? ConstantFoldFP(NativeFP: sqrt, V, Ty)
1078 : ConstantFoldFP(NativeFP: log2, V, Ty);
1079 if (!C)
1080 return std::nullopt;
1081 return cast<ConstantFP>(Val: C)->getValueAPF();
1082 }
1083 default:
1084 llvm_unreachable("unexpected opcode in ConstantFoldUnaryFPOp");
1085 }
1086 };
1087
1088 auto tryFoldScalar = [&](Register R) -> std::optional<APFloat> {
1089 if (const ConstantFP *Cst = getConstantFPVRegVal(VReg: R, MRI))
1090 return Fold(Cst->getValueAPF());
1091 return std::nullopt;
1092 };
1093 if (MRI.getType(Reg: Src).isVector()) {
1094 auto *BV = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
1095 if (!BV)
1096 return {};
1097 SmallVector<APFloat> Folded;
1098 for (unsigned SrcIdx = 0; SrcIdx < BV->getNumSources(); ++SrcIdx) {
1099 if (auto MaybeFold = tryFoldScalar(BV->getSourceReg(I: SrcIdx))) {
1100 Folded.emplace_back(Args: std::move(*MaybeFold));
1101 continue;
1102 }
1103 return {};
1104 }
1105 return Folded;
1106 }
1107 if (auto MaybeCst = tryFoldScalar(Src))
1108 return {std::move(*MaybeCst)};
1109 return {};
1110}
1111
1112std::optional<SmallVector<APInt>>
1113llvm::ConstantFoldICmp(unsigned Pred, const Register Op1, const Register Op2,
1114 unsigned DstScalarSizeInBits, unsigned ExtOp,
1115 const MachineRegisterInfo &MRI) {
1116 assert(ExtOp == TargetOpcode::G_SEXT || ExtOp == TargetOpcode::G_ZEXT ||
1117 ExtOp == TargetOpcode::G_ANYEXT);
1118
1119 const LLT Ty = MRI.getType(Reg: Op1);
1120
1121 auto GetICmpResultCst = [&](bool IsTrue) {
1122 if (IsTrue)
1123 return ExtOp == TargetOpcode::G_SEXT
1124 ? APInt::getAllOnes(numBits: DstScalarSizeInBits)
1125 : APInt::getOneBitSet(numBits: DstScalarSizeInBits, BitNo: 0);
1126 return APInt::getZero(numBits: DstScalarSizeInBits);
1127 };
1128
1129 auto TryFoldScalar = [&](Register LHS, Register RHS) -> std::optional<APInt> {
1130 auto RHSCst = getIConstantVRegVal(VReg: RHS, MRI);
1131 if (!RHSCst)
1132 return std::nullopt;
1133 auto LHSCst = getIConstantVRegVal(VReg: LHS, MRI);
1134 if (!LHSCst)
1135 return std::nullopt;
1136
1137 switch (Pred) {
1138 case CmpInst::Predicate::ICMP_EQ:
1139 return GetICmpResultCst(LHSCst->eq(RHS: *RHSCst));
1140 case CmpInst::Predicate::ICMP_NE:
1141 return GetICmpResultCst(LHSCst->ne(RHS: *RHSCst));
1142 case CmpInst::Predicate::ICMP_UGT:
1143 return GetICmpResultCst(LHSCst->ugt(RHS: *RHSCst));
1144 case CmpInst::Predicate::ICMP_UGE:
1145 return GetICmpResultCst(LHSCst->uge(RHS: *RHSCst));
1146 case CmpInst::Predicate::ICMP_ULT:
1147 return GetICmpResultCst(LHSCst->ult(RHS: *RHSCst));
1148 case CmpInst::Predicate::ICMP_ULE:
1149 return GetICmpResultCst(LHSCst->ule(RHS: *RHSCst));
1150 case CmpInst::Predicate::ICMP_SGT:
1151 return GetICmpResultCst(LHSCst->sgt(RHS: *RHSCst));
1152 case CmpInst::Predicate::ICMP_SGE:
1153 return GetICmpResultCst(LHSCst->sge(RHS: *RHSCst));
1154 case CmpInst::Predicate::ICMP_SLT:
1155 return GetICmpResultCst(LHSCst->slt(RHS: *RHSCst));
1156 case CmpInst::Predicate::ICMP_SLE:
1157 return GetICmpResultCst(LHSCst->sle(RHS: *RHSCst));
1158 default:
1159 return std::nullopt;
1160 }
1161 };
1162
1163 SmallVector<APInt> FoldedICmps;
1164
1165 if (Ty.isVector()) {
1166 // Try to constant fold each element.
1167 auto *BV1 = getOpcodeDef<GBuildVector>(Reg: Op1, MRI);
1168 auto *BV2 = getOpcodeDef<GBuildVector>(Reg: Op2, MRI);
1169 if (!BV1 || !BV2)
1170 return std::nullopt;
1171 assert(BV1->getNumSources() == BV2->getNumSources() && "Invalid vectors");
1172 for (unsigned I = 0; I < BV1->getNumSources(); ++I) {
1173 if (auto MaybeFold =
1174 TryFoldScalar(BV1->getSourceReg(I), BV2->getSourceReg(I))) {
1175 FoldedICmps.emplace_back(Args&: *MaybeFold);
1176 continue;
1177 }
1178 return std::nullopt;
1179 }
1180 return FoldedICmps;
1181 }
1182
1183 if (auto MaybeCst = TryFoldScalar(Op1, Op2)) {
1184 FoldedICmps.emplace_back(Args&: *MaybeCst);
1185 return FoldedICmps;
1186 }
1187
1188 return std::nullopt;
1189}
1190
1191bool llvm::isKnownToBeAPowerOfTwo(Register Reg, const MachineRegisterInfo &MRI,
1192 GISelValueTracking *VT, bool OrNegative,
1193 bool OrZero) {
1194 std::optional<DefinitionAndSourceRegister> DefSrcReg =
1195 getDefSrcRegIgnoringCopies(Reg, MRI);
1196 if (!DefSrcReg)
1197 return false;
1198
1199 const MachineInstr &MI = *DefSrcReg->MI;
1200 const LLT Ty = MRI.getType(Reg);
1201
1202 auto IsPow2 = [OrNegative, OrZero](const APInt &V) {
1203 return V.isPowerOf2() || (OrNegative && V.isNegatedPowerOf2()) ||
1204 (OrZero && V.isZero());
1205 };
1206
1207 switch (MI.getOpcode()) {
1208 case TargetOpcode::G_CONSTANT: {
1209 unsigned BitWidth = Ty.getScalarSizeInBits();
1210 const ConstantInt *CI = MI.getOperand(i: 1).getCImm();
1211 return IsPow2(CI->getValue().zextOrTrunc(width: BitWidth));
1212 }
1213 case TargetOpcode::G_SHL: {
1214 // A left-shift of a constant one will have exactly one bit set because
1215 // shifting the bit off the end is undefined. Otherwise, shifting a power
1216 // of two (or zero) yields a power of two, or zero, since the bit moves or
1217 // is shifted off the end. That is sufficient with OrZero set, otherwise
1218 // the result must also be known never to be zero.
1219
1220 // TODO: Constant splat
1221 if (auto ConstLHS = getIConstantVRegVal(VReg: MI.getOperand(i: 1).getReg(), MRI)) {
1222 if (*ConstLHS == 1)
1223 return true;
1224 }
1225 if (OrZero || (VT && VT->isKnownNeverZero(R: Reg))) {
1226 if (isKnownToBeAPowerOfTwo(Reg: MI.getOperand(i: 1).getReg(), MRI, VT, OrNegative,
1227 OrZero))
1228 return true;
1229 }
1230
1231 break;
1232 }
1233 case TargetOpcode::G_LSHR: {
1234 if (auto ConstLHS = getIConstantVRegVal(VReg: MI.getOperand(i: 1).getReg(), MRI)) {
1235 if (ConstLHS->isSignMask())
1236 return true;
1237 }
1238 // As with G_SHL, the set bit is either moved or shifted off the end.
1239 // OrNegative is not propagated because a logical right shift fills with
1240 // zeros and thus does not preserve negated powers of two.
1241 if (OrZero || (VT && VT->isKnownNeverZero(R: Reg))) {
1242 if (isKnownToBeAPowerOfTwo(Reg: MI.getOperand(i: 1).getReg(), MRI, VT,
1243 /*OrNegative=*/false, OrZero))
1244 return true;
1245 }
1246
1247 break;
1248 }
1249 case TargetOpcode::G_TRUNC: {
1250 // Truncation preserves the lower bits of the operand, so the set bit either
1251 // survives or is cut off, yielding zero.
1252 if (OrZero || (VT && VT->isKnownNeverZero(R: Reg))) {
1253 if (isKnownToBeAPowerOfTwo(Reg: MI.getOperand(i: 1).getReg(), MRI, VT,
1254 /*OrNegative=*/false, OrZero))
1255 return true;
1256 }
1257
1258 break;
1259 }
1260 case TargetOpcode::G_BUILD_VECTOR: {
1261 // TODO: Probably should have a recursion depth guard since you could have
1262 // bitcasted vector elements.
1263 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands()))
1264 if (!isKnownToBeAPowerOfTwo(Reg: MO.getReg(), MRI, VT, OrNegative, OrZero))
1265 return false;
1266
1267 return true;
1268 }
1269 case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1270 // Only handle constants since we would need to know if number of leading
1271 // zeros is greater than the truncation amount.
1272 const unsigned BitWidth = Ty.getScalarSizeInBits();
1273 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
1274 auto Const = getIConstantVRegVal(VReg: MO.getReg(), MRI);
1275 if (!Const || !IsPow2(Const->zextOrTrunc(width: BitWidth)))
1276 return false;
1277 }
1278
1279 return true;
1280 }
1281 default:
1282 break;
1283 }
1284
1285 if (!VT)
1286 return false;
1287
1288 // More could be done here, though the above checks are enough
1289 // to handle some common cases.
1290
1291 // Fall back to computeKnownBits to catch other known cases.
1292 KnownBits Known = VT->getKnownBits(R: Reg);
1293 return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);
1294}
1295
1296void llvm::getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU) {
1297 AU.addPreserved<StackProtector>();
1298}
1299
1300LLT llvm::getLCMType(LLT OrigTy, LLT TargetTy) {
1301 if (OrigTy.getSizeInBits() == TargetTy.getSizeInBits())
1302 return OrigTy;
1303
1304 if (OrigTy.isVector() && TargetTy.isVector()) {
1305 LLT OrigElt = OrigTy.getElementType();
1306 LLT TargetElt = TargetTy.getElementType();
1307
1308 // TODO: The docstring for this function says the intention is to use this
1309 // function to build MERGE/UNMERGE instructions. It won't be the case that
1310 // we generate a MERGE/UNMERGE between fixed and scalable vector types. We
1311 // could implement getLCMType between the two in the future if there was a
1312 // need, but it is not worth it now as this function should not be used in
1313 // that way.
1314 assert(((OrigTy.isScalableVector() && !TargetTy.isFixedVector()) ||
1315 (OrigTy.isFixedVector() && !TargetTy.isScalableVector())) &&
1316 "getLCMType not implemented between fixed and scalable vectors.");
1317
1318 if (OrigElt.getSizeInBits() == TargetElt.getSizeInBits()) {
1319 int GCDMinElts = std::gcd(m: OrigTy.getElementCount().getKnownMinValue(),
1320 n: TargetTy.getElementCount().getKnownMinValue());
1321 // Prefer the original element type.
1322 ElementCount Mul = OrigTy.getElementCount() *
1323 TargetTy.getElementCount().getKnownMinValue();
1324 return LLT::vector(EC: Mul.divideCoefficientBy(RHS: GCDMinElts),
1325 ScalarTy: OrigTy.getElementType());
1326 }
1327 unsigned LCM = std::lcm(m: OrigTy.getSizeInBits().getKnownMinValue(),
1328 n: TargetTy.getSizeInBits().getKnownMinValue());
1329 return LLT::vector(
1330 EC: ElementCount::get(MinVal: LCM / OrigElt.getSizeInBits(), Scalable: OrigTy.isScalable()),
1331 ScalarTy: OrigElt);
1332 }
1333
1334 // One type is scalar, one type is vector
1335 if (OrigTy.isVector() || TargetTy.isVector()) {
1336 LLT VecTy = OrigTy.isVector() ? OrigTy : TargetTy;
1337 LLT ScalarTy = OrigTy.isVector() ? TargetTy : OrigTy;
1338 LLT EltTy = VecTy.getElementType();
1339 LLT OrigEltTy = OrigTy.isVector() ? OrigTy.getElementType() : OrigTy;
1340
1341 // Prefer scalar type from OrigTy.
1342 if (EltTy.getSizeInBits() == ScalarTy.getSizeInBits())
1343 return LLT::vector(EC: VecTy.getElementCount(), ScalarTy: OrigEltTy);
1344
1345 // Different size scalars. Create vector with the same total size.
1346 // LCM will take fixed/scalable from VecTy.
1347 unsigned LCM = std::lcm(m: EltTy.getSizeInBits().getFixedValue() *
1348 VecTy.getElementCount().getKnownMinValue(),
1349 n: ScalarTy.getSizeInBits().getFixedValue());
1350 // Prefer type from OrigTy
1351 return LLT::vector(EC: ElementCount::get(MinVal: LCM / OrigEltTy.getSizeInBits(),
1352 Scalable: VecTy.getElementCount().isScalable()),
1353 ScalarTy: OrigEltTy);
1354 }
1355
1356 // At this point, both types are scalars of different size
1357 unsigned LCM = std::lcm(m: OrigTy.getSizeInBits().getFixedValue(),
1358 n: TargetTy.getSizeInBits().getFixedValue());
1359 // Preserve pointer types.
1360 if (LCM == OrigTy.getSizeInBits())
1361 return OrigTy;
1362 if (LCM == TargetTy.getSizeInBits())
1363 return TargetTy;
1364 return LLT::scalar(SizeInBits: LCM);
1365}
1366
1367LLT llvm::getCoverTy(LLT OrigTy, LLT TargetTy) {
1368
1369 if ((OrigTy.isScalableVector() && TargetTy.isFixedVector()) ||
1370 (OrigTy.isFixedVector() && TargetTy.isScalableVector()))
1371 llvm_unreachable(
1372 "getCoverTy not implemented between fixed and scalable vectors.");
1373
1374 if (!OrigTy.isVector() || !TargetTy.isVector() || OrigTy == TargetTy ||
1375 (OrigTy.getScalarSizeInBits() != TargetTy.getScalarSizeInBits()))
1376 return getLCMType(OrigTy, TargetTy);
1377
1378 unsigned OrigTyNumElts = OrigTy.getElementCount().getKnownMinValue();
1379 unsigned TargetTyNumElts = TargetTy.getElementCount().getKnownMinValue();
1380 if (OrigTyNumElts % TargetTyNumElts == 0)
1381 return OrigTy;
1382
1383 unsigned NumElts = alignTo(Value: OrigTyNumElts, Align: TargetTyNumElts);
1384 return LLT::scalarOrVector(EC: ElementCount::getFixed(MinVal: NumElts),
1385 ScalarTy: OrigTy.getElementType());
1386}
1387
1388LLT llvm::getGCDType(LLT OrigTy, LLT TargetTy) {
1389 if (OrigTy.getSizeInBits() == TargetTy.getSizeInBits())
1390 return OrigTy;
1391
1392 if (OrigTy.isVector() && TargetTy.isVector()) {
1393 LLT OrigElt = OrigTy.getElementType();
1394
1395 // TODO: The docstring for this function says the intention is to use this
1396 // function to build MERGE/UNMERGE instructions. It won't be the case that
1397 // we generate a MERGE/UNMERGE between fixed and scalable vector types. We
1398 // could implement getGCDType between the two in the future if there was a
1399 // need, but it is not worth it now as this function should not be used in
1400 // that way.
1401 assert(((OrigTy.isScalableVector() && !TargetTy.isFixedVector()) ||
1402 (OrigTy.isFixedVector() && !TargetTy.isScalableVector())) &&
1403 "getGCDType not implemented between fixed and scalable vectors.");
1404
1405 unsigned GCD = std::gcd(m: OrigTy.getSizeInBits().getKnownMinValue(),
1406 n: TargetTy.getSizeInBits().getKnownMinValue());
1407 if (GCD == OrigElt.getSizeInBits())
1408 return LLT::scalarOrVector(EC: ElementCount::get(MinVal: 1, Scalable: OrigTy.isScalable()),
1409 ScalarTy: OrigElt);
1410
1411 // Cannot produce original element type, but both have vscale in common.
1412 if (GCD < OrigElt.getSizeInBits())
1413 return LLT::scalarOrVector(EC: ElementCount::get(MinVal: 1, Scalable: OrigTy.isScalable()),
1414 ScalarSize: GCD);
1415
1416 return LLT::vector(
1417 EC: ElementCount::get(MinVal: GCD / OrigElt.getSizeInBits().getFixedValue(),
1418 Scalable: OrigTy.isScalable()),
1419 ScalarTy: OrigElt);
1420 }
1421
1422 // If one type is vector and the element size matches the scalar size, then
1423 // the gcd is the scalar type.
1424 if (OrigTy.isVector() &&
1425 OrigTy.getElementType().getSizeInBits() == TargetTy.getSizeInBits())
1426 return OrigTy.getElementType();
1427 if (TargetTy.isVector() &&
1428 TargetTy.getElementType().getSizeInBits() == OrigTy.getSizeInBits())
1429 return OrigTy;
1430
1431 // At this point, both types are either scalars of different type or one is a
1432 // vector and one is a scalar. If both types are scalars, the GCD type is the
1433 // GCD between the two scalar sizes. If one is vector and one is scalar, then
1434 // the GCD type is the GCD between the scalar and the vector element size.
1435 LLT OrigScalar = OrigTy.getScalarType();
1436 LLT TargetScalar = TargetTy.getScalarType();
1437 unsigned GCD = std::gcd(m: OrigScalar.getSizeInBits().getFixedValue(),
1438 n: TargetScalar.getSizeInBits().getFixedValue());
1439 return LLT::integer(SizeInBits: GCD);
1440}
1441
1442std::optional<int> llvm::getSplatIndex(MachineInstr &MI) {
1443 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
1444 "Only G_SHUFFLE_VECTOR can have a splat index!");
1445 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
1446 auto FirstDefinedIdx = find_if(Range&: Mask, P: [](int Elt) { return Elt >= 0; });
1447
1448 // If all elements are undefined, this shuffle can be considered a splat.
1449 // Return 0 for better potential for callers to simplify.
1450 if (FirstDefinedIdx == Mask.end())
1451 return 0;
1452
1453 // Make sure all remaining elements are either undef or the same
1454 // as the first non-undef value.
1455 int SplatValue = *FirstDefinedIdx;
1456 if (any_of(Range: make_range(x: std::next(x: FirstDefinedIdx), y: Mask.end()),
1457 P: [&SplatValue](int Elt) { return Elt >= 0 && Elt != SplatValue; }))
1458 return std::nullopt;
1459
1460 return SplatValue;
1461}
1462
1463static bool isBuildVectorOp(unsigned Opcode) {
1464 return Opcode == TargetOpcode::G_BUILD_VECTOR ||
1465 Opcode == TargetOpcode::G_BUILD_VECTOR_TRUNC;
1466}
1467
1468namespace {
1469
1470std::optional<ValueAndVReg> getAnyConstantSplat(Register VReg,
1471 const MachineRegisterInfo &MRI,
1472 bool AllowUndef) {
1473 MachineInstr *MI = getDefIgnoringCopies(Reg: VReg, MRI);
1474 if (!MI)
1475 return std::nullopt;
1476
1477 bool isConcatVectorsOp = MI->getOpcode() == TargetOpcode::G_CONCAT_VECTORS;
1478 if (!isBuildVectorOp(Opcode: MI->getOpcode()) && !isConcatVectorsOp)
1479 return std::nullopt;
1480
1481 std::optional<ValueAndVReg> SplatValAndReg;
1482 for (MachineOperand &Op : MI->uses()) {
1483 Register Element = Op.getReg();
1484 // If we have a G_CONCAT_VECTOR, we recursively look into the
1485 // vectors that we're concatenating to see if they're splats.
1486 auto ElementValAndReg =
1487 isConcatVectorsOp
1488 ? getAnyConstantSplat(VReg: Element, MRI, AllowUndef)
1489 : getAnyConstantVRegValWithLookThrough(VReg: Element, MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
1490
1491 // If AllowUndef, treat undef as value that will result in a constant splat.
1492 if (!ElementValAndReg) {
1493 if (AllowUndef && mi_match(R: Element, MRI, P: m_GImplicitDef()))
1494 continue;
1495 return std::nullopt;
1496 }
1497
1498 // Record splat value
1499 if (!SplatValAndReg)
1500 SplatValAndReg = ElementValAndReg;
1501
1502 // Different constant than the one already recorded, not a constant splat.
1503 if (SplatValAndReg->Value != ElementValAndReg->Value)
1504 return std::nullopt;
1505 }
1506
1507 return SplatValAndReg;
1508}
1509
1510} // end anonymous namespace
1511
1512bool llvm::isBuildVectorConstantSplat(const Register Reg,
1513 const MachineRegisterInfo &MRI,
1514 int64_t SplatValue, bool AllowUndef) {
1515 if (auto SplatValAndReg = getAnyConstantSplat(VReg: Reg, MRI, AllowUndef))
1516 return SplatValAndReg->Value.getSExtValue() == SplatValue;
1517
1518 return false;
1519}
1520
1521bool llvm::isBuildVectorConstantSplat(const Register Reg,
1522 const MachineRegisterInfo &MRI,
1523 const APInt &SplatValue,
1524 bool AllowUndef) {
1525 if (auto SplatValAndReg = getAnyConstantSplat(VReg: Reg, MRI, AllowUndef)) {
1526 if (SplatValAndReg->Value.getBitWidth() < SplatValue.getBitWidth())
1527 return APInt::isSameValue(
1528 I1: SplatValAndReg->Value.sext(width: SplatValue.getBitWidth()), I2: SplatValue);
1529 return APInt::isSameValue(
1530 I1: SplatValAndReg->Value,
1531 I2: SplatValue.sext(width: SplatValAndReg->Value.getBitWidth()));
1532 }
1533
1534 return false;
1535}
1536
1537bool llvm::isBuildVectorConstantSplat(const MachineInstr &MI,
1538 const MachineRegisterInfo &MRI,
1539 int64_t SplatValue, bool AllowUndef) {
1540 return isBuildVectorConstantSplat(Reg: MI.getOperand(i: 0).getReg(), MRI, SplatValue,
1541 AllowUndef);
1542}
1543
1544bool llvm::isBuildVectorConstantSplat(const MachineInstr &MI,
1545 const MachineRegisterInfo &MRI,
1546 const APInt &SplatValue,
1547 bool AllowUndef) {
1548 return isBuildVectorConstantSplat(Reg: MI.getOperand(i: 0).getReg(), MRI, SplatValue,
1549 AllowUndef);
1550}
1551
1552std::optional<APInt>
1553llvm::getIConstantSplatVal(const Register Reg, const MachineRegisterInfo &MRI) {
1554 if (auto SplatValAndReg =
1555 getAnyConstantSplat(VReg: Reg, MRI, /* AllowUndef */ false)) {
1556 if (std::optional<ValueAndVReg> ValAndVReg =
1557 getIConstantVRegValWithLookThrough(VReg: SplatValAndReg->VReg, MRI))
1558 return ValAndVReg->Value;
1559 }
1560
1561 return std::nullopt;
1562}
1563
1564std::optional<APInt>
1565llvm::getIConstantSplatVal(const MachineInstr &MI,
1566 const MachineRegisterInfo &MRI) {
1567 return getIConstantSplatVal(Reg: MI.getOperand(i: 0).getReg(), MRI);
1568}
1569
1570std::optional<int64_t>
1571llvm::getIConstantSplatSExtVal(const Register Reg,
1572 const MachineRegisterInfo &MRI) {
1573 if (auto SplatValAndReg =
1574 getAnyConstantSplat(VReg: Reg, MRI, /* AllowUndef */ false))
1575 return getIConstantVRegSExtVal(VReg: SplatValAndReg->VReg, MRI);
1576 return std::nullopt;
1577}
1578
1579std::optional<int64_t>
1580llvm::getIConstantSplatSExtVal(const MachineInstr &MI,
1581 const MachineRegisterInfo &MRI) {
1582 return getIConstantSplatSExtVal(Reg: MI.getOperand(i: 0).getReg(), MRI);
1583}
1584
1585std::optional<FPValueAndVReg>
1586llvm::getFConstantSplat(Register VReg, const MachineRegisterInfo &MRI,
1587 bool AllowUndef) {
1588 if (auto SplatValAndReg = getAnyConstantSplat(VReg, MRI, AllowUndef))
1589 return getFConstantVRegValWithLookThrough(VReg: SplatValAndReg->VReg, MRI);
1590 return std::nullopt;
1591}
1592
1593bool llvm::isBuildVectorAllZeros(const MachineInstr &MI,
1594 const MachineRegisterInfo &MRI,
1595 bool AllowUndef) {
1596 return isBuildVectorConstantSplat(MI, MRI, SplatValue: 0, AllowUndef);
1597}
1598
1599bool llvm::isBuildVectorAllOnes(const MachineInstr &MI,
1600 const MachineRegisterInfo &MRI,
1601 bool AllowUndef) {
1602 return isBuildVectorConstantSplat(MI, MRI, SplatValue: -1, AllowUndef);
1603}
1604
1605std::optional<Register> llvm::getVectorSplat(const MachineInstr &MI,
1606 const MachineRegisterInfo &MRI) {
1607 unsigned Opc = MI.getOpcode();
1608 if (Opc == TargetOpcode::G_SPLAT_VECTOR)
1609 return MI.getOperand(i: 1).getReg();
1610
1611 if (!isBuildVectorOp(Opcode: Opc))
1612 return std::nullopt;
1613 auto Reg = MI.getOperand(i: 1).getReg();
1614 if (any_of(Range: drop_begin(RangeOrContainer: MI.operands(), N: 2),
1615 P: [&Reg](const MachineOperand &Op) { return Op.getReg() != Reg; }))
1616 return std::nullopt;
1617 return Reg;
1618}
1619
1620static bool isConstantScalar(const MachineInstr &MI,
1621 const MachineRegisterInfo &MRI,
1622 bool AllowFP = true,
1623 bool AllowOpaqueConstants = true) {
1624 switch (MI.getOpcode()) {
1625 case TargetOpcode::G_CONSTANT:
1626 case TargetOpcode::G_IMPLICIT_DEF:
1627 return true;
1628 case TargetOpcode::G_FCONSTANT:
1629 return AllowFP;
1630 case TargetOpcode::G_GLOBAL_VALUE:
1631 case TargetOpcode::G_FRAME_INDEX:
1632 case TargetOpcode::G_BLOCK_ADDR:
1633 case TargetOpcode::G_JUMP_TABLE:
1634 return AllowOpaqueConstants;
1635 default:
1636 return false;
1637 }
1638}
1639
1640bool llvm::isConstantOrConstantVector(MachineInstr &MI,
1641 const MachineRegisterInfo &MRI) {
1642 Register Def = MI.getOperand(i: 0).getReg();
1643 if (auto C = getIConstantVRegValWithLookThrough(VReg: Def, MRI))
1644 return true;
1645 GBuildVector *BV = dyn_cast<GBuildVector>(Val: &MI);
1646 if (!BV)
1647 return false;
1648 for (unsigned SrcIdx = 0; SrcIdx < BV->getNumSources(); ++SrcIdx) {
1649 if (getIConstantVRegValWithLookThrough(VReg: BV->getSourceReg(I: SrcIdx), MRI) ||
1650 getOpcodeDef<GImplicitDef>(Reg: BV->getSourceReg(I: SrcIdx), MRI))
1651 continue;
1652 return false;
1653 }
1654 return true;
1655}
1656
1657bool llvm::isConstantOrConstantVector(const MachineInstr &MI,
1658 const MachineRegisterInfo &MRI,
1659 bool AllowFP, bool AllowOpaqueConstants) {
1660 if (isConstantScalar(MI, MRI, AllowFP, AllowOpaqueConstants))
1661 return true;
1662
1663 if (!isBuildVectorOp(Opcode: MI.getOpcode()))
1664 return false;
1665
1666 const unsigned NumOps = MI.getNumOperands();
1667 for (unsigned I = 1; I != NumOps; ++I) {
1668 const MachineInstr *ElementDef = MRI.getVRegDef(Reg: MI.getOperand(i: I).getReg());
1669 if (!isConstantScalar(MI: *ElementDef, MRI, AllowFP, AllowOpaqueConstants))
1670 return false;
1671 }
1672
1673 return true;
1674}
1675
1676std::optional<APInt>
1677llvm::isConstantOrConstantSplatVector(Register Def,
1678 const MachineRegisterInfo &MRI) {
1679 if (auto C = getIConstantVRegValWithLookThrough(VReg: Def, MRI))
1680 return C->Value;
1681 auto MaybeCst = getIConstantSplatSExtVal(Reg: Def, MRI);
1682 if (!MaybeCst)
1683 return std::nullopt;
1684 const unsigned ScalarSize = MRI.getType(Reg: Def).getScalarSizeInBits();
1685 return APInt(ScalarSize, *MaybeCst, true);
1686}
1687
1688std::optional<APFloat>
1689llvm::isConstantOrConstantSplatVectorFP(Register Def,
1690 const MachineRegisterInfo &MRI) {
1691 if (auto FpConst = getFConstantVRegValWithLookThrough(VReg: Def, MRI))
1692 return FpConst->Value;
1693 auto MaybeCstFP = getFConstantSplat(VReg: Def, MRI, /*allowUndef=*/AllowUndef: false);
1694 if (!MaybeCstFP)
1695 return std::nullopt;
1696 return MaybeCstFP->Value;
1697}
1698
1699bool llvm::isNullOrNullSplat(const MachineInstr &MI,
1700 const MachineRegisterInfo &MRI, bool AllowUndefs) {
1701 switch (MI.getOpcode()) {
1702 case TargetOpcode::G_IMPLICIT_DEF:
1703 return AllowUndefs;
1704 case TargetOpcode::G_CONSTANT:
1705 return MI.getOperand(i: 1).getCImm()->isNullValue();
1706 case TargetOpcode::G_FCONSTANT: {
1707 const ConstantFP *FPImm = MI.getOperand(i: 1).getFPImm();
1708 return FPImm->isZero() && !FPImm->isNegative();
1709 }
1710 default:
1711 if (!AllowUndefs) // TODO: isBuildVectorAllZeros assumes undef is OK already
1712 return false;
1713 return isBuildVectorAllZeros(MI, MRI);
1714 }
1715}
1716
1717bool llvm::isAllOnesOrAllOnesSplat(const MachineInstr &MI,
1718 const MachineRegisterInfo &MRI,
1719 bool AllowUndefs) {
1720 switch (MI.getOpcode()) {
1721 case TargetOpcode::G_IMPLICIT_DEF:
1722 return AllowUndefs;
1723 case TargetOpcode::G_CONSTANT:
1724 return MI.getOperand(i: 1).getCImm()->isAllOnesValue();
1725 default:
1726 if (!AllowUndefs) // TODO: isBuildVectorAllOnes assumes undef is OK already
1727 return false;
1728 return isBuildVectorAllOnes(MI, MRI);
1729 }
1730}
1731
1732bool llvm::matchUnaryPredicate(
1733 const MachineRegisterInfo &MRI, Register Reg,
1734 llvm::function_ref<bool(const Constant *ConstVal)> Match,
1735 bool AllowUndefs) {
1736
1737 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);
1738 if (AllowUndefs && Def->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1739 return Match(nullptr);
1740
1741 // TODO: Also handle fconstant
1742 if (Def->getOpcode() == TargetOpcode::G_CONSTANT)
1743 return Match(Def->getOperand(i: 1).getCImm());
1744
1745 if (Def->getOpcode() != TargetOpcode::G_BUILD_VECTOR)
1746 return false;
1747
1748 for (unsigned I = 1, E = Def->getNumOperands(); I != E; ++I) {
1749 Register SrcElt = Def->getOperand(i: I).getReg();
1750 const MachineInstr *SrcDef = getDefIgnoringCopies(Reg: SrcElt, MRI);
1751 if (AllowUndefs && SrcDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) {
1752 if (!Match(nullptr))
1753 return false;
1754 continue;
1755 }
1756
1757 if (SrcDef->getOpcode() != TargetOpcode::G_CONSTANT ||
1758 !Match(SrcDef->getOperand(i: 1).getCImm()))
1759 return false;
1760 }
1761
1762 return true;
1763}
1764
1765bool llvm::isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector,
1766 bool IsFP) {
1767 switch (TLI.getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
1768 case TargetLowering::UndefinedBooleanContent:
1769 return Val & 0x1;
1770 case TargetLowering::ZeroOrOneBooleanContent:
1771 return Val == 1;
1772 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1773 return Val == -1;
1774 }
1775 llvm_unreachable("Invalid boolean contents");
1776}
1777
1778bool llvm::isConstFalseVal(const TargetLowering &TLI, int64_t Val,
1779 bool IsVector, bool IsFP) {
1780 switch (TLI.getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
1781 case TargetLowering::UndefinedBooleanContent:
1782 return ~Val & 0x1;
1783 case TargetLowering::ZeroOrOneBooleanContent:
1784 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1785 return Val == 0;
1786 }
1787 llvm_unreachable("Invalid boolean contents");
1788}
1789
1790int64_t llvm::getICmpTrueVal(const TargetLowering &TLI, bool IsVector,
1791 bool IsFP) {
1792 switch (TLI.getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
1793 case TargetLowering::UndefinedBooleanContent:
1794 case TargetLowering::ZeroOrOneBooleanContent:
1795 return 1;
1796 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1797 return -1;
1798 }
1799 llvm_unreachable("Invalid boolean contents");
1800}
1801
1802void llvm::saveUsesAndErase(MachineInstr &MI, MachineRegisterInfo &MRI,
1803 LostDebugLocObserver *LocObserver,
1804 SmallInstListTy &DeadInstChain) {
1805 for (MachineOperand &Op : MI.uses()) {
1806 if (Op.isReg() && Op.getReg().isVirtual())
1807 DeadInstChain.insert(I: MRI.getVRegDef(Reg: Op.getReg()));
1808 }
1809 LLVM_DEBUG(dbgs() << MI << "Is dead; erasing.\n");
1810 DeadInstChain.remove(I: &MI);
1811 salvageDebugInfo(MRI, MI);
1812 MI.eraseFromParent();
1813 if (LocObserver)
1814 LocObserver->checkpoint(CheckDebugLocs: false);
1815}
1816
1817void llvm::eraseInstrs(ArrayRef<MachineInstr *> DeadInstrs,
1818 MachineRegisterInfo &MRI,
1819 LostDebugLocObserver *LocObserver) {
1820 SmallInstListTy DeadInstChain;
1821 for (MachineInstr *MI : DeadInstrs)
1822 saveUsesAndErase(MI&: *MI, MRI, LocObserver, DeadInstChain);
1823
1824 while (!DeadInstChain.empty()) {
1825 MachineInstr *Inst = DeadInstChain.pop_back_val();
1826 if (!isTriviallyDead(MI: *Inst, MRI))
1827 continue;
1828 saveUsesAndErase(MI&: *Inst, MRI, LocObserver, DeadInstChain);
1829 }
1830}
1831
1832void llvm::eraseInstr(MachineInstr &MI, MachineRegisterInfo &MRI,
1833 LostDebugLocObserver *LocObserver) {
1834 return eraseInstrs(DeadInstrs: {&MI}, MRI, LocObserver);
1835}
1836
1837void llvm::salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI) {
1838 for (auto &Def : MI.defs()) {
1839 assert(Def.isReg() && "Must be a reg");
1840
1841 SmallVector<MachineOperand *, 16> DbgUsers;
1842 for (auto &MOUse : MRI.use_operands(Reg: Def.getReg())) {
1843 MachineInstr *DbgValue = MOUse.getParent();
1844 // Ignore partially formed DBG_VALUEs (a complete one has 4 operands:
1845 // location, offset, variable and expression).
1846 if (!DbgValue->isDebugValueList() &&
1847 !(DbgValue->isNonListDebugValue() && DbgValue->getNumOperands() == 4))
1848 continue;
1849 DbgUsers.push_back(Elt: &MOUse);
1850 }
1851
1852 if (!DbgUsers.empty()) {
1853 salvageDebugInfoForDbgValue(MRI, MI, DbgUsers);
1854 }
1855
1856 // MI is about to go away, so whatever could not be salvaged would be left
1857 // referring to a register without a definition.
1858 MRI.markUsesInDebugValueAsUndef(Reg: Def.getReg());
1859 }
1860}
1861
1862bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) {
1863 switch (Opc) {
1864 case TargetOpcode::G_FABS:
1865 case TargetOpcode::G_FADD:
1866 case TargetOpcode::G_FCANONICALIZE:
1867 case TargetOpcode::G_FCEIL:
1868 case TargetOpcode::G_FCONSTANT:
1869 case TargetOpcode::G_FCOPYSIGN:
1870 case TargetOpcode::G_FCOS:
1871 case TargetOpcode::G_FDIV:
1872 case TargetOpcode::G_FEXP2:
1873 case TargetOpcode::G_FEXP:
1874 case TargetOpcode::G_FFLOOR:
1875 case TargetOpcode::G_FLOG10:
1876 case TargetOpcode::G_FLOG2:
1877 case TargetOpcode::G_FLOG:
1878 case TargetOpcode::G_FMA:
1879 case TargetOpcode::G_FMAD:
1880 case TargetOpcode::G_FMAXIMUM:
1881 case TargetOpcode::G_FMAXIMUMNUM:
1882 case TargetOpcode::G_FMAXNUM:
1883 case TargetOpcode::G_FMAXNUM_IEEE:
1884 case TargetOpcode::G_FMINIMUM:
1885 case TargetOpcode::G_FMINIMUMNUM:
1886 case TargetOpcode::G_FMINNUM:
1887 case TargetOpcode::G_FMINNUM_IEEE:
1888 case TargetOpcode::G_FMUL:
1889 case TargetOpcode::G_FNEARBYINT:
1890 case TargetOpcode::G_FNEG:
1891 case TargetOpcode::G_FPEXT:
1892 case TargetOpcode::G_FPEXTLOAD:
1893 case TargetOpcode::G_FPOW:
1894 case TargetOpcode::G_FPTRUNC:
1895 case TargetOpcode::G_FPTRUNCSTORE:
1896 case TargetOpcode::G_FREM:
1897 case TargetOpcode::G_FRINT:
1898 case TargetOpcode::G_FSIN:
1899 case TargetOpcode::G_FTAN:
1900 case TargetOpcode::G_FACOS:
1901 case TargetOpcode::G_FASIN:
1902 case TargetOpcode::G_FATAN:
1903 case TargetOpcode::G_FATAN2:
1904 case TargetOpcode::G_FCOSH:
1905 case TargetOpcode::G_FSINH:
1906 case TargetOpcode::G_FTANH:
1907 case TargetOpcode::G_FSQRT:
1908 case TargetOpcode::G_FSUB:
1909 case TargetOpcode::G_INTRINSIC_ROUND:
1910 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
1911 case TargetOpcode::G_INTRINSIC_TRUNC:
1912 return true;
1913 default:
1914 return false;
1915 }
1916}
1917
1918/// Shifts return poison if shiftwidth is larger than the bitwidth.
1919static bool shiftAmountKnownInRange(Register ShiftAmount,
1920 const MachineRegisterInfo &MRI) {
1921 LLT Ty = MRI.getType(Reg: ShiftAmount);
1922
1923 if (Ty.isScalableVector())
1924 return false; // Can't tell, just return false to be safe
1925
1926 if (Ty.isScalar()) {
1927 std::optional<ValueAndVReg> Val =
1928 getIConstantVRegValWithLookThrough(VReg: ShiftAmount, MRI);
1929 if (!Val)
1930 return false;
1931 return Val->Value.ult(RHS: Ty.getScalarSizeInBits());
1932 }
1933
1934 GBuildVector *BV = getOpcodeDef<GBuildVector>(Reg: ShiftAmount, MRI);
1935 if (!BV)
1936 return false;
1937
1938 unsigned Sources = BV->getNumSources();
1939 for (unsigned I = 0; I < Sources; ++I) {
1940 std::optional<ValueAndVReg> Val =
1941 getIConstantVRegValWithLookThrough(VReg: BV->getSourceReg(I), MRI);
1942 if (!Val)
1943 return false;
1944 if (!Val->Value.ult(RHS: Ty.getScalarSizeInBits()))
1945 return false;
1946 }
1947
1948 return true;
1949}
1950
1951static bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
1952 bool ConsiderFlagsAndMetadata,
1953 UndefPoisonKind Kind) {
1954 MachineInstr *RegDef = MRI.getVRegDef(Reg);
1955
1956 if (ConsiderFlagsAndMetadata && includesPoison(Kind))
1957 if (auto *GMI = dyn_cast<GenericMachineInstr>(Val: RegDef))
1958 if (GMI->hasPoisonGeneratingFlags())
1959 return true;
1960
1961 // Check whether opcode is a poison/undef-generating operation.
1962 switch (RegDef->getOpcode()) {
1963 case TargetOpcode::G_BUILD_VECTOR:
1964 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
1965 return false;
1966 case TargetOpcode::G_SHL:
1967 case TargetOpcode::G_ASHR:
1968 case TargetOpcode::G_LSHR:
1969 return includesPoison(Kind) &&
1970 !shiftAmountKnownInRange(ShiftAmount: RegDef->getOperand(i: 2).getReg(), MRI);
1971 case TargetOpcode::G_FPTOSI:
1972 case TargetOpcode::G_FPTOUI:
1973 // fptosi/ui yields poison if the resulting value does not fit in the
1974 // destination type.
1975 return true;
1976 case TargetOpcode::G_CTLZ:
1977 case TargetOpcode::G_CTTZ:
1978 case TargetOpcode::G_CTLS:
1979 case TargetOpcode::G_ABS:
1980 case TargetOpcode::G_CTPOP:
1981 case TargetOpcode::G_BSWAP:
1982 case TargetOpcode::G_BITREVERSE:
1983 case TargetOpcode::G_FSHL:
1984 case TargetOpcode::G_FSHR:
1985 case TargetOpcode::G_SMAX:
1986 case TargetOpcode::G_SMIN:
1987 case TargetOpcode::G_SCMP:
1988 case TargetOpcode::G_UMAX:
1989 case TargetOpcode::G_UMIN:
1990 case TargetOpcode::G_UCMP:
1991 case TargetOpcode::G_PTRMASK:
1992 case TargetOpcode::G_SADDO:
1993 case TargetOpcode::G_SSUBO:
1994 case TargetOpcode::G_UADDO:
1995 case TargetOpcode::G_USUBO:
1996 case TargetOpcode::G_SMULO:
1997 case TargetOpcode::G_UMULO:
1998 case TargetOpcode::G_SADDSAT:
1999 case TargetOpcode::G_UADDSAT:
2000 case TargetOpcode::G_SSUBSAT:
2001 case TargetOpcode::G_USUBSAT:
2002 case TargetOpcode::G_SBFX:
2003 case TargetOpcode::G_UBFX:
2004 return false;
2005 case TargetOpcode::G_SSHLSAT:
2006 case TargetOpcode::G_USHLSAT:
2007 return includesPoison(Kind) &&
2008 !shiftAmountKnownInRange(ShiftAmount: RegDef->getOperand(i: 2).getReg(), MRI);
2009 case TargetOpcode::G_INSERT_VECTOR_ELT: {
2010 GInsertVectorElement *Insert = cast<GInsertVectorElement>(Val: RegDef);
2011 if (includesPoison(Kind)) {
2012 std::optional<ValueAndVReg> Index =
2013 getIConstantVRegValWithLookThrough(VReg: Insert->getIndexReg(), MRI);
2014 if (!Index)
2015 return true;
2016 LLT VecTy = MRI.getType(Reg: Insert->getVectorReg());
2017 return Index->Value.uge(RHS: VecTy.getElementCount().getKnownMinValue());
2018 }
2019 return false;
2020 }
2021 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
2022 GExtractVectorElement *Extract = cast<GExtractVectorElement>(Val: RegDef);
2023 if (includesPoison(Kind)) {
2024 std::optional<ValueAndVReg> Index =
2025 getIConstantVRegValWithLookThrough(VReg: Extract->getIndexReg(), MRI);
2026 if (!Index)
2027 return true;
2028 LLT VecTy = MRI.getType(Reg: Extract->getVectorReg());
2029 return Index->Value.uge(RHS: VecTy.getElementCount().getKnownMinValue());
2030 }
2031 return false;
2032 }
2033 case TargetOpcode::G_SHUFFLE_VECTOR: {
2034 GShuffleVector *Shuffle = cast<GShuffleVector>(Val: RegDef);
2035 ArrayRef<int> Mask = Shuffle->getMask();
2036 return includesPoison(Kind) && is_contained(Range&: Mask, Element: -1);
2037 }
2038 case TargetOpcode::G_FNEG:
2039 case TargetOpcode::G_PHI:
2040 case TargetOpcode::G_SELECT:
2041 case TargetOpcode::G_UREM:
2042 case TargetOpcode::G_SREM:
2043 case TargetOpcode::G_FREEZE:
2044 case TargetOpcode::G_ICMP:
2045 case TargetOpcode::G_FCMP:
2046 case TargetOpcode::G_FADD:
2047 case TargetOpcode::G_FSUB:
2048 case TargetOpcode::G_FMUL:
2049 case TargetOpcode::G_FDIV:
2050 case TargetOpcode::G_FREM:
2051 case TargetOpcode::G_PTR_ADD:
2052 return false;
2053 default:
2054 return !isa<GCastOp>(Val: RegDef) && !isa<GBinOp>(Val: RegDef);
2055 }
2056}
2057
2058static bool isGuaranteedNotToBeUndefOrPoison(Register Reg,
2059 const MachineRegisterInfo &MRI,
2060 unsigned Depth,
2061 UndefPoisonKind Kind) {
2062 if (Depth >= MaxAnalysisRecursionDepth)
2063 return false;
2064
2065 MachineInstr *RegDef = MRI.getVRegDef(Reg);
2066
2067 switch (RegDef->getOpcode()) {
2068 case TargetOpcode::G_FREEZE:
2069 return true;
2070 case TargetOpcode::G_IMPLICIT_DEF:
2071 return !includesUndef(Kind);
2072 case TargetOpcode::G_CONSTANT:
2073 case TargetOpcode::G_FCONSTANT:
2074 return true;
2075 case TargetOpcode::G_BUILD_VECTOR: {
2076 GBuildVector *BV = cast<GBuildVector>(Val: RegDef);
2077 unsigned NumSources = BV->getNumSources();
2078 for (unsigned I = 0; I < NumSources; ++I)
2079 if (!::isGuaranteedNotToBeUndefOrPoison(Reg: BV->getSourceReg(I), MRI,
2080 Depth: Depth + 1, Kind))
2081 return false;
2082 return true;
2083 }
2084 case TargetOpcode::G_PHI: {
2085 GPhi *Phi = cast<GPhi>(Val: RegDef);
2086 unsigned NumIncoming = Phi->getNumIncomingValues();
2087 for (unsigned I = 0; I < NumIncoming; ++I)
2088 if (!::isGuaranteedNotToBeUndefOrPoison(Reg: Phi->getIncomingValue(I), MRI,
2089 Depth: Depth + 1, Kind))
2090 return false;
2091 return true;
2092 }
2093 default: {
2094 auto MOCheck = [&](const MachineOperand &MO) {
2095 if (!MO.isReg())
2096 return true;
2097 return ::isGuaranteedNotToBeUndefOrPoison(Reg: MO.getReg(), MRI, Depth: Depth + 1,
2098 Kind);
2099 };
2100 return !::canCreateUndefOrPoison(Reg, MRI,
2101 /*ConsiderFlagsAndMetadata=*/true, Kind) &&
2102 all_of(Range: RegDef->uses(), P: MOCheck);
2103 }
2104 }
2105}
2106
2107bool llvm::canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
2108 bool ConsiderFlagsAndMetadata) {
2109 return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata,
2110 Kind: UndefPoisonKind::UndefOrPoison);
2111}
2112
2113bool canCreatePoison(Register Reg, const MachineRegisterInfo &MRI,
2114 bool ConsiderFlagsAndMetadata = true) {
2115 return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata,
2116 Kind: UndefPoisonKind::PoisonOnly);
2117}
2118
2119bool llvm::isGuaranteedNotToBeUndefOrPoison(Register Reg,
2120 const MachineRegisterInfo &MRI,
2121 unsigned Depth) {
2122 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
2123 Kind: UndefPoisonKind::UndefOrPoison);
2124}
2125
2126bool llvm::isGuaranteedNotToBePoison(Register Reg,
2127 const MachineRegisterInfo &MRI,
2128 unsigned Depth) {
2129 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
2130 Kind: UndefPoisonKind::PoisonOnly);
2131}
2132
2133bool llvm::isGuaranteedNotToBeUndef(Register Reg,
2134 const MachineRegisterInfo &MRI,
2135 unsigned Depth) {
2136 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
2137 Kind: UndefPoisonKind::UndefOnly);
2138}
2139
2140Type *llvm::getTypeForLLT(LLT Ty, LLVMContext &C) {
2141 if (Ty.isVector())
2142 return VectorType::get(ElementType: IntegerType::get(C, NumBits: Ty.getScalarSizeInBits()),
2143 EC: Ty.getElementCount());
2144 return IntegerType::get(C, NumBits: Ty.getSizeInBits());
2145}
2146
2147bool llvm::isAssertMI(const MachineInstr &MI) {
2148 switch (MI.getOpcode()) {
2149 default:
2150 return false;
2151 case TargetOpcode::G_ASSERT_ALIGN:
2152 case TargetOpcode::G_ASSERT_SEXT:
2153 case TargetOpcode::G_ASSERT_ZEXT:
2154 return true;
2155 }
2156}
2157
2158APInt llvm::GIConstant::getScalarValue() const {
2159 assert(Kind == GIConstantKind::Scalar && "Expected scalar constant");
2160
2161 return Value;
2162}
2163
2164std::optional<GIConstant>
2165llvm::GIConstant::getConstant(Register Const, const MachineRegisterInfo &MRI) {
2166 MachineInstr *Constant = getDefIgnoringCopies(Reg: Const, MRI);
2167
2168 if (GSplatVector *Splat = dyn_cast<GSplatVector>(Val: Constant)) {
2169 std::optional<ValueAndVReg> MayBeConstant =
2170 getIConstantVRegValWithLookThrough(VReg: Splat->getScalarReg(), MRI);
2171 if (!MayBeConstant)
2172 return std::nullopt;
2173 return GIConstant(MayBeConstant->Value, GIConstantKind::ScalableVector);
2174 }
2175
2176 if (GBuildVector *Build = dyn_cast<GBuildVector>(Val: Constant)) {
2177 SmallVector<APInt> Values;
2178 unsigned NumSources = Build->getNumSources();
2179 for (unsigned I = 0; I < NumSources; ++I) {
2180 Register SrcReg = Build->getSourceReg(I);
2181 std::optional<ValueAndVReg> MayBeConstant =
2182 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI);
2183 if (!MayBeConstant)
2184 return std::nullopt;
2185 Values.push_back(Elt: MayBeConstant->Value);
2186 }
2187 return GIConstant(Values);
2188 }
2189
2190 std::optional<ValueAndVReg> MayBeConstant =
2191 getIConstantVRegValWithLookThrough(VReg: Const, MRI);
2192 if (!MayBeConstant)
2193 return std::nullopt;
2194
2195 return GIConstant(MayBeConstant->Value, GIConstantKind::Scalar);
2196}
2197
2198APFloat llvm::GFConstant::getScalarValue() const {
2199 assert(Kind == GFConstantKind::Scalar && "Expected scalar constant");
2200
2201 return Values[0];
2202}
2203
2204std::optional<GFConstant>
2205llvm::GFConstant::getConstant(Register Const, const MachineRegisterInfo &MRI) {
2206 MachineInstr *Constant = getDefIgnoringCopies(Reg: Const, MRI);
2207
2208 if (GSplatVector *Splat = dyn_cast<GSplatVector>(Val: Constant)) {
2209 std::optional<FPValueAndVReg> MayBeConstant =
2210 getFConstantVRegValWithLookThrough(VReg: Splat->getScalarReg(), MRI);
2211 if (!MayBeConstant)
2212 return std::nullopt;
2213 return GFConstant(MayBeConstant->Value, GFConstantKind::ScalableVector);
2214 }
2215
2216 if (GBuildVector *Build = dyn_cast<GBuildVector>(Val: Constant)) {
2217 SmallVector<APFloat> Values;
2218 unsigned NumSources = Build->getNumSources();
2219 for (unsigned I = 0; I < NumSources; ++I) {
2220 Register SrcReg = Build->getSourceReg(I);
2221 std::optional<FPValueAndVReg> MayBeConstant =
2222 getFConstantVRegValWithLookThrough(VReg: SrcReg, MRI);
2223 if (!MayBeConstant)
2224 return std::nullopt;
2225 Values.push_back(Elt: MayBeConstant->Value);
2226 }
2227 return GFConstant(Values);
2228 }
2229
2230 std::optional<FPValueAndVReg> MayBeConstant =
2231 getFConstantVRegValWithLookThrough(VReg: Const, MRI);
2232 if (!MayBeConstant)
2233 return std::nullopt;
2234
2235 return GFConstant(MayBeConstant->Value, GFConstantKind::Scalar);
2236}
2237
2238// Returns a list of types to use for memory op lowering in MemOps. A partial
2239// port of findOptimalMemOpLowering in TargetLowering.
2240static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
2241 unsigned Limit, const MemOp &Op,
2242 unsigned DstAS, unsigned SrcAS,
2243 const AttributeList &FuncAttributes,
2244 const TargetLowering &TLI) {
2245 if (Op.isMemcpyOrMemmoveWithFixedDstAlign() &&
2246 Op.getSrcAlign() < Op.getDstAlign())
2247 return false;
2248
2249 LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes);
2250
2251 if (Ty == LLT()) {
2252 // Use the largest scalar type whose alignment constraints are satisfied.
2253 // We only need to check DstAlign here as SrcAlign is always greater or
2254 // equal to DstAlign (or zero).
2255 Ty = LLT::integer(SizeInBits: 64);
2256 if (Op.isFixedDstAlign())
2257 while (Op.getDstAlign() < Ty.getSizeInBytes() &&
2258 !TLI.allowsMisalignedMemoryAccesses(Ty, AddrSpace: DstAS, Alignment: Op.getDstAlign()))
2259 Ty = LLT::integer(SizeInBits: Ty.getSizeInBytes());
2260 assert(Ty.getSizeInBits() > 0 && "Could not find valid type");
2261 // FIXME: check for the largest legal type we can load/store to.
2262 }
2263
2264 unsigned NumMemOps = 0;
2265 uint64_t Size = Op.size();
2266 while (Size) {
2267 unsigned TySize = Ty.getSizeInBytes();
2268 while (TySize > Size) {
2269 // For now, only use non-vector load / store's for the left-over pieces.
2270 LLT NewTy = Ty;
2271 // FIXME: check for mem op safety and legality of the types. Not all of
2272 // SDAGisms map cleanly to GISel concepts.
2273 if (NewTy.isVector())
2274 NewTy =
2275 NewTy.getSizeInBits() > 64 ? LLT::integer(SizeInBits: 64) : LLT::integer(SizeInBits: 32);
2276 NewTy = LLT::integer(SizeInBits: llvm::bit_floor(Value: NewTy.getSizeInBits() - 1));
2277 unsigned NewTySize = NewTy.getSizeInBytes();
2278 assert(NewTySize > 0 && "Could not find appropriate type");
2279
2280 // If the new LLT cannot cover all of the remaining bits, then consider
2281 // issuing a (or a pair of) unaligned and overlapping load / store.
2282 unsigned Fast;
2283 // Need to get a VT equivalent for allowMisalignedMemoryAccesses().
2284 MVT VT = getMVTForLLT(Ty);
2285 if (NumMemOps && !Op.isVolatile() && NewTySize < Size &&
2286 TLI.allowsMisalignedMemoryAccesses(
2287 VT, AddrSpace: DstAS, Alignment: Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
2288 Flags: MachineMemOperand::MONone, &Fast) &&
2289 Fast)
2290 TySize = Size;
2291 else {
2292 Ty = NewTy;
2293 TySize = NewTySize;
2294 }
2295 }
2296
2297 if (++NumMemOps > Limit)
2298 return false;
2299
2300 MemOps.push_back(x: Ty);
2301 Size -= TySize;
2302 }
2303
2304 return true;
2305}
2306
2307bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
2308 const MachineRegisterInfo &MRI, unsigned MaxLen,
2309 Register &Dst, Register &Src,
2310 uint64_t &KnownLen, Align &Alignment,
2311 bool &DstAlignCanChange,
2312 std::vector<LLT> &MemOps) {
2313 const unsigned Opc = MI.getOpcode();
2314 assert((Opc == TargetOpcode::G_MEMCPY ||
2315 Opc == TargetOpcode::G_MEMCPY_INLINE ||
2316 Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET ||
2317 Opc == TargetOpcode::G_MEMSET_INLINE) &&
2318 "Expected memcpy like instruction");
2319
2320 auto MMOIt = MI.memoperands_begin();
2321 const MachineMemOperand *MemOp = *MMOIt;
2322
2323 Align DstAlign = MemOp->getBaseAlign();
2324 Align SrcAlign;
2325 Alignment = DstAlign;
2326 Register Len;
2327 std::tie(args&: Dst, args&: Src, args&: Len) = MI.getFirst3Regs();
2328
2329 if (Opc != TargetOpcode::G_MEMSET && Opc != TargetOpcode::G_MEMSET_INLINE) {
2330 assert(MMOIt != MI.memoperands_end() && "Expected a second MMO on MI");
2331 MemOp = *(++MMOIt);
2332 SrcAlign = MemOp->getBaseAlign();
2333 Alignment = std::min(a: DstAlign, b: SrcAlign);
2334 }
2335
2336 // See if this is a constant length copy.
2337 auto LenVRegAndVal = getIConstantVRegValWithLookThrough(VReg: Len, MRI);
2338 if (!LenVRegAndVal) {
2339 // FIXME: support dynamically sized G_MEMCPY_INLINE and G_MEMSET_INLINE
2340 assert(Opc != TargetOpcode::G_MEMCPY_INLINE &&
2341 Opc != TargetOpcode::G_MEMSET_INLINE &&
2342 "inline memcpy and memset with dynamic size are not yet supported");
2343 return false;
2344 }
2345
2346 KnownLen = LenVRegAndVal->Value.getZExtValue();
2347 DstAlignCanChange = false;
2348
2349 if (KnownLen == 0)
2350 return true;
2351
2352 if (Opc != TargetOpcode::G_MEMCPY_INLINE &&
2353 Opc != TargetOpcode::G_MEMSET_INLINE && MaxLen && KnownLen > MaxLen)
2354 return false;
2355
2356 bool IsVolatile = MemOp->isVolatile();
2357 const MachineFunction &MF = *MI.getParent()->getParent();
2358 const auto &TLI = *MF.getSubtarget().getTargetLowering();
2359 // On Darwin, -Os means optimize for size without hurting performance, so
2360 // only really optimize for size when -Oz (MinSize) is used.
2361 bool OptSize = MF.getTarget().getTargetTriple().isOSDarwin()
2362 ? MF.getFunction().hasMinSize()
2363 : MF.getFunction().hasOptSize();
2364
2365 const MachineFrameInfo &MFI = MF.getFrameInfo();
2366 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
2367 if (FIDef && !MFI.isFixedObjectIndex(ObjectIdx: FIDef->getOperand(i: 1).getIndex()))
2368 DstAlignCanChange = true;
2369
2370 const auto &DstMMO = **MI.memoperands_begin();
2371 MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo();
2372
2373 switch (Opc) {
2374 case TargetOpcode::G_MEMCPY_INLINE:
2375 case TargetOpcode::G_MEMCPY: {
2376 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
2377 MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
2378 uint64_t Limit = Opc == TargetOpcode::G_MEMCPY_INLINE
2379 ? std::numeric_limits<uint64_t>::max()
2380 : TLI.getMaxStoresPerMemcpy(OptSize);
2381 return findGISelOptimalMemOpLowering(
2382 MemOps, Limit,
2383 Op: MemOp::Copy(Size: KnownLen, DstAlignCanChange, DstAlign: std::min(a: DstAlign, b: SrcAlign),
2384 SrcAlign, IsVolatile),
2385 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
2386 FuncAttributes: MF.getFunction().getAttributes(), TLI);
2387 }
2388 case TargetOpcode::G_MEMMOVE: {
2389 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
2390 MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
2391 unsigned Limit = TLI.getMaxStoresPerMemmove(OptSize);
2392 return findGISelOptimalMemOpLowering(
2393 MemOps, Limit,
2394 Op: MemOp::Move(Size: KnownLen, DstAlignCanChange, DstAlign: std::min(a: DstAlign, b: SrcAlign),
2395 SrcAlign, IsVolatile),
2396 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
2397 FuncAttributes: MF.getFunction().getAttributes(), TLI);
2398 }
2399 case TargetOpcode::G_MEMSET:
2400 case TargetOpcode::G_MEMSET_INLINE: {
2401 unsigned Limit = Opc == TargetOpcode::G_MEMSET_INLINE
2402 ? std::numeric_limits<unsigned>::max()
2403 : TLI.getMaxStoresPerMemset(OptSize);
2404 auto ValVRegAndVal = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
2405 bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0;
2406 return findGISelOptimalMemOpLowering(
2407 MemOps, Limit,
2408 Op: MemOp::Set(Size: KnownLen, DstAlignCanChange, DstAlign,
2409 /*IsZeroMemset=*/IsZeroVal,
2410 /*IsVolatile=*/IsVolatile),
2411 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes(), TLI);
2412 }
2413 default:
2414 llvm_unreachable("Unexpected memcpy-family opcode");
2415 }
2416}
2417