1//===- llvm/CodeGen/GlobalISel/Utils.cpp -------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file This file implements the utility functions used by the GlobalISel
9/// pipeline.
10//===----------------------------------------------------------------------===//
11
12#include "llvm/CodeGen/GlobalISel/Utils.h"
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/Analysis/ValueTracking.h"
16#include "llvm/CodeGen/CodeGenCommonISel.h"
17#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
18#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
19#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
20#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
21#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
22#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
23#include "llvm/CodeGen/LowLevelTypeUtils.h"
24#include "llvm/CodeGen/MachineInstr.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/MachineSizeOpts.h"
29#include "llvm/CodeGen/RegisterBankInfo.h"
30#include "llvm/CodeGen/StackProtector.h"
31#include "llvm/CodeGen/TargetInstrInfo.h"
32#include "llvm/CodeGen/TargetLowering.h"
33#include "llvm/CodeGen/TargetOpcodes.h"
34#include "llvm/CodeGen/TargetPassConfig.h"
35#include "llvm/CodeGen/TargetRegisterInfo.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/Support/UndefPoison.h"
38#include "llvm/Target/TargetMachine.h"
39#include "llvm/Transforms/Utils/SizeOpts.h"
40#include <limits>
41#include <numeric>
42#include <optional>
43#include <tuple>
44
45#define DEBUG_TYPE "globalisel-utils"
46
47using namespace llvm;
48using namespace MIPatternMatch;
49
50Register llvm::constrainRegToClass(MachineRegisterInfo &MRI,
51 const TargetInstrInfo &TII,
52 const RegisterBankInfo &RBI, Register Reg,
53 const TargetRegisterClass &RegClass) {
54 if (!RBI.constrainGenericRegister(Reg, RC: RegClass, MRI))
55 return MRI.createVirtualRegister(RegClass: &RegClass);
56
57 return Reg;
58}
59
60Register llvm::constrainOperandRegClass(
61 const MachineFunction &MF, const TargetRegisterInfo &TRI,
62 MachineRegisterInfo &MRI, const TargetInstrInfo &TII,
63 const RegisterBankInfo &RBI, MachineInstr &InsertPt,
64 const TargetRegisterClass &RegClass, MachineOperand &RegMO) {
65 Register Reg = RegMO.getReg();
66 // Assume physical registers are properly constrained.
67 assert(Reg.isVirtual() && "PhysReg not implemented");
68
69 // Save the old register class to check whether
70 // the change notifications will be required.
71 // TODO: A better approach would be to pass
72 // the observers to constrainRegToClass().
73 auto *OldRegClass = MRI.getRegClassOrNull(Reg);
74 Register ConstrainedReg = constrainRegToClass(MRI, TII, RBI, Reg, RegClass);
75 // If we created a new virtual register because the class is not compatible
76 // then create a copy between the new and the old register.
77 if (ConstrainedReg != Reg) {
78 MachineBasicBlock::iterator InsertIt(&InsertPt);
79 MachineBasicBlock &MBB = *InsertPt.getParent();
80 // FIXME: The copy needs to have the classes constrained for its operands.
81 // Use operand's regbank to get the class for old register (Reg).
82 if (RegMO.isUse()) {
83 BuildMI(BB&: MBB, I: InsertIt, MIMD: InsertPt.getDebugLoc(),
84 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ConstrainedReg)
85 .addReg(RegNo: Reg);
86 } else {
87 assert(RegMO.isDef() && "Must be a definition");
88 BuildMI(BB&: MBB, I: std::next(x: InsertIt), MIMD: InsertPt.getDebugLoc(),
89 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Reg)
90 .addReg(RegNo: ConstrainedReg);
91 }
92 if (GISelChangeObserver *Observer = MF.getObserver()) {
93 Observer->changingInstr(MI&: *RegMO.getParent());
94 }
95 RegMO.setReg(ConstrainedReg);
96 if (GISelChangeObserver *Observer = MF.getObserver()) {
97 Observer->changedInstr(MI&: *RegMO.getParent());
98 }
99 } else if (OldRegClass != MRI.getRegClassOrNull(Reg)) {
100 if (GISelChangeObserver *Observer = MF.getObserver()) {
101 if (!RegMO.isDef()) {
102 MachineInstr *RegDef = MRI.getVRegDef(Reg);
103 Observer->changedInstr(MI&: *RegDef);
104 }
105 Observer->changingAllUsesOfReg(MRI, Reg);
106 Observer->finishedChangingAllUsesOfReg();
107 }
108 }
109 return ConstrainedReg;
110}
111
112Register llvm::constrainOperandRegClass(
113 const MachineFunction &MF, const TargetRegisterInfo &TRI,
114 MachineRegisterInfo &MRI, const TargetInstrInfo &TII,
115 const RegisterBankInfo &RBI, MachineInstr &InsertPt, const MCInstrDesc &II,
116 MachineOperand &RegMO, unsigned OpIdx) {
117 Register Reg = RegMO.getReg();
118 // Assume physical registers are properly constrained.
119 assert(Reg.isVirtual() && "PhysReg not implemented");
120
121 const TargetRegisterClass *OpRC = TII.getRegClass(MCID: II, OpNum: OpIdx);
122 // Some of the target independent instructions, like COPY, may not impose any
123 // register class constraints on some of their operands: If it's a use, we can
124 // skip constraining as the instruction defining the register would constrain
125 // it.
126
127 if (OpRC) {
128 // Obtain the RC from incoming regbank if it is a proper sub-class. Operands
129 // can have multiple regbanks for a superclass that combine different
130 // register types (E.g., AMDGPU's VGPR and AGPR). The regbank ambiguity
131 // resolved by targets during regbankselect should not be overridden.
132 if (const auto *SubRC = TRI.getCommonSubClass(
133 A: OpRC, B: TRI.getConstrainedRegClassForOperand(MO: RegMO, MRI)))
134 OpRC = SubRC;
135
136 OpRC = TRI.getAllocatableClass(RC: OpRC);
137 }
138
139 if (!OpRC) {
140 assert((!isTargetSpecificOpcode(II.getOpcode()) || RegMO.isUse()) &&
141 "Register class constraint is required unless either the "
142 "instruction is target independent or the operand is a use");
143 // FIXME: Just bailing out like this here could be not enough, unless we
144 // expect the users of this function to do the right thing for PHIs and
145 // COPY:
146 // v1 = COPY v0
147 // v2 = COPY v1
148 // v1 here may end up not being constrained at all. Please notice that to
149 // reproduce the issue we likely need a destination pattern of a selection
150 // rule producing such extra copies, not just an input GMIR with them as
151 // every existing target using selectImpl handles copies before calling it
152 // and they never reach this function.
153 return Reg;
154 }
155 return constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt, RegClass: *OpRC,
156 RegMO);
157}
158
159void llvm::constrainSelectedInstRegOperands(MachineInstr &I,
160 const TargetInstrInfo &TII,
161 const TargetRegisterInfo &TRI,
162 const RegisterBankInfo &RBI) {
163 assert(!isPreISelGenericOpcode(I.getOpcode()) &&
164 "A selected instruction is expected");
165 MachineBasicBlock &MBB = *I.getParent();
166 MachineFunction &MF = *MBB.getParent();
167 MachineRegisterInfo &MRI = MF.getRegInfo();
168
169 for (unsigned OpI = 0, OpE = I.getNumExplicitOperands(); OpI != OpE; ++OpI) {
170 MachineOperand &MO = I.getOperand(i: OpI);
171
172 // There's nothing to be done on non-register operands.
173 if (!MO.isReg())
174 continue;
175
176 LLVM_DEBUG(dbgs() << "Converting operand: " << MO << '\n');
177
178 Register Reg = MO.getReg();
179 // Physical registers don't need to be constrained.
180 if (Reg.isPhysical())
181 continue;
182
183 // Register operands with a value of 0 (e.g. predicate operands) don't need
184 // to be constrained.
185 if (Reg == 0)
186 continue;
187
188 // If the operand is a vreg, we should constrain its regclass, and only
189 // insert COPYs if that's impossible.
190 // constrainOperandRegClass does that for us.
191 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt&: I, II: I.getDesc(), RegMO&: MO, OpIdx: OpI);
192
193 // Tie uses to defs as indicated in MCInstrDesc if this hasn't already been
194 // done.
195 if (MO.isUse()) {
196 int DefIdx = I.getDesc().getOperandConstraint(OpNum: OpI, Constraint: MCOI::TIED_TO);
197 if (DefIdx != -1 && !I.isRegTiedToUseOperand(DefOpIdx: DefIdx))
198 I.tieOperands(DefIdx, UseIdx: OpI);
199 }
200 }
201}
202
203bool llvm::canReplaceReg(Register DstReg, Register SrcReg,
204 MachineRegisterInfo &MRI) {
205 // Give up if either DstReg or SrcReg is a physical register.
206 if (DstReg.isPhysical() || SrcReg.isPhysical())
207 return false;
208 // Give up if the types don't match.
209 if (MRI.getType(Reg: DstReg) != MRI.getType(Reg: SrcReg))
210 return false;
211 // Replace if either DstReg has no constraints or the register
212 // constraints match.
213 const auto &DstRBC = MRI.getRegClassOrRegBank(Reg: DstReg);
214 if (!DstRBC || DstRBC == MRI.getRegClassOrRegBank(Reg: SrcReg))
215 return true;
216
217 // Otherwise match if the Src is already a regclass that is covered by the Dst
218 // RegBank.
219 return isa<const RegisterBank *>(Val: DstRBC) && MRI.getRegClassOrNull(Reg: SrcReg) &&
220 cast<const RegisterBank *>(Val: DstRBC)->covers(
221 RC: *MRI.getRegClassOrNull(Reg: SrcReg));
222}
223
224bool llvm::isTriviallyDead(const MachineInstr &MI,
225 const MachineRegisterInfo &MRI) {
226 // Instructions without side-effects are dead iff they only define dead regs.
227 // This function is hot and this loop returns early in the common case,
228 // so only perform additional checks before this if absolutely necessary.
229 for (const auto &MO : MI.all_defs()) {
230 Register Reg = MO.getReg();
231 if (Reg.isPhysical() || !MRI.use_nodbg_empty(RegNo: Reg))
232 return false;
233 }
234 return MI.wouldBeTriviallyDead();
235}
236
237static void reportGISelDiagnostic(DiagnosticSeverity Severity,
238 MachineFunction &MF,
239 MachineOptimizationRemarkEmitter &MORE,
240 MachineOptimizationRemarkMissed &R) {
241 bool IsGlobalISelAbortEnabled =
242 MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
243 bool IsFatal = Severity == DS_Error && IsGlobalISelAbortEnabled;
244 // Print the function name explicitly if we don't have a debug location (which
245 // makes the diagnostic less useful) or if we're going to emit a raw error.
246 if (!R.getLocation().isValid() || IsFatal)
247 R << (" (in function: " + MF.getName() + ")").str();
248
249 if (IsFatal)
250 reportFatalUsageError(reason: Twine(R.getMsg()));
251 else
252 MORE.emit(OptDiag&: R);
253}
254
255void llvm::reportGISelWarning(MachineFunction &MF,
256 MachineOptimizationRemarkEmitter &MORE,
257 MachineOptimizationRemarkMissed &R) {
258 reportGISelDiagnostic(Severity: DS_Warning, MF, MORE, R);
259}
260
261void llvm::reportGISelFailure(MachineFunction &MF,
262 MachineOptimizationRemarkEmitter &MORE,
263 MachineOptimizationRemarkMissed &R) {
264 MF.getProperties().setFailedISel();
265 reportGISelDiagnostic(Severity: DS_Error, MF, MORE, R);
266}
267
268void llvm::reportGISelFailure(MachineFunction &MF,
269 MachineOptimizationRemarkEmitter &MORE,
270 const char *PassName, StringRef Msg,
271 const MachineInstr &MI) {
272 MachineOptimizationRemarkMissed R(PassName, "GISelFailure: ",
273 MI.getDebugLoc(), MI.getParent());
274 R << Msg;
275 // Printing MI is expensive; only do it if expensive remarks are enabled.
276 if (MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable ||
277 MORE.allowExtraAnalysis(PassName))
278 R << ": " << ore::MNV("Inst", MI);
279 reportGISelFailure(MF, MORE, R);
280}
281
282unsigned llvm::getInverseGMinMaxOpcode(unsigned MinMaxOpc) {
283 switch (MinMaxOpc) {
284 case TargetOpcode::G_SMIN:
285 return TargetOpcode::G_SMAX;
286 case TargetOpcode::G_SMAX:
287 return TargetOpcode::G_SMIN;
288 case TargetOpcode::G_UMIN:
289 return TargetOpcode::G_UMAX;
290 case TargetOpcode::G_UMAX:
291 return TargetOpcode::G_UMIN;
292 default:
293 llvm_unreachable("unrecognized opcode");
294 }
295}
296
297std::optional<APInt> llvm::getIConstantVRegVal(Register VReg,
298 const MachineRegisterInfo &MRI) {
299 std::optional<ValueAndVReg> ValAndVReg = getIConstantVRegValWithLookThrough(
300 VReg, MRI, /*LookThroughInstrs*/ false);
301 assert((!ValAndVReg || ValAndVReg->VReg == VReg) &&
302 "Value found while looking through instrs");
303 if (!ValAndVReg)
304 return std::nullopt;
305 return ValAndVReg->Value;
306}
307
308const APInt &llvm::getIConstantFromReg(Register Reg,
309 const MachineRegisterInfo &MRI) {
310 MachineInstr *Const = MRI.getVRegDef(Reg);
311 assert((Const && Const->getOpcode() == TargetOpcode::G_CONSTANT) &&
312 "expected a G_CONSTANT on Reg");
313 return Const->getOperand(i: 1).getCImm()->getValue();
314}
315
316std::optional<int64_t>
317llvm::getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI) {
318 std::optional<APInt> Val = getIConstantVRegVal(VReg, MRI);
319 if (Val && Val->getBitWidth() <= 64)
320 return Val->getSExtValue();
321 return std::nullopt;
322}
323
324namespace {
325
326// This function is used in many places, and as such, it has some
327// micro-optimizations to try and make it as fast as it can be.
328//
329// - We use template arguments to avoid an indirect call caused by passing a
330// function_ref/std::function
331// - GetAPCstValue does not return std::optional<APInt> as that's expensive.
332// Instead it returns true/false and places the result in a pre-constructed
333// APInt.
334//
335// Please change this function carefully and benchmark your changes.
336template <bool (*IsConstantOpcode)(const MachineInstr *),
337 bool (*GetAPCstValue)(const MachineInstr *MI, APInt &)>
338std::optional<ValueAndVReg>
339getConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI,
340 bool LookThroughInstrs = true,
341 bool LookThroughAnyExt = false) {
342 SmallVector<std::pair<unsigned, unsigned>, 4> SeenOpcodes;
343 MachineInstr *MI;
344
345 while ((MI = MRI.getVRegDef(Reg: VReg)) && !IsConstantOpcode(MI) &&
346 LookThroughInstrs) {
347 switch (MI->getOpcode()) {
348 case TargetOpcode::G_ANYEXT:
349 if (!LookThroughAnyExt)
350 return std::nullopt;
351 [[fallthrough]];
352 case TargetOpcode::G_TRUNC:
353 case TargetOpcode::G_SEXT:
354 case TargetOpcode::G_ZEXT:
355 SeenOpcodes.push_back(Elt: std::make_pair(
356 x: MI->getOpcode(),
357 y: MRI.getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits()));
358 VReg = MI->getOperand(i: 1).getReg();
359 break;
360 case TargetOpcode::COPY:
361 VReg = MI->getOperand(i: 1).getReg();
362 if (VReg.isPhysical())
363 return std::nullopt;
364 break;
365 case TargetOpcode::G_INTTOPTR:
366 VReg = MI->getOperand(i: 1).getReg();
367 break;
368 default:
369 return std::nullopt;
370 }
371 }
372 if (!MI || !IsConstantOpcode(MI))
373 return std::nullopt;
374
375 APInt Val;
376 if (!GetAPCstValue(MI, Val))
377 return std::nullopt;
378 for (auto &Pair : reverse(C&: SeenOpcodes)) {
379 switch (Pair.first) {
380 case TargetOpcode::G_TRUNC:
381 Val = Val.trunc(width: Pair.second);
382 break;
383 case TargetOpcode::G_ANYEXT:
384 case TargetOpcode::G_SEXT:
385 Val = Val.sext(width: Pair.second);
386 break;
387 case TargetOpcode::G_ZEXT:
388 Val = Val.zext(width: Pair.second);
389 break;
390 }
391 }
392
393 return ValueAndVReg{.Value: std::move(Val), .VReg: VReg};
394}
395
396bool isIConstant(const MachineInstr *MI) {
397 if (!MI)
398 return false;
399 return MI->getOpcode() == TargetOpcode::G_CONSTANT;
400}
401
402bool isFConstant(const MachineInstr *MI) {
403 if (!MI)
404 return false;
405 return MI->getOpcode() == TargetOpcode::G_FCONSTANT;
406}
407
408bool isAnyConstant(const MachineInstr *MI) {
409 if (!MI)
410 return false;
411 unsigned Opc = MI->getOpcode();
412 return Opc == TargetOpcode::G_CONSTANT || Opc == TargetOpcode::G_FCONSTANT;
413}
414
415bool getCImmAsAPInt(const MachineInstr *MI, APInt &Result) {
416 const MachineOperand &CstVal = MI->getOperand(i: 1);
417 if (!CstVal.isCImm())
418 return false;
419 Result = CstVal.getCImm()->getValue();
420 return true;
421}
422
423bool getCImmOrFPImmAsAPInt(const MachineInstr *MI, APInt &Result) {
424 const MachineOperand &CstVal = MI->getOperand(i: 1);
425 if (CstVal.isCImm())
426 Result = CstVal.getCImm()->getValue();
427 else if (CstVal.isFPImm())
428 Result = CstVal.getFPImm()->getValueAPF().bitcastToAPInt();
429 else
430 return false;
431 return true;
432}
433
434} // end anonymous namespace
435
436std::optional<ValueAndVReg> llvm::getIConstantVRegValWithLookThrough(
437 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs) {
438 return getConstantVRegValWithLookThrough<isIConstant, getCImmAsAPInt>(
439 VReg, MRI, LookThroughInstrs);
440}
441
442std::optional<ValueAndVReg> llvm::getAnyConstantVRegValWithLookThrough(
443 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs,
444 bool LookThroughAnyExt) {
445 return getConstantVRegValWithLookThrough<isAnyConstant,
446 getCImmOrFPImmAsAPInt>(
447 VReg, MRI, LookThroughInstrs, LookThroughAnyExt);
448}
449
450std::optional<FPValueAndVReg> llvm::getFConstantVRegValWithLookThrough(
451 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs) {
452 auto Reg =
453 getConstantVRegValWithLookThrough<isFConstant, getCImmOrFPImmAsAPInt>(
454 VReg, MRI, LookThroughInstrs);
455 if (!Reg)
456 return std::nullopt;
457
458 APFloat FloatVal(getFltSemanticForLLT(Ty: LLT::scalar(SizeInBits: Reg->Value.getBitWidth())),
459 Reg->Value);
460 return FPValueAndVReg{.Value: FloatVal, .VReg: Reg->VReg};
461}
462
463const ConstantFP *
464llvm::getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI) {
465 MachineInstr *MI = MRI.getVRegDef(Reg: VReg);
466 if (TargetOpcode::G_FCONSTANT != MI->getOpcode())
467 return nullptr;
468 return MI->getOperand(i: 1).getFPImm();
469}
470
471std::optional<DefinitionAndSourceRegister>
472llvm::getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI) {
473 Register DefSrcReg = Reg;
474 // This assumes that the code is in SSA form, so there should only be one
475 // definition.
476 auto DefIt = MRI.def_begin(RegNo: Reg);
477 if (DefIt == MRI.def_end())
478 return {};
479 MachineOperand &DefOpnd = *DefIt;
480 MachineInstr *DefMI = DefOpnd.getParent();
481 auto DstTy = MRI.getType(Reg: DefOpnd.getReg());
482 if (!DstTy.isValid())
483 return std::nullopt;
484 unsigned Opc = DefMI->getOpcode();
485 while (Opc == TargetOpcode::COPY || isPreISelGenericOptimizationHint(Opcode: Opc)) {
486 Register SrcReg = DefMI->getOperand(i: 1).getReg();
487 auto SrcTy = MRI.getType(Reg: SrcReg);
488 if (!SrcTy.isValid())
489 break;
490 DefMI = MRI.getVRegDef(Reg: SrcReg);
491 DefSrcReg = SrcReg;
492 Opc = DefMI->getOpcode();
493 }
494 return DefinitionAndSourceRegister{.MI: DefMI, .Reg: DefSrcReg};
495}
496
497MachineInstr *llvm::getDefIgnoringCopies(Register Reg,
498 const MachineRegisterInfo &MRI) {
499 std::optional<DefinitionAndSourceRegister> DefSrcReg =
500 getDefSrcRegIgnoringCopies(Reg, MRI);
501 return DefSrcReg ? DefSrcReg->MI : nullptr;
502}
503
504Register llvm::getSrcRegIgnoringCopies(Register Reg,
505 const MachineRegisterInfo &MRI) {
506 std::optional<DefinitionAndSourceRegister> DefSrcReg =
507 getDefSrcRegIgnoringCopies(Reg, MRI);
508 return DefSrcReg ? DefSrcReg->Reg : Register();
509}
510
511void llvm::extractParts(Register Reg, LLT Ty, int NumParts,
512 SmallVectorImpl<Register> &VRegs,
513 MachineIRBuilder &MIRBuilder,
514 MachineRegisterInfo &MRI) {
515 for (int i = 0; i < NumParts; ++i)
516 VRegs.push_back(Elt: MRI.createGenericVirtualRegister(Ty));
517 MIRBuilder.buildUnmerge(Res: VRegs, Op: Reg);
518}
519
520bool llvm::extractParts(Register Reg, LLT RegTy, LLT MainTy, LLT &LeftoverTy,
521 SmallVectorImpl<Register> &VRegs,
522 SmallVectorImpl<Register> &LeftoverRegs,
523 MachineIRBuilder &MIRBuilder,
524 MachineRegisterInfo &MRI) {
525 assert(!LeftoverTy.isValid() && "this is an out argument");
526
527 unsigned RegSize = RegTy.getSizeInBits();
528 unsigned MainSize = MainTy.getSizeInBits();
529 unsigned NumParts = RegSize / MainSize;
530 unsigned LeftoverSize = RegSize - NumParts * MainSize;
531
532 // Use an unmerge when possible.
533 if (LeftoverSize == 0) {
534 for (unsigned I = 0; I < NumParts; ++I)
535 VRegs.push_back(Elt: MRI.createGenericVirtualRegister(Ty: MainTy));
536 MIRBuilder.buildUnmerge(Res: VRegs, Op: Reg);
537 return true;
538 }
539
540 // Try to use unmerge for irregular vector split where possible
541 // For example when splitting a <6 x i32> into <4 x i32> with <2 x i32>
542 // leftover, it becomes:
543 // <2 x i32> %2, <2 x i32>%3, <2 x i32> %4 = G_UNMERGE_VALUE <6 x i32> %1
544 // <4 x i32> %5 = G_CONCAT_VECTOR <2 x i32> %2, <2 x i32> %3
545 if (RegTy.isVector() && MainTy.isVector()) {
546 unsigned RegNumElts = RegTy.getNumElements();
547 unsigned MainNumElts = MainTy.getNumElements();
548 unsigned LeftoverNumElts = RegNumElts % MainNumElts;
549 // If can unmerge to LeftoverTy, do it
550 if (MainNumElts % LeftoverNumElts == 0 &&
551 RegNumElts % LeftoverNumElts == 0 &&
552 RegTy.getScalarSizeInBits() == MainTy.getScalarSizeInBits() &&
553 LeftoverNumElts > 1) {
554 LeftoverTy = LLT::fixed_vector(NumElements: LeftoverNumElts, ScalarTy: RegTy.getElementType());
555
556 // Unmerge the SrcReg to LeftoverTy vectors
557 SmallVector<Register, 4> UnmergeValues;
558 extractParts(Reg, Ty: LeftoverTy, NumParts: RegNumElts / LeftoverNumElts, VRegs&: UnmergeValues,
559 MIRBuilder, MRI);
560
561 // Find how many LeftoverTy makes one MainTy
562 unsigned LeftoverPerMain = MainNumElts / LeftoverNumElts;
563 unsigned NumOfLeftoverVal =
564 ((RegNumElts % MainNumElts) / LeftoverNumElts);
565
566 // Create as many MainTy as possible using unmerged value
567 SmallVector<Register, 4> MergeValues;
568 for (unsigned I = 0; I < UnmergeValues.size() - NumOfLeftoverVal; I++) {
569 MergeValues.push_back(Elt: UnmergeValues[I]);
570 if (MergeValues.size() == LeftoverPerMain) {
571 VRegs.push_back(
572 Elt: MIRBuilder.buildMergeLikeInstr(Res: MainTy, Ops: MergeValues).getReg(Idx: 0));
573 MergeValues.clear();
574 }
575 }
576 // Populate LeftoverRegs with the leftovers
577 for (unsigned I = UnmergeValues.size() - NumOfLeftoverVal;
578 I < UnmergeValues.size(); I++) {
579 LeftoverRegs.push_back(Elt: UnmergeValues[I]);
580 }
581 return true;
582 }
583 }
584 // Perform irregular split. Leftover is last element of RegPieces.
585 if (MainTy.isVector()) {
586 SmallVector<Register, 8> RegPieces;
587 extractVectorParts(Reg, NumElts: MainTy.getNumElements(), VRegs&: RegPieces, MIRBuilder,
588 MRI);
589 for (unsigned i = 0; i < RegPieces.size() - 1; ++i)
590 VRegs.push_back(Elt: RegPieces[i]);
591 LeftoverRegs.push_back(Elt: RegPieces[RegPieces.size() - 1]);
592 LeftoverTy = MRI.getType(Reg: LeftoverRegs[0]);
593 return true;
594 }
595
596 LeftoverTy = LLT::integer(SizeInBits: LeftoverSize);
597 // For irregular sizes, extract the individual parts.
598 for (unsigned I = 0; I != NumParts; ++I) {
599 Register NewReg = MRI.createGenericVirtualRegister(Ty: MainTy);
600 VRegs.push_back(Elt: NewReg);
601 MIRBuilder.buildExtract(Res: NewReg, Src: Reg, Index: MainSize * I);
602 }
603
604 for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
605 Offset += LeftoverSize) {
606 Register NewReg = MRI.createGenericVirtualRegister(Ty: LeftoverTy);
607 LeftoverRegs.push_back(Elt: NewReg);
608 MIRBuilder.buildExtract(Res: NewReg, Src: Reg, Index: Offset);
609 }
610
611 return true;
612}
613
614void llvm::extractVectorParts(Register Reg, unsigned NumElts,
615 SmallVectorImpl<Register> &VRegs,
616 MachineIRBuilder &MIRBuilder,
617 MachineRegisterInfo &MRI) {
618 LLT RegTy = MRI.getType(Reg);
619 assert(RegTy.isVector() && "Expected a vector type");
620
621 LLT EltTy = RegTy.getElementType();
622 LLT NarrowTy = (NumElts == 1) ? EltTy : LLT::fixed_vector(NumElements: NumElts, ScalarTy: EltTy);
623 unsigned RegNumElts = RegTy.getNumElements();
624 unsigned LeftoverNumElts = RegNumElts % NumElts;
625 unsigned NumNarrowTyPieces = RegNumElts / NumElts;
626
627 // Perfect split without leftover
628 if (LeftoverNumElts == 0)
629 return extractParts(Reg, Ty: NarrowTy, NumParts: NumNarrowTyPieces, VRegs, MIRBuilder,
630 MRI);
631
632 // Irregular split. Provide direct access to all elements for artifact
633 // combiner using unmerge to elements. Then build vectors with NumElts
634 // elements. Remaining element(s) will be (used to build vector) Leftover.
635 SmallVector<Register, 8> Elts;
636 extractParts(Reg, Ty: EltTy, NumParts: RegNumElts, VRegs&: Elts, MIRBuilder, MRI);
637
638 unsigned Offset = 0;
639 // Requested sub-vectors of NarrowTy.
640 for (unsigned i = 0; i < NumNarrowTyPieces; ++i, Offset += NumElts) {
641 ArrayRef<Register> Pieces(&Elts[Offset], NumElts);
642 VRegs.push_back(Elt: MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: Pieces).getReg(Idx: 0));
643 }
644
645 // Leftover element(s).
646 if (LeftoverNumElts == 1) {
647 VRegs.push_back(Elt: Elts[Offset]);
648 } else {
649 LLT LeftoverTy = LLT::fixed_vector(NumElements: LeftoverNumElts, ScalarTy: EltTy);
650 ArrayRef<Register> Pieces(&Elts[Offset], LeftoverNumElts);
651 VRegs.push_back(
652 Elt: MIRBuilder.buildMergeLikeInstr(Res: LeftoverTy, Ops: Pieces).getReg(Idx: 0));
653 }
654}
655
656MachineInstr *llvm::getOpcodeDef(unsigned Opcode, Register Reg,
657 const MachineRegisterInfo &MRI) {
658 MachineInstr *DefMI = getDefIgnoringCopies(Reg, MRI);
659 return DefMI && DefMI->getOpcode() == Opcode ? DefMI : nullptr;
660}
661
662std::optional<APInt> llvm::ConstantFoldBinOp(unsigned Opcode,
663 const Register Op1,
664 const Register Op2,
665 const MachineRegisterInfo &MRI) {
666 auto MaybeOp2Cst = getAnyConstantVRegValWithLookThrough(VReg: Op2, MRI, LookThroughInstrs: false);
667 if (!MaybeOp2Cst)
668 return std::nullopt;
669
670 auto MaybeOp1Cst = getAnyConstantVRegValWithLookThrough(VReg: Op1, MRI, LookThroughInstrs: false);
671 if (!MaybeOp1Cst)
672 return std::nullopt;
673
674 const APInt &C1 = MaybeOp1Cst->Value;
675 const APInt &C2 = MaybeOp2Cst->Value;
676 switch (Opcode) {
677 default:
678 break;
679 case TargetOpcode::G_ADD:
680 return C1 + C2;
681 case TargetOpcode::G_PTR_ADD:
682 // Types can be of different width here.
683 // Result needs to be the same width as C1, so trunc or sext C2.
684 return C1 + C2.sextOrTrunc(width: C1.getBitWidth());
685 case TargetOpcode::G_AND:
686 return C1 & C2;
687 case TargetOpcode::G_ASHR:
688 return C1.ashr(ShiftAmt: C2);
689 case TargetOpcode::G_LSHR:
690 return C1.lshr(ShiftAmt: C2);
691 case TargetOpcode::G_MUL:
692 return C1 * C2;
693 case TargetOpcode::G_OR:
694 return C1 | C2;
695 case TargetOpcode::G_SHL:
696 return C1 << C2;
697 case TargetOpcode::G_SUB:
698 return C1 - C2;
699 case TargetOpcode::G_XOR:
700 return C1 ^ C2;
701 case TargetOpcode::G_UDIV:
702 if (!C2.getBoolValue())
703 break;
704 return C1.udiv(RHS: C2);
705 case TargetOpcode::G_SDIV:
706 if (!C2.getBoolValue())
707 break;
708 return C1.sdiv(RHS: C2);
709 case TargetOpcode::G_UREM:
710 if (!C2.getBoolValue())
711 break;
712 return C1.urem(RHS: C2);
713 case TargetOpcode::G_SREM:
714 if (!C2.getBoolValue())
715 break;
716 return C1.srem(RHS: C2);
717 case TargetOpcode::G_SMIN:
718 return APIntOps::smin(A: C1, B: C2);
719 case TargetOpcode::G_SMAX:
720 return APIntOps::smax(A: C1, B: C2);
721 case TargetOpcode::G_UMIN:
722 return APIntOps::umin(A: C1, B: C2);
723 case TargetOpcode::G_UMAX:
724 return APIntOps::umax(A: C1, B: C2);
725 }
726
727 return std::nullopt;
728}
729
730std::optional<APFloat>
731llvm::ConstantFoldFPBinOp(unsigned Opcode, const Register Op1,
732 const Register Op2, const MachineRegisterInfo &MRI) {
733 const ConstantFP *Op2Cst = getConstantFPVRegVal(VReg: Op2, MRI);
734 if (!Op2Cst)
735 return std::nullopt;
736
737 const ConstantFP *Op1Cst = getConstantFPVRegVal(VReg: Op1, MRI);
738 if (!Op1Cst)
739 return std::nullopt;
740
741 APFloat C1 = Op1Cst->getValueAPF();
742 const APFloat &C2 = Op2Cst->getValueAPF();
743 switch (Opcode) {
744 case TargetOpcode::G_FADD:
745 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
746 return C1;
747 case TargetOpcode::G_FSUB:
748 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
749 return C1;
750 case TargetOpcode::G_FMUL:
751 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
752 return C1;
753 case TargetOpcode::G_FDIV:
754 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
755 return C1;
756 case TargetOpcode::G_FREM:
757 C1.mod(RHS: C2);
758 return C1;
759 case TargetOpcode::G_FCOPYSIGN:
760 C1.copySign(RHS: C2);
761 return C1;
762 case TargetOpcode::G_FMINNUM:
763 return minnum(A: C1, B: C2);
764 case TargetOpcode::G_FMAXNUM:
765 return maxnum(A: C1, B: C2);
766 case TargetOpcode::G_FMINIMUM:
767 return minimum(A: C1, B: C2);
768 case TargetOpcode::G_FMAXIMUM:
769 return maximum(A: C1, B: C2);
770 case TargetOpcode::G_FMINIMUMNUM:
771 return minimumnum(A: C1, B: C2);
772 case TargetOpcode::G_FMAXIMUMNUM:
773 return maximumnum(A: C1, B: C2);
774 case TargetOpcode::G_FMINNUM_IEEE:
775 case TargetOpcode::G_FMAXNUM_IEEE:
776 // FIXME: These operations were unfortunately named. fminnum/fmaxnum do not
777 // follow the IEEE behavior for signaling nans and follow libm's fmin/fmax,
778 // and currently there isn't a nice wrapper in APFloat for the version with
779 // correct snan handling.
780 break;
781 default:
782 break;
783 }
784
785 return std::nullopt;
786}
787
788static GBuildVector *getBuildVectorLikeDef(Register Reg,
789 const MachineRegisterInfo &MRI) {
790 if (auto *BV = getOpcodeDef<GBuildVector>(Reg, MRI))
791 return BV;
792
793 auto *Bitcast = getOpcodeDef(Opcode: TargetOpcode::G_BITCAST, Reg, MRI);
794 if (!Bitcast)
795 return nullptr;
796
797 auto [Dst, DstTy, Src, SrcTy] = Bitcast->getFirst2RegLLTs();
798 if (!SrcTy.isVector() || !DstTy.isVector())
799 return nullptr;
800 if (SrcTy.getElementCount() != DstTy.getElementCount())
801 return nullptr;
802 if (SrcTy.getScalarSizeInBits() != DstTy.getScalarSizeInBits())
803 return nullptr;
804
805 return getOpcodeDef<GBuildVector>(Reg: Src, MRI);
806}
807
808SmallVector<APInt>
809llvm::ConstantFoldVectorBinop(unsigned Opcode, const Register Op1,
810 const Register Op2,
811 const MachineRegisterInfo &MRI) {
812 auto *SrcVec2 = getBuildVectorLikeDef(Reg: Op2, MRI);
813 if (!SrcVec2)
814 return SmallVector<APInt>();
815
816 auto *SrcVec1 = getBuildVectorLikeDef(Reg: Op1, MRI);
817 if (!SrcVec1)
818 return SmallVector<APInt>();
819
820 SmallVector<APInt> FoldedElements;
821 for (unsigned Idx = 0, E = SrcVec1->getNumSources(); Idx < E; ++Idx) {
822 auto MaybeCst = ConstantFoldBinOp(Opcode, Op1: SrcVec1->getSourceReg(I: Idx),
823 Op2: SrcVec2->getSourceReg(I: Idx), MRI);
824 if (!MaybeCst)
825 return SmallVector<APInt>();
826 FoldedElements.push_back(Elt: *MaybeCst);
827 }
828 return FoldedElements;
829}
830
831Align llvm::inferAlignFromPtrInfo(MachineFunction &MF,
832 const MachinePointerInfo &MPO) {
833 auto PSV = dyn_cast_if_present<const PseudoSourceValue *>(Val: MPO.V);
834 if (auto FSPV = dyn_cast_or_null<FixedStackPseudoSourceValue>(Val: PSV)) {
835 MachineFrameInfo &MFI = MF.getFrameInfo();
836 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FSPV->getFrameIndex()),
837 Offset: MPO.Offset);
838 }
839
840 if (const Value *V = dyn_cast_if_present<const Value *>(Val: MPO.V)) {
841 const Module *M = MF.getFunction().getParent();
842 return V->getPointerAlignment(DL: M->getDataLayout());
843 }
844
845 return Align(1);
846}
847
848Register llvm::getFunctionLiveInPhysReg(MachineFunction &MF,
849 const TargetInstrInfo &TII,
850 MCRegister PhysReg,
851 const TargetRegisterClass &RC,
852 const DebugLoc &DL, LLT RegTy) {
853 MachineBasicBlock &EntryMBB = MF.front();
854 MachineRegisterInfo &MRI = MF.getRegInfo();
855 Register LiveIn = MRI.getLiveInVirtReg(PReg: PhysReg);
856 if (LiveIn) {
857 MachineInstr *Def = MRI.getVRegDef(Reg: LiveIn);
858 if (Def) {
859 // FIXME: Should the verifier check this is in the entry block?
860 assert(Def->getParent() == &EntryMBB && "live-in copy not in entry block");
861 return LiveIn;
862 }
863
864 // It's possible the incoming argument register and copy was added during
865 // lowering, but later deleted due to being/becoming dead. If this happens,
866 // re-insert the copy.
867 } else {
868 // The live in register was not present, so add it.
869 LiveIn = MF.addLiveIn(PReg: PhysReg, RC: &RC);
870 if (RegTy.isValid())
871 MRI.setType(VReg: LiveIn, Ty: RegTy);
872 }
873
874 BuildMI(BB&: EntryMBB, I: EntryMBB.begin(), MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: LiveIn)
875 .addReg(RegNo: PhysReg);
876 if (!EntryMBB.isLiveIn(Reg: PhysReg))
877 EntryMBB.addLiveIn(PhysReg);
878 return LiveIn;
879}
880
881std::optional<APInt> llvm::ConstantFoldExtOp(unsigned Opcode,
882 const Register Op1, uint64_t Imm,
883 const MachineRegisterInfo &MRI) {
884 auto MaybeOp1Cst = getIConstantVRegVal(VReg: Op1, MRI);
885 if (MaybeOp1Cst) {
886 switch (Opcode) {
887 default:
888 break;
889 case TargetOpcode::G_SEXT_INREG: {
890 LLT Ty = MRI.getType(Reg: Op1);
891 return MaybeOp1Cst->trunc(width: Imm).sext(width: Ty.getScalarSizeInBits());
892 }
893 }
894 }
895 return std::nullopt;
896}
897
898std::optional<APInt> llvm::ConstantFoldCastOp(unsigned Opcode, LLT DstTy,
899 const Register Op0,
900 const MachineRegisterInfo &MRI) {
901 std::optional<APInt> Val = getIConstantVRegVal(VReg: Op0, MRI);
902 if (!Val)
903 return Val;
904
905 const unsigned DstSize = DstTy.getScalarSizeInBits();
906
907 switch (Opcode) {
908 case TargetOpcode::G_SEXT:
909 return Val->sext(width: DstSize);
910 case TargetOpcode::G_ZEXT:
911 case TargetOpcode::G_ANYEXT:
912 // TODO: DAG considers target preference when constant folding any_extend.
913 return Val->zext(width: DstSize);
914 default:
915 break;
916 }
917
918 llvm_unreachable("unexpected cast opcode to constant fold");
919}
920
921std::optional<APFloat>
922llvm::ConstantFoldIntToFloat(unsigned Opcode, LLT DstTy, Register Src,
923 const MachineRegisterInfo &MRI) {
924 assert(Opcode == TargetOpcode::G_SITOFP || Opcode == TargetOpcode::G_UITOFP);
925 if (auto MaybeSrcVal = getIConstantVRegVal(VReg: Src, MRI)) {
926 APFloat DstVal(getFltSemanticForLLT(Ty: DstTy));
927 DstVal.convertFromAPInt(Input: *MaybeSrcVal, IsSigned: Opcode == TargetOpcode::G_SITOFP,
928 RM: APFloat::rmNearestTiesToEven);
929 return DstVal;
930 }
931 return std::nullopt;
932}
933
934SmallVector<APInt>
935llvm::ConstantFoldUnaryIntOp(unsigned Opcode, LLT DstTy, Register Src,
936 const MachineRegisterInfo &MRI) {
937 unsigned EltBits = DstTy.getScalarSizeInBits();
938 auto Fold = [Opcode, EltBits](const APInt &V) -> APInt {
939 switch (Opcode) {
940 case TargetOpcode::G_CTLZ:
941 case TargetOpcode::G_CTLZ_ZERO_POISON:
942 return APInt(EltBits, V.countl_zero());
943 case TargetOpcode::G_CTTZ:
944 case TargetOpcode::G_CTTZ_ZERO_POISON:
945 return APInt(EltBits, V.countr_zero());
946 case TargetOpcode::G_CTPOP:
947 return APInt(EltBits, V.popcount());
948 case TargetOpcode::G_ABS:
949 return V.abs();
950 case TargetOpcode::G_BSWAP:
951 return V.byteSwap();
952 case TargetOpcode::G_BITREVERSE:
953 return V.reverseBits();
954 }
955 llvm_unreachable("unexpected opcode in ConstantFoldUnaryIntOp");
956 };
957
958 auto tryFoldScalar = [&](Register R) -> std::optional<APInt> {
959 if (auto MaybeCst = getIConstantVRegVal(VReg: R, MRI))
960 return Fold(*MaybeCst);
961 return std::nullopt;
962 };
963 if (MRI.getType(Reg: Src).isVector()) {
964 auto *BV = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
965 if (!BV)
966 return {};
967 SmallVector<APInt> Folded;
968 for (unsigned SrcIdx = 0; SrcIdx < BV->getNumSources(); ++SrcIdx) {
969 if (auto MaybeFold = tryFoldScalar(BV->getSourceReg(I: SrcIdx))) {
970 Folded.emplace_back(Args: std::move(*MaybeFold));
971 continue;
972 }
973 return {};
974 }
975 return Folded;
976 }
977 if (auto MaybeCst = tryFoldScalar(Src))
978 return {std::move(*MaybeCst)};
979 return {};
980}
981
982std::optional<SmallVector<APInt>>
983llvm::ConstantFoldICmp(unsigned Pred, const Register Op1, const Register Op2,
984 unsigned DstScalarSizeInBits, unsigned ExtOp,
985 const MachineRegisterInfo &MRI) {
986 assert(ExtOp == TargetOpcode::G_SEXT || ExtOp == TargetOpcode::G_ZEXT ||
987 ExtOp == TargetOpcode::G_ANYEXT);
988
989 const LLT Ty = MRI.getType(Reg: Op1);
990
991 auto GetICmpResultCst = [&](bool IsTrue) {
992 if (IsTrue)
993 return ExtOp == TargetOpcode::G_SEXT
994 ? APInt::getAllOnes(numBits: DstScalarSizeInBits)
995 : APInt::getOneBitSet(numBits: DstScalarSizeInBits, BitNo: 0);
996 return APInt::getZero(numBits: DstScalarSizeInBits);
997 };
998
999 auto TryFoldScalar = [&](Register LHS, Register RHS) -> std::optional<APInt> {
1000 auto RHSCst = getIConstantVRegVal(VReg: RHS, MRI);
1001 if (!RHSCst)
1002 return std::nullopt;
1003 auto LHSCst = getIConstantVRegVal(VReg: LHS, MRI);
1004 if (!LHSCst)
1005 return std::nullopt;
1006
1007 switch (Pred) {
1008 case CmpInst::Predicate::ICMP_EQ:
1009 return GetICmpResultCst(LHSCst->eq(RHS: *RHSCst));
1010 case CmpInst::Predicate::ICMP_NE:
1011 return GetICmpResultCst(LHSCst->ne(RHS: *RHSCst));
1012 case CmpInst::Predicate::ICMP_UGT:
1013 return GetICmpResultCst(LHSCst->ugt(RHS: *RHSCst));
1014 case CmpInst::Predicate::ICMP_UGE:
1015 return GetICmpResultCst(LHSCst->uge(RHS: *RHSCst));
1016 case CmpInst::Predicate::ICMP_ULT:
1017 return GetICmpResultCst(LHSCst->ult(RHS: *RHSCst));
1018 case CmpInst::Predicate::ICMP_ULE:
1019 return GetICmpResultCst(LHSCst->ule(RHS: *RHSCst));
1020 case CmpInst::Predicate::ICMP_SGT:
1021 return GetICmpResultCst(LHSCst->sgt(RHS: *RHSCst));
1022 case CmpInst::Predicate::ICMP_SGE:
1023 return GetICmpResultCst(LHSCst->sge(RHS: *RHSCst));
1024 case CmpInst::Predicate::ICMP_SLT:
1025 return GetICmpResultCst(LHSCst->slt(RHS: *RHSCst));
1026 case CmpInst::Predicate::ICMP_SLE:
1027 return GetICmpResultCst(LHSCst->sle(RHS: *RHSCst));
1028 default:
1029 return std::nullopt;
1030 }
1031 };
1032
1033 SmallVector<APInt> FoldedICmps;
1034
1035 if (Ty.isVector()) {
1036 // Try to constant fold each element.
1037 auto *BV1 = getOpcodeDef<GBuildVector>(Reg: Op1, MRI);
1038 auto *BV2 = getOpcodeDef<GBuildVector>(Reg: Op2, MRI);
1039 if (!BV1 || !BV2)
1040 return std::nullopt;
1041 assert(BV1->getNumSources() == BV2->getNumSources() && "Invalid vectors");
1042 for (unsigned I = 0; I < BV1->getNumSources(); ++I) {
1043 if (auto MaybeFold =
1044 TryFoldScalar(BV1->getSourceReg(I), BV2->getSourceReg(I))) {
1045 FoldedICmps.emplace_back(Args&: *MaybeFold);
1046 continue;
1047 }
1048 return std::nullopt;
1049 }
1050 return FoldedICmps;
1051 }
1052
1053 if (auto MaybeCst = TryFoldScalar(Op1, Op2)) {
1054 FoldedICmps.emplace_back(Args&: *MaybeCst);
1055 return FoldedICmps;
1056 }
1057
1058 return std::nullopt;
1059}
1060
1061bool llvm::isKnownToBeAPowerOfTwo(Register Reg, const MachineRegisterInfo &MRI,
1062 GISelValueTracking *VT, bool OrNegative) {
1063 std::optional<DefinitionAndSourceRegister> DefSrcReg =
1064 getDefSrcRegIgnoringCopies(Reg, MRI);
1065 if (!DefSrcReg)
1066 return false;
1067
1068 const MachineInstr &MI = *DefSrcReg->MI;
1069 const LLT Ty = MRI.getType(Reg);
1070
1071 auto IsPow2 = [OrNegative](const APInt &V) {
1072 return V.isPowerOf2() || (OrNegative && V.isNegatedPowerOf2());
1073 };
1074
1075 switch (MI.getOpcode()) {
1076 case TargetOpcode::G_CONSTANT: {
1077 unsigned BitWidth = Ty.getScalarSizeInBits();
1078 const ConstantInt *CI = MI.getOperand(i: 1).getCImm();
1079 return IsPow2(CI->getValue().zextOrTrunc(width: BitWidth));
1080 }
1081 case TargetOpcode::G_SHL: {
1082 // A left-shift of a constant one will have exactly one bit set because
1083 // shifting the bit off the end is undefined.
1084
1085 // TODO: Constant splat
1086 if (auto ConstLHS = getIConstantVRegVal(VReg: MI.getOperand(i: 1).getReg(), MRI)) {
1087 if (*ConstLHS == 1)
1088 return true;
1089 }
1090
1091 break;
1092 }
1093 case TargetOpcode::G_LSHR: {
1094 if (auto ConstLHS = getIConstantVRegVal(VReg: MI.getOperand(i: 1).getReg(), MRI)) {
1095 if (ConstLHS->isSignMask())
1096 return true;
1097 }
1098
1099 break;
1100 }
1101 case TargetOpcode::G_BUILD_VECTOR: {
1102 // TODO: Probably should have a recursion depth guard since you could have
1103 // bitcasted vector elements.
1104 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands()))
1105 if (!isKnownToBeAPowerOfTwo(Reg: MO.getReg(), MRI, VT, OrNegative))
1106 return false;
1107
1108 return true;
1109 }
1110 case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1111 // Only handle constants since we would need to know if number of leading
1112 // zeros is greater than the truncation amount.
1113 const unsigned BitWidth = Ty.getScalarSizeInBits();
1114 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
1115 auto Const = getIConstantVRegVal(VReg: MO.getReg(), MRI);
1116 if (!Const || !IsPow2(Const->zextOrTrunc(width: BitWidth)))
1117 return false;
1118 }
1119
1120 return true;
1121 }
1122 default:
1123 break;
1124 }
1125
1126 if (!VT)
1127 return false;
1128
1129 // More could be done here, though the above checks are enough
1130 // to handle some common cases.
1131
1132 // Fall back to computeKnownBits to catch other known cases.
1133 KnownBits Known = VT->getKnownBits(R: Reg);
1134 return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);
1135}
1136
1137void llvm::getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU) {
1138 AU.addPreserved<StackProtector>();
1139}
1140
1141LLT llvm::getLCMType(LLT OrigTy, LLT TargetTy) {
1142 if (OrigTy.getSizeInBits() == TargetTy.getSizeInBits())
1143 return OrigTy;
1144
1145 if (OrigTy.isVector() && TargetTy.isVector()) {
1146 LLT OrigElt = OrigTy.getElementType();
1147 LLT TargetElt = TargetTy.getElementType();
1148
1149 // TODO: The docstring for this function says the intention is to use this
1150 // function to build MERGE/UNMERGE instructions. It won't be the case that
1151 // we generate a MERGE/UNMERGE between fixed and scalable vector types. We
1152 // could implement getLCMType between the two in the future if there was a
1153 // need, but it is not worth it now as this function should not be used in
1154 // that way.
1155 assert(((OrigTy.isScalableVector() && !TargetTy.isFixedVector()) ||
1156 (OrigTy.isFixedVector() && !TargetTy.isScalableVector())) &&
1157 "getLCMType not implemented between fixed and scalable vectors.");
1158
1159 if (OrigElt.getSizeInBits() == TargetElt.getSizeInBits()) {
1160 int GCDMinElts = std::gcd(m: OrigTy.getElementCount().getKnownMinValue(),
1161 n: TargetTy.getElementCount().getKnownMinValue());
1162 // Prefer the original element type.
1163 ElementCount Mul = OrigTy.getElementCount().multiplyCoefficientBy(
1164 RHS: TargetTy.getElementCount().getKnownMinValue());
1165 return LLT::vector(EC: Mul.divideCoefficientBy(RHS: GCDMinElts),
1166 ScalarTy: OrigTy.getElementType());
1167 }
1168 unsigned LCM = std::lcm(m: OrigTy.getSizeInBits().getKnownMinValue(),
1169 n: TargetTy.getSizeInBits().getKnownMinValue());
1170 return LLT::vector(
1171 EC: ElementCount::get(MinVal: LCM / OrigElt.getSizeInBits(), Scalable: OrigTy.isScalable()),
1172 ScalarTy: OrigElt);
1173 }
1174
1175 // One type is scalar, one type is vector
1176 if (OrigTy.isVector() || TargetTy.isVector()) {
1177 LLT VecTy = OrigTy.isVector() ? OrigTy : TargetTy;
1178 LLT ScalarTy = OrigTy.isVector() ? TargetTy : OrigTy;
1179 LLT EltTy = VecTy.getElementType();
1180 LLT OrigEltTy = OrigTy.isVector() ? OrigTy.getElementType() : OrigTy;
1181
1182 // Prefer scalar type from OrigTy.
1183 if (EltTy.getSizeInBits() == ScalarTy.getSizeInBits())
1184 return LLT::vector(EC: VecTy.getElementCount(), ScalarTy: OrigEltTy);
1185
1186 // Different size scalars. Create vector with the same total size.
1187 // LCM will take fixed/scalable from VecTy.
1188 unsigned LCM = std::lcm(m: EltTy.getSizeInBits().getFixedValue() *
1189 VecTy.getElementCount().getKnownMinValue(),
1190 n: ScalarTy.getSizeInBits().getFixedValue());
1191 // Prefer type from OrigTy
1192 return LLT::vector(EC: ElementCount::get(MinVal: LCM / OrigEltTy.getSizeInBits(),
1193 Scalable: VecTy.getElementCount().isScalable()),
1194 ScalarTy: OrigEltTy);
1195 }
1196
1197 // At this point, both types are scalars of different size
1198 unsigned LCM = std::lcm(m: OrigTy.getSizeInBits().getFixedValue(),
1199 n: TargetTy.getSizeInBits().getFixedValue());
1200 // Preserve pointer types.
1201 if (LCM == OrigTy.getSizeInBits())
1202 return OrigTy;
1203 if (LCM == TargetTy.getSizeInBits())
1204 return TargetTy;
1205 return LLT::scalar(SizeInBits: LCM);
1206}
1207
1208LLT llvm::getCoverTy(LLT OrigTy, LLT TargetTy) {
1209
1210 if ((OrigTy.isScalableVector() && TargetTy.isFixedVector()) ||
1211 (OrigTy.isFixedVector() && TargetTy.isScalableVector()))
1212 llvm_unreachable(
1213 "getCoverTy not implemented between fixed and scalable vectors.");
1214
1215 if (!OrigTy.isVector() || !TargetTy.isVector() || OrigTy == TargetTy ||
1216 (OrigTy.getScalarSizeInBits() != TargetTy.getScalarSizeInBits()))
1217 return getLCMType(OrigTy, TargetTy);
1218
1219 unsigned OrigTyNumElts = OrigTy.getElementCount().getKnownMinValue();
1220 unsigned TargetTyNumElts = TargetTy.getElementCount().getKnownMinValue();
1221 if (OrigTyNumElts % TargetTyNumElts == 0)
1222 return OrigTy;
1223
1224 unsigned NumElts = alignTo(Value: OrigTyNumElts, Align: TargetTyNumElts);
1225 return LLT::scalarOrVector(EC: ElementCount::getFixed(MinVal: NumElts),
1226 ScalarTy: OrigTy.getElementType());
1227}
1228
1229LLT llvm::getGCDType(LLT OrigTy, LLT TargetTy) {
1230 if (OrigTy.getSizeInBits() == TargetTy.getSizeInBits())
1231 return OrigTy;
1232
1233 if (OrigTy.isVector() && TargetTy.isVector()) {
1234 LLT OrigElt = OrigTy.getElementType();
1235
1236 // TODO: The docstring for this function says the intention is to use this
1237 // function to build MERGE/UNMERGE instructions. It won't be the case that
1238 // we generate a MERGE/UNMERGE between fixed and scalable vector types. We
1239 // could implement getGCDType between the two in the future if there was a
1240 // need, but it is not worth it now as this function should not be used in
1241 // that way.
1242 assert(((OrigTy.isScalableVector() && !TargetTy.isFixedVector()) ||
1243 (OrigTy.isFixedVector() && !TargetTy.isScalableVector())) &&
1244 "getGCDType not implemented between fixed and scalable vectors.");
1245
1246 unsigned GCD = std::gcd(m: OrigTy.getSizeInBits().getKnownMinValue(),
1247 n: TargetTy.getSizeInBits().getKnownMinValue());
1248 if (GCD == OrigElt.getSizeInBits())
1249 return LLT::scalarOrVector(EC: ElementCount::get(MinVal: 1, Scalable: OrigTy.isScalable()),
1250 ScalarTy: OrigElt);
1251
1252 // Cannot produce original element type, but both have vscale in common.
1253 if (GCD < OrigElt.getSizeInBits())
1254 return LLT::scalarOrVector(EC: ElementCount::get(MinVal: 1, Scalable: OrigTy.isScalable()),
1255 ScalarSize: GCD);
1256
1257 return LLT::vector(
1258 EC: ElementCount::get(MinVal: GCD / OrigElt.getSizeInBits().getFixedValue(),
1259 Scalable: OrigTy.isScalable()),
1260 ScalarTy: OrigElt);
1261 }
1262
1263 // If one type is vector and the element size matches the scalar size, then
1264 // the gcd is the scalar type.
1265 if (OrigTy.isVector() &&
1266 OrigTy.getElementType().getSizeInBits() == TargetTy.getSizeInBits())
1267 return OrigTy.getElementType();
1268 if (TargetTy.isVector() &&
1269 TargetTy.getElementType().getSizeInBits() == OrigTy.getSizeInBits())
1270 return OrigTy;
1271
1272 // At this point, both types are either scalars of different type or one is a
1273 // vector and one is a scalar. If both types are scalars, the GCD type is the
1274 // GCD between the two scalar sizes. If one is vector and one is scalar, then
1275 // the GCD type is the GCD between the scalar and the vector element size.
1276 LLT OrigScalar = OrigTy.getScalarType();
1277 LLT TargetScalar = TargetTy.getScalarType();
1278 unsigned GCD = std::gcd(m: OrigScalar.getSizeInBits().getFixedValue(),
1279 n: TargetScalar.getSizeInBits().getFixedValue());
1280 return LLT::integer(SizeInBits: GCD);
1281}
1282
1283std::optional<int> llvm::getSplatIndex(MachineInstr &MI) {
1284 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
1285 "Only G_SHUFFLE_VECTOR can have a splat index!");
1286 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
1287 auto FirstDefinedIdx = find_if(Range&: Mask, P: [](int Elt) { return Elt >= 0; });
1288
1289 // If all elements are undefined, this shuffle can be considered a splat.
1290 // Return 0 for better potential for callers to simplify.
1291 if (FirstDefinedIdx == Mask.end())
1292 return 0;
1293
1294 // Make sure all remaining elements are either undef or the same
1295 // as the first non-undef value.
1296 int SplatValue = *FirstDefinedIdx;
1297 if (any_of(Range: make_range(x: std::next(x: FirstDefinedIdx), y: Mask.end()),
1298 P: [&SplatValue](int Elt) { return Elt >= 0 && Elt != SplatValue; }))
1299 return std::nullopt;
1300
1301 return SplatValue;
1302}
1303
1304static bool isBuildVectorOp(unsigned Opcode) {
1305 return Opcode == TargetOpcode::G_BUILD_VECTOR ||
1306 Opcode == TargetOpcode::G_BUILD_VECTOR_TRUNC;
1307}
1308
1309namespace {
1310
1311std::optional<ValueAndVReg> getAnyConstantSplat(Register VReg,
1312 const MachineRegisterInfo &MRI,
1313 bool AllowUndef) {
1314 MachineInstr *MI = getDefIgnoringCopies(Reg: VReg, MRI);
1315 if (!MI)
1316 return std::nullopt;
1317
1318 bool isConcatVectorsOp = MI->getOpcode() == TargetOpcode::G_CONCAT_VECTORS;
1319 if (!isBuildVectorOp(Opcode: MI->getOpcode()) && !isConcatVectorsOp)
1320 return std::nullopt;
1321
1322 std::optional<ValueAndVReg> SplatValAndReg;
1323 for (MachineOperand &Op : MI->uses()) {
1324 Register Element = Op.getReg();
1325 // If we have a G_CONCAT_VECTOR, we recursively look into the
1326 // vectors that we're concatenating to see if they're splats.
1327 auto ElementValAndReg =
1328 isConcatVectorsOp
1329 ? getAnyConstantSplat(VReg: Element, MRI, AllowUndef)
1330 : getAnyConstantVRegValWithLookThrough(VReg: Element, MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
1331
1332 // If AllowUndef, treat undef as value that will result in a constant splat.
1333 if (!ElementValAndReg) {
1334 if (AllowUndef && isa<GImplicitDef>(Val: MRI.getVRegDef(Reg: Element)))
1335 continue;
1336 return std::nullopt;
1337 }
1338
1339 // Record splat value
1340 if (!SplatValAndReg)
1341 SplatValAndReg = ElementValAndReg;
1342
1343 // Different constant than the one already recorded, not a constant splat.
1344 if (SplatValAndReg->Value != ElementValAndReg->Value)
1345 return std::nullopt;
1346 }
1347
1348 return SplatValAndReg;
1349}
1350
1351} // end anonymous namespace
1352
1353bool llvm::isBuildVectorConstantSplat(const Register Reg,
1354 const MachineRegisterInfo &MRI,
1355 int64_t SplatValue, bool AllowUndef) {
1356 if (auto SplatValAndReg = getAnyConstantSplat(VReg: Reg, MRI, AllowUndef))
1357 return SplatValAndReg->Value.getSExtValue() == SplatValue;
1358
1359 return false;
1360}
1361
1362bool llvm::isBuildVectorConstantSplat(const Register Reg,
1363 const MachineRegisterInfo &MRI,
1364 const APInt &SplatValue,
1365 bool AllowUndef) {
1366 if (auto SplatValAndReg = getAnyConstantSplat(VReg: Reg, MRI, AllowUndef)) {
1367 if (SplatValAndReg->Value.getBitWidth() < SplatValue.getBitWidth())
1368 return APInt::isSameValue(
1369 I1: SplatValAndReg->Value.sext(width: SplatValue.getBitWidth()), I2: SplatValue);
1370 return APInt::isSameValue(
1371 I1: SplatValAndReg->Value,
1372 I2: SplatValue.sext(width: SplatValAndReg->Value.getBitWidth()));
1373 }
1374
1375 return false;
1376}
1377
1378bool llvm::isBuildVectorConstantSplat(const MachineInstr &MI,
1379 const MachineRegisterInfo &MRI,
1380 int64_t SplatValue, bool AllowUndef) {
1381 return isBuildVectorConstantSplat(Reg: MI.getOperand(i: 0).getReg(), MRI, SplatValue,
1382 AllowUndef);
1383}
1384
1385bool llvm::isBuildVectorConstantSplat(const MachineInstr &MI,
1386 const MachineRegisterInfo &MRI,
1387 const APInt &SplatValue,
1388 bool AllowUndef) {
1389 return isBuildVectorConstantSplat(Reg: MI.getOperand(i: 0).getReg(), MRI, SplatValue,
1390 AllowUndef);
1391}
1392
1393std::optional<APInt>
1394llvm::getIConstantSplatVal(const Register Reg, const MachineRegisterInfo &MRI) {
1395 if (auto SplatValAndReg =
1396 getAnyConstantSplat(VReg: Reg, MRI, /* AllowUndef */ false)) {
1397 if (std::optional<ValueAndVReg> ValAndVReg =
1398 getIConstantVRegValWithLookThrough(VReg: SplatValAndReg->VReg, MRI))
1399 return ValAndVReg->Value;
1400 }
1401
1402 return std::nullopt;
1403}
1404
1405std::optional<APInt>
1406llvm::getIConstantSplatVal(const MachineInstr &MI,
1407 const MachineRegisterInfo &MRI) {
1408 return getIConstantSplatVal(Reg: MI.getOperand(i: 0).getReg(), MRI);
1409}
1410
1411std::optional<int64_t>
1412llvm::getIConstantSplatSExtVal(const Register Reg,
1413 const MachineRegisterInfo &MRI) {
1414 if (auto SplatValAndReg =
1415 getAnyConstantSplat(VReg: Reg, MRI, /* AllowUndef */ false))
1416 return getIConstantVRegSExtVal(VReg: SplatValAndReg->VReg, MRI);
1417 return std::nullopt;
1418}
1419
1420std::optional<int64_t>
1421llvm::getIConstantSplatSExtVal(const MachineInstr &MI,
1422 const MachineRegisterInfo &MRI) {
1423 return getIConstantSplatSExtVal(Reg: MI.getOperand(i: 0).getReg(), MRI);
1424}
1425
1426std::optional<FPValueAndVReg>
1427llvm::getFConstantSplat(Register VReg, const MachineRegisterInfo &MRI,
1428 bool AllowUndef) {
1429 if (auto SplatValAndReg = getAnyConstantSplat(VReg, MRI, AllowUndef))
1430 return getFConstantVRegValWithLookThrough(VReg: SplatValAndReg->VReg, MRI);
1431 return std::nullopt;
1432}
1433
1434bool llvm::isBuildVectorAllZeros(const MachineInstr &MI,
1435 const MachineRegisterInfo &MRI,
1436 bool AllowUndef) {
1437 return isBuildVectorConstantSplat(MI, MRI, SplatValue: 0, AllowUndef);
1438}
1439
1440bool llvm::isBuildVectorAllOnes(const MachineInstr &MI,
1441 const MachineRegisterInfo &MRI,
1442 bool AllowUndef) {
1443 return isBuildVectorConstantSplat(MI, MRI, SplatValue: -1, AllowUndef);
1444}
1445
1446std::optional<RegOrConstant>
1447llvm::getVectorSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI) {
1448 unsigned Opc = MI.getOpcode();
1449 if (!isBuildVectorOp(Opcode: Opc))
1450 return std::nullopt;
1451 if (auto Splat = getIConstantSplatSExtVal(MI, MRI))
1452 return RegOrConstant(*Splat);
1453 auto Reg = MI.getOperand(i: 1).getReg();
1454 if (any_of(Range: drop_begin(RangeOrContainer: MI.operands(), N: 2),
1455 P: [&Reg](const MachineOperand &Op) { return Op.getReg() != Reg; }))
1456 return std::nullopt;
1457 return RegOrConstant(Reg);
1458}
1459
1460static bool isConstantScalar(const MachineInstr &MI,
1461 const MachineRegisterInfo &MRI,
1462 bool AllowFP = true,
1463 bool AllowOpaqueConstants = true) {
1464 switch (MI.getOpcode()) {
1465 case TargetOpcode::G_CONSTANT:
1466 case TargetOpcode::G_IMPLICIT_DEF:
1467 return true;
1468 case TargetOpcode::G_FCONSTANT:
1469 return AllowFP;
1470 case TargetOpcode::G_GLOBAL_VALUE:
1471 case TargetOpcode::G_FRAME_INDEX:
1472 case TargetOpcode::G_BLOCK_ADDR:
1473 case TargetOpcode::G_JUMP_TABLE:
1474 return AllowOpaqueConstants;
1475 default:
1476 return false;
1477 }
1478}
1479
1480bool llvm::isConstantOrConstantVector(MachineInstr &MI,
1481 const MachineRegisterInfo &MRI) {
1482 Register Def = MI.getOperand(i: 0).getReg();
1483 if (auto C = getIConstantVRegValWithLookThrough(VReg: Def, MRI))
1484 return true;
1485 GBuildVector *BV = dyn_cast<GBuildVector>(Val: &MI);
1486 if (!BV)
1487 return false;
1488 for (unsigned SrcIdx = 0; SrcIdx < BV->getNumSources(); ++SrcIdx) {
1489 if (getIConstantVRegValWithLookThrough(VReg: BV->getSourceReg(I: SrcIdx), MRI) ||
1490 getOpcodeDef<GImplicitDef>(Reg: BV->getSourceReg(I: SrcIdx), MRI))
1491 continue;
1492 return false;
1493 }
1494 return true;
1495}
1496
1497bool llvm::isConstantOrConstantVector(const MachineInstr &MI,
1498 const MachineRegisterInfo &MRI,
1499 bool AllowFP, bool AllowOpaqueConstants) {
1500 if (isConstantScalar(MI, MRI, AllowFP, AllowOpaqueConstants))
1501 return true;
1502
1503 if (!isBuildVectorOp(Opcode: MI.getOpcode()))
1504 return false;
1505
1506 const unsigned NumOps = MI.getNumOperands();
1507 for (unsigned I = 1; I != NumOps; ++I) {
1508 const MachineInstr *ElementDef = MRI.getVRegDef(Reg: MI.getOperand(i: I).getReg());
1509 if (!isConstantScalar(MI: *ElementDef, MRI, AllowFP, AllowOpaqueConstants))
1510 return false;
1511 }
1512
1513 return true;
1514}
1515
1516std::optional<APInt>
1517llvm::isConstantOrConstantSplatVector(Register Def,
1518 const MachineRegisterInfo &MRI) {
1519 if (auto C = getIConstantVRegValWithLookThrough(VReg: Def, MRI))
1520 return C->Value;
1521 auto MaybeCst = getIConstantSplatSExtVal(Reg: Def, MRI);
1522 if (!MaybeCst)
1523 return std::nullopt;
1524 const unsigned ScalarSize = MRI.getType(Reg: Def).getScalarSizeInBits();
1525 return APInt(ScalarSize, *MaybeCst, true);
1526}
1527
1528std::optional<APFloat>
1529llvm::isConstantOrConstantSplatVectorFP(Register Def,
1530 const MachineRegisterInfo &MRI) {
1531 if (auto FpConst = getFConstantVRegValWithLookThrough(VReg: Def, MRI))
1532 return FpConst->Value;
1533 auto MaybeCstFP = getFConstantSplat(VReg: Def, MRI, /*allowUndef=*/AllowUndef: false);
1534 if (!MaybeCstFP)
1535 return std::nullopt;
1536 return MaybeCstFP->Value;
1537}
1538
1539bool llvm::isNullOrNullSplat(const MachineInstr &MI,
1540 const MachineRegisterInfo &MRI, bool AllowUndefs) {
1541 switch (MI.getOpcode()) {
1542 case TargetOpcode::G_IMPLICIT_DEF:
1543 return AllowUndefs;
1544 case TargetOpcode::G_CONSTANT:
1545 return MI.getOperand(i: 1).getCImm()->isNullValue();
1546 case TargetOpcode::G_FCONSTANT: {
1547 const ConstantFP *FPImm = MI.getOperand(i: 1).getFPImm();
1548 return FPImm->isZero() && !FPImm->isNegative();
1549 }
1550 default:
1551 if (!AllowUndefs) // TODO: isBuildVectorAllZeros assumes undef is OK already
1552 return false;
1553 return isBuildVectorAllZeros(MI, MRI);
1554 }
1555}
1556
1557bool llvm::isAllOnesOrAllOnesSplat(const MachineInstr &MI,
1558 const MachineRegisterInfo &MRI,
1559 bool AllowUndefs) {
1560 switch (MI.getOpcode()) {
1561 case TargetOpcode::G_IMPLICIT_DEF:
1562 return AllowUndefs;
1563 case TargetOpcode::G_CONSTANT:
1564 return MI.getOperand(i: 1).getCImm()->isAllOnesValue();
1565 default:
1566 if (!AllowUndefs) // TODO: isBuildVectorAllOnes assumes undef is OK already
1567 return false;
1568 return isBuildVectorAllOnes(MI, MRI);
1569 }
1570}
1571
1572bool llvm::matchUnaryPredicate(
1573 const MachineRegisterInfo &MRI, Register Reg,
1574 std::function<bool(const Constant *ConstVal)> Match, bool AllowUndefs) {
1575
1576 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);
1577 if (AllowUndefs && Def->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1578 return Match(nullptr);
1579
1580 // TODO: Also handle fconstant
1581 if (Def->getOpcode() == TargetOpcode::G_CONSTANT)
1582 return Match(Def->getOperand(i: 1).getCImm());
1583
1584 if (Def->getOpcode() != TargetOpcode::G_BUILD_VECTOR)
1585 return false;
1586
1587 for (unsigned I = 1, E = Def->getNumOperands(); I != E; ++I) {
1588 Register SrcElt = Def->getOperand(i: I).getReg();
1589 const MachineInstr *SrcDef = getDefIgnoringCopies(Reg: SrcElt, MRI);
1590 if (AllowUndefs && SrcDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) {
1591 if (!Match(nullptr))
1592 return false;
1593 continue;
1594 }
1595
1596 if (SrcDef->getOpcode() != TargetOpcode::G_CONSTANT ||
1597 !Match(SrcDef->getOperand(i: 1).getCImm()))
1598 return false;
1599 }
1600
1601 return true;
1602}
1603
1604bool llvm::isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector,
1605 bool IsFP) {
1606 switch (TLI.getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
1607 case TargetLowering::UndefinedBooleanContent:
1608 return Val & 0x1;
1609 case TargetLowering::ZeroOrOneBooleanContent:
1610 return Val == 1;
1611 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1612 return Val == -1;
1613 }
1614 llvm_unreachable("Invalid boolean contents");
1615}
1616
1617bool llvm::isConstFalseVal(const TargetLowering &TLI, int64_t Val,
1618 bool IsVector, bool IsFP) {
1619 switch (TLI.getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
1620 case TargetLowering::UndefinedBooleanContent:
1621 return ~Val & 0x1;
1622 case TargetLowering::ZeroOrOneBooleanContent:
1623 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1624 return Val == 0;
1625 }
1626 llvm_unreachable("Invalid boolean contents");
1627}
1628
1629int64_t llvm::getICmpTrueVal(const TargetLowering &TLI, bool IsVector,
1630 bool IsFP) {
1631 switch (TLI.getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
1632 case TargetLowering::UndefinedBooleanContent:
1633 case TargetLowering::ZeroOrOneBooleanContent:
1634 return 1;
1635 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1636 return -1;
1637 }
1638 llvm_unreachable("Invalid boolean contents");
1639}
1640
1641void llvm::saveUsesAndErase(MachineInstr &MI, MachineRegisterInfo &MRI,
1642 LostDebugLocObserver *LocObserver,
1643 SmallInstListTy &DeadInstChain) {
1644 for (MachineOperand &Op : MI.uses()) {
1645 if (Op.isReg() && Op.getReg().isVirtual())
1646 DeadInstChain.insert(I: MRI.getVRegDef(Reg: Op.getReg()));
1647 }
1648 LLVM_DEBUG(dbgs() << MI << "Is dead; erasing.\n");
1649 DeadInstChain.remove(I: &MI);
1650 MI.eraseFromParent();
1651 if (LocObserver)
1652 LocObserver->checkpoint(CheckDebugLocs: false);
1653}
1654
1655void llvm::eraseInstrs(ArrayRef<MachineInstr *> DeadInstrs,
1656 MachineRegisterInfo &MRI,
1657 LostDebugLocObserver *LocObserver) {
1658 SmallInstListTy DeadInstChain;
1659 for (MachineInstr *MI : DeadInstrs)
1660 saveUsesAndErase(MI&: *MI, MRI, LocObserver, DeadInstChain);
1661
1662 while (!DeadInstChain.empty()) {
1663 MachineInstr *Inst = DeadInstChain.pop_back_val();
1664 if (!isTriviallyDead(MI: *Inst, MRI))
1665 continue;
1666 saveUsesAndErase(MI&: *Inst, MRI, LocObserver, DeadInstChain);
1667 }
1668}
1669
1670void llvm::eraseInstr(MachineInstr &MI, MachineRegisterInfo &MRI,
1671 LostDebugLocObserver *LocObserver) {
1672 return eraseInstrs(DeadInstrs: {&MI}, MRI, LocObserver);
1673}
1674
1675void llvm::salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI) {
1676 for (auto &Def : MI.defs()) {
1677 assert(Def.isReg() && "Must be a reg");
1678
1679 SmallVector<MachineOperand *, 16> DbgUsers;
1680 for (auto &MOUse : MRI.use_operands(Reg: Def.getReg())) {
1681 MachineInstr *DbgValue = MOUse.getParent();
1682 // Ignore partially formed DBG_VALUEs.
1683 if (DbgValue->isNonListDebugValue() && DbgValue->getNumOperands() == 4) {
1684 DbgUsers.push_back(Elt: &MOUse);
1685 }
1686 }
1687
1688 if (!DbgUsers.empty()) {
1689 salvageDebugInfoForDbgValue(MRI, MI, DbgUsers);
1690 }
1691 }
1692}
1693
1694bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) {
1695 switch (Opc) {
1696 case TargetOpcode::G_FABS:
1697 case TargetOpcode::G_FADD:
1698 case TargetOpcode::G_FCANONICALIZE:
1699 case TargetOpcode::G_FCEIL:
1700 case TargetOpcode::G_FCONSTANT:
1701 case TargetOpcode::G_FCOPYSIGN:
1702 case TargetOpcode::G_FCOS:
1703 case TargetOpcode::G_FDIV:
1704 case TargetOpcode::G_FEXP2:
1705 case TargetOpcode::G_FEXP:
1706 case TargetOpcode::G_FFLOOR:
1707 case TargetOpcode::G_FLOG10:
1708 case TargetOpcode::G_FLOG2:
1709 case TargetOpcode::G_FLOG:
1710 case TargetOpcode::G_FMA:
1711 case TargetOpcode::G_FMAD:
1712 case TargetOpcode::G_FMAXIMUM:
1713 case TargetOpcode::G_FMAXIMUMNUM:
1714 case TargetOpcode::G_FMAXNUM:
1715 case TargetOpcode::G_FMAXNUM_IEEE:
1716 case TargetOpcode::G_FMINIMUM:
1717 case TargetOpcode::G_FMINIMUMNUM:
1718 case TargetOpcode::G_FMINNUM:
1719 case TargetOpcode::G_FMINNUM_IEEE:
1720 case TargetOpcode::G_FMUL:
1721 case TargetOpcode::G_FNEARBYINT:
1722 case TargetOpcode::G_FNEG:
1723 case TargetOpcode::G_FPEXT:
1724 case TargetOpcode::G_FPEXTLOAD:
1725 case TargetOpcode::G_FPOW:
1726 case TargetOpcode::G_FPTRUNC:
1727 case TargetOpcode::G_FPTRUNCSTORE:
1728 case TargetOpcode::G_FREM:
1729 case TargetOpcode::G_FRINT:
1730 case TargetOpcode::G_FSIN:
1731 case TargetOpcode::G_FTAN:
1732 case TargetOpcode::G_FACOS:
1733 case TargetOpcode::G_FASIN:
1734 case TargetOpcode::G_FATAN:
1735 case TargetOpcode::G_FATAN2:
1736 case TargetOpcode::G_FCOSH:
1737 case TargetOpcode::G_FSINH:
1738 case TargetOpcode::G_FTANH:
1739 case TargetOpcode::G_FSQRT:
1740 case TargetOpcode::G_FSUB:
1741 case TargetOpcode::G_INTRINSIC_ROUND:
1742 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
1743 case TargetOpcode::G_INTRINSIC_TRUNC:
1744 return true;
1745 default:
1746 return false;
1747 }
1748}
1749
1750/// Shifts return poison if shiftwidth is larger than the bitwidth.
1751static bool shiftAmountKnownInRange(Register ShiftAmount,
1752 const MachineRegisterInfo &MRI) {
1753 LLT Ty = MRI.getType(Reg: ShiftAmount);
1754
1755 if (Ty.isScalableVector())
1756 return false; // Can't tell, just return false to be safe
1757
1758 if (Ty.isScalar()) {
1759 std::optional<ValueAndVReg> Val =
1760 getIConstantVRegValWithLookThrough(VReg: ShiftAmount, MRI);
1761 if (!Val)
1762 return false;
1763 return Val->Value.ult(RHS: Ty.getScalarSizeInBits());
1764 }
1765
1766 GBuildVector *BV = getOpcodeDef<GBuildVector>(Reg: ShiftAmount, MRI);
1767 if (!BV)
1768 return false;
1769
1770 unsigned Sources = BV->getNumSources();
1771 for (unsigned I = 0; I < Sources; ++I) {
1772 std::optional<ValueAndVReg> Val =
1773 getIConstantVRegValWithLookThrough(VReg: BV->getSourceReg(I), MRI);
1774 if (!Val)
1775 return false;
1776 if (!Val->Value.ult(RHS: Ty.getScalarSizeInBits()))
1777 return false;
1778 }
1779
1780 return true;
1781}
1782
1783static bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
1784 bool ConsiderFlagsAndMetadata,
1785 UndefPoisonKind Kind) {
1786 MachineInstr *RegDef = MRI.getVRegDef(Reg);
1787
1788 if (ConsiderFlagsAndMetadata && includesPoison(Kind))
1789 if (auto *GMI = dyn_cast<GenericMachineInstr>(Val: RegDef))
1790 if (GMI->hasPoisonGeneratingFlags())
1791 return true;
1792
1793 // Check whether opcode is a poison/undef-generating operation.
1794 switch (RegDef->getOpcode()) {
1795 case TargetOpcode::G_BUILD_VECTOR:
1796 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
1797 return false;
1798 case TargetOpcode::G_SHL:
1799 case TargetOpcode::G_ASHR:
1800 case TargetOpcode::G_LSHR:
1801 return includesPoison(Kind) &&
1802 !shiftAmountKnownInRange(ShiftAmount: RegDef->getOperand(i: 2).getReg(), MRI);
1803 case TargetOpcode::G_FPTOSI:
1804 case TargetOpcode::G_FPTOUI:
1805 // fptosi/ui yields poison if the resulting value does not fit in the
1806 // destination type.
1807 return true;
1808 case TargetOpcode::G_CTLZ:
1809 case TargetOpcode::G_CTTZ:
1810 case TargetOpcode::G_CTLS:
1811 case TargetOpcode::G_ABS:
1812 case TargetOpcode::G_CTPOP:
1813 case TargetOpcode::G_BSWAP:
1814 case TargetOpcode::G_BITREVERSE:
1815 case TargetOpcode::G_FSHL:
1816 case TargetOpcode::G_FSHR:
1817 case TargetOpcode::G_SMAX:
1818 case TargetOpcode::G_SMIN:
1819 case TargetOpcode::G_SCMP:
1820 case TargetOpcode::G_UMAX:
1821 case TargetOpcode::G_UMIN:
1822 case TargetOpcode::G_UCMP:
1823 case TargetOpcode::G_PTRMASK:
1824 case TargetOpcode::G_SADDO:
1825 case TargetOpcode::G_SSUBO:
1826 case TargetOpcode::G_UADDO:
1827 case TargetOpcode::G_USUBO:
1828 case TargetOpcode::G_SMULO:
1829 case TargetOpcode::G_UMULO:
1830 case TargetOpcode::G_SADDSAT:
1831 case TargetOpcode::G_UADDSAT:
1832 case TargetOpcode::G_SSUBSAT:
1833 case TargetOpcode::G_USUBSAT:
1834 case TargetOpcode::G_SBFX:
1835 case TargetOpcode::G_UBFX:
1836 return false;
1837 case TargetOpcode::G_SSHLSAT:
1838 case TargetOpcode::G_USHLSAT:
1839 return includesPoison(Kind) &&
1840 !shiftAmountKnownInRange(ShiftAmount: RegDef->getOperand(i: 2).getReg(), MRI);
1841 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1842 GInsertVectorElement *Insert = cast<GInsertVectorElement>(Val: RegDef);
1843 if (includesPoison(Kind)) {
1844 std::optional<ValueAndVReg> Index =
1845 getIConstantVRegValWithLookThrough(VReg: Insert->getIndexReg(), MRI);
1846 if (!Index)
1847 return true;
1848 LLT VecTy = MRI.getType(Reg: Insert->getVectorReg());
1849 return Index->Value.uge(RHS: VecTy.getElementCount().getKnownMinValue());
1850 }
1851 return false;
1852 }
1853 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1854 GExtractVectorElement *Extract = cast<GExtractVectorElement>(Val: RegDef);
1855 if (includesPoison(Kind)) {
1856 std::optional<ValueAndVReg> Index =
1857 getIConstantVRegValWithLookThrough(VReg: Extract->getIndexReg(), MRI);
1858 if (!Index)
1859 return true;
1860 LLT VecTy = MRI.getType(Reg: Extract->getVectorReg());
1861 return Index->Value.uge(RHS: VecTy.getElementCount().getKnownMinValue());
1862 }
1863 return false;
1864 }
1865 case TargetOpcode::G_SHUFFLE_VECTOR: {
1866 GShuffleVector *Shuffle = cast<GShuffleVector>(Val: RegDef);
1867 ArrayRef<int> Mask = Shuffle->getMask();
1868 return includesPoison(Kind) && is_contained(Range&: Mask, Element: -1);
1869 }
1870 case TargetOpcode::G_FNEG:
1871 case TargetOpcode::G_PHI:
1872 case TargetOpcode::G_SELECT:
1873 case TargetOpcode::G_UREM:
1874 case TargetOpcode::G_SREM:
1875 case TargetOpcode::G_FREEZE:
1876 case TargetOpcode::G_ICMP:
1877 case TargetOpcode::G_FCMP:
1878 case TargetOpcode::G_FADD:
1879 case TargetOpcode::G_FSUB:
1880 case TargetOpcode::G_FMUL:
1881 case TargetOpcode::G_FDIV:
1882 case TargetOpcode::G_FREM:
1883 case TargetOpcode::G_PTR_ADD:
1884 return false;
1885 default:
1886 return !isa<GCastOp>(Val: RegDef) && !isa<GBinOp>(Val: RegDef);
1887 }
1888}
1889
1890static bool isGuaranteedNotToBeUndefOrPoison(Register Reg,
1891 const MachineRegisterInfo &MRI,
1892 unsigned Depth,
1893 UndefPoisonKind Kind) {
1894 if (Depth >= MaxAnalysisRecursionDepth)
1895 return false;
1896
1897 MachineInstr *RegDef = MRI.getVRegDef(Reg);
1898
1899 switch (RegDef->getOpcode()) {
1900 case TargetOpcode::G_FREEZE:
1901 return true;
1902 case TargetOpcode::G_IMPLICIT_DEF:
1903 return !includesUndef(Kind);
1904 case TargetOpcode::G_CONSTANT:
1905 case TargetOpcode::G_FCONSTANT:
1906 return true;
1907 case TargetOpcode::G_BUILD_VECTOR: {
1908 GBuildVector *BV = cast<GBuildVector>(Val: RegDef);
1909 unsigned NumSources = BV->getNumSources();
1910 for (unsigned I = 0; I < NumSources; ++I)
1911 if (!::isGuaranteedNotToBeUndefOrPoison(Reg: BV->getSourceReg(I), MRI,
1912 Depth: Depth + 1, Kind))
1913 return false;
1914 return true;
1915 }
1916 case TargetOpcode::G_PHI: {
1917 GPhi *Phi = cast<GPhi>(Val: RegDef);
1918 unsigned NumIncoming = Phi->getNumIncomingValues();
1919 for (unsigned I = 0; I < NumIncoming; ++I)
1920 if (!::isGuaranteedNotToBeUndefOrPoison(Reg: Phi->getIncomingValue(I), MRI,
1921 Depth: Depth + 1, Kind))
1922 return false;
1923 return true;
1924 }
1925 default: {
1926 auto MOCheck = [&](const MachineOperand &MO) {
1927 if (!MO.isReg())
1928 return true;
1929 return ::isGuaranteedNotToBeUndefOrPoison(Reg: MO.getReg(), MRI, Depth: Depth + 1,
1930 Kind);
1931 };
1932 return !::canCreateUndefOrPoison(Reg, MRI,
1933 /*ConsiderFlagsAndMetadata=*/true, Kind) &&
1934 all_of(Range: RegDef->uses(), P: MOCheck);
1935 }
1936 }
1937}
1938
1939bool llvm::canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
1940 bool ConsiderFlagsAndMetadata) {
1941 return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata,
1942 Kind: UndefPoisonKind::UndefOrPoison);
1943}
1944
1945bool canCreatePoison(Register Reg, const MachineRegisterInfo &MRI,
1946 bool ConsiderFlagsAndMetadata = true) {
1947 return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata,
1948 Kind: UndefPoisonKind::PoisonOnly);
1949}
1950
1951bool llvm::isGuaranteedNotToBeUndefOrPoison(Register Reg,
1952 const MachineRegisterInfo &MRI,
1953 unsigned Depth) {
1954 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
1955 Kind: UndefPoisonKind::UndefOrPoison);
1956}
1957
1958bool llvm::isGuaranteedNotToBePoison(Register Reg,
1959 const MachineRegisterInfo &MRI,
1960 unsigned Depth) {
1961 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
1962 Kind: UndefPoisonKind::PoisonOnly);
1963}
1964
1965bool llvm::isGuaranteedNotToBeUndef(Register Reg,
1966 const MachineRegisterInfo &MRI,
1967 unsigned Depth) {
1968 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
1969 Kind: UndefPoisonKind::UndefOnly);
1970}
1971
1972Type *llvm::getTypeForLLT(LLT Ty, LLVMContext &C) {
1973 if (Ty.isVector())
1974 return VectorType::get(ElementType: IntegerType::get(C, NumBits: Ty.getScalarSizeInBits()),
1975 EC: Ty.getElementCount());
1976 return IntegerType::get(C, NumBits: Ty.getSizeInBits());
1977}
1978
1979bool llvm::isAssertMI(const MachineInstr &MI) {
1980 switch (MI.getOpcode()) {
1981 default:
1982 return false;
1983 case TargetOpcode::G_ASSERT_ALIGN:
1984 case TargetOpcode::G_ASSERT_SEXT:
1985 case TargetOpcode::G_ASSERT_ZEXT:
1986 return true;
1987 }
1988}
1989
1990APInt llvm::GIConstant::getScalarValue() const {
1991 assert(Kind == GIConstantKind::Scalar && "Expected scalar constant");
1992
1993 return Value;
1994}
1995
1996std::optional<GIConstant>
1997llvm::GIConstant::getConstant(Register Const, const MachineRegisterInfo &MRI) {
1998 MachineInstr *Constant = getDefIgnoringCopies(Reg: Const, MRI);
1999
2000 if (GSplatVector *Splat = dyn_cast<GSplatVector>(Val: Constant)) {
2001 std::optional<ValueAndVReg> MayBeConstant =
2002 getIConstantVRegValWithLookThrough(VReg: Splat->getScalarReg(), MRI);
2003 if (!MayBeConstant)
2004 return std::nullopt;
2005 return GIConstant(MayBeConstant->Value, GIConstantKind::ScalableVector);
2006 }
2007
2008 if (GBuildVector *Build = dyn_cast<GBuildVector>(Val: Constant)) {
2009 SmallVector<APInt> Values;
2010 unsigned NumSources = Build->getNumSources();
2011 for (unsigned I = 0; I < NumSources; ++I) {
2012 Register SrcReg = Build->getSourceReg(I);
2013 std::optional<ValueAndVReg> MayBeConstant =
2014 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI);
2015 if (!MayBeConstant)
2016 return std::nullopt;
2017 Values.push_back(Elt: MayBeConstant->Value);
2018 }
2019 return GIConstant(Values);
2020 }
2021
2022 std::optional<ValueAndVReg> MayBeConstant =
2023 getIConstantVRegValWithLookThrough(VReg: Const, MRI);
2024 if (!MayBeConstant)
2025 return std::nullopt;
2026
2027 return GIConstant(MayBeConstant->Value, GIConstantKind::Scalar);
2028}
2029
2030APFloat llvm::GFConstant::getScalarValue() const {
2031 assert(Kind == GFConstantKind::Scalar && "Expected scalar constant");
2032
2033 return Values[0];
2034}
2035
2036std::optional<GFConstant>
2037llvm::GFConstant::getConstant(Register Const, const MachineRegisterInfo &MRI) {
2038 MachineInstr *Constant = getDefIgnoringCopies(Reg: Const, MRI);
2039
2040 if (GSplatVector *Splat = dyn_cast<GSplatVector>(Val: Constant)) {
2041 std::optional<FPValueAndVReg> MayBeConstant =
2042 getFConstantVRegValWithLookThrough(VReg: Splat->getScalarReg(), MRI);
2043 if (!MayBeConstant)
2044 return std::nullopt;
2045 return GFConstant(MayBeConstant->Value, GFConstantKind::ScalableVector);
2046 }
2047
2048 if (GBuildVector *Build = dyn_cast<GBuildVector>(Val: Constant)) {
2049 SmallVector<APFloat> Values;
2050 unsigned NumSources = Build->getNumSources();
2051 for (unsigned I = 0; I < NumSources; ++I) {
2052 Register SrcReg = Build->getSourceReg(I);
2053 std::optional<FPValueAndVReg> MayBeConstant =
2054 getFConstantVRegValWithLookThrough(VReg: SrcReg, MRI);
2055 if (!MayBeConstant)
2056 return std::nullopt;
2057 Values.push_back(Elt: MayBeConstant->Value);
2058 }
2059 return GFConstant(Values);
2060 }
2061
2062 std::optional<FPValueAndVReg> MayBeConstant =
2063 getFConstantVRegValWithLookThrough(VReg: Const, MRI);
2064 if (!MayBeConstant)
2065 return std::nullopt;
2066
2067 return GFConstant(MayBeConstant->Value, GFConstantKind::Scalar);
2068}
2069
2070// Returns a list of types to use for memory op lowering in MemOps. A partial
2071// port of findOptimalMemOpLowering in TargetLowering.
2072static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
2073 unsigned Limit, const MemOp &Op,
2074 unsigned DstAS, unsigned SrcAS,
2075 const AttributeList &FuncAttributes,
2076 const TargetLowering &TLI) {
2077 if (Op.isMemcpyOrMemmoveWithFixedDstAlign() &&
2078 Op.getSrcAlign() < Op.getDstAlign())
2079 return false;
2080
2081 LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes);
2082
2083 if (Ty == LLT()) {
2084 // Use the largest scalar type whose alignment constraints are satisfied.
2085 // We only need to check DstAlign here as SrcAlign is always greater or
2086 // equal to DstAlign (or zero).
2087 Ty = LLT::integer(SizeInBits: 64);
2088 if (Op.isFixedDstAlign())
2089 while (Op.getDstAlign() < Ty.getSizeInBytes() &&
2090 !TLI.allowsMisalignedMemoryAccesses(Ty, AddrSpace: DstAS, Alignment: Op.getDstAlign()))
2091 Ty = LLT::integer(SizeInBits: Ty.getSizeInBytes());
2092 assert(Ty.getSizeInBits() > 0 && "Could not find valid type");
2093 // FIXME: check for the largest legal type we can load/store to.
2094 }
2095
2096 unsigned NumMemOps = 0;
2097 uint64_t Size = Op.size();
2098 while (Size) {
2099 unsigned TySize = Ty.getSizeInBytes();
2100 while (TySize > Size) {
2101 // For now, only use non-vector load / store's for the left-over pieces.
2102 LLT NewTy = Ty;
2103 // FIXME: check for mem op safety and legality of the types. Not all of
2104 // SDAGisms map cleanly to GISel concepts.
2105 if (NewTy.isVector())
2106 NewTy =
2107 NewTy.getSizeInBits() > 64 ? LLT::integer(SizeInBits: 64) : LLT::integer(SizeInBits: 32);
2108 NewTy = LLT::integer(SizeInBits: llvm::bit_floor(Value: NewTy.getSizeInBits() - 1));
2109 unsigned NewTySize = NewTy.getSizeInBytes();
2110 assert(NewTySize > 0 && "Could not find appropriate type");
2111
2112 // If the new LLT cannot cover all of the remaining bits, then consider
2113 // issuing a (or a pair of) unaligned and overlapping load / store.
2114 unsigned Fast;
2115 // Need to get a VT equivalent for allowMisalignedMemoryAccesses().
2116 MVT VT = getMVTForLLT(Ty);
2117 if (NumMemOps && !Op.isVolatile() && NewTySize < Size &&
2118 TLI.allowsMisalignedMemoryAccesses(
2119 VT, AddrSpace: DstAS, Alignment: Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
2120 Flags: MachineMemOperand::MONone, &Fast) &&
2121 Fast)
2122 TySize = Size;
2123 else {
2124 Ty = NewTy;
2125 TySize = NewTySize;
2126 }
2127 }
2128
2129 if (++NumMemOps > Limit)
2130 return false;
2131
2132 MemOps.push_back(x: Ty);
2133 Size -= TySize;
2134 }
2135
2136 return true;
2137}
2138
2139bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
2140 const MachineRegisterInfo &MRI, unsigned MaxLen,
2141 Register &Dst, Register &Src,
2142 uint64_t &KnownLen, Align &Alignment,
2143 bool &DstAlignCanChange,
2144 std::vector<LLT> &MemOps) {
2145 const unsigned Opc = MI.getOpcode();
2146 assert((Opc == TargetOpcode::G_MEMCPY ||
2147 Opc == TargetOpcode::G_MEMCPY_INLINE ||
2148 Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET ||
2149 Opc == TargetOpcode::G_MEMSET_INLINE) &&
2150 "Expected memcpy like instruction");
2151
2152 auto MMOIt = MI.memoperands_begin();
2153 const MachineMemOperand *MemOp = *MMOIt;
2154
2155 Align DstAlign = MemOp->getBaseAlign();
2156 Align SrcAlign;
2157 Alignment = DstAlign;
2158 Register Len;
2159 std::tie(args&: Dst, args&: Src, args&: Len) = MI.getFirst3Regs();
2160
2161 if (Opc != TargetOpcode::G_MEMSET && Opc != TargetOpcode::G_MEMSET_INLINE) {
2162 assert(MMOIt != MI.memoperands_end() && "Expected a second MMO on MI");
2163 MemOp = *(++MMOIt);
2164 SrcAlign = MemOp->getBaseAlign();
2165 Alignment = std::min(a: DstAlign, b: SrcAlign);
2166 }
2167
2168 // See if this is a constant length copy.
2169 auto LenVRegAndVal = getIConstantVRegValWithLookThrough(VReg: Len, MRI);
2170 if (!LenVRegAndVal) {
2171 // FIXME: support dynamically sized G_MEMCPY_INLINE and G_MEMSET_INLINE
2172 assert(Opc != TargetOpcode::G_MEMCPY_INLINE &&
2173 Opc != TargetOpcode::G_MEMSET_INLINE &&
2174 "inline memcpy and memset with dynamic size are not yet supported");
2175 return false;
2176 }
2177
2178 KnownLen = LenVRegAndVal->Value.getZExtValue();
2179 DstAlignCanChange = false;
2180
2181 if (KnownLen == 0)
2182 return true;
2183
2184 if (Opc != TargetOpcode::G_MEMCPY_INLINE &&
2185 Opc != TargetOpcode::G_MEMSET_INLINE && MaxLen && KnownLen > MaxLen)
2186 return false;
2187
2188 bool IsVolatile = MemOp->isVolatile();
2189 const MachineFunction &MF = *MI.getParent()->getParent();
2190 const auto &TLI = *MF.getSubtarget().getTargetLowering();
2191 // On Darwin, -Os means optimize for size without hurting performance, so
2192 // only really optimize for size when -Oz (MinSize) is used.
2193 bool OptSize = MF.getTarget().getTargetTriple().isOSDarwin()
2194 ? MF.getFunction().hasMinSize()
2195 : MF.getFunction().hasOptSize();
2196
2197 const MachineFrameInfo &MFI = MF.getFrameInfo();
2198 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
2199 if (FIDef && !MFI.isFixedObjectIndex(ObjectIdx: FIDef->getOperand(i: 1).getIndex()))
2200 DstAlignCanChange = true;
2201
2202 const auto &DstMMO = **MI.memoperands_begin();
2203 MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo();
2204
2205 switch (Opc) {
2206 case TargetOpcode::G_MEMCPY_INLINE:
2207 case TargetOpcode::G_MEMCPY: {
2208 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
2209 MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
2210 uint64_t Limit = Opc == TargetOpcode::G_MEMCPY_INLINE
2211 ? std::numeric_limits<uint64_t>::max()
2212 : TLI.getMaxStoresPerMemcpy(OptSize);
2213 return findGISelOptimalMemOpLowering(
2214 MemOps, Limit,
2215 Op: MemOp::Copy(Size: KnownLen, DstAlignCanChange, DstAlign: std::min(a: DstAlign, b: SrcAlign),
2216 SrcAlign, IsVolatile),
2217 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
2218 FuncAttributes: MF.getFunction().getAttributes(), TLI);
2219 }
2220 case TargetOpcode::G_MEMMOVE: {
2221 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
2222 MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
2223 unsigned Limit = TLI.getMaxStoresPerMemmove(OptSize);
2224 // FIXME: SelectionDAG always passes true for 'IsVolatile', apparently
2225 // due to a bug in it's findOptimalMemOpLowering implementation. For now do
2226 // the same thing here.
2227 return findGISelOptimalMemOpLowering(
2228 MemOps, Limit,
2229 Op: MemOp::Move(Size: KnownLen, DstAlignCanChange, DstAlign: std::min(a: DstAlign, b: SrcAlign),
2230 SrcAlign, /*IsVolatile=*/true),
2231 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
2232 FuncAttributes: MF.getFunction().getAttributes(), TLI);
2233 }
2234 case TargetOpcode::G_MEMSET:
2235 case TargetOpcode::G_MEMSET_INLINE: {
2236 unsigned Limit = Opc == TargetOpcode::G_MEMSET_INLINE
2237 ? std::numeric_limits<unsigned>::max()
2238 : TLI.getMaxStoresPerMemset(OptSize);
2239 auto ValVRegAndVal = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
2240 bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0;
2241 return findGISelOptimalMemOpLowering(
2242 MemOps, Limit,
2243 Op: MemOp::Set(Size: KnownLen, DstAlignCanChange, DstAlign,
2244 /*IsZeroMemset=*/IsZeroVal,
2245 /*IsVolatile=*/IsVolatile),
2246 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes(), TLI);
2247 }
2248 default:
2249 llvm_unreachable("Unexpected memcpy-family opcode");
2250 }
2251}
2252