1//===- llvm/CodeGen/GlobalISel/Utils.cpp -------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file This file implements the utility functions used by the GlobalISel
9/// pipeline.
10//===----------------------------------------------------------------------===//
11
12#include "llvm/CodeGen/GlobalISel/Utils.h"
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/Analysis/ValueTracking.h"
16#include "llvm/CodeGen/CodeGenCommonISel.h"
17#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
18#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
19#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
20#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
21#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
22#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
23#include "llvm/CodeGen/LowLevelTypeUtils.h"
24#include "llvm/CodeGen/MachineInstr.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/MachineSizeOpts.h"
29#include "llvm/CodeGen/RegisterBankInfo.h"
30#include "llvm/CodeGen/StackProtector.h"
31#include "llvm/CodeGen/TargetInstrInfo.h"
32#include "llvm/CodeGen/TargetLowering.h"
33#include "llvm/CodeGen/TargetOpcodes.h"
34#include "llvm/CodeGen/TargetPassConfig.h"
35#include "llvm/CodeGen/TargetRegisterInfo.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/Support/UndefPoison.h"
38#include "llvm/Target/TargetMachine.h"
39#include "llvm/Transforms/Utils/SizeOpts.h"
40#include <limits>
41#include <numeric>
42#include <optional>
43#include <tuple>
44
45#define DEBUG_TYPE "globalisel-utils"
46
47using namespace llvm;
48using namespace MIPatternMatch;
49
50Register llvm::constrainRegToClass(MachineRegisterInfo &MRI,
51 const TargetInstrInfo &TII,
52 const RegisterBankInfo &RBI, Register Reg,
53 const TargetRegisterClass &RegClass) {
54 if (!RBI.constrainGenericRegister(Reg, RC: RegClass, MRI))
55 return MRI.createVirtualRegister(RegClass: &RegClass);
56
57 return Reg;
58}
59
60Register llvm::constrainOperandRegClass(
61 const MachineFunction &MF, const TargetRegisterInfo &TRI,
62 MachineRegisterInfo &MRI, const TargetInstrInfo &TII,
63 const RegisterBankInfo &RBI, MachineInstr &InsertPt,
64 const TargetRegisterClass &RegClass, MachineOperand &RegMO) {
65 Register Reg = RegMO.getReg();
66 // Assume physical registers are properly constrained.
67 assert(Reg.isVirtual() && "PhysReg not implemented");
68
69 // Save the old register class to check whether
70 // the change notifications will be required.
71 // TODO: A better approach would be to pass
72 // the observers to constrainRegToClass().
73 auto *OldRegClass = MRI.getRegClassOrNull(Reg);
74 Register ConstrainedReg = constrainRegToClass(MRI, TII, RBI, Reg, RegClass);
75 // If we created a new virtual register because the class is not compatible
76 // then create a copy between the new and the old register.
77 if (ConstrainedReg != Reg) {
78 MachineBasicBlock::iterator InsertIt(&InsertPt);
79 MachineBasicBlock &MBB = *InsertPt.getParent();
80 // FIXME: The copy needs to have the classes constrained for its operands.
81 // Use operand's regbank to get the class for old register (Reg).
82 if (RegMO.isUse()) {
83 BuildMI(BB&: MBB, I: InsertIt, MIMD: InsertPt.getDebugLoc(),
84 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ConstrainedReg)
85 .addReg(RegNo: Reg);
86 } else {
87 assert(RegMO.isDef() && "Must be a definition");
88 BuildMI(BB&: MBB, I: std::next(x: InsertIt), MIMD: InsertPt.getDebugLoc(),
89 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Reg)
90 .addReg(RegNo: ConstrainedReg);
91 }
92 if (GISelChangeObserver *Observer = MF.getObserver()) {
93 Observer->changingInstr(MI&: *RegMO.getParent());
94 }
95 RegMO.setReg(ConstrainedReg);
96 if (GISelChangeObserver *Observer = MF.getObserver()) {
97 Observer->changedInstr(MI&: *RegMO.getParent());
98 }
99 } else if (OldRegClass != MRI.getRegClassOrNull(Reg)) {
100 if (GISelChangeObserver *Observer = MF.getObserver()) {
101 if (!RegMO.isDef()) {
102 MachineInstr *RegDef = MRI.getVRegDef(Reg);
103 Observer->changedInstr(MI&: *RegDef);
104 }
105 Observer->changingAllUsesOfReg(MRI, Reg);
106 Observer->finishedChangingAllUsesOfReg();
107 }
108 }
109 return ConstrainedReg;
110}
111
112Register llvm::constrainOperandRegClass(
113 const MachineFunction &MF, const TargetRegisterInfo &TRI,
114 MachineRegisterInfo &MRI, const TargetInstrInfo &TII,
115 const RegisterBankInfo &RBI, MachineInstr &InsertPt, const MCInstrDesc &II,
116 MachineOperand &RegMO, unsigned OpIdx) {
117 Register Reg = RegMO.getReg();
118 // Assume physical registers are properly constrained.
119 assert(Reg.isVirtual() && "PhysReg not implemented");
120
121 const TargetRegisterClass *OpRC = TII.getRegClass(MCID: II, OpNum: OpIdx);
122 // Some of the target independent instructions, like COPY, may not impose any
123 // register class constraints on some of their operands: If it's a use, we can
124 // skip constraining as the instruction defining the register would constrain
125 // it.
126
127 if (OpRC) {
128 // Obtain the RC from incoming regbank if it is a proper sub-class. Operands
129 // can have multiple regbanks for a superclass that combine different
130 // register types (E.g., AMDGPU's VGPR and AGPR). The regbank ambiguity
131 // resolved by targets during regbankselect should not be overridden.
132 if (const auto *SubRC = TRI.getCommonSubClass(
133 A: OpRC, B: TRI.getConstrainedRegClassForReg(Reg, MRI)))
134 OpRC = SubRC;
135
136 OpRC = TRI.getAllocatableClass(RC: OpRC);
137 }
138
139 if (!OpRC) {
140 assert((!isTargetSpecificOpcode(II.getOpcode()) || RegMO.isUse()) &&
141 "Register class constraint is required unless either the "
142 "instruction is target independent or the operand is a use");
143 // FIXME: Just bailing out like this here could be not enough, unless we
144 // expect the users of this function to do the right thing for PHIs and
145 // COPY:
146 // v1 = COPY v0
147 // v2 = COPY v1
148 // v1 here may end up not being constrained at all. Please notice that to
149 // reproduce the issue we likely need a destination pattern of a selection
150 // rule producing such extra copies, not just an input GMIR with them as
151 // every existing target using selectImpl handles copies before calling it
152 // and they never reach this function.
153 return Reg;
154 }
155 return constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt, RegClass: *OpRC,
156 RegMO);
157}
158
159void llvm::constrainSelectedInstRegOperands(MachineInstr &I,
160 const TargetInstrInfo &TII,
161 const TargetRegisterInfo &TRI,
162 const RegisterBankInfo &RBI) {
163 assert(!isPreISelGenericOpcode(I.getOpcode()) &&
164 "A selected instruction is expected");
165 MachineBasicBlock &MBB = *I.getParent();
166 MachineFunction &MF = *MBB.getParent();
167 MachineRegisterInfo &MRI = MF.getRegInfo();
168
169 for (unsigned OpI = 0, OpE = I.getNumExplicitOperands(); OpI != OpE; ++OpI) {
170 MachineOperand &MO = I.getOperand(i: OpI);
171
172 // There's nothing to be done on non-register operands.
173 if (!MO.isReg())
174 continue;
175
176 LLVM_DEBUG(dbgs() << "Converting operand: " << MO << '\n');
177
178 Register Reg = MO.getReg();
179 // Physical registers don't need to be constrained.
180 if (Reg.isPhysical())
181 continue;
182
183 // Register operands with a value of 0 (e.g. predicate operands) don't need
184 // to be constrained.
185 if (Reg == 0)
186 continue;
187
188 // If the operand is a vreg, we should constrain its regclass, and only
189 // insert COPYs if that's impossible.
190 // constrainOperandRegClass does that for us.
191 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt&: I, II: I.getDesc(), RegMO&: MO, OpIdx: OpI);
192
193 // Tie uses to defs as indicated in MCInstrDesc if this hasn't already been
194 // done.
195 if (MO.isUse()) {
196 int DefIdx = I.getDesc().getOperandConstraint(OpNum: OpI, Constraint: MCOI::TIED_TO);
197 if (DefIdx != -1 && !I.isRegTiedToUseOperand(DefOpIdx: DefIdx))
198 I.tieOperands(DefIdx, UseIdx: OpI);
199 }
200 }
201}
202
203bool llvm::canReplaceReg(Register DstReg, Register SrcReg,
204 MachineRegisterInfo &MRI) {
205 // Give up if either DstReg or SrcReg is a physical register.
206 if (DstReg.isPhysical() || SrcReg.isPhysical())
207 return false;
208 // Give up if the types don't match.
209 if (MRI.getType(Reg: DstReg) != MRI.getType(Reg: SrcReg))
210 return false;
211 // Replace if either DstReg has no constraints or the register
212 // constraints match.
213 const auto &DstRBC = MRI.getRegClassOrRegBank(Reg: DstReg);
214 if (!DstRBC || DstRBC == MRI.getRegClassOrRegBank(Reg: SrcReg))
215 return true;
216
217 // Otherwise match if the Src is already a regclass that is covered by the Dst
218 // RegBank.
219 return isa<const RegisterBank *>(Val: DstRBC) && MRI.getRegClassOrNull(Reg: SrcReg) &&
220 cast<const RegisterBank *>(Val: DstRBC)->covers(
221 RC: *MRI.getRegClassOrNull(Reg: SrcReg));
222}
223
224bool llvm::isTriviallyDead(const MachineInstr &MI,
225 const MachineRegisterInfo &MRI) {
226 // Instructions without side-effects are dead iff they only define dead regs.
227 // This function is hot and this loop returns early in the common case,
228 // so only perform additional checks before this if absolutely necessary.
229 for (const auto &MO : MI.all_defs()) {
230 Register Reg = MO.getReg();
231 if (Reg.isPhysical() || !MRI.use_nodbg_empty(RegNo: Reg))
232 return false;
233 }
234 return MI.wouldBeTriviallyDead();
235}
236
237static void reportGISelDiagnostic(DiagnosticSeverity Severity,
238 MachineFunction &MF,
239 MachineOptimizationRemarkEmitter &MORE,
240 MachineOptimizationRemarkMissed &R) {
241 bool IsGlobalISelAbortEnabled =
242 MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
243 bool IsFatal = Severity == DS_Error && IsGlobalISelAbortEnabled;
244 // Print the function name explicitly if we don't have a debug location (which
245 // makes the diagnostic less useful) or if we're going to emit a raw error.
246 if (!R.getLocation().isValid() || IsFatal)
247 R << (" (in function: " + MF.getName() + ")").str();
248
249 if (IsFatal)
250 reportFatalUsageError(reason: Twine(R.getMsg()));
251 else
252 MORE.emit(OptDiag&: R);
253}
254
255void llvm::reportGISelWarning(MachineFunction &MF,
256 MachineOptimizationRemarkEmitter &MORE,
257 MachineOptimizationRemarkMissed &R) {
258 reportGISelDiagnostic(Severity: DS_Warning, MF, MORE, R);
259}
260
261void llvm::reportGISelFailure(MachineFunction &MF,
262 MachineOptimizationRemarkEmitter &MORE,
263 MachineOptimizationRemarkMissed &R) {
264 MF.getProperties().setFailedISel();
265 reportGISelDiagnostic(Severity: DS_Error, MF, MORE, R);
266}
267
268void llvm::reportGISelFailure(MachineFunction &MF,
269 MachineOptimizationRemarkEmitter &MORE,
270 const char *PassName, StringRef Msg,
271 const MachineInstr &MI) {
272 MachineOptimizationRemarkMissed R(PassName, "GISelFailure: ",
273 MI.getDebugLoc(), MI.getParent());
274 R << Msg;
275 // Printing MI is expensive; only do it if expensive remarks are enabled.
276 if (MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable ||
277 MORE.allowExtraAnalysis(PassName))
278 R << ": " << ore::MNV("Inst", MI);
279 reportGISelFailure(MF, MORE, R);
280}
281
282unsigned llvm::getInverseGMinMaxOpcode(unsigned MinMaxOpc) {
283 switch (MinMaxOpc) {
284 case TargetOpcode::G_SMIN:
285 return TargetOpcode::G_SMAX;
286 case TargetOpcode::G_SMAX:
287 return TargetOpcode::G_SMIN;
288 case TargetOpcode::G_UMIN:
289 return TargetOpcode::G_UMAX;
290 case TargetOpcode::G_UMAX:
291 return TargetOpcode::G_UMIN;
292 default:
293 llvm_unreachable("unrecognized opcode");
294 }
295}
296
297std::optional<APInt> llvm::getIConstantVRegVal(Register VReg,
298 const MachineRegisterInfo &MRI) {
299 std::optional<ValueAndVReg> ValAndVReg = getIConstantVRegValWithLookThrough(
300 VReg, MRI, /*LookThroughInstrs*/ false);
301 assert((!ValAndVReg || ValAndVReg->VReg == VReg) &&
302 "Value found while looking through instrs");
303 if (!ValAndVReg)
304 return std::nullopt;
305 return ValAndVReg->Value;
306}
307
308const APInt &llvm::getIConstantFromReg(Register Reg,
309 const MachineRegisterInfo &MRI) {
310 MachineInstr *Const = MRI.getVRegDef(Reg);
311 assert((Const && Const->getOpcode() == TargetOpcode::G_CONSTANT) &&
312 "expected a G_CONSTANT on Reg");
313 return Const->getOperand(i: 1).getCImm()->getValue();
314}
315
316std::optional<int64_t>
317llvm::getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI) {
318 std::optional<APInt> Val = getIConstantVRegVal(VReg, MRI);
319 if (Val && Val->getBitWidth() <= 64)
320 return Val->getSExtValue();
321 return std::nullopt;
322}
323
324namespace {
325
326// This function is used in many places, and as such, it has some
327// micro-optimizations to try and make it as fast as it can be.
328//
329// - We use template arguments to avoid an indirect call caused by passing a
330// function_ref/std::function
331// - GetAPCstValue does not return std::optional<APInt> as that's expensive.
332// Instead it returns true/false and places the result in a pre-constructed
333// APInt.
334//
335// Please change this function carefully and benchmark your changes.
336template <bool (*IsConstantOpcode)(const MachineInstr *),
337 bool (*GetAPCstValue)(const MachineInstr *MI, APInt &)>
338std::optional<ValueAndVReg>
339getConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI,
340 bool LookThroughInstrs = true,
341 bool LookThroughAnyExt = false) {
342 SmallVector<std::pair<unsigned, unsigned>, 4> SeenOpcodes;
343 MachineInstr *MI;
344
345 while ((MI = MRI.getVRegDef(Reg: VReg)) && !IsConstantOpcode(MI) &&
346 LookThroughInstrs) {
347 switch (MI->getOpcode()) {
348 case TargetOpcode::G_ANYEXT:
349 if (!LookThroughAnyExt)
350 return std::nullopt;
351 [[fallthrough]];
352 case TargetOpcode::G_TRUNC:
353 case TargetOpcode::G_SEXT:
354 case TargetOpcode::G_ZEXT:
355 SeenOpcodes.push_back(Elt: std::make_pair(
356 x: MI->getOpcode(),
357 y: MRI.getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits()));
358 VReg = MI->getOperand(i: 1).getReg();
359 break;
360 case TargetOpcode::COPY:
361 VReg = MI->getOperand(i: 1).getReg();
362 if (VReg.isPhysical())
363 return std::nullopt;
364 break;
365 case TargetOpcode::G_INTTOPTR:
366 VReg = MI->getOperand(i: 1).getReg();
367 break;
368 default:
369 return std::nullopt;
370 }
371 }
372 if (!MI || !IsConstantOpcode(MI))
373 return std::nullopt;
374
375 APInt Val;
376 if (!GetAPCstValue(MI, Val))
377 return std::nullopt;
378 for (auto &Pair : reverse(C&: SeenOpcodes)) {
379 switch (Pair.first) {
380 case TargetOpcode::G_TRUNC:
381 Val = Val.trunc(width: Pair.second);
382 break;
383 case TargetOpcode::G_ANYEXT:
384 case TargetOpcode::G_SEXT:
385 Val = Val.sext(width: Pair.second);
386 break;
387 case TargetOpcode::G_ZEXT:
388 Val = Val.zext(width: Pair.second);
389 break;
390 }
391 }
392
393 return ValueAndVReg{.Value: std::move(Val), .VReg: VReg};
394}
395
396bool isIConstant(const MachineInstr *MI) {
397 if (!MI)
398 return false;
399 return MI->getOpcode() == TargetOpcode::G_CONSTANT;
400}
401
402bool isFConstant(const MachineInstr *MI) {
403 if (!MI)
404 return false;
405 return MI->getOpcode() == TargetOpcode::G_FCONSTANT;
406}
407
408bool isAnyConstant(const MachineInstr *MI) {
409 if (!MI)
410 return false;
411 unsigned Opc = MI->getOpcode();
412 return Opc == TargetOpcode::G_CONSTANT || Opc == TargetOpcode::G_FCONSTANT;
413}
414
415bool getCImmAsAPInt(const MachineInstr *MI, APInt &Result) {
416 const MachineOperand &CstVal = MI->getOperand(i: 1);
417 if (!CstVal.isCImm())
418 return false;
419 Result = CstVal.getCImm()->getValue();
420 return true;
421}
422
423bool getCImmOrFPImmAsAPInt(const MachineInstr *MI, APInt &Result) {
424 const MachineOperand &CstVal = MI->getOperand(i: 1);
425 if (CstVal.isCImm())
426 Result = CstVal.getCImm()->getValue();
427 else if (CstVal.isFPImm())
428 Result = CstVal.getFPImm()->getValueAPF().bitcastToAPInt();
429 else
430 return false;
431 return true;
432}
433
434} // end anonymous namespace
435
436std::optional<ValueAndVReg> llvm::getIConstantVRegValWithLookThrough(
437 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs) {
438 return getConstantVRegValWithLookThrough<isIConstant, getCImmAsAPInt>(
439 VReg, MRI, LookThroughInstrs);
440}
441
442std::optional<ValueAndVReg> llvm::getAnyConstantVRegValWithLookThrough(
443 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs,
444 bool LookThroughAnyExt) {
445 return getConstantVRegValWithLookThrough<isAnyConstant,
446 getCImmOrFPImmAsAPInt>(
447 VReg, MRI, LookThroughInstrs, LookThroughAnyExt);
448}
449
450std::optional<FPValueAndVReg> llvm::getFConstantVRegValWithLookThrough(
451 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs) {
452 auto Reg =
453 getConstantVRegValWithLookThrough<isFConstant, getCImmOrFPImmAsAPInt>(
454 VReg, MRI, LookThroughInstrs);
455 if (!Reg)
456 return std::nullopt;
457
458 APFloat FloatVal(getFltSemanticForLLT(Ty: LLT::scalar(SizeInBits: Reg->Value.getBitWidth())),
459 Reg->Value);
460 return FPValueAndVReg{.Value: FloatVal, .VReg: Reg->VReg};
461}
462
463const ConstantFP *
464llvm::getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI) {
465 MachineInstr *MI = MRI.getVRegDef(Reg: VReg);
466 if (TargetOpcode::G_FCONSTANT != MI->getOpcode())
467 return nullptr;
468 return MI->getOperand(i: 1).getFPImm();
469}
470
471std::optional<DefinitionAndSourceRegister>
472llvm::getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI) {
473 Register DefSrcReg = Reg;
474 // This assumes that the code is in SSA form, so there should only be one
475 // definition.
476 auto DefIt = MRI.def_begin(RegNo: Reg);
477 if (DefIt == MRI.def_end())
478 return {};
479 MachineOperand &DefOpnd = *DefIt;
480 MachineInstr *DefMI = DefOpnd.getParent();
481 auto DstTy = MRI.getType(Reg: DefOpnd.getReg());
482 if (!DstTy.isValid())
483 return std::nullopt;
484 unsigned Opc = DefMI->getOpcode();
485 while (Opc == TargetOpcode::COPY || isPreISelGenericOptimizationHint(Opcode: Opc)) {
486 Register SrcReg = DefMI->getOperand(i: 1).getReg();
487 auto SrcTy = MRI.getType(Reg: SrcReg);
488 if (!SrcTy.isValid())
489 break;
490 DefMI = MRI.getVRegDef(Reg: SrcReg);
491 DefSrcReg = SrcReg;
492 Opc = DefMI->getOpcode();
493 }
494 return DefinitionAndSourceRegister{.MI: DefMI, .Reg: DefSrcReg};
495}
496
497MachineInstr *llvm::getDefIgnoringCopies(Register Reg,
498 const MachineRegisterInfo &MRI) {
499 std::optional<DefinitionAndSourceRegister> DefSrcReg =
500 getDefSrcRegIgnoringCopies(Reg, MRI);
501 return DefSrcReg ? DefSrcReg->MI : nullptr;
502}
503
504Register llvm::getSrcRegIgnoringCopies(Register Reg,
505 const MachineRegisterInfo &MRI) {
506 std::optional<DefinitionAndSourceRegister> DefSrcReg =
507 getDefSrcRegIgnoringCopies(Reg, MRI);
508 return DefSrcReg ? DefSrcReg->Reg : Register();
509}
510
511void llvm::extractParts(Register Reg, LLT Ty, int NumParts,
512 SmallVectorImpl<Register> &VRegs,
513 MachineIRBuilder &MIRBuilder,
514 MachineRegisterInfo &MRI) {
515 for (int i = 0; i < NumParts; ++i)
516 VRegs.push_back(Elt: MRI.createGenericVirtualRegister(Ty));
517 MIRBuilder.buildUnmerge(Res: VRegs, Op: Reg);
518}
519
520bool llvm::extractParts(Register Reg, LLT RegTy, LLT MainTy, LLT &LeftoverTy,
521 SmallVectorImpl<Register> &VRegs,
522 SmallVectorImpl<Register> &LeftoverRegs,
523 MachineIRBuilder &MIRBuilder,
524 MachineRegisterInfo &MRI) {
525 assert(!LeftoverTy.isValid() && "this is an out argument");
526
527 unsigned RegSize = RegTy.getSizeInBits();
528 unsigned MainSize = MainTy.getSizeInBits();
529 unsigned NumParts = RegSize / MainSize;
530 unsigned LeftoverSize = RegSize - NumParts * MainSize;
531
532 // Use an unmerge when possible.
533 if (LeftoverSize == 0) {
534 for (unsigned I = 0; I < NumParts; ++I)
535 VRegs.push_back(Elt: MRI.createGenericVirtualRegister(Ty: MainTy));
536 MIRBuilder.buildUnmerge(Res: VRegs, Op: Reg);
537 return true;
538 }
539
540 // Try to use unmerge for irregular vector split where possible
541 // For example when splitting a <6 x i32> into <4 x i32> with <2 x i32>
542 // leftover, it becomes:
543 // <2 x i32> %2, <2 x i32>%3, <2 x i32> %4 = G_UNMERGE_VALUE <6 x i32> %1
544 // <4 x i32> %5 = G_CONCAT_VECTOR <2 x i32> %2, <2 x i32> %3
545 if (RegTy.isVector() && MainTy.isVector()) {
546 unsigned RegNumElts = RegTy.getNumElements();
547 unsigned MainNumElts = MainTy.getNumElements();
548 unsigned LeftoverNumElts = RegNumElts % MainNumElts;
549 // If can unmerge to LeftoverTy, do it
550 if (MainNumElts % LeftoverNumElts == 0 &&
551 RegNumElts % LeftoverNumElts == 0 &&
552 RegTy.getScalarSizeInBits() == MainTy.getScalarSizeInBits() &&
553 LeftoverNumElts > 1) {
554 LeftoverTy = LLT::fixed_vector(NumElements: LeftoverNumElts, ScalarTy: RegTy.getElementType());
555
556 // Unmerge the SrcReg to LeftoverTy vectors
557 SmallVector<Register, 4> UnmergeValues;
558 extractParts(Reg, Ty: LeftoverTy, NumParts: RegNumElts / LeftoverNumElts, VRegs&: UnmergeValues,
559 MIRBuilder, MRI);
560
561 // Find how many LeftoverTy makes one MainTy
562 unsigned LeftoverPerMain = MainNumElts / LeftoverNumElts;
563 unsigned NumOfLeftoverVal =
564 ((RegNumElts % MainNumElts) / LeftoverNumElts);
565
566 // Create as many MainTy as possible using unmerged value
567 SmallVector<Register, 4> MergeValues;
568 for (unsigned I = 0; I < UnmergeValues.size() - NumOfLeftoverVal; I++) {
569 MergeValues.push_back(Elt: UnmergeValues[I]);
570 if (MergeValues.size() == LeftoverPerMain) {
571 VRegs.push_back(
572 Elt: MIRBuilder.buildMergeLikeInstr(Res: MainTy, Ops: MergeValues).getReg(Idx: 0));
573 MergeValues.clear();
574 }
575 }
576 // Populate LeftoverRegs with the leftovers
577 for (unsigned I = UnmergeValues.size() - NumOfLeftoverVal;
578 I < UnmergeValues.size(); I++) {
579 LeftoverRegs.push_back(Elt: UnmergeValues[I]);
580 }
581 return true;
582 }
583 }
584 // Perform irregular split. Leftover is last element of RegPieces.
585 if (MainTy.isVector()) {
586 SmallVector<Register, 8> RegPieces;
587 extractVectorParts(Reg, NumElts: MainTy.getNumElements(), VRegs&: RegPieces, MIRBuilder,
588 MRI);
589 for (unsigned i = 0; i < RegPieces.size() - 1; ++i)
590 VRegs.push_back(Elt: RegPieces[i]);
591 LeftoverRegs.push_back(Elt: RegPieces[RegPieces.size() - 1]);
592 LeftoverTy = MRI.getType(Reg: LeftoverRegs[0]);
593 return true;
594 }
595
596 LeftoverTy = LLT::integer(SizeInBits: LeftoverSize);
597 // For irregular sizes, extract the individual parts.
598 for (unsigned I = 0; I != NumParts; ++I) {
599 Register NewReg = MRI.createGenericVirtualRegister(Ty: MainTy);
600 VRegs.push_back(Elt: NewReg);
601 MIRBuilder.buildExtract(Res: NewReg, Src: Reg, Index: MainSize * I);
602 }
603
604 for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
605 Offset += LeftoverSize) {
606 Register NewReg = MRI.createGenericVirtualRegister(Ty: LeftoverTy);
607 LeftoverRegs.push_back(Elt: NewReg);
608 MIRBuilder.buildExtract(Res: NewReg, Src: Reg, Index: Offset);
609 }
610
611 return true;
612}
613
614void llvm::extractVectorParts(Register Reg, unsigned NumElts,
615 SmallVectorImpl<Register> &VRegs,
616 MachineIRBuilder &MIRBuilder,
617 MachineRegisterInfo &MRI) {
618 LLT RegTy = MRI.getType(Reg);
619 assert(RegTy.isVector() && "Expected a vector type");
620
621 LLT EltTy = RegTy.getElementType();
622 LLT NarrowTy = (NumElts == 1) ? EltTy : LLT::fixed_vector(NumElements: NumElts, ScalarTy: EltTy);
623 unsigned RegNumElts = RegTy.getNumElements();
624 unsigned LeftoverNumElts = RegNumElts % NumElts;
625 unsigned NumNarrowTyPieces = RegNumElts / NumElts;
626
627 // Perfect split without leftover
628 if (LeftoverNumElts == 0)
629 return extractParts(Reg, Ty: NarrowTy, NumParts: NumNarrowTyPieces, VRegs, MIRBuilder,
630 MRI);
631
632 // Irregular split. Provide direct access to all elements for artifact
633 // combiner using unmerge to elements. Then build vectors with NumElts
634 // elements. Remaining element(s) will be (used to build vector) Leftover.
635 SmallVector<Register, 8> Elts;
636 extractParts(Reg, Ty: EltTy, NumParts: RegNumElts, VRegs&: Elts, MIRBuilder, MRI);
637
638 unsigned Offset = 0;
639 // Requested sub-vectors of NarrowTy.
640 for (unsigned i = 0; i < NumNarrowTyPieces; ++i, Offset += NumElts) {
641 ArrayRef<Register> Pieces(&Elts[Offset], NumElts);
642 VRegs.push_back(Elt: MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: Pieces).getReg(Idx: 0));
643 }
644
645 // Leftover element(s).
646 if (LeftoverNumElts == 1) {
647 VRegs.push_back(Elt: Elts[Offset]);
648 } else {
649 LLT LeftoverTy = LLT::fixed_vector(NumElements: LeftoverNumElts, ScalarTy: EltTy);
650 ArrayRef<Register> Pieces(&Elts[Offset], LeftoverNumElts);
651 VRegs.push_back(
652 Elt: MIRBuilder.buildMergeLikeInstr(Res: LeftoverTy, Ops: Pieces).getReg(Idx: 0));
653 }
654}
655
656MachineInstr *llvm::getOpcodeDef(unsigned Opcode, Register Reg,
657 const MachineRegisterInfo &MRI) {
658 MachineInstr *DefMI = getDefIgnoringCopies(Reg, MRI);
659 return DefMI && DefMI->getOpcode() == Opcode ? DefMI : nullptr;
660}
661
662std::optional<APInt> llvm::ConstantFoldBinOp(unsigned Opcode,
663 const Register Op1,
664 const Register Op2,
665 const MachineRegisterInfo &MRI) {
666 auto MaybeOp2Cst = getAnyConstantVRegValWithLookThrough(VReg: Op2, MRI, LookThroughInstrs: false);
667 if (!MaybeOp2Cst)
668 return std::nullopt;
669
670 auto MaybeOp1Cst = getAnyConstantVRegValWithLookThrough(VReg: Op1, MRI, LookThroughInstrs: false);
671 if (!MaybeOp1Cst)
672 return std::nullopt;
673
674 const APInt &C1 = MaybeOp1Cst->Value;
675 const APInt &C2 = MaybeOp2Cst->Value;
676 switch (Opcode) {
677 default:
678 break;
679 case TargetOpcode::G_ADD:
680 return C1 + C2;
681 case TargetOpcode::G_PTR_ADD:
682 // Types can be of different width here.
683 // Result needs to be the same width as C1, so trunc or sext C2.
684 return C1 + C2.sextOrTrunc(width: C1.getBitWidth());
685 case TargetOpcode::G_AND:
686 return C1 & C2;
687 case TargetOpcode::G_ASHR:
688 return C1.ashr(ShiftAmt: C2);
689 case TargetOpcode::G_LSHR:
690 return C1.lshr(ShiftAmt: C2);
691 case TargetOpcode::G_MUL:
692 return C1 * C2;
693 case TargetOpcode::G_OR:
694 return C1 | C2;
695 case TargetOpcode::G_SHL:
696 return C1 << C2;
697 case TargetOpcode::G_SUB:
698 return C1 - C2;
699 case TargetOpcode::G_XOR:
700 return C1 ^ C2;
701 case TargetOpcode::G_UDIV:
702 if (!C2.getBoolValue())
703 break;
704 return C1.udiv(RHS: C2);
705 case TargetOpcode::G_SDIV:
706 if (!C2.getBoolValue())
707 break;
708 return C1.sdiv(RHS: C2);
709 case TargetOpcode::G_UREM:
710 if (!C2.getBoolValue())
711 break;
712 return C1.urem(RHS: C2);
713 case TargetOpcode::G_SREM:
714 if (!C2.getBoolValue())
715 break;
716 return C1.srem(RHS: C2);
717 case TargetOpcode::G_SMIN:
718 return APIntOps::smin(A: C1, B: C2);
719 case TargetOpcode::G_SMAX:
720 return APIntOps::smax(A: C1, B: C2);
721 case TargetOpcode::G_UMIN:
722 return APIntOps::umin(A: C1, B: C2);
723 case TargetOpcode::G_UMAX:
724 return APIntOps::umax(A: C1, B: C2);
725 }
726
727 return std::nullopt;
728}
729
730std::optional<APFloat>
731llvm::ConstantFoldFPBinOp(unsigned Opcode, const Register Op1,
732 const Register Op2, const MachineRegisterInfo &MRI) {
733 const ConstantFP *Op2Cst = getConstantFPVRegVal(VReg: Op2, MRI);
734 if (!Op2Cst)
735 return std::nullopt;
736
737 const ConstantFP *Op1Cst = getConstantFPVRegVal(VReg: Op1, MRI);
738 if (!Op1Cst)
739 return std::nullopt;
740
741 APFloat C1 = Op1Cst->getValueAPF();
742 const APFloat &C2 = Op2Cst->getValueAPF();
743 switch (Opcode) {
744 case TargetOpcode::G_FADD:
745 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
746 return C1;
747 case TargetOpcode::G_FSUB:
748 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
749 return C1;
750 case TargetOpcode::G_FMUL:
751 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
752 return C1;
753 case TargetOpcode::G_FDIV:
754 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
755 return C1;
756 case TargetOpcode::G_FREM:
757 C1.mod(RHS: C2);
758 return C1;
759 case TargetOpcode::G_FCOPYSIGN:
760 C1.copySign(RHS: C2);
761 return C1;
762 case TargetOpcode::G_FMINNUM:
763 return minnum(A: C1, B: C2);
764 case TargetOpcode::G_FMAXNUM:
765 return maxnum(A: C1, B: C2);
766 case TargetOpcode::G_FMINIMUM:
767 return minimum(A: C1, B: C2);
768 case TargetOpcode::G_FMAXIMUM:
769 return maximum(A: C1, B: C2);
770 case TargetOpcode::G_FMINIMUMNUM:
771 return minimumnum(A: C1, B: C2);
772 case TargetOpcode::G_FMAXIMUMNUM:
773 return maximumnum(A: C1, B: C2);
774 case TargetOpcode::G_FMINNUM_IEEE:
775 case TargetOpcode::G_FMAXNUM_IEEE:
776 // FIXME: These operations were unfortunately named. fminnum/fmaxnum do not
777 // follow the IEEE behavior for signaling nans and follow libm's fmin/fmax,
778 // and currently there isn't a nice wrapper in APFloat for the version with
779 // correct snan handling.
780 break;
781 default:
782 break;
783 }
784
785 return std::nullopt;
786}
787
788static GBuildVector *getBuildVectorLikeDef(Register Reg,
789 const MachineRegisterInfo &MRI) {
790 if (auto *BV = getOpcodeDef<GBuildVector>(Reg, MRI))
791 return BV;
792
793 auto *Bitcast = getOpcodeDef(Opcode: TargetOpcode::G_BITCAST, Reg, MRI);
794 if (!Bitcast)
795 return nullptr;
796
797 auto [Dst, DstTy, Src, SrcTy] = Bitcast->getFirst2RegLLTs();
798 if (!SrcTy.isVector() || !DstTy.isVector())
799 return nullptr;
800 if (SrcTy.getElementCount() != DstTy.getElementCount())
801 return nullptr;
802 if (SrcTy.getScalarSizeInBits() != DstTy.getScalarSizeInBits())
803 return nullptr;
804
805 return getOpcodeDef<GBuildVector>(Reg: Src, MRI);
806}
807
808SmallVector<APInt>
809llvm::ConstantFoldVectorBinop(unsigned Opcode, const Register Op1,
810 const Register Op2,
811 const MachineRegisterInfo &MRI) {
812 auto *SrcVec2 = getBuildVectorLikeDef(Reg: Op2, MRI);
813 if (!SrcVec2)
814 return SmallVector<APInt>();
815
816 auto *SrcVec1 = getBuildVectorLikeDef(Reg: Op1, MRI);
817 if (!SrcVec1)
818 return SmallVector<APInt>();
819
820 SmallVector<APInt> FoldedElements;
821 for (unsigned Idx = 0, E = SrcVec1->getNumSources(); Idx < E; ++Idx) {
822 auto MaybeCst = ConstantFoldBinOp(Opcode, Op1: SrcVec1->getSourceReg(I: Idx),
823 Op2: SrcVec2->getSourceReg(I: Idx), MRI);
824 if (!MaybeCst)
825 return SmallVector<APInt>();
826 FoldedElements.push_back(Elt: *MaybeCst);
827 }
828 return FoldedElements;
829}
830
831Align llvm::inferAlignFromPtrInfo(MachineFunction &MF,
832 const MachinePointerInfo &MPO) {
833 auto PSV = dyn_cast_if_present<const PseudoSourceValue *>(Val: MPO.V);
834 if (auto FSPV = dyn_cast_or_null<FixedStackPseudoSourceValue>(Val: PSV)) {
835 MachineFrameInfo &MFI = MF.getFrameInfo();
836 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FSPV->getFrameIndex()),
837 Offset: MPO.Offset);
838 }
839
840 if (const Value *V = dyn_cast_if_present<const Value *>(Val: MPO.V)) {
841 const Module *M = MF.getFunction().getParent();
842 return V->getPointerAlignment(DL: M->getDataLayout());
843 }
844
845 return Align(1);
846}
847
848Register llvm::getFunctionLiveInPhysReg(MachineFunction &MF,
849 const TargetInstrInfo &TII,
850 MCRegister PhysReg,
851 const TargetRegisterClass &RC,
852 const DebugLoc &DL, LLT RegTy) {
853 MachineBasicBlock &EntryMBB = MF.front();
854 MachineRegisterInfo &MRI = MF.getRegInfo();
855 Register LiveIn = MRI.getLiveInVirtReg(PReg: PhysReg);
856 if (LiveIn) {
857 MachineInstr *Def = MRI.getVRegDef(Reg: LiveIn);
858 if (Def) {
859 // FIXME: Should the verifier check this is in the entry block?
860 assert(Def->getParent() == &EntryMBB && "live-in copy not in entry block");
861 return LiveIn;
862 }
863
864 // It's possible the incoming argument register and copy was added during
865 // lowering, but later deleted due to being/becoming dead. If this happens,
866 // re-insert the copy.
867 } else {
868 // The live in register was not present, so add it.
869 LiveIn = MF.addLiveIn(PReg: PhysReg, RC: &RC);
870 if (RegTy.isValid())
871 MRI.setType(VReg: LiveIn, Ty: RegTy);
872 }
873
874 BuildMI(BB&: EntryMBB, I: EntryMBB.begin(), MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: LiveIn)
875 .addReg(RegNo: PhysReg);
876 if (!EntryMBB.isLiveIn(Reg: PhysReg))
877 EntryMBB.addLiveIn(PhysReg);
878 return LiveIn;
879}
880
881std::optional<APInt> llvm::ConstantFoldExtOp(unsigned Opcode,
882 const Register Op1, uint64_t Imm,
883 const MachineRegisterInfo &MRI) {
884 auto MaybeOp1Cst = getIConstantVRegVal(VReg: Op1, MRI);
885 if (MaybeOp1Cst) {
886 switch (Opcode) {
887 default:
888 break;
889 case TargetOpcode::G_SEXT_INREG: {
890 LLT Ty = MRI.getType(Reg: Op1);
891 return MaybeOp1Cst->trunc(width: Imm).sext(width: Ty.getScalarSizeInBits());
892 }
893 }
894 }
895 return std::nullopt;
896}
897
898std::optional<APInt> llvm::ConstantFoldCastOp(unsigned Opcode, LLT DstTy,
899 const Register Op0,
900 const MachineRegisterInfo &MRI) {
901 std::optional<APInt> Val = getIConstantVRegVal(VReg: Op0, MRI);
902 if (!Val)
903 return Val;
904
905 const unsigned DstSize = DstTy.getScalarSizeInBits();
906
907 switch (Opcode) {
908 case TargetOpcode::G_SEXT:
909 return Val->sext(width: DstSize);
910 case TargetOpcode::G_ZEXT:
911 case TargetOpcode::G_ANYEXT:
912 // TODO: DAG considers target preference when constant folding any_extend.
913 return Val->zext(width: DstSize);
914 default:
915 break;
916 }
917
918 llvm_unreachable("unexpected cast opcode to constant fold");
919}
920
921std::optional<APFloat>
922llvm::ConstantFoldIntToFloat(unsigned Opcode, LLT DstTy, Register Src,
923 const MachineRegisterInfo &MRI) {
924 assert(Opcode == TargetOpcode::G_SITOFP || Opcode == TargetOpcode::G_UITOFP);
925 if (auto MaybeSrcVal = getIConstantVRegVal(VReg: Src, MRI)) {
926 APFloat DstVal(getFltSemanticForLLT(Ty: DstTy));
927 DstVal.convertFromAPInt(Input: *MaybeSrcVal, IsSigned: Opcode == TargetOpcode::G_SITOFP,
928 RM: APFloat::rmNearestTiesToEven);
929 return DstVal;
930 }
931 return std::nullopt;
932}
933
934SmallVector<APInt>
935llvm::ConstantFoldUnaryIntOp(unsigned Opcode, LLT DstTy, Register Src,
936 const MachineRegisterInfo &MRI) {
937 unsigned EltBits = DstTy.getScalarSizeInBits();
938 auto Fold = [Opcode, EltBits](const APInt &V) -> APInt {
939 switch (Opcode) {
940 case TargetOpcode::G_CTLZ:
941 case TargetOpcode::G_CTLZ_ZERO_POISON:
942 return APInt(EltBits, V.countl_zero());
943 case TargetOpcode::G_CTTZ:
944 case TargetOpcode::G_CTTZ_ZERO_POISON:
945 return APInt(EltBits, V.countr_zero());
946 case TargetOpcode::G_CTPOP:
947 return APInt(EltBits, V.popcount());
948 case TargetOpcode::G_ABS:
949 return V.abs();
950 case TargetOpcode::G_BSWAP:
951 return V.byteSwap();
952 case TargetOpcode::G_BITREVERSE:
953 return V.reverseBits();
954 }
955 llvm_unreachable("unexpected opcode in ConstantFoldUnaryIntOp");
956 };
957
958 auto tryFoldScalar = [&](Register R) -> std::optional<APInt> {
959 if (auto MaybeCst = getIConstantVRegVal(VReg: R, MRI))
960 return Fold(*MaybeCst);
961 return std::nullopt;
962 };
963 if (MRI.getType(Reg: Src).isVector()) {
964 auto *BV = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
965 if (!BV)
966 return {};
967 SmallVector<APInt> Folded;
968 for (unsigned SrcIdx = 0; SrcIdx < BV->getNumSources(); ++SrcIdx) {
969 if (auto MaybeFold = tryFoldScalar(BV->getSourceReg(I: SrcIdx))) {
970 Folded.emplace_back(Args: std::move(*MaybeFold));
971 continue;
972 }
973 return {};
974 }
975 return Folded;
976 }
977 if (auto MaybeCst = tryFoldScalar(Src))
978 return {std::move(*MaybeCst)};
979 return {};
980}
981
982std::optional<SmallVector<APInt>>
983llvm::ConstantFoldICmp(unsigned Pred, const Register Op1, const Register Op2,
984 unsigned DstScalarSizeInBits, unsigned ExtOp,
985 const MachineRegisterInfo &MRI) {
986 assert(ExtOp == TargetOpcode::G_SEXT || ExtOp == TargetOpcode::G_ZEXT ||
987 ExtOp == TargetOpcode::G_ANYEXT);
988
989 const LLT Ty = MRI.getType(Reg: Op1);
990
991 auto GetICmpResultCst = [&](bool IsTrue) {
992 if (IsTrue)
993 return ExtOp == TargetOpcode::G_SEXT
994 ? APInt::getAllOnes(numBits: DstScalarSizeInBits)
995 : APInt::getOneBitSet(numBits: DstScalarSizeInBits, BitNo: 0);
996 return APInt::getZero(numBits: DstScalarSizeInBits);
997 };
998
999 auto TryFoldScalar = [&](Register LHS, Register RHS) -> std::optional<APInt> {
1000 auto RHSCst = getIConstantVRegVal(VReg: RHS, MRI);
1001 if (!RHSCst)
1002 return std::nullopt;
1003 auto LHSCst = getIConstantVRegVal(VReg: LHS, MRI);
1004 if (!LHSCst)
1005 return std::nullopt;
1006
1007 switch (Pred) {
1008 case CmpInst::Predicate::ICMP_EQ:
1009 return GetICmpResultCst(LHSCst->eq(RHS: *RHSCst));
1010 case CmpInst::Predicate::ICMP_NE:
1011 return GetICmpResultCst(LHSCst->ne(RHS: *RHSCst));
1012 case CmpInst::Predicate::ICMP_UGT:
1013 return GetICmpResultCst(LHSCst->ugt(RHS: *RHSCst));
1014 case CmpInst::Predicate::ICMP_UGE:
1015 return GetICmpResultCst(LHSCst->uge(RHS: *RHSCst));
1016 case CmpInst::Predicate::ICMP_ULT:
1017 return GetICmpResultCst(LHSCst->ult(RHS: *RHSCst));
1018 case CmpInst::Predicate::ICMP_ULE:
1019 return GetICmpResultCst(LHSCst->ule(RHS: *RHSCst));
1020 case CmpInst::Predicate::ICMP_SGT:
1021 return GetICmpResultCst(LHSCst->sgt(RHS: *RHSCst));
1022 case CmpInst::Predicate::ICMP_SGE:
1023 return GetICmpResultCst(LHSCst->sge(RHS: *RHSCst));
1024 case CmpInst::Predicate::ICMP_SLT:
1025 return GetICmpResultCst(LHSCst->slt(RHS: *RHSCst));
1026 case CmpInst::Predicate::ICMP_SLE:
1027 return GetICmpResultCst(LHSCst->sle(RHS: *RHSCst));
1028 default:
1029 return std::nullopt;
1030 }
1031 };
1032
1033 SmallVector<APInt> FoldedICmps;
1034
1035 if (Ty.isVector()) {
1036 // Try to constant fold each element.
1037 auto *BV1 = getOpcodeDef<GBuildVector>(Reg: Op1, MRI);
1038 auto *BV2 = getOpcodeDef<GBuildVector>(Reg: Op2, MRI);
1039 if (!BV1 || !BV2)
1040 return std::nullopt;
1041 assert(BV1->getNumSources() == BV2->getNumSources() && "Invalid vectors");
1042 for (unsigned I = 0; I < BV1->getNumSources(); ++I) {
1043 if (auto MaybeFold =
1044 TryFoldScalar(BV1->getSourceReg(I), BV2->getSourceReg(I))) {
1045 FoldedICmps.emplace_back(Args&: *MaybeFold);
1046 continue;
1047 }
1048 return std::nullopt;
1049 }
1050 return FoldedICmps;
1051 }
1052
1053 if (auto MaybeCst = TryFoldScalar(Op1, Op2)) {
1054 FoldedICmps.emplace_back(Args&: *MaybeCst);
1055 return FoldedICmps;
1056 }
1057
1058 return std::nullopt;
1059}
1060
1061bool llvm::isKnownToBeAPowerOfTwo(Register Reg, const MachineRegisterInfo &MRI,
1062 GISelValueTracking *VT, bool OrNegative) {
1063 std::optional<DefinitionAndSourceRegister> DefSrcReg =
1064 getDefSrcRegIgnoringCopies(Reg, MRI);
1065 if (!DefSrcReg)
1066 return false;
1067
1068 const MachineInstr &MI = *DefSrcReg->MI;
1069 const LLT Ty = MRI.getType(Reg);
1070
1071 auto IsPow2 = [OrNegative](const APInt &V) {
1072 return V.isPowerOf2() || (OrNegative && V.isNegatedPowerOf2());
1073 };
1074
1075 switch (MI.getOpcode()) {
1076 case TargetOpcode::G_CONSTANT: {
1077 unsigned BitWidth = Ty.getScalarSizeInBits();
1078 const ConstantInt *CI = MI.getOperand(i: 1).getCImm();
1079 return IsPow2(CI->getValue().zextOrTrunc(width: BitWidth));
1080 }
1081 case TargetOpcode::G_SHL: {
1082 // A left-shift of a constant one will have exactly one bit set because
1083 // shifting the bit off the end is undefined.
1084
1085 // TODO: Constant splat
1086 if (auto ConstLHS = getIConstantVRegVal(VReg: MI.getOperand(i: 1).getReg(), MRI)) {
1087 if (*ConstLHS == 1)
1088 return true;
1089 }
1090
1091 break;
1092 }
1093 case TargetOpcode::G_LSHR: {
1094 if (auto ConstLHS = getIConstantVRegVal(VReg: MI.getOperand(i: 1).getReg(), MRI)) {
1095 if (ConstLHS->isSignMask())
1096 return true;
1097 }
1098
1099 break;
1100 }
1101 case TargetOpcode::G_BUILD_VECTOR: {
1102 // TODO: Probably should have a recursion depth guard since you could have
1103 // bitcasted vector elements.
1104 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands()))
1105 if (!isKnownToBeAPowerOfTwo(Reg: MO.getReg(), MRI, VT, OrNegative))
1106 return false;
1107
1108 return true;
1109 }
1110 case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1111 // Only handle constants since we would need to know if number of leading
1112 // zeros is greater than the truncation amount.
1113 const unsigned BitWidth = Ty.getScalarSizeInBits();
1114 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
1115 auto Const = getIConstantVRegVal(VReg: MO.getReg(), MRI);
1116 if (!Const || !IsPow2(Const->zextOrTrunc(width: BitWidth)))
1117 return false;
1118 }
1119
1120 return true;
1121 }
1122 default:
1123 break;
1124 }
1125
1126 if (!VT)
1127 return false;
1128
1129 // More could be done here, though the above checks are enough
1130 // to handle some common cases.
1131
1132 // Fall back to computeKnownBits to catch other known cases.
1133 KnownBits Known = VT->getKnownBits(R: Reg);
1134 return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);
1135}
1136
1137void llvm::getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU) {
1138 AU.addPreserved<StackProtector>();
1139}
1140
1141LLT llvm::getLCMType(LLT OrigTy, LLT TargetTy) {
1142 if (OrigTy.getSizeInBits() == TargetTy.getSizeInBits())
1143 return OrigTy;
1144
1145 if (OrigTy.isVector() && TargetTy.isVector()) {
1146 LLT OrigElt = OrigTy.getElementType();
1147 LLT TargetElt = TargetTy.getElementType();
1148
1149 // TODO: The docstring for this function says the intention is to use this
1150 // function to build MERGE/UNMERGE instructions. It won't be the case that
1151 // we generate a MERGE/UNMERGE between fixed and scalable vector types. We
1152 // could implement getLCMType between the two in the future if there was a
1153 // need, but it is not worth it now as this function should not be used in
1154 // that way.
1155 assert(((OrigTy.isScalableVector() && !TargetTy.isFixedVector()) ||
1156 (OrigTy.isFixedVector() && !TargetTy.isScalableVector())) &&
1157 "getLCMType not implemented between fixed and scalable vectors.");
1158
1159 if (OrigElt.getSizeInBits() == TargetElt.getSizeInBits()) {
1160 int GCDMinElts = std::gcd(m: OrigTy.getElementCount().getKnownMinValue(),
1161 n: TargetTy.getElementCount().getKnownMinValue());
1162 // Prefer the original element type.
1163 ElementCount Mul = OrigTy.getElementCount() *
1164 TargetTy.getElementCount().getKnownMinValue();
1165 return LLT::vector(EC: Mul.divideCoefficientBy(RHS: GCDMinElts),
1166 ScalarTy: OrigTy.getElementType());
1167 }
1168 unsigned LCM = std::lcm(m: OrigTy.getSizeInBits().getKnownMinValue(),
1169 n: TargetTy.getSizeInBits().getKnownMinValue());
1170 return LLT::vector(
1171 EC: ElementCount::get(MinVal: LCM / OrigElt.getSizeInBits(), Scalable: OrigTy.isScalable()),
1172 ScalarTy: OrigElt);
1173 }
1174
1175 // One type is scalar, one type is vector
1176 if (OrigTy.isVector() || TargetTy.isVector()) {
1177 LLT VecTy = OrigTy.isVector() ? OrigTy : TargetTy;
1178 LLT ScalarTy = OrigTy.isVector() ? TargetTy : OrigTy;
1179 LLT EltTy = VecTy.getElementType();
1180 LLT OrigEltTy = OrigTy.isVector() ? OrigTy.getElementType() : OrigTy;
1181
1182 // Prefer scalar type from OrigTy.
1183 if (EltTy.getSizeInBits() == ScalarTy.getSizeInBits())
1184 return LLT::vector(EC: VecTy.getElementCount(), ScalarTy: OrigEltTy);
1185
1186 // Different size scalars. Create vector with the same total size.
1187 // LCM will take fixed/scalable from VecTy.
1188 unsigned LCM = std::lcm(m: EltTy.getSizeInBits().getFixedValue() *
1189 VecTy.getElementCount().getKnownMinValue(),
1190 n: ScalarTy.getSizeInBits().getFixedValue());
1191 // Prefer type from OrigTy
1192 return LLT::vector(EC: ElementCount::get(MinVal: LCM / OrigEltTy.getSizeInBits(),
1193 Scalable: VecTy.getElementCount().isScalable()),
1194 ScalarTy: OrigEltTy);
1195 }
1196
1197 // At this point, both types are scalars of different size
1198 unsigned LCM = std::lcm(m: OrigTy.getSizeInBits().getFixedValue(),
1199 n: TargetTy.getSizeInBits().getFixedValue());
1200 // Preserve pointer types.
1201 if (LCM == OrigTy.getSizeInBits())
1202 return OrigTy;
1203 if (LCM == TargetTy.getSizeInBits())
1204 return TargetTy;
1205 return LLT::scalar(SizeInBits: LCM);
1206}
1207
1208LLT llvm::getCoverTy(LLT OrigTy, LLT TargetTy) {
1209
1210 if ((OrigTy.isScalableVector() && TargetTy.isFixedVector()) ||
1211 (OrigTy.isFixedVector() && TargetTy.isScalableVector()))
1212 llvm_unreachable(
1213 "getCoverTy not implemented between fixed and scalable vectors.");
1214
1215 if (!OrigTy.isVector() || !TargetTy.isVector() || OrigTy == TargetTy ||
1216 (OrigTy.getScalarSizeInBits() != TargetTy.getScalarSizeInBits()))
1217 return getLCMType(OrigTy, TargetTy);
1218
1219 unsigned OrigTyNumElts = OrigTy.getElementCount().getKnownMinValue();
1220 unsigned TargetTyNumElts = TargetTy.getElementCount().getKnownMinValue();
1221 if (OrigTyNumElts % TargetTyNumElts == 0)
1222 return OrigTy;
1223
1224 unsigned NumElts = alignTo(Value: OrigTyNumElts, Align: TargetTyNumElts);
1225 return LLT::scalarOrVector(EC: ElementCount::getFixed(MinVal: NumElts),
1226 ScalarTy: OrigTy.getElementType());
1227}
1228
1229LLT llvm::getGCDType(LLT OrigTy, LLT TargetTy) {
1230 if (OrigTy.getSizeInBits() == TargetTy.getSizeInBits())
1231 return OrigTy;
1232
1233 if (OrigTy.isVector() && TargetTy.isVector()) {
1234 LLT OrigElt = OrigTy.getElementType();
1235
1236 // TODO: The docstring for this function says the intention is to use this
1237 // function to build MERGE/UNMERGE instructions. It won't be the case that
1238 // we generate a MERGE/UNMERGE between fixed and scalable vector types. We
1239 // could implement getGCDType between the two in the future if there was a
1240 // need, but it is not worth it now as this function should not be used in
1241 // that way.
1242 assert(((OrigTy.isScalableVector() && !TargetTy.isFixedVector()) ||
1243 (OrigTy.isFixedVector() && !TargetTy.isScalableVector())) &&
1244 "getGCDType not implemented between fixed and scalable vectors.");
1245
1246 unsigned GCD = std::gcd(m: OrigTy.getSizeInBits().getKnownMinValue(),
1247 n: TargetTy.getSizeInBits().getKnownMinValue());
1248 if (GCD == OrigElt.getSizeInBits())
1249 return LLT::scalarOrVector(EC: ElementCount::get(MinVal: 1, Scalable: OrigTy.isScalable()),
1250 ScalarTy: OrigElt);
1251
1252 // Cannot produce original element type, but both have vscale in common.
1253 if (GCD < OrigElt.getSizeInBits())
1254 return LLT::scalarOrVector(EC: ElementCount::get(MinVal: 1, Scalable: OrigTy.isScalable()),
1255 ScalarSize: GCD);
1256
1257 return LLT::vector(
1258 EC: ElementCount::get(MinVal: GCD / OrigElt.getSizeInBits().getFixedValue(),
1259 Scalable: OrigTy.isScalable()),
1260 ScalarTy: OrigElt);
1261 }
1262
1263 // If one type is vector and the element size matches the scalar size, then
1264 // the gcd is the scalar type.
1265 if (OrigTy.isVector() &&
1266 OrigTy.getElementType().getSizeInBits() == TargetTy.getSizeInBits())
1267 return OrigTy.getElementType();
1268 if (TargetTy.isVector() &&
1269 TargetTy.getElementType().getSizeInBits() == OrigTy.getSizeInBits())
1270 return OrigTy;
1271
1272 // At this point, both types are either scalars of different type or one is a
1273 // vector and one is a scalar. If both types are scalars, the GCD type is the
1274 // GCD between the two scalar sizes. If one is vector and one is scalar, then
1275 // the GCD type is the GCD between the scalar and the vector element size.
1276 LLT OrigScalar = OrigTy.getScalarType();
1277 LLT TargetScalar = TargetTy.getScalarType();
1278 unsigned GCD = std::gcd(m: OrigScalar.getSizeInBits().getFixedValue(),
1279 n: TargetScalar.getSizeInBits().getFixedValue());
1280 return LLT::integer(SizeInBits: GCD);
1281}
1282
1283std::optional<int> llvm::getSplatIndex(MachineInstr &MI) {
1284 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
1285 "Only G_SHUFFLE_VECTOR can have a splat index!");
1286 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
1287 auto FirstDefinedIdx = find_if(Range&: Mask, P: [](int Elt) { return Elt >= 0; });
1288
1289 // If all elements are undefined, this shuffle can be considered a splat.
1290 // Return 0 for better potential for callers to simplify.
1291 if (FirstDefinedIdx == Mask.end())
1292 return 0;
1293
1294 // Make sure all remaining elements are either undef or the same
1295 // as the first non-undef value.
1296 int SplatValue = *FirstDefinedIdx;
1297 if (any_of(Range: make_range(x: std::next(x: FirstDefinedIdx), y: Mask.end()),
1298 P: [&SplatValue](int Elt) { return Elt >= 0 && Elt != SplatValue; }))
1299 return std::nullopt;
1300
1301 return SplatValue;
1302}
1303
1304static bool isBuildVectorOp(unsigned Opcode) {
1305 return Opcode == TargetOpcode::G_BUILD_VECTOR ||
1306 Opcode == TargetOpcode::G_BUILD_VECTOR_TRUNC;
1307}
1308
1309namespace {
1310
1311std::optional<ValueAndVReg> getAnyConstantSplat(Register VReg,
1312 const MachineRegisterInfo &MRI,
1313 bool AllowUndef) {
1314 MachineInstr *MI = getDefIgnoringCopies(Reg: VReg, MRI);
1315 if (!MI)
1316 return std::nullopt;
1317
1318 bool isConcatVectorsOp = MI->getOpcode() == TargetOpcode::G_CONCAT_VECTORS;
1319 if (!isBuildVectorOp(Opcode: MI->getOpcode()) && !isConcatVectorsOp)
1320 return std::nullopt;
1321
1322 std::optional<ValueAndVReg> SplatValAndReg;
1323 for (MachineOperand &Op : MI->uses()) {
1324 Register Element = Op.getReg();
1325 // If we have a G_CONCAT_VECTOR, we recursively look into the
1326 // vectors that we're concatenating to see if they're splats.
1327 auto ElementValAndReg =
1328 isConcatVectorsOp
1329 ? getAnyConstantSplat(VReg: Element, MRI, AllowUndef)
1330 : getAnyConstantVRegValWithLookThrough(VReg: Element, MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
1331
1332 // If AllowUndef, treat undef as value that will result in a constant splat.
1333 if (!ElementValAndReg) {
1334 if (AllowUndef && mi_match(R: Element, MRI, P: m_GImplicitDef()))
1335 continue;
1336 return std::nullopt;
1337 }
1338
1339 // Record splat value
1340 if (!SplatValAndReg)
1341 SplatValAndReg = ElementValAndReg;
1342
1343 // Different constant than the one already recorded, not a constant splat.
1344 if (SplatValAndReg->Value != ElementValAndReg->Value)
1345 return std::nullopt;
1346 }
1347
1348 return SplatValAndReg;
1349}
1350
1351} // end anonymous namespace
1352
1353bool llvm::isBuildVectorConstantSplat(const Register Reg,
1354 const MachineRegisterInfo &MRI,
1355 int64_t SplatValue, bool AllowUndef) {
1356 if (auto SplatValAndReg = getAnyConstantSplat(VReg: Reg, MRI, AllowUndef))
1357 return SplatValAndReg->Value.getSExtValue() == SplatValue;
1358
1359 return false;
1360}
1361
1362bool llvm::isBuildVectorConstantSplat(const Register Reg,
1363 const MachineRegisterInfo &MRI,
1364 const APInt &SplatValue,
1365 bool AllowUndef) {
1366 if (auto SplatValAndReg = getAnyConstantSplat(VReg: Reg, MRI, AllowUndef)) {
1367 if (SplatValAndReg->Value.getBitWidth() < SplatValue.getBitWidth())
1368 return APInt::isSameValue(
1369 I1: SplatValAndReg->Value.sext(width: SplatValue.getBitWidth()), I2: SplatValue);
1370 return APInt::isSameValue(
1371 I1: SplatValAndReg->Value,
1372 I2: SplatValue.sext(width: SplatValAndReg->Value.getBitWidth()));
1373 }
1374
1375 return false;
1376}
1377
1378bool llvm::isBuildVectorConstantSplat(const MachineInstr &MI,
1379 const MachineRegisterInfo &MRI,
1380 int64_t SplatValue, bool AllowUndef) {
1381 return isBuildVectorConstantSplat(Reg: MI.getOperand(i: 0).getReg(), MRI, SplatValue,
1382 AllowUndef);
1383}
1384
1385bool llvm::isBuildVectorConstantSplat(const MachineInstr &MI,
1386 const MachineRegisterInfo &MRI,
1387 const APInt &SplatValue,
1388 bool AllowUndef) {
1389 return isBuildVectorConstantSplat(Reg: MI.getOperand(i: 0).getReg(), MRI, SplatValue,
1390 AllowUndef);
1391}
1392
1393std::optional<APInt>
1394llvm::getIConstantSplatVal(const Register Reg, const MachineRegisterInfo &MRI) {
1395 if (auto SplatValAndReg =
1396 getAnyConstantSplat(VReg: Reg, MRI, /* AllowUndef */ false)) {
1397 if (std::optional<ValueAndVReg> ValAndVReg =
1398 getIConstantVRegValWithLookThrough(VReg: SplatValAndReg->VReg, MRI))
1399 return ValAndVReg->Value;
1400 }
1401
1402 return std::nullopt;
1403}
1404
1405std::optional<APInt>
1406llvm::getIConstantSplatVal(const MachineInstr &MI,
1407 const MachineRegisterInfo &MRI) {
1408 return getIConstantSplatVal(Reg: MI.getOperand(i: 0).getReg(), MRI);
1409}
1410
1411std::optional<int64_t>
1412llvm::getIConstantSplatSExtVal(const Register Reg,
1413 const MachineRegisterInfo &MRI) {
1414 if (auto SplatValAndReg =
1415 getAnyConstantSplat(VReg: Reg, MRI, /* AllowUndef */ false))
1416 return getIConstantVRegSExtVal(VReg: SplatValAndReg->VReg, MRI);
1417 return std::nullopt;
1418}
1419
1420std::optional<int64_t>
1421llvm::getIConstantSplatSExtVal(const MachineInstr &MI,
1422 const MachineRegisterInfo &MRI) {
1423 return getIConstantSplatSExtVal(Reg: MI.getOperand(i: 0).getReg(), MRI);
1424}
1425
1426std::optional<FPValueAndVReg>
1427llvm::getFConstantSplat(Register VReg, const MachineRegisterInfo &MRI,
1428 bool AllowUndef) {
1429 if (auto SplatValAndReg = getAnyConstantSplat(VReg, MRI, AllowUndef))
1430 return getFConstantVRegValWithLookThrough(VReg: SplatValAndReg->VReg, MRI);
1431 return std::nullopt;
1432}
1433
1434bool llvm::isBuildVectorAllZeros(const MachineInstr &MI,
1435 const MachineRegisterInfo &MRI,
1436 bool AllowUndef) {
1437 return isBuildVectorConstantSplat(MI, MRI, SplatValue: 0, AllowUndef);
1438}
1439
1440bool llvm::isBuildVectorAllOnes(const MachineInstr &MI,
1441 const MachineRegisterInfo &MRI,
1442 bool AllowUndef) {
1443 return isBuildVectorConstantSplat(MI, MRI, SplatValue: -1, AllowUndef);
1444}
1445
1446std::optional<Register> llvm::getVectorSplat(const MachineInstr &MI,
1447 const MachineRegisterInfo &MRI) {
1448 unsigned Opc = MI.getOpcode();
1449 if (Opc == TargetOpcode::G_SPLAT_VECTOR)
1450 return MI.getOperand(i: 1).getReg();
1451
1452 if (!isBuildVectorOp(Opcode: Opc))
1453 return std::nullopt;
1454 auto Reg = MI.getOperand(i: 1).getReg();
1455 if (any_of(Range: drop_begin(RangeOrContainer: MI.operands(), N: 2),
1456 P: [&Reg](const MachineOperand &Op) { return Op.getReg() != Reg; }))
1457 return std::nullopt;
1458 return Reg;
1459}
1460
1461static bool isConstantScalar(const MachineInstr &MI,
1462 const MachineRegisterInfo &MRI,
1463 bool AllowFP = true,
1464 bool AllowOpaqueConstants = true) {
1465 switch (MI.getOpcode()) {
1466 case TargetOpcode::G_CONSTANT:
1467 case TargetOpcode::G_IMPLICIT_DEF:
1468 return true;
1469 case TargetOpcode::G_FCONSTANT:
1470 return AllowFP;
1471 case TargetOpcode::G_GLOBAL_VALUE:
1472 case TargetOpcode::G_FRAME_INDEX:
1473 case TargetOpcode::G_BLOCK_ADDR:
1474 case TargetOpcode::G_JUMP_TABLE:
1475 return AllowOpaqueConstants;
1476 default:
1477 return false;
1478 }
1479}
1480
1481bool llvm::isConstantOrConstantVector(MachineInstr &MI,
1482 const MachineRegisterInfo &MRI) {
1483 Register Def = MI.getOperand(i: 0).getReg();
1484 if (auto C = getIConstantVRegValWithLookThrough(VReg: Def, MRI))
1485 return true;
1486 GBuildVector *BV = dyn_cast<GBuildVector>(Val: &MI);
1487 if (!BV)
1488 return false;
1489 for (unsigned SrcIdx = 0; SrcIdx < BV->getNumSources(); ++SrcIdx) {
1490 if (getIConstantVRegValWithLookThrough(VReg: BV->getSourceReg(I: SrcIdx), MRI) ||
1491 getOpcodeDef<GImplicitDef>(Reg: BV->getSourceReg(I: SrcIdx), MRI))
1492 continue;
1493 return false;
1494 }
1495 return true;
1496}
1497
1498bool llvm::isConstantOrConstantVector(const MachineInstr &MI,
1499 const MachineRegisterInfo &MRI,
1500 bool AllowFP, bool AllowOpaqueConstants) {
1501 if (isConstantScalar(MI, MRI, AllowFP, AllowOpaqueConstants))
1502 return true;
1503
1504 if (!isBuildVectorOp(Opcode: MI.getOpcode()))
1505 return false;
1506
1507 const unsigned NumOps = MI.getNumOperands();
1508 for (unsigned I = 1; I != NumOps; ++I) {
1509 const MachineInstr *ElementDef = MRI.getVRegDef(Reg: MI.getOperand(i: I).getReg());
1510 if (!isConstantScalar(MI: *ElementDef, MRI, AllowFP, AllowOpaqueConstants))
1511 return false;
1512 }
1513
1514 return true;
1515}
1516
1517std::optional<APInt>
1518llvm::isConstantOrConstantSplatVector(Register Def,
1519 const MachineRegisterInfo &MRI) {
1520 if (auto C = getIConstantVRegValWithLookThrough(VReg: Def, MRI))
1521 return C->Value;
1522 auto MaybeCst = getIConstantSplatSExtVal(Reg: Def, MRI);
1523 if (!MaybeCst)
1524 return std::nullopt;
1525 const unsigned ScalarSize = MRI.getType(Reg: Def).getScalarSizeInBits();
1526 return APInt(ScalarSize, *MaybeCst, true);
1527}
1528
1529std::optional<APFloat>
1530llvm::isConstantOrConstantSplatVectorFP(Register Def,
1531 const MachineRegisterInfo &MRI) {
1532 if (auto FpConst = getFConstantVRegValWithLookThrough(VReg: Def, MRI))
1533 return FpConst->Value;
1534 auto MaybeCstFP = getFConstantSplat(VReg: Def, MRI, /*allowUndef=*/AllowUndef: false);
1535 if (!MaybeCstFP)
1536 return std::nullopt;
1537 return MaybeCstFP->Value;
1538}
1539
1540bool llvm::isNullOrNullSplat(const MachineInstr &MI,
1541 const MachineRegisterInfo &MRI, bool AllowUndefs) {
1542 switch (MI.getOpcode()) {
1543 case TargetOpcode::G_IMPLICIT_DEF:
1544 return AllowUndefs;
1545 case TargetOpcode::G_CONSTANT:
1546 return MI.getOperand(i: 1).getCImm()->isNullValue();
1547 case TargetOpcode::G_FCONSTANT: {
1548 const ConstantFP *FPImm = MI.getOperand(i: 1).getFPImm();
1549 return FPImm->isZero() && !FPImm->isNegative();
1550 }
1551 default:
1552 if (!AllowUndefs) // TODO: isBuildVectorAllZeros assumes undef is OK already
1553 return false;
1554 return isBuildVectorAllZeros(MI, MRI);
1555 }
1556}
1557
1558bool llvm::isAllOnesOrAllOnesSplat(const MachineInstr &MI,
1559 const MachineRegisterInfo &MRI,
1560 bool AllowUndefs) {
1561 switch (MI.getOpcode()) {
1562 case TargetOpcode::G_IMPLICIT_DEF:
1563 return AllowUndefs;
1564 case TargetOpcode::G_CONSTANT:
1565 return MI.getOperand(i: 1).getCImm()->isAllOnesValue();
1566 default:
1567 if (!AllowUndefs) // TODO: isBuildVectorAllOnes assumes undef is OK already
1568 return false;
1569 return isBuildVectorAllOnes(MI, MRI);
1570 }
1571}
1572
1573bool llvm::matchUnaryPredicate(
1574 const MachineRegisterInfo &MRI, Register Reg,
1575 llvm::function_ref<bool(const Constant *ConstVal)> Match,
1576 bool AllowUndefs) {
1577
1578 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);
1579 if (AllowUndefs && Def->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1580 return Match(nullptr);
1581
1582 // TODO: Also handle fconstant
1583 if (Def->getOpcode() == TargetOpcode::G_CONSTANT)
1584 return Match(Def->getOperand(i: 1).getCImm());
1585
1586 if (Def->getOpcode() != TargetOpcode::G_BUILD_VECTOR)
1587 return false;
1588
1589 for (unsigned I = 1, E = Def->getNumOperands(); I != E; ++I) {
1590 Register SrcElt = Def->getOperand(i: I).getReg();
1591 const MachineInstr *SrcDef = getDefIgnoringCopies(Reg: SrcElt, MRI);
1592 if (AllowUndefs && SrcDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) {
1593 if (!Match(nullptr))
1594 return false;
1595 continue;
1596 }
1597
1598 if (SrcDef->getOpcode() != TargetOpcode::G_CONSTANT ||
1599 !Match(SrcDef->getOperand(i: 1).getCImm()))
1600 return false;
1601 }
1602
1603 return true;
1604}
1605
1606bool llvm::isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector,
1607 bool IsFP) {
1608 switch (TLI.getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
1609 case TargetLowering::UndefinedBooleanContent:
1610 return Val & 0x1;
1611 case TargetLowering::ZeroOrOneBooleanContent:
1612 return Val == 1;
1613 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1614 return Val == -1;
1615 }
1616 llvm_unreachable("Invalid boolean contents");
1617}
1618
1619bool llvm::isConstFalseVal(const TargetLowering &TLI, int64_t Val,
1620 bool IsVector, bool IsFP) {
1621 switch (TLI.getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
1622 case TargetLowering::UndefinedBooleanContent:
1623 return ~Val & 0x1;
1624 case TargetLowering::ZeroOrOneBooleanContent:
1625 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1626 return Val == 0;
1627 }
1628 llvm_unreachable("Invalid boolean contents");
1629}
1630
1631int64_t llvm::getICmpTrueVal(const TargetLowering &TLI, bool IsVector,
1632 bool IsFP) {
1633 switch (TLI.getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
1634 case TargetLowering::UndefinedBooleanContent:
1635 case TargetLowering::ZeroOrOneBooleanContent:
1636 return 1;
1637 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1638 return -1;
1639 }
1640 llvm_unreachable("Invalid boolean contents");
1641}
1642
1643void llvm::saveUsesAndErase(MachineInstr &MI, MachineRegisterInfo &MRI,
1644 LostDebugLocObserver *LocObserver,
1645 SmallInstListTy &DeadInstChain) {
1646 for (MachineOperand &Op : MI.uses()) {
1647 if (Op.isReg() && Op.getReg().isVirtual())
1648 DeadInstChain.insert(I: MRI.getVRegDef(Reg: Op.getReg()));
1649 }
1650 LLVM_DEBUG(dbgs() << MI << "Is dead; erasing.\n");
1651 DeadInstChain.remove(I: &MI);
1652 salvageDebugInfo(MRI, MI);
1653 MI.eraseFromParent();
1654 if (LocObserver)
1655 LocObserver->checkpoint(CheckDebugLocs: false);
1656}
1657
1658void llvm::eraseInstrs(ArrayRef<MachineInstr *> DeadInstrs,
1659 MachineRegisterInfo &MRI,
1660 LostDebugLocObserver *LocObserver) {
1661 SmallInstListTy DeadInstChain;
1662 for (MachineInstr *MI : DeadInstrs)
1663 saveUsesAndErase(MI&: *MI, MRI, LocObserver, DeadInstChain);
1664
1665 while (!DeadInstChain.empty()) {
1666 MachineInstr *Inst = DeadInstChain.pop_back_val();
1667 if (!isTriviallyDead(MI: *Inst, MRI))
1668 continue;
1669 saveUsesAndErase(MI&: *Inst, MRI, LocObserver, DeadInstChain);
1670 }
1671}
1672
1673void llvm::eraseInstr(MachineInstr &MI, MachineRegisterInfo &MRI,
1674 LostDebugLocObserver *LocObserver) {
1675 return eraseInstrs(DeadInstrs: {&MI}, MRI, LocObserver);
1676}
1677
1678void llvm::salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI) {
1679 for (auto &Def : MI.defs()) {
1680 assert(Def.isReg() && "Must be a reg");
1681
1682 SmallVector<MachineOperand *, 16> DbgUsers;
1683 for (auto &MOUse : MRI.use_operands(Reg: Def.getReg())) {
1684 MachineInstr *DbgValue = MOUse.getParent();
1685 // Ignore partially formed DBG_VALUEs.
1686 if (DbgValue->isNonListDebugValue() && DbgValue->getNumOperands() == 4) {
1687 DbgUsers.push_back(Elt: &MOUse);
1688 }
1689 }
1690
1691 if (!DbgUsers.empty()) {
1692 salvageDebugInfoForDbgValue(MRI, MI, DbgUsers);
1693 }
1694
1695 // MI is about to go away, so whatever could not be salvaged would be left
1696 // referring to a register without a definition.
1697 MRI.markUsesInDebugValueAsUndef(Reg: Def.getReg());
1698 }
1699}
1700
1701bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) {
1702 switch (Opc) {
1703 case TargetOpcode::G_FABS:
1704 case TargetOpcode::G_FADD:
1705 case TargetOpcode::G_FCANONICALIZE:
1706 case TargetOpcode::G_FCEIL:
1707 case TargetOpcode::G_FCONSTANT:
1708 case TargetOpcode::G_FCOPYSIGN:
1709 case TargetOpcode::G_FCOS:
1710 case TargetOpcode::G_FDIV:
1711 case TargetOpcode::G_FEXP2:
1712 case TargetOpcode::G_FEXP:
1713 case TargetOpcode::G_FFLOOR:
1714 case TargetOpcode::G_FLOG10:
1715 case TargetOpcode::G_FLOG2:
1716 case TargetOpcode::G_FLOG:
1717 case TargetOpcode::G_FMA:
1718 case TargetOpcode::G_FMAD:
1719 case TargetOpcode::G_FMAXIMUM:
1720 case TargetOpcode::G_FMAXIMUMNUM:
1721 case TargetOpcode::G_FMAXNUM:
1722 case TargetOpcode::G_FMAXNUM_IEEE:
1723 case TargetOpcode::G_FMINIMUM:
1724 case TargetOpcode::G_FMINIMUMNUM:
1725 case TargetOpcode::G_FMINNUM:
1726 case TargetOpcode::G_FMINNUM_IEEE:
1727 case TargetOpcode::G_FMUL:
1728 case TargetOpcode::G_FNEARBYINT:
1729 case TargetOpcode::G_FNEG:
1730 case TargetOpcode::G_FPEXT:
1731 case TargetOpcode::G_FPEXTLOAD:
1732 case TargetOpcode::G_FPOW:
1733 case TargetOpcode::G_FPTRUNC:
1734 case TargetOpcode::G_FPTRUNCSTORE:
1735 case TargetOpcode::G_FREM:
1736 case TargetOpcode::G_FRINT:
1737 case TargetOpcode::G_FSIN:
1738 case TargetOpcode::G_FTAN:
1739 case TargetOpcode::G_FACOS:
1740 case TargetOpcode::G_FASIN:
1741 case TargetOpcode::G_FATAN:
1742 case TargetOpcode::G_FATAN2:
1743 case TargetOpcode::G_FCOSH:
1744 case TargetOpcode::G_FSINH:
1745 case TargetOpcode::G_FTANH:
1746 case TargetOpcode::G_FSQRT:
1747 case TargetOpcode::G_FSUB:
1748 case TargetOpcode::G_INTRINSIC_ROUND:
1749 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
1750 case TargetOpcode::G_INTRINSIC_TRUNC:
1751 return true;
1752 default:
1753 return false;
1754 }
1755}
1756
1757/// Shifts return poison if shiftwidth is larger than the bitwidth.
1758static bool shiftAmountKnownInRange(Register ShiftAmount,
1759 const MachineRegisterInfo &MRI) {
1760 LLT Ty = MRI.getType(Reg: ShiftAmount);
1761
1762 if (Ty.isScalableVector())
1763 return false; // Can't tell, just return false to be safe
1764
1765 if (Ty.isScalar()) {
1766 std::optional<ValueAndVReg> Val =
1767 getIConstantVRegValWithLookThrough(VReg: ShiftAmount, MRI);
1768 if (!Val)
1769 return false;
1770 return Val->Value.ult(RHS: Ty.getScalarSizeInBits());
1771 }
1772
1773 GBuildVector *BV = getOpcodeDef<GBuildVector>(Reg: ShiftAmount, MRI);
1774 if (!BV)
1775 return false;
1776
1777 unsigned Sources = BV->getNumSources();
1778 for (unsigned I = 0; I < Sources; ++I) {
1779 std::optional<ValueAndVReg> Val =
1780 getIConstantVRegValWithLookThrough(VReg: BV->getSourceReg(I), MRI);
1781 if (!Val)
1782 return false;
1783 if (!Val->Value.ult(RHS: Ty.getScalarSizeInBits()))
1784 return false;
1785 }
1786
1787 return true;
1788}
1789
1790static bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
1791 bool ConsiderFlagsAndMetadata,
1792 UndefPoisonKind Kind) {
1793 MachineInstr *RegDef = MRI.getVRegDef(Reg);
1794
1795 if (ConsiderFlagsAndMetadata && includesPoison(Kind))
1796 if (auto *GMI = dyn_cast<GenericMachineInstr>(Val: RegDef))
1797 if (GMI->hasPoisonGeneratingFlags())
1798 return true;
1799
1800 // Check whether opcode is a poison/undef-generating operation.
1801 switch (RegDef->getOpcode()) {
1802 case TargetOpcode::G_BUILD_VECTOR:
1803 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
1804 return false;
1805 case TargetOpcode::G_SHL:
1806 case TargetOpcode::G_ASHR:
1807 case TargetOpcode::G_LSHR:
1808 return includesPoison(Kind) &&
1809 !shiftAmountKnownInRange(ShiftAmount: RegDef->getOperand(i: 2).getReg(), MRI);
1810 case TargetOpcode::G_FPTOSI:
1811 case TargetOpcode::G_FPTOUI:
1812 // fptosi/ui yields poison if the resulting value does not fit in the
1813 // destination type.
1814 return true;
1815 case TargetOpcode::G_CTLZ:
1816 case TargetOpcode::G_CTTZ:
1817 case TargetOpcode::G_CTLS:
1818 case TargetOpcode::G_ABS:
1819 case TargetOpcode::G_CTPOP:
1820 case TargetOpcode::G_BSWAP:
1821 case TargetOpcode::G_BITREVERSE:
1822 case TargetOpcode::G_FSHL:
1823 case TargetOpcode::G_FSHR:
1824 case TargetOpcode::G_SMAX:
1825 case TargetOpcode::G_SMIN:
1826 case TargetOpcode::G_SCMP:
1827 case TargetOpcode::G_UMAX:
1828 case TargetOpcode::G_UMIN:
1829 case TargetOpcode::G_UCMP:
1830 case TargetOpcode::G_PTRMASK:
1831 case TargetOpcode::G_SADDO:
1832 case TargetOpcode::G_SSUBO:
1833 case TargetOpcode::G_UADDO:
1834 case TargetOpcode::G_USUBO:
1835 case TargetOpcode::G_SMULO:
1836 case TargetOpcode::G_UMULO:
1837 case TargetOpcode::G_SADDSAT:
1838 case TargetOpcode::G_UADDSAT:
1839 case TargetOpcode::G_SSUBSAT:
1840 case TargetOpcode::G_USUBSAT:
1841 case TargetOpcode::G_SBFX:
1842 case TargetOpcode::G_UBFX:
1843 return false;
1844 case TargetOpcode::G_SSHLSAT:
1845 case TargetOpcode::G_USHLSAT:
1846 return includesPoison(Kind) &&
1847 !shiftAmountKnownInRange(ShiftAmount: RegDef->getOperand(i: 2).getReg(), MRI);
1848 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1849 GInsertVectorElement *Insert = cast<GInsertVectorElement>(Val: RegDef);
1850 if (includesPoison(Kind)) {
1851 std::optional<ValueAndVReg> Index =
1852 getIConstantVRegValWithLookThrough(VReg: Insert->getIndexReg(), MRI);
1853 if (!Index)
1854 return true;
1855 LLT VecTy = MRI.getType(Reg: Insert->getVectorReg());
1856 return Index->Value.uge(RHS: VecTy.getElementCount().getKnownMinValue());
1857 }
1858 return false;
1859 }
1860 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1861 GExtractVectorElement *Extract = cast<GExtractVectorElement>(Val: RegDef);
1862 if (includesPoison(Kind)) {
1863 std::optional<ValueAndVReg> Index =
1864 getIConstantVRegValWithLookThrough(VReg: Extract->getIndexReg(), MRI);
1865 if (!Index)
1866 return true;
1867 LLT VecTy = MRI.getType(Reg: Extract->getVectorReg());
1868 return Index->Value.uge(RHS: VecTy.getElementCount().getKnownMinValue());
1869 }
1870 return false;
1871 }
1872 case TargetOpcode::G_SHUFFLE_VECTOR: {
1873 GShuffleVector *Shuffle = cast<GShuffleVector>(Val: RegDef);
1874 ArrayRef<int> Mask = Shuffle->getMask();
1875 return includesPoison(Kind) && is_contained(Range&: Mask, Element: -1);
1876 }
1877 case TargetOpcode::G_FNEG:
1878 case TargetOpcode::G_PHI:
1879 case TargetOpcode::G_SELECT:
1880 case TargetOpcode::G_UREM:
1881 case TargetOpcode::G_SREM:
1882 case TargetOpcode::G_FREEZE:
1883 case TargetOpcode::G_ICMP:
1884 case TargetOpcode::G_FCMP:
1885 case TargetOpcode::G_FADD:
1886 case TargetOpcode::G_FSUB:
1887 case TargetOpcode::G_FMUL:
1888 case TargetOpcode::G_FDIV:
1889 case TargetOpcode::G_FREM:
1890 case TargetOpcode::G_PTR_ADD:
1891 return false;
1892 default:
1893 return !isa<GCastOp>(Val: RegDef) && !isa<GBinOp>(Val: RegDef);
1894 }
1895}
1896
1897static bool isGuaranteedNotToBeUndefOrPoison(Register Reg,
1898 const MachineRegisterInfo &MRI,
1899 unsigned Depth,
1900 UndefPoisonKind Kind) {
1901 if (Depth >= MaxAnalysisRecursionDepth)
1902 return false;
1903
1904 MachineInstr *RegDef = MRI.getVRegDef(Reg);
1905
1906 switch (RegDef->getOpcode()) {
1907 case TargetOpcode::G_FREEZE:
1908 return true;
1909 case TargetOpcode::G_IMPLICIT_DEF:
1910 return !includesUndef(Kind);
1911 case TargetOpcode::G_CONSTANT:
1912 case TargetOpcode::G_FCONSTANT:
1913 return true;
1914 case TargetOpcode::G_BUILD_VECTOR: {
1915 GBuildVector *BV = cast<GBuildVector>(Val: RegDef);
1916 unsigned NumSources = BV->getNumSources();
1917 for (unsigned I = 0; I < NumSources; ++I)
1918 if (!::isGuaranteedNotToBeUndefOrPoison(Reg: BV->getSourceReg(I), MRI,
1919 Depth: Depth + 1, Kind))
1920 return false;
1921 return true;
1922 }
1923 case TargetOpcode::G_PHI: {
1924 GPhi *Phi = cast<GPhi>(Val: RegDef);
1925 unsigned NumIncoming = Phi->getNumIncomingValues();
1926 for (unsigned I = 0; I < NumIncoming; ++I)
1927 if (!::isGuaranteedNotToBeUndefOrPoison(Reg: Phi->getIncomingValue(I), MRI,
1928 Depth: Depth + 1, Kind))
1929 return false;
1930 return true;
1931 }
1932 default: {
1933 auto MOCheck = [&](const MachineOperand &MO) {
1934 if (!MO.isReg())
1935 return true;
1936 return ::isGuaranteedNotToBeUndefOrPoison(Reg: MO.getReg(), MRI, Depth: Depth + 1,
1937 Kind);
1938 };
1939 return !::canCreateUndefOrPoison(Reg, MRI,
1940 /*ConsiderFlagsAndMetadata=*/true, Kind) &&
1941 all_of(Range: RegDef->uses(), P: MOCheck);
1942 }
1943 }
1944}
1945
1946bool llvm::canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
1947 bool ConsiderFlagsAndMetadata) {
1948 return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata,
1949 Kind: UndefPoisonKind::UndefOrPoison);
1950}
1951
1952bool canCreatePoison(Register Reg, const MachineRegisterInfo &MRI,
1953 bool ConsiderFlagsAndMetadata = true) {
1954 return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata,
1955 Kind: UndefPoisonKind::PoisonOnly);
1956}
1957
1958bool llvm::isGuaranteedNotToBeUndefOrPoison(Register Reg,
1959 const MachineRegisterInfo &MRI,
1960 unsigned Depth) {
1961 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
1962 Kind: UndefPoisonKind::UndefOrPoison);
1963}
1964
1965bool llvm::isGuaranteedNotToBePoison(Register Reg,
1966 const MachineRegisterInfo &MRI,
1967 unsigned Depth) {
1968 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
1969 Kind: UndefPoisonKind::PoisonOnly);
1970}
1971
1972bool llvm::isGuaranteedNotToBeUndef(Register Reg,
1973 const MachineRegisterInfo &MRI,
1974 unsigned Depth) {
1975 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
1976 Kind: UndefPoisonKind::UndefOnly);
1977}
1978
1979Type *llvm::getTypeForLLT(LLT Ty, LLVMContext &C) {
1980 if (Ty.isVector())
1981 return VectorType::get(ElementType: IntegerType::get(C, NumBits: Ty.getScalarSizeInBits()),
1982 EC: Ty.getElementCount());
1983 return IntegerType::get(C, NumBits: Ty.getSizeInBits());
1984}
1985
1986bool llvm::isAssertMI(const MachineInstr &MI) {
1987 switch (MI.getOpcode()) {
1988 default:
1989 return false;
1990 case TargetOpcode::G_ASSERT_ALIGN:
1991 case TargetOpcode::G_ASSERT_SEXT:
1992 case TargetOpcode::G_ASSERT_ZEXT:
1993 return true;
1994 }
1995}
1996
1997APInt llvm::GIConstant::getScalarValue() const {
1998 assert(Kind == GIConstantKind::Scalar && "Expected scalar constant");
1999
2000 return Value;
2001}
2002
2003std::optional<GIConstant>
2004llvm::GIConstant::getConstant(Register Const, const MachineRegisterInfo &MRI) {
2005 MachineInstr *Constant = getDefIgnoringCopies(Reg: Const, MRI);
2006
2007 if (GSplatVector *Splat = dyn_cast<GSplatVector>(Val: Constant)) {
2008 std::optional<ValueAndVReg> MayBeConstant =
2009 getIConstantVRegValWithLookThrough(VReg: Splat->getScalarReg(), MRI);
2010 if (!MayBeConstant)
2011 return std::nullopt;
2012 return GIConstant(MayBeConstant->Value, GIConstantKind::ScalableVector);
2013 }
2014
2015 if (GBuildVector *Build = dyn_cast<GBuildVector>(Val: Constant)) {
2016 SmallVector<APInt> Values;
2017 unsigned NumSources = Build->getNumSources();
2018 for (unsigned I = 0; I < NumSources; ++I) {
2019 Register SrcReg = Build->getSourceReg(I);
2020 std::optional<ValueAndVReg> MayBeConstant =
2021 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI);
2022 if (!MayBeConstant)
2023 return std::nullopt;
2024 Values.push_back(Elt: MayBeConstant->Value);
2025 }
2026 return GIConstant(Values);
2027 }
2028
2029 std::optional<ValueAndVReg> MayBeConstant =
2030 getIConstantVRegValWithLookThrough(VReg: Const, MRI);
2031 if (!MayBeConstant)
2032 return std::nullopt;
2033
2034 return GIConstant(MayBeConstant->Value, GIConstantKind::Scalar);
2035}
2036
2037APFloat llvm::GFConstant::getScalarValue() const {
2038 assert(Kind == GFConstantKind::Scalar && "Expected scalar constant");
2039
2040 return Values[0];
2041}
2042
2043std::optional<GFConstant>
2044llvm::GFConstant::getConstant(Register Const, const MachineRegisterInfo &MRI) {
2045 MachineInstr *Constant = getDefIgnoringCopies(Reg: Const, MRI);
2046
2047 if (GSplatVector *Splat = dyn_cast<GSplatVector>(Val: Constant)) {
2048 std::optional<FPValueAndVReg> MayBeConstant =
2049 getFConstantVRegValWithLookThrough(VReg: Splat->getScalarReg(), MRI);
2050 if (!MayBeConstant)
2051 return std::nullopt;
2052 return GFConstant(MayBeConstant->Value, GFConstantKind::ScalableVector);
2053 }
2054
2055 if (GBuildVector *Build = dyn_cast<GBuildVector>(Val: Constant)) {
2056 SmallVector<APFloat> Values;
2057 unsigned NumSources = Build->getNumSources();
2058 for (unsigned I = 0; I < NumSources; ++I) {
2059 Register SrcReg = Build->getSourceReg(I);
2060 std::optional<FPValueAndVReg> MayBeConstant =
2061 getFConstantVRegValWithLookThrough(VReg: SrcReg, MRI);
2062 if (!MayBeConstant)
2063 return std::nullopt;
2064 Values.push_back(Elt: MayBeConstant->Value);
2065 }
2066 return GFConstant(Values);
2067 }
2068
2069 std::optional<FPValueAndVReg> MayBeConstant =
2070 getFConstantVRegValWithLookThrough(VReg: Const, MRI);
2071 if (!MayBeConstant)
2072 return std::nullopt;
2073
2074 return GFConstant(MayBeConstant->Value, GFConstantKind::Scalar);
2075}
2076
2077// Returns a list of types to use for memory op lowering in MemOps. A partial
2078// port of findOptimalMemOpLowering in TargetLowering.
2079static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
2080 unsigned Limit, const MemOp &Op,
2081 unsigned DstAS, unsigned SrcAS,
2082 const AttributeList &FuncAttributes,
2083 const TargetLowering &TLI) {
2084 if (Op.isMemcpyOrMemmoveWithFixedDstAlign() &&
2085 Op.getSrcAlign() < Op.getDstAlign())
2086 return false;
2087
2088 LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes);
2089
2090 if (Ty == LLT()) {
2091 // Use the largest scalar type whose alignment constraints are satisfied.
2092 // We only need to check DstAlign here as SrcAlign is always greater or
2093 // equal to DstAlign (or zero).
2094 Ty = LLT::integer(SizeInBits: 64);
2095 if (Op.isFixedDstAlign())
2096 while (Op.getDstAlign() < Ty.getSizeInBytes() &&
2097 !TLI.allowsMisalignedMemoryAccesses(Ty, AddrSpace: DstAS, Alignment: Op.getDstAlign()))
2098 Ty = LLT::integer(SizeInBits: Ty.getSizeInBytes());
2099 assert(Ty.getSizeInBits() > 0 && "Could not find valid type");
2100 // FIXME: check for the largest legal type we can load/store to.
2101 }
2102
2103 unsigned NumMemOps = 0;
2104 uint64_t Size = Op.size();
2105 while (Size) {
2106 unsigned TySize = Ty.getSizeInBytes();
2107 while (TySize > Size) {
2108 // For now, only use non-vector load / store's for the left-over pieces.
2109 LLT NewTy = Ty;
2110 // FIXME: check for mem op safety and legality of the types. Not all of
2111 // SDAGisms map cleanly to GISel concepts.
2112 if (NewTy.isVector())
2113 NewTy =
2114 NewTy.getSizeInBits() > 64 ? LLT::integer(SizeInBits: 64) : LLT::integer(SizeInBits: 32);
2115 NewTy = LLT::integer(SizeInBits: llvm::bit_floor(Value: NewTy.getSizeInBits() - 1));
2116 unsigned NewTySize = NewTy.getSizeInBytes();
2117 assert(NewTySize > 0 && "Could not find appropriate type");
2118
2119 // If the new LLT cannot cover all of the remaining bits, then consider
2120 // issuing a (or a pair of) unaligned and overlapping load / store.
2121 unsigned Fast;
2122 // Need to get a VT equivalent for allowMisalignedMemoryAccesses().
2123 MVT VT = getMVTForLLT(Ty);
2124 if (NumMemOps && !Op.isVolatile() && NewTySize < Size &&
2125 TLI.allowsMisalignedMemoryAccesses(
2126 VT, AddrSpace: DstAS, Alignment: Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
2127 Flags: MachineMemOperand::MONone, &Fast) &&
2128 Fast)
2129 TySize = Size;
2130 else {
2131 Ty = NewTy;
2132 TySize = NewTySize;
2133 }
2134 }
2135
2136 if (++NumMemOps > Limit)
2137 return false;
2138
2139 MemOps.push_back(x: Ty);
2140 Size -= TySize;
2141 }
2142
2143 return true;
2144}
2145
2146bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
2147 const MachineRegisterInfo &MRI, unsigned MaxLen,
2148 Register &Dst, Register &Src,
2149 uint64_t &KnownLen, Align &Alignment,
2150 bool &DstAlignCanChange,
2151 std::vector<LLT> &MemOps) {
2152 const unsigned Opc = MI.getOpcode();
2153 assert((Opc == TargetOpcode::G_MEMCPY ||
2154 Opc == TargetOpcode::G_MEMCPY_INLINE ||
2155 Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET ||
2156 Opc == TargetOpcode::G_MEMSET_INLINE) &&
2157 "Expected memcpy like instruction");
2158
2159 auto MMOIt = MI.memoperands_begin();
2160 const MachineMemOperand *MemOp = *MMOIt;
2161
2162 Align DstAlign = MemOp->getBaseAlign();
2163 Align SrcAlign;
2164 Alignment = DstAlign;
2165 Register Len;
2166 std::tie(args&: Dst, args&: Src, args&: Len) = MI.getFirst3Regs();
2167
2168 if (Opc != TargetOpcode::G_MEMSET && Opc != TargetOpcode::G_MEMSET_INLINE) {
2169 assert(MMOIt != MI.memoperands_end() && "Expected a second MMO on MI");
2170 MemOp = *(++MMOIt);
2171 SrcAlign = MemOp->getBaseAlign();
2172 Alignment = std::min(a: DstAlign, b: SrcAlign);
2173 }
2174
2175 // See if this is a constant length copy.
2176 auto LenVRegAndVal = getIConstantVRegValWithLookThrough(VReg: Len, MRI);
2177 if (!LenVRegAndVal) {
2178 // FIXME: support dynamically sized G_MEMCPY_INLINE and G_MEMSET_INLINE
2179 assert(Opc != TargetOpcode::G_MEMCPY_INLINE &&
2180 Opc != TargetOpcode::G_MEMSET_INLINE &&
2181 "inline memcpy and memset with dynamic size are not yet supported");
2182 return false;
2183 }
2184
2185 KnownLen = LenVRegAndVal->Value.getZExtValue();
2186 DstAlignCanChange = false;
2187
2188 if (KnownLen == 0)
2189 return true;
2190
2191 if (Opc != TargetOpcode::G_MEMCPY_INLINE &&
2192 Opc != TargetOpcode::G_MEMSET_INLINE && MaxLen && KnownLen > MaxLen)
2193 return false;
2194
2195 bool IsVolatile = MemOp->isVolatile();
2196 const MachineFunction &MF = *MI.getParent()->getParent();
2197 const auto &TLI = *MF.getSubtarget().getTargetLowering();
2198 // On Darwin, -Os means optimize for size without hurting performance, so
2199 // only really optimize for size when -Oz (MinSize) is used.
2200 bool OptSize = MF.getTarget().getTargetTriple().isOSDarwin()
2201 ? MF.getFunction().hasMinSize()
2202 : MF.getFunction().hasOptSize();
2203
2204 const MachineFrameInfo &MFI = MF.getFrameInfo();
2205 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
2206 if (FIDef && !MFI.isFixedObjectIndex(ObjectIdx: FIDef->getOperand(i: 1).getIndex()))
2207 DstAlignCanChange = true;
2208
2209 const auto &DstMMO = **MI.memoperands_begin();
2210 MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo();
2211
2212 switch (Opc) {
2213 case TargetOpcode::G_MEMCPY_INLINE:
2214 case TargetOpcode::G_MEMCPY: {
2215 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
2216 MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
2217 uint64_t Limit = Opc == TargetOpcode::G_MEMCPY_INLINE
2218 ? std::numeric_limits<uint64_t>::max()
2219 : TLI.getMaxStoresPerMemcpy(OptSize);
2220 return findGISelOptimalMemOpLowering(
2221 MemOps, Limit,
2222 Op: MemOp::Copy(Size: KnownLen, DstAlignCanChange, DstAlign: std::min(a: DstAlign, b: SrcAlign),
2223 SrcAlign, IsVolatile),
2224 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
2225 FuncAttributes: MF.getFunction().getAttributes(), TLI);
2226 }
2227 case TargetOpcode::G_MEMMOVE: {
2228 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
2229 MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
2230 unsigned Limit = TLI.getMaxStoresPerMemmove(OptSize);
2231 return findGISelOptimalMemOpLowering(
2232 MemOps, Limit,
2233 Op: MemOp::Move(Size: KnownLen, DstAlignCanChange, DstAlign: std::min(a: DstAlign, b: SrcAlign),
2234 SrcAlign, IsVolatile),
2235 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
2236 FuncAttributes: MF.getFunction().getAttributes(), TLI);
2237 }
2238 case TargetOpcode::G_MEMSET:
2239 case TargetOpcode::G_MEMSET_INLINE: {
2240 unsigned Limit = Opc == TargetOpcode::G_MEMSET_INLINE
2241 ? std::numeric_limits<unsigned>::max()
2242 : TLI.getMaxStoresPerMemset(OptSize);
2243 auto ValVRegAndVal = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
2244 bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0;
2245 return findGISelOptimalMemOpLowering(
2246 MemOps, Limit,
2247 Op: MemOp::Set(Size: KnownLen, DstAlignCanChange, DstAlign,
2248 /*IsZeroMemset=*/IsZeroVal,
2249 /*IsVolatile=*/IsVolatile),
2250 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes(), TLI);
2251 }
2252 default:
2253 llvm_unreachable("Unexpected memcpy-family opcode");
2254 }
2255}
2256