1//===- llvm/CodeGen/GlobalISel/Utils.cpp -------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file This file implements the utility functions used by the GlobalISel
9/// pipeline.
10//===----------------------------------------------------------------------===//
11
12#include "llvm/CodeGen/GlobalISel/Utils.h"
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/Analysis/ValueTracking.h"
16#include "llvm/CodeGen/CodeGenCommonISel.h"
17#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
18#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
19#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
20#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
21#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
22#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
23#include "llvm/CodeGen/LowLevelTypeUtils.h"
24#include "llvm/CodeGen/MachineInstr.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/MachineSizeOpts.h"
29#include "llvm/CodeGen/RegisterBankInfo.h"
30#include "llvm/CodeGen/StackProtector.h"
31#include "llvm/CodeGen/TargetInstrInfo.h"
32#include "llvm/CodeGen/TargetLowering.h"
33#include "llvm/CodeGen/TargetOpcodes.h"
34#include "llvm/CodeGen/TargetPassConfig.h"
35#include "llvm/CodeGen/TargetRegisterInfo.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/Support/UndefPoison.h"
38#include "llvm/Target/TargetMachine.h"
39#include "llvm/Transforms/Utils/SizeOpts.h"
40#include <limits>
41#include <numeric>
42#include <optional>
43#include <tuple>
44
45#define DEBUG_TYPE "globalisel-utils"
46
47using namespace llvm;
48using namespace MIPatternMatch;
49
50Register llvm::constrainRegToClass(MachineRegisterInfo &MRI,
51 const TargetInstrInfo &TII,
52 const RegisterBankInfo &RBI, Register Reg,
53 const TargetRegisterClass &RegClass) {
54 if (!RBI.constrainGenericRegister(Reg, RC: RegClass, MRI))
55 return MRI.createVirtualRegister(RegClass: &RegClass);
56
57 return Reg;
58}
59
60Register llvm::constrainOperandRegClass(
61 const MachineFunction &MF, const TargetRegisterInfo &TRI,
62 MachineRegisterInfo &MRI, const TargetInstrInfo &TII,
63 const RegisterBankInfo &RBI, MachineInstr &InsertPt,
64 const TargetRegisterClass &RegClass, MachineOperand &RegMO) {
65 Register Reg = RegMO.getReg();
66 // Assume physical registers are properly constrained.
67 assert(Reg.isVirtual() && "PhysReg not implemented");
68
69 // Save the old register class to check whether
70 // the change notifications will be required.
71 // TODO: A better approach would be to pass
72 // the observers to constrainRegToClass().
73 auto *OldRegClass = MRI.getRegClassOrNull(Reg);
74 Register ConstrainedReg = constrainRegToClass(MRI, TII, RBI, Reg, RegClass);
75 // If we created a new virtual register because the class is not compatible
76 // then create a copy between the new and the old register.
77 if (ConstrainedReg != Reg) {
78 MachineBasicBlock::iterator InsertIt(&InsertPt);
79 MachineBasicBlock &MBB = *InsertPt.getParent();
80 // FIXME: The copy needs to have the classes constrained for its operands.
81 // Use operand's regbank to get the class for old register (Reg).
82 if (RegMO.isUse()) {
83 BuildMI(BB&: MBB, I: InsertIt, MIMD: InsertPt.getDebugLoc(),
84 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ConstrainedReg)
85 .addReg(RegNo: Reg);
86 } else {
87 assert(RegMO.isDef() && "Must be a definition");
88 BuildMI(BB&: MBB, I: std::next(x: InsertIt), MIMD: InsertPt.getDebugLoc(),
89 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Reg)
90 .addReg(RegNo: ConstrainedReg);
91 }
92 if (GISelChangeObserver *Observer = MF.getObserver()) {
93 Observer->changingInstr(MI&: *RegMO.getParent());
94 }
95 RegMO.setReg(ConstrainedReg);
96 if (GISelChangeObserver *Observer = MF.getObserver()) {
97 Observer->changedInstr(MI&: *RegMO.getParent());
98 }
99 } else if (OldRegClass != MRI.getRegClassOrNull(Reg)) {
100 if (GISelChangeObserver *Observer = MF.getObserver()) {
101 if (!RegMO.isDef()) {
102 MachineInstr *RegDef = MRI.getVRegDef(Reg);
103 Observer->changedInstr(MI&: *RegDef);
104 }
105 Observer->changingAllUsesOfReg(MRI, Reg);
106 Observer->finishedChangingAllUsesOfReg();
107 }
108 }
109 return ConstrainedReg;
110}
111
112Register llvm::constrainOperandRegClass(
113 const MachineFunction &MF, const TargetRegisterInfo &TRI,
114 MachineRegisterInfo &MRI, const TargetInstrInfo &TII,
115 const RegisterBankInfo &RBI, MachineInstr &InsertPt, const MCInstrDesc &II,
116 MachineOperand &RegMO, unsigned OpIdx) {
117 Register Reg = RegMO.getReg();
118 // Assume physical registers are properly constrained.
119 assert(Reg.isVirtual() && "PhysReg not implemented");
120
121 const TargetRegisterClass *OpRC = TII.getRegClass(MCID: II, OpNum: OpIdx);
122 // Some of the target independent instructions, like COPY, may not impose any
123 // register class constraints on some of their operands: If it's a use, we can
124 // skip constraining as the instruction defining the register would constrain
125 // it.
126
127 if (OpRC) {
128 // Obtain the RC from incoming regbank if it is a proper sub-class. Operands
129 // can have multiple regbanks for a superclass that combine different
130 // register types (E.g., AMDGPU's VGPR and AGPR). The regbank ambiguity
131 // resolved by targets during regbankselect should not be overridden.
132 if (const auto *SubRC = TRI.getCommonSubClass(
133 A: OpRC, B: TRI.getConstrainedRegClassForOperand(MO: RegMO, MRI)))
134 OpRC = SubRC;
135
136 OpRC = TRI.getAllocatableClass(RC: OpRC);
137 }
138
139 if (!OpRC) {
140 assert((!isTargetSpecificOpcode(II.getOpcode()) || RegMO.isUse()) &&
141 "Register class constraint is required unless either the "
142 "instruction is target independent or the operand is a use");
143 // FIXME: Just bailing out like this here could be not enough, unless we
144 // expect the users of this function to do the right thing for PHIs and
145 // COPY:
146 // v1 = COPY v0
147 // v2 = COPY v1
148 // v1 here may end up not being constrained at all. Please notice that to
149 // reproduce the issue we likely need a destination pattern of a selection
150 // rule producing such extra copies, not just an input GMIR with them as
151 // every existing target using selectImpl handles copies before calling it
152 // and they never reach this function.
153 return Reg;
154 }
155 return constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt, RegClass: *OpRC,
156 RegMO);
157}
158
159void llvm::constrainSelectedInstRegOperands(MachineInstr &I,
160 const TargetInstrInfo &TII,
161 const TargetRegisterInfo &TRI,
162 const RegisterBankInfo &RBI) {
163 assert(!isPreISelGenericOpcode(I.getOpcode()) &&
164 "A selected instruction is expected");
165 MachineBasicBlock &MBB = *I.getParent();
166 MachineFunction &MF = *MBB.getParent();
167 MachineRegisterInfo &MRI = MF.getRegInfo();
168
169 for (unsigned OpI = 0, OpE = I.getNumExplicitOperands(); OpI != OpE; ++OpI) {
170 MachineOperand &MO = I.getOperand(i: OpI);
171
172 // There's nothing to be done on non-register operands.
173 if (!MO.isReg())
174 continue;
175
176 LLVM_DEBUG(dbgs() << "Converting operand: " << MO << '\n');
177
178 Register Reg = MO.getReg();
179 // Physical registers don't need to be constrained.
180 if (Reg.isPhysical())
181 continue;
182
183 // Register operands with a value of 0 (e.g. predicate operands) don't need
184 // to be constrained.
185 if (Reg == 0)
186 continue;
187
188 // If the operand is a vreg, we should constrain its regclass, and only
189 // insert COPYs if that's impossible.
190 // constrainOperandRegClass does that for us.
191 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt&: I, II: I.getDesc(), RegMO&: MO, OpIdx: OpI);
192
193 // Tie uses to defs as indicated in MCInstrDesc if this hasn't already been
194 // done.
195 if (MO.isUse()) {
196 int DefIdx = I.getDesc().getOperandConstraint(OpNum: OpI, Constraint: MCOI::TIED_TO);
197 if (DefIdx != -1 && !I.isRegTiedToUseOperand(DefOpIdx: DefIdx))
198 I.tieOperands(DefIdx, UseIdx: OpI);
199 }
200 }
201}
202
203bool llvm::canReplaceReg(Register DstReg, Register SrcReg,
204 MachineRegisterInfo &MRI) {
205 // Give up if either DstReg or SrcReg is a physical register.
206 if (DstReg.isPhysical() || SrcReg.isPhysical())
207 return false;
208 // Give up if the types don't match.
209 if (MRI.getType(Reg: DstReg) != MRI.getType(Reg: SrcReg))
210 return false;
211 // Replace if either DstReg has no constraints or the register
212 // constraints match.
213 const auto &DstRBC = MRI.getRegClassOrRegBank(Reg: DstReg);
214 if (!DstRBC || DstRBC == MRI.getRegClassOrRegBank(Reg: SrcReg))
215 return true;
216
217 // Otherwise match if the Src is already a regclass that is covered by the Dst
218 // RegBank.
219 return isa<const RegisterBank *>(Val: DstRBC) && MRI.getRegClassOrNull(Reg: SrcReg) &&
220 cast<const RegisterBank *>(Val: DstRBC)->covers(
221 RC: *MRI.getRegClassOrNull(Reg: SrcReg));
222}
223
224bool llvm::isTriviallyDead(const MachineInstr &MI,
225 const MachineRegisterInfo &MRI) {
226 // Instructions without side-effects are dead iff they only define dead regs.
227 // This function is hot and this loop returns early in the common case,
228 // so only perform additional checks before this if absolutely necessary.
229 for (const auto &MO : MI.all_defs()) {
230 Register Reg = MO.getReg();
231 if (Reg.isPhysical() || !MRI.use_nodbg_empty(RegNo: Reg))
232 return false;
233 }
234 return MI.wouldBeTriviallyDead();
235}
236
237static void reportGISelDiagnostic(DiagnosticSeverity Severity,
238 MachineFunction &MF,
239 MachineOptimizationRemarkEmitter &MORE,
240 MachineOptimizationRemarkMissed &R) {
241 bool IsGlobalISelAbortEnabled =
242 MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
243 bool IsFatal = Severity == DS_Error && IsGlobalISelAbortEnabled;
244 // Print the function name explicitly if we don't have a debug location (which
245 // makes the diagnostic less useful) or if we're going to emit a raw error.
246 if (!R.getLocation().isValid() || IsFatal)
247 R << (" (in function: " + MF.getName() + ")").str();
248
249 if (IsFatal)
250 reportFatalUsageError(reason: Twine(R.getMsg()));
251 else
252 MORE.emit(OptDiag&: R);
253}
254
255void llvm::reportGISelWarning(MachineFunction &MF,
256 MachineOptimizationRemarkEmitter &MORE,
257 MachineOptimizationRemarkMissed &R) {
258 reportGISelDiagnostic(Severity: DS_Warning, MF, MORE, R);
259}
260
261void llvm::reportGISelFailure(MachineFunction &MF,
262 MachineOptimizationRemarkEmitter &MORE,
263 MachineOptimizationRemarkMissed &R) {
264 MF.getProperties().setFailedISel();
265 reportGISelDiagnostic(Severity: DS_Error, MF, MORE, R);
266}
267
268void llvm::reportGISelFailure(MachineFunction &MF,
269 MachineOptimizationRemarkEmitter &MORE,
270 const char *PassName, StringRef Msg,
271 const MachineInstr &MI) {
272 MachineOptimizationRemarkMissed R(PassName, "GISelFailure: ",
273 MI.getDebugLoc(), MI.getParent());
274 R << Msg;
275 // Printing MI is expensive; only do it if expensive remarks are enabled.
276 if (MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable ||
277 MORE.allowExtraAnalysis(PassName))
278 R << ": " << ore::MNV("Inst", MI);
279 reportGISelFailure(MF, MORE, R);
280}
281
282unsigned llvm::getInverseGMinMaxOpcode(unsigned MinMaxOpc) {
283 switch (MinMaxOpc) {
284 case TargetOpcode::G_SMIN:
285 return TargetOpcode::G_SMAX;
286 case TargetOpcode::G_SMAX:
287 return TargetOpcode::G_SMIN;
288 case TargetOpcode::G_UMIN:
289 return TargetOpcode::G_UMAX;
290 case TargetOpcode::G_UMAX:
291 return TargetOpcode::G_UMIN;
292 default:
293 llvm_unreachable("unrecognized opcode");
294 }
295}
296
297std::optional<APInt> llvm::getIConstantVRegVal(Register VReg,
298 const MachineRegisterInfo &MRI) {
299 std::optional<ValueAndVReg> ValAndVReg = getIConstantVRegValWithLookThrough(
300 VReg, MRI, /*LookThroughInstrs*/ false);
301 assert((!ValAndVReg || ValAndVReg->VReg == VReg) &&
302 "Value found while looking through instrs");
303 if (!ValAndVReg)
304 return std::nullopt;
305 return ValAndVReg->Value;
306}
307
308const APInt &llvm::getIConstantFromReg(Register Reg,
309 const MachineRegisterInfo &MRI) {
310 MachineInstr *Const = MRI.getVRegDef(Reg);
311 assert((Const && Const->getOpcode() == TargetOpcode::G_CONSTANT) &&
312 "expected a G_CONSTANT on Reg");
313 return Const->getOperand(i: 1).getCImm()->getValue();
314}
315
316std::optional<int64_t>
317llvm::getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI) {
318 std::optional<APInt> Val = getIConstantVRegVal(VReg, MRI);
319 if (Val && Val->getBitWidth() <= 64)
320 return Val->getSExtValue();
321 return std::nullopt;
322}
323
324namespace {
325
326// This function is used in many places, and as such, it has some
327// micro-optimizations to try and make it as fast as it can be.
328//
329// - We use template arguments to avoid an indirect call caused by passing a
330// function_ref/std::function
331// - GetAPCstValue does not return std::optional<APInt> as that's expensive.
332// Instead it returns true/false and places the result in a pre-constructed
333// APInt.
334//
335// Please change this function carefully and benchmark your changes.
336template <bool (*IsConstantOpcode)(const MachineInstr *),
337 bool (*GetAPCstValue)(const MachineInstr *MI, APInt &)>
338std::optional<ValueAndVReg>
339getConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI,
340 bool LookThroughInstrs = true,
341 bool LookThroughAnyExt = false) {
342 SmallVector<std::pair<unsigned, unsigned>, 4> SeenOpcodes;
343 MachineInstr *MI;
344
345 while ((MI = MRI.getVRegDef(Reg: VReg)) && !IsConstantOpcode(MI) &&
346 LookThroughInstrs) {
347 switch (MI->getOpcode()) {
348 case TargetOpcode::G_ANYEXT:
349 if (!LookThroughAnyExt)
350 return std::nullopt;
351 [[fallthrough]];
352 case TargetOpcode::G_TRUNC:
353 case TargetOpcode::G_SEXT:
354 case TargetOpcode::G_ZEXT:
355 SeenOpcodes.push_back(Elt: std::make_pair(
356 x: MI->getOpcode(),
357 y: MRI.getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits()));
358 VReg = MI->getOperand(i: 1).getReg();
359 break;
360 case TargetOpcode::COPY:
361 VReg = MI->getOperand(i: 1).getReg();
362 if (VReg.isPhysical())
363 return std::nullopt;
364 break;
365 case TargetOpcode::G_INTTOPTR:
366 VReg = MI->getOperand(i: 1).getReg();
367 break;
368 default:
369 return std::nullopt;
370 }
371 }
372 if (!MI || !IsConstantOpcode(MI))
373 return std::nullopt;
374
375 APInt Val;
376 if (!GetAPCstValue(MI, Val))
377 return std::nullopt;
378 for (auto &Pair : reverse(C&: SeenOpcodes)) {
379 switch (Pair.first) {
380 case TargetOpcode::G_TRUNC:
381 Val = Val.trunc(width: Pair.second);
382 break;
383 case TargetOpcode::G_ANYEXT:
384 case TargetOpcode::G_SEXT:
385 Val = Val.sext(width: Pair.second);
386 break;
387 case TargetOpcode::G_ZEXT:
388 Val = Val.zext(width: Pair.second);
389 break;
390 }
391 }
392
393 return ValueAndVReg{.Value: std::move(Val), .VReg: VReg};
394}
395
396bool isIConstant(const MachineInstr *MI) {
397 if (!MI)
398 return false;
399 return MI->getOpcode() == TargetOpcode::G_CONSTANT;
400}
401
402bool isFConstant(const MachineInstr *MI) {
403 if (!MI)
404 return false;
405 return MI->getOpcode() == TargetOpcode::G_FCONSTANT;
406}
407
408bool isAnyConstant(const MachineInstr *MI) {
409 if (!MI)
410 return false;
411 unsigned Opc = MI->getOpcode();
412 return Opc == TargetOpcode::G_CONSTANT || Opc == TargetOpcode::G_FCONSTANT;
413}
414
415bool getCImmAsAPInt(const MachineInstr *MI, APInt &Result) {
416 const MachineOperand &CstVal = MI->getOperand(i: 1);
417 if (!CstVal.isCImm())
418 return false;
419 Result = CstVal.getCImm()->getValue();
420 return true;
421}
422
423bool getCImmOrFPImmAsAPInt(const MachineInstr *MI, APInt &Result) {
424 const MachineOperand &CstVal = MI->getOperand(i: 1);
425 if (CstVal.isCImm())
426 Result = CstVal.getCImm()->getValue();
427 else if (CstVal.isFPImm())
428 Result = CstVal.getFPImm()->getValueAPF().bitcastToAPInt();
429 else
430 return false;
431 return true;
432}
433
434} // end anonymous namespace
435
436std::optional<ValueAndVReg> llvm::getIConstantVRegValWithLookThrough(
437 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs) {
438 return getConstantVRegValWithLookThrough<isIConstant, getCImmAsAPInt>(
439 VReg, MRI, LookThroughInstrs);
440}
441
442std::optional<ValueAndVReg> llvm::getAnyConstantVRegValWithLookThrough(
443 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs,
444 bool LookThroughAnyExt) {
445 return getConstantVRegValWithLookThrough<isAnyConstant,
446 getCImmOrFPImmAsAPInt>(
447 VReg, MRI, LookThroughInstrs, LookThroughAnyExt);
448}
449
450std::optional<FPValueAndVReg> llvm::getFConstantVRegValWithLookThrough(
451 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs) {
452 auto Reg =
453 getConstantVRegValWithLookThrough<isFConstant, getCImmOrFPImmAsAPInt>(
454 VReg, MRI, LookThroughInstrs);
455 if (!Reg)
456 return std::nullopt;
457
458 APFloat FloatVal(getFltSemanticForLLT(Ty: LLT::scalar(SizeInBits: Reg->Value.getBitWidth())),
459 Reg->Value);
460 return FPValueAndVReg{.Value: FloatVal, .VReg: Reg->VReg};
461}
462
463const ConstantFP *
464llvm::getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI) {
465 MachineInstr *MI = MRI.getVRegDef(Reg: VReg);
466 if (TargetOpcode::G_FCONSTANT != MI->getOpcode())
467 return nullptr;
468 return MI->getOperand(i: 1).getFPImm();
469}
470
471std::optional<DefinitionAndSourceRegister>
472llvm::getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI) {
473 Register DefSrcReg = Reg;
474 // This assumes that the code is in SSA form, so there should only be one
475 // definition.
476 auto DefIt = MRI.def_begin(RegNo: Reg);
477 if (DefIt == MRI.def_end())
478 return {};
479 MachineOperand &DefOpnd = *DefIt;
480 MachineInstr *DefMI = DefOpnd.getParent();
481 auto DstTy = MRI.getType(Reg: DefOpnd.getReg());
482 if (!DstTy.isValid())
483 return std::nullopt;
484 unsigned Opc = DefMI->getOpcode();
485 while (Opc == TargetOpcode::COPY || isPreISelGenericOptimizationHint(Opcode: Opc)) {
486 Register SrcReg = DefMI->getOperand(i: 1).getReg();
487 auto SrcTy = MRI.getType(Reg: SrcReg);
488 if (!SrcTy.isValid())
489 break;
490 DefMI = MRI.getVRegDef(Reg: SrcReg);
491 DefSrcReg = SrcReg;
492 Opc = DefMI->getOpcode();
493 }
494 return DefinitionAndSourceRegister{.MI: DefMI, .Reg: DefSrcReg};
495}
496
497MachineInstr *llvm::getDefIgnoringCopies(Register Reg,
498 const MachineRegisterInfo &MRI) {
499 std::optional<DefinitionAndSourceRegister> DefSrcReg =
500 getDefSrcRegIgnoringCopies(Reg, MRI);
501 return DefSrcReg ? DefSrcReg->MI : nullptr;
502}
503
504Register llvm::getSrcRegIgnoringCopies(Register Reg,
505 const MachineRegisterInfo &MRI) {
506 std::optional<DefinitionAndSourceRegister> DefSrcReg =
507 getDefSrcRegIgnoringCopies(Reg, MRI);
508 return DefSrcReg ? DefSrcReg->Reg : Register();
509}
510
511void llvm::extractParts(Register Reg, LLT Ty, int NumParts,
512 SmallVectorImpl<Register> &VRegs,
513 MachineIRBuilder &MIRBuilder,
514 MachineRegisterInfo &MRI) {
515 for (int i = 0; i < NumParts; ++i)
516 VRegs.push_back(Elt: MRI.createGenericVirtualRegister(Ty));
517 MIRBuilder.buildUnmerge(Res: VRegs, Op: Reg);
518}
519
520bool llvm::extractParts(Register Reg, LLT RegTy, LLT MainTy, LLT &LeftoverTy,
521 SmallVectorImpl<Register> &VRegs,
522 SmallVectorImpl<Register> &LeftoverRegs,
523 MachineIRBuilder &MIRBuilder,
524 MachineRegisterInfo &MRI) {
525 assert(!LeftoverTy.isValid() && "this is an out argument");
526
527 unsigned RegSize = RegTy.getSizeInBits();
528 unsigned MainSize = MainTy.getSizeInBits();
529 unsigned NumParts = RegSize / MainSize;
530 unsigned LeftoverSize = RegSize - NumParts * MainSize;
531
532 // Use an unmerge when possible.
533 if (LeftoverSize == 0) {
534 for (unsigned I = 0; I < NumParts; ++I)
535 VRegs.push_back(Elt: MRI.createGenericVirtualRegister(Ty: MainTy));
536 MIRBuilder.buildUnmerge(Res: VRegs, Op: Reg);
537 return true;
538 }
539
540 // Try to use unmerge for irregular vector split where possible
541 // For example when splitting a <6 x i32> into <4 x i32> with <2 x i32>
542 // leftover, it becomes:
543 // <2 x i32> %2, <2 x i32>%3, <2 x i32> %4 = G_UNMERGE_VALUE <6 x i32> %1
544 // <4 x i32> %5 = G_CONCAT_VECTOR <2 x i32> %2, <2 x i32> %3
545 if (RegTy.isVector() && MainTy.isVector()) {
546 unsigned RegNumElts = RegTy.getNumElements();
547 unsigned MainNumElts = MainTy.getNumElements();
548 unsigned LeftoverNumElts = RegNumElts % MainNumElts;
549 // If can unmerge to LeftoverTy, do it
550 if (MainNumElts % LeftoverNumElts == 0 &&
551 RegNumElts % LeftoverNumElts == 0 &&
552 RegTy.getScalarSizeInBits() == MainTy.getScalarSizeInBits() &&
553 LeftoverNumElts > 1) {
554 LeftoverTy = LLT::fixed_vector(NumElements: LeftoverNumElts, ScalarTy: RegTy.getElementType());
555
556 // Unmerge the SrcReg to LeftoverTy vectors
557 SmallVector<Register, 4> UnmergeValues;
558 extractParts(Reg, Ty: LeftoverTy, NumParts: RegNumElts / LeftoverNumElts, VRegs&: UnmergeValues,
559 MIRBuilder, MRI);
560
561 // Find how many LeftoverTy makes one MainTy
562 unsigned LeftoverPerMain = MainNumElts / LeftoverNumElts;
563 unsigned NumOfLeftoverVal =
564 ((RegNumElts % MainNumElts) / LeftoverNumElts);
565
566 // Create as many MainTy as possible using unmerged value
567 SmallVector<Register, 4> MergeValues;
568 for (unsigned I = 0; I < UnmergeValues.size() - NumOfLeftoverVal; I++) {
569 MergeValues.push_back(Elt: UnmergeValues[I]);
570 if (MergeValues.size() == LeftoverPerMain) {
571 VRegs.push_back(
572 Elt: MIRBuilder.buildMergeLikeInstr(Res: MainTy, Ops: MergeValues).getReg(Idx: 0));
573 MergeValues.clear();
574 }
575 }
576 // Populate LeftoverRegs with the leftovers
577 for (unsigned I = UnmergeValues.size() - NumOfLeftoverVal;
578 I < UnmergeValues.size(); I++) {
579 LeftoverRegs.push_back(Elt: UnmergeValues[I]);
580 }
581 return true;
582 }
583 }
584 // Perform irregular split. Leftover is last element of RegPieces.
585 if (MainTy.isVector()) {
586 SmallVector<Register, 8> RegPieces;
587 extractVectorParts(Reg, NumElts: MainTy.getNumElements(), VRegs&: RegPieces, MIRBuilder,
588 MRI);
589 for (unsigned i = 0; i < RegPieces.size() - 1; ++i)
590 VRegs.push_back(Elt: RegPieces[i]);
591 LeftoverRegs.push_back(Elt: RegPieces[RegPieces.size() - 1]);
592 LeftoverTy = MRI.getType(Reg: LeftoverRegs[0]);
593 return true;
594 }
595
596 LeftoverTy = LLT::integer(SizeInBits: LeftoverSize);
597 // For irregular sizes, extract the individual parts.
598 for (unsigned I = 0; I != NumParts; ++I) {
599 Register NewReg = MRI.createGenericVirtualRegister(Ty: MainTy);
600 VRegs.push_back(Elt: NewReg);
601 MIRBuilder.buildExtract(Res: NewReg, Src: Reg, Index: MainSize * I);
602 }
603
604 for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
605 Offset += LeftoverSize) {
606 Register NewReg = MRI.createGenericVirtualRegister(Ty: LeftoverTy);
607 LeftoverRegs.push_back(Elt: NewReg);
608 MIRBuilder.buildExtract(Res: NewReg, Src: Reg, Index: Offset);
609 }
610
611 return true;
612}
613
614void llvm::extractVectorParts(Register Reg, unsigned NumElts,
615 SmallVectorImpl<Register> &VRegs,
616 MachineIRBuilder &MIRBuilder,
617 MachineRegisterInfo &MRI) {
618 LLT RegTy = MRI.getType(Reg);
619 assert(RegTy.isVector() && "Expected a vector type");
620
621 LLT EltTy = RegTy.getElementType();
622 LLT NarrowTy = (NumElts == 1) ? EltTy : LLT::fixed_vector(NumElements: NumElts, ScalarTy: EltTy);
623 unsigned RegNumElts = RegTy.getNumElements();
624 unsigned LeftoverNumElts = RegNumElts % NumElts;
625 unsigned NumNarrowTyPieces = RegNumElts / NumElts;
626
627 // Perfect split without leftover
628 if (LeftoverNumElts == 0)
629 return extractParts(Reg, Ty: NarrowTy, NumParts: NumNarrowTyPieces, VRegs, MIRBuilder,
630 MRI);
631
632 // Irregular split. Provide direct access to all elements for artifact
633 // combiner using unmerge to elements. Then build vectors with NumElts
634 // elements. Remaining element(s) will be (used to build vector) Leftover.
635 SmallVector<Register, 8> Elts;
636 extractParts(Reg, Ty: EltTy, NumParts: RegNumElts, VRegs&: Elts, MIRBuilder, MRI);
637
638 unsigned Offset = 0;
639 // Requested sub-vectors of NarrowTy.
640 for (unsigned i = 0; i < NumNarrowTyPieces; ++i, Offset += NumElts) {
641 ArrayRef<Register> Pieces(&Elts[Offset], NumElts);
642 VRegs.push_back(Elt: MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: Pieces).getReg(Idx: 0));
643 }
644
645 // Leftover element(s).
646 if (LeftoverNumElts == 1) {
647 VRegs.push_back(Elt: Elts[Offset]);
648 } else {
649 LLT LeftoverTy = LLT::fixed_vector(NumElements: LeftoverNumElts, ScalarTy: EltTy);
650 ArrayRef<Register> Pieces(&Elts[Offset], LeftoverNumElts);
651 VRegs.push_back(
652 Elt: MIRBuilder.buildMergeLikeInstr(Res: LeftoverTy, Ops: Pieces).getReg(Idx: 0));
653 }
654}
655
656MachineInstr *llvm::getOpcodeDef(unsigned Opcode, Register Reg,
657 const MachineRegisterInfo &MRI) {
658 MachineInstr *DefMI = getDefIgnoringCopies(Reg, MRI);
659 return DefMI && DefMI->getOpcode() == Opcode ? DefMI : nullptr;
660}
661
662APFloat llvm::getAPFloatFromSize(double Val, unsigned Size) {
663 if (Size == 32)
664 return APFloat(float(Val));
665 if (Size == 64)
666 return APFloat(Val);
667 if (Size != 16)
668 llvm_unreachable("Unsupported FPConstant size");
669 bool Ignored;
670 APFloat APF(Val);
671 APF.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
672 return APF;
673}
674
675std::optional<APInt> llvm::ConstantFoldBinOp(unsigned Opcode,
676 const Register Op1,
677 const Register Op2,
678 const MachineRegisterInfo &MRI) {
679 auto MaybeOp2Cst = getAnyConstantVRegValWithLookThrough(VReg: Op2, MRI, LookThroughInstrs: false);
680 if (!MaybeOp2Cst)
681 return std::nullopt;
682
683 auto MaybeOp1Cst = getAnyConstantVRegValWithLookThrough(VReg: Op1, MRI, LookThroughInstrs: false);
684 if (!MaybeOp1Cst)
685 return std::nullopt;
686
687 const APInt &C1 = MaybeOp1Cst->Value;
688 const APInt &C2 = MaybeOp2Cst->Value;
689 switch (Opcode) {
690 default:
691 break;
692 case TargetOpcode::G_ADD:
693 return C1 + C2;
694 case TargetOpcode::G_PTR_ADD:
695 // Types can be of different width here.
696 // Result needs to be the same width as C1, so trunc or sext C2.
697 return C1 + C2.sextOrTrunc(width: C1.getBitWidth());
698 case TargetOpcode::G_AND:
699 return C1 & C2;
700 case TargetOpcode::G_ASHR:
701 return C1.ashr(ShiftAmt: C2);
702 case TargetOpcode::G_LSHR:
703 return C1.lshr(ShiftAmt: C2);
704 case TargetOpcode::G_MUL:
705 return C1 * C2;
706 case TargetOpcode::G_OR:
707 return C1 | C2;
708 case TargetOpcode::G_SHL:
709 return C1 << C2;
710 case TargetOpcode::G_SUB:
711 return C1 - C2;
712 case TargetOpcode::G_XOR:
713 return C1 ^ C2;
714 case TargetOpcode::G_UDIV:
715 if (!C2.getBoolValue())
716 break;
717 return C1.udiv(RHS: C2);
718 case TargetOpcode::G_SDIV:
719 if (!C2.getBoolValue())
720 break;
721 return C1.sdiv(RHS: C2);
722 case TargetOpcode::G_UREM:
723 if (!C2.getBoolValue())
724 break;
725 return C1.urem(RHS: C2);
726 case TargetOpcode::G_SREM:
727 if (!C2.getBoolValue())
728 break;
729 return C1.srem(RHS: C2);
730 case TargetOpcode::G_SMIN:
731 return APIntOps::smin(A: C1, B: C2);
732 case TargetOpcode::G_SMAX:
733 return APIntOps::smax(A: C1, B: C2);
734 case TargetOpcode::G_UMIN:
735 return APIntOps::umin(A: C1, B: C2);
736 case TargetOpcode::G_UMAX:
737 return APIntOps::umax(A: C1, B: C2);
738 }
739
740 return std::nullopt;
741}
742
743std::optional<APFloat>
744llvm::ConstantFoldFPBinOp(unsigned Opcode, const Register Op1,
745 const Register Op2, const MachineRegisterInfo &MRI) {
746 const ConstantFP *Op2Cst = getConstantFPVRegVal(VReg: Op2, MRI);
747 if (!Op2Cst)
748 return std::nullopt;
749
750 const ConstantFP *Op1Cst = getConstantFPVRegVal(VReg: Op1, MRI);
751 if (!Op1Cst)
752 return std::nullopt;
753
754 APFloat C1 = Op1Cst->getValueAPF();
755 const APFloat &C2 = Op2Cst->getValueAPF();
756 switch (Opcode) {
757 case TargetOpcode::G_FADD:
758 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
759 return C1;
760 case TargetOpcode::G_FSUB:
761 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
762 return C1;
763 case TargetOpcode::G_FMUL:
764 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
765 return C1;
766 case TargetOpcode::G_FDIV:
767 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
768 return C1;
769 case TargetOpcode::G_FREM:
770 C1.mod(RHS: C2);
771 return C1;
772 case TargetOpcode::G_FCOPYSIGN:
773 C1.copySign(RHS: C2);
774 return C1;
775 case TargetOpcode::G_FMINNUM:
776 return minnum(A: C1, B: C2);
777 case TargetOpcode::G_FMAXNUM:
778 return maxnum(A: C1, B: C2);
779 case TargetOpcode::G_FMINIMUM:
780 return minimum(A: C1, B: C2);
781 case TargetOpcode::G_FMAXIMUM:
782 return maximum(A: C1, B: C2);
783 case TargetOpcode::G_FMINIMUMNUM:
784 return minimumnum(A: C1, B: C2);
785 case TargetOpcode::G_FMAXIMUMNUM:
786 return maximumnum(A: C1, B: C2);
787 case TargetOpcode::G_FMINNUM_IEEE:
788 case TargetOpcode::G_FMAXNUM_IEEE:
789 // FIXME: These operations were unfortunately named. fminnum/fmaxnum do not
790 // follow the IEEE behavior for signaling nans and follow libm's fmin/fmax,
791 // and currently there isn't a nice wrapper in APFloat for the version with
792 // correct snan handling.
793 break;
794 default:
795 break;
796 }
797
798 return std::nullopt;
799}
800
801static GBuildVector *getBuildVectorLikeDef(Register Reg,
802 const MachineRegisterInfo &MRI) {
803 if (auto *BV = getOpcodeDef<GBuildVector>(Reg, MRI))
804 return BV;
805
806 auto *Bitcast = getOpcodeDef(Opcode: TargetOpcode::G_BITCAST, Reg, MRI);
807 if (!Bitcast)
808 return nullptr;
809
810 auto [Dst, DstTy, Src, SrcTy] = Bitcast->getFirst2RegLLTs();
811 if (!SrcTy.isVector() || !DstTy.isVector())
812 return nullptr;
813 if (SrcTy.getElementCount() != DstTy.getElementCount())
814 return nullptr;
815 if (SrcTy.getScalarSizeInBits() != DstTy.getScalarSizeInBits())
816 return nullptr;
817
818 return getOpcodeDef<GBuildVector>(Reg: Src, MRI);
819}
820
821SmallVector<APInt>
822llvm::ConstantFoldVectorBinop(unsigned Opcode, const Register Op1,
823 const Register Op2,
824 const MachineRegisterInfo &MRI) {
825 auto *SrcVec2 = getBuildVectorLikeDef(Reg: Op2, MRI);
826 if (!SrcVec2)
827 return SmallVector<APInt>();
828
829 auto *SrcVec1 = getBuildVectorLikeDef(Reg: Op1, MRI);
830 if (!SrcVec1)
831 return SmallVector<APInt>();
832
833 SmallVector<APInt> FoldedElements;
834 for (unsigned Idx = 0, E = SrcVec1->getNumSources(); Idx < E; ++Idx) {
835 auto MaybeCst = ConstantFoldBinOp(Opcode, Op1: SrcVec1->getSourceReg(I: Idx),
836 Op2: SrcVec2->getSourceReg(I: Idx), MRI);
837 if (!MaybeCst)
838 return SmallVector<APInt>();
839 FoldedElements.push_back(Elt: *MaybeCst);
840 }
841 return FoldedElements;
842}
843
844Align llvm::inferAlignFromPtrInfo(MachineFunction &MF,
845 const MachinePointerInfo &MPO) {
846 auto PSV = dyn_cast_if_present<const PseudoSourceValue *>(Val: MPO.V);
847 if (auto FSPV = dyn_cast_or_null<FixedStackPseudoSourceValue>(Val: PSV)) {
848 MachineFrameInfo &MFI = MF.getFrameInfo();
849 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FSPV->getFrameIndex()),
850 Offset: MPO.Offset);
851 }
852
853 if (const Value *V = dyn_cast_if_present<const Value *>(Val: MPO.V)) {
854 const Module *M = MF.getFunction().getParent();
855 return V->getPointerAlignment(DL: M->getDataLayout());
856 }
857
858 return Align(1);
859}
860
861Register llvm::getFunctionLiveInPhysReg(MachineFunction &MF,
862 const TargetInstrInfo &TII,
863 MCRegister PhysReg,
864 const TargetRegisterClass &RC,
865 const DebugLoc &DL, LLT RegTy) {
866 MachineBasicBlock &EntryMBB = MF.front();
867 MachineRegisterInfo &MRI = MF.getRegInfo();
868 Register LiveIn = MRI.getLiveInVirtReg(PReg: PhysReg);
869 if (LiveIn) {
870 MachineInstr *Def = MRI.getVRegDef(Reg: LiveIn);
871 if (Def) {
872 // FIXME: Should the verifier check this is in the entry block?
873 assert(Def->getParent() == &EntryMBB && "live-in copy not in entry block");
874 return LiveIn;
875 }
876
877 // It's possible the incoming argument register and copy was added during
878 // lowering, but later deleted due to being/becoming dead. If this happens,
879 // re-insert the copy.
880 } else {
881 // The live in register was not present, so add it.
882 LiveIn = MF.addLiveIn(PReg: PhysReg, RC: &RC);
883 if (RegTy.isValid())
884 MRI.setType(VReg: LiveIn, Ty: RegTy);
885 }
886
887 BuildMI(BB&: EntryMBB, I: EntryMBB.begin(), MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: LiveIn)
888 .addReg(RegNo: PhysReg);
889 if (!EntryMBB.isLiveIn(Reg: PhysReg))
890 EntryMBB.addLiveIn(PhysReg);
891 return LiveIn;
892}
893
894std::optional<APInt> llvm::ConstantFoldExtOp(unsigned Opcode,
895 const Register Op1, uint64_t Imm,
896 const MachineRegisterInfo &MRI) {
897 auto MaybeOp1Cst = getIConstantVRegVal(VReg: Op1, MRI);
898 if (MaybeOp1Cst) {
899 switch (Opcode) {
900 default:
901 break;
902 case TargetOpcode::G_SEXT_INREG: {
903 LLT Ty = MRI.getType(Reg: Op1);
904 return MaybeOp1Cst->trunc(width: Imm).sext(width: Ty.getScalarSizeInBits());
905 }
906 }
907 }
908 return std::nullopt;
909}
910
911std::optional<APInt> llvm::ConstantFoldCastOp(unsigned Opcode, LLT DstTy,
912 const Register Op0,
913 const MachineRegisterInfo &MRI) {
914 std::optional<APInt> Val = getIConstantVRegVal(VReg: Op0, MRI);
915 if (!Val)
916 return Val;
917
918 const unsigned DstSize = DstTy.getScalarSizeInBits();
919
920 switch (Opcode) {
921 case TargetOpcode::G_SEXT:
922 return Val->sext(width: DstSize);
923 case TargetOpcode::G_ZEXT:
924 case TargetOpcode::G_ANYEXT:
925 // TODO: DAG considers target preference when constant folding any_extend.
926 return Val->zext(width: DstSize);
927 default:
928 break;
929 }
930
931 llvm_unreachable("unexpected cast opcode to constant fold");
932}
933
934std::optional<APFloat>
935llvm::ConstantFoldIntToFloat(unsigned Opcode, LLT DstTy, Register Src,
936 const MachineRegisterInfo &MRI) {
937 assert(Opcode == TargetOpcode::G_SITOFP || Opcode == TargetOpcode::G_UITOFP);
938 if (auto MaybeSrcVal = getIConstantVRegVal(VReg: Src, MRI)) {
939 APFloat DstVal(getFltSemanticForLLT(Ty: DstTy));
940 DstVal.convertFromAPInt(Input: *MaybeSrcVal, IsSigned: Opcode == TargetOpcode::G_SITOFP,
941 RM: APFloat::rmNearestTiesToEven);
942 return DstVal;
943 }
944 return std::nullopt;
945}
946
947SmallVector<APInt>
948llvm::ConstantFoldUnaryIntOp(unsigned Opcode, LLT DstTy, Register Src,
949 const MachineRegisterInfo &MRI) {
950 unsigned EltBits = DstTy.getScalarSizeInBits();
951 auto Fold = [Opcode, EltBits](const APInt &V) -> APInt {
952 switch (Opcode) {
953 case TargetOpcode::G_CTLZ:
954 case TargetOpcode::G_CTLZ_ZERO_POISON:
955 return APInt(EltBits, V.countl_zero());
956 case TargetOpcode::G_CTTZ:
957 case TargetOpcode::G_CTTZ_ZERO_POISON:
958 return APInt(EltBits, V.countr_zero());
959 case TargetOpcode::G_CTPOP:
960 return APInt(EltBits, V.popcount());
961 case TargetOpcode::G_ABS:
962 return V.abs();
963 case TargetOpcode::G_BSWAP:
964 return V.byteSwap();
965 case TargetOpcode::G_BITREVERSE:
966 return V.reverseBits();
967 }
968 llvm_unreachable("unexpected opcode in ConstantFoldUnaryIntOp");
969 };
970
971 auto tryFoldScalar = [&](Register R) -> std::optional<APInt> {
972 if (auto MaybeCst = getIConstantVRegVal(VReg: R, MRI))
973 return Fold(*MaybeCst);
974 return std::nullopt;
975 };
976 if (MRI.getType(Reg: Src).isVector()) {
977 auto *BV = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
978 if (!BV)
979 return {};
980 SmallVector<APInt> Folded;
981 for (unsigned SrcIdx = 0; SrcIdx < BV->getNumSources(); ++SrcIdx) {
982 if (auto MaybeFold = tryFoldScalar(BV->getSourceReg(I: SrcIdx))) {
983 Folded.emplace_back(Args: std::move(*MaybeFold));
984 continue;
985 }
986 return {};
987 }
988 return Folded;
989 }
990 if (auto MaybeCst = tryFoldScalar(Src))
991 return {std::move(*MaybeCst)};
992 return {};
993}
994
995std::optional<SmallVector<APInt>>
996llvm::ConstantFoldICmp(unsigned Pred, const Register Op1, const Register Op2,
997 unsigned DstScalarSizeInBits, unsigned ExtOp,
998 const MachineRegisterInfo &MRI) {
999 assert(ExtOp == TargetOpcode::G_SEXT || ExtOp == TargetOpcode::G_ZEXT ||
1000 ExtOp == TargetOpcode::G_ANYEXT);
1001
1002 const LLT Ty = MRI.getType(Reg: Op1);
1003
1004 auto GetICmpResultCst = [&](bool IsTrue) {
1005 if (IsTrue)
1006 return ExtOp == TargetOpcode::G_SEXT
1007 ? APInt::getAllOnes(numBits: DstScalarSizeInBits)
1008 : APInt::getOneBitSet(numBits: DstScalarSizeInBits, BitNo: 0);
1009 return APInt::getZero(numBits: DstScalarSizeInBits);
1010 };
1011
1012 auto TryFoldScalar = [&](Register LHS, Register RHS) -> std::optional<APInt> {
1013 auto RHSCst = getIConstantVRegVal(VReg: RHS, MRI);
1014 if (!RHSCst)
1015 return std::nullopt;
1016 auto LHSCst = getIConstantVRegVal(VReg: LHS, MRI);
1017 if (!LHSCst)
1018 return std::nullopt;
1019
1020 switch (Pred) {
1021 case CmpInst::Predicate::ICMP_EQ:
1022 return GetICmpResultCst(LHSCst->eq(RHS: *RHSCst));
1023 case CmpInst::Predicate::ICMP_NE:
1024 return GetICmpResultCst(LHSCst->ne(RHS: *RHSCst));
1025 case CmpInst::Predicate::ICMP_UGT:
1026 return GetICmpResultCst(LHSCst->ugt(RHS: *RHSCst));
1027 case CmpInst::Predicate::ICMP_UGE:
1028 return GetICmpResultCst(LHSCst->uge(RHS: *RHSCst));
1029 case CmpInst::Predicate::ICMP_ULT:
1030 return GetICmpResultCst(LHSCst->ult(RHS: *RHSCst));
1031 case CmpInst::Predicate::ICMP_ULE:
1032 return GetICmpResultCst(LHSCst->ule(RHS: *RHSCst));
1033 case CmpInst::Predicate::ICMP_SGT:
1034 return GetICmpResultCst(LHSCst->sgt(RHS: *RHSCst));
1035 case CmpInst::Predicate::ICMP_SGE:
1036 return GetICmpResultCst(LHSCst->sge(RHS: *RHSCst));
1037 case CmpInst::Predicate::ICMP_SLT:
1038 return GetICmpResultCst(LHSCst->slt(RHS: *RHSCst));
1039 case CmpInst::Predicate::ICMP_SLE:
1040 return GetICmpResultCst(LHSCst->sle(RHS: *RHSCst));
1041 default:
1042 return std::nullopt;
1043 }
1044 };
1045
1046 SmallVector<APInt> FoldedICmps;
1047
1048 if (Ty.isVector()) {
1049 // Try to constant fold each element.
1050 auto *BV1 = getOpcodeDef<GBuildVector>(Reg: Op1, MRI);
1051 auto *BV2 = getOpcodeDef<GBuildVector>(Reg: Op2, MRI);
1052 if (!BV1 || !BV2)
1053 return std::nullopt;
1054 assert(BV1->getNumSources() == BV2->getNumSources() && "Invalid vectors");
1055 for (unsigned I = 0; I < BV1->getNumSources(); ++I) {
1056 if (auto MaybeFold =
1057 TryFoldScalar(BV1->getSourceReg(I), BV2->getSourceReg(I))) {
1058 FoldedICmps.emplace_back(Args&: *MaybeFold);
1059 continue;
1060 }
1061 return std::nullopt;
1062 }
1063 return FoldedICmps;
1064 }
1065
1066 if (auto MaybeCst = TryFoldScalar(Op1, Op2)) {
1067 FoldedICmps.emplace_back(Args&: *MaybeCst);
1068 return FoldedICmps;
1069 }
1070
1071 return std::nullopt;
1072}
1073
1074bool llvm::isKnownToBeAPowerOfTwo(Register Reg, const MachineRegisterInfo &MRI,
1075 GISelValueTracking *VT, bool OrNegative) {
1076 std::optional<DefinitionAndSourceRegister> DefSrcReg =
1077 getDefSrcRegIgnoringCopies(Reg, MRI);
1078 if (!DefSrcReg)
1079 return false;
1080
1081 const MachineInstr &MI = *DefSrcReg->MI;
1082 const LLT Ty = MRI.getType(Reg);
1083
1084 auto IsPow2 = [OrNegative](const APInt &V) {
1085 return V.isPowerOf2() || (OrNegative && V.isNegatedPowerOf2());
1086 };
1087
1088 switch (MI.getOpcode()) {
1089 case TargetOpcode::G_CONSTANT: {
1090 unsigned BitWidth = Ty.getScalarSizeInBits();
1091 const ConstantInt *CI = MI.getOperand(i: 1).getCImm();
1092 return IsPow2(CI->getValue().zextOrTrunc(width: BitWidth));
1093 }
1094 case TargetOpcode::G_SHL: {
1095 // A left-shift of a constant one will have exactly one bit set because
1096 // shifting the bit off the end is undefined.
1097
1098 // TODO: Constant splat
1099 if (auto ConstLHS = getIConstantVRegVal(VReg: MI.getOperand(i: 1).getReg(), MRI)) {
1100 if (*ConstLHS == 1)
1101 return true;
1102 }
1103
1104 break;
1105 }
1106 case TargetOpcode::G_LSHR: {
1107 if (auto ConstLHS = getIConstantVRegVal(VReg: MI.getOperand(i: 1).getReg(), MRI)) {
1108 if (ConstLHS->isSignMask())
1109 return true;
1110 }
1111
1112 break;
1113 }
1114 case TargetOpcode::G_BUILD_VECTOR: {
1115 // TODO: Probably should have a recursion depth guard since you could have
1116 // bitcasted vector elements.
1117 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands()))
1118 if (!isKnownToBeAPowerOfTwo(Reg: MO.getReg(), MRI, VT, OrNegative))
1119 return false;
1120
1121 return true;
1122 }
1123 case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1124 // Only handle constants since we would need to know if number of leading
1125 // zeros is greater than the truncation amount.
1126 const unsigned BitWidth = Ty.getScalarSizeInBits();
1127 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
1128 auto Const = getIConstantVRegVal(VReg: MO.getReg(), MRI);
1129 if (!Const || !IsPow2(Const->zextOrTrunc(width: BitWidth)))
1130 return false;
1131 }
1132
1133 return true;
1134 }
1135 default:
1136 break;
1137 }
1138
1139 if (!VT)
1140 return false;
1141
1142 // More could be done here, though the above checks are enough
1143 // to handle some common cases.
1144
1145 // Fall back to computeKnownBits to catch other known cases.
1146 KnownBits Known = VT->getKnownBits(R: Reg);
1147 return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);
1148}
1149
1150void llvm::getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU) {
1151 AU.addPreserved<StackProtector>();
1152}
1153
1154LLT llvm::getLCMType(LLT OrigTy, LLT TargetTy) {
1155 if (OrigTy.getSizeInBits() == TargetTy.getSizeInBits())
1156 return OrigTy;
1157
1158 if (OrigTy.isVector() && TargetTy.isVector()) {
1159 LLT OrigElt = OrigTy.getElementType();
1160 LLT TargetElt = TargetTy.getElementType();
1161
1162 // TODO: The docstring for this function says the intention is to use this
1163 // function to build MERGE/UNMERGE instructions. It won't be the case that
1164 // we generate a MERGE/UNMERGE between fixed and scalable vector types. We
1165 // could implement getLCMType between the two in the future if there was a
1166 // need, but it is not worth it now as this function should not be used in
1167 // that way.
1168 assert(((OrigTy.isScalableVector() && !TargetTy.isFixedVector()) ||
1169 (OrigTy.isFixedVector() && !TargetTy.isScalableVector())) &&
1170 "getLCMType not implemented between fixed and scalable vectors.");
1171
1172 if (OrigElt.getSizeInBits() == TargetElt.getSizeInBits()) {
1173 int GCDMinElts = std::gcd(m: OrigTy.getElementCount().getKnownMinValue(),
1174 n: TargetTy.getElementCount().getKnownMinValue());
1175 // Prefer the original element type.
1176 ElementCount Mul = OrigTy.getElementCount().multiplyCoefficientBy(
1177 RHS: TargetTy.getElementCount().getKnownMinValue());
1178 return LLT::vector(EC: Mul.divideCoefficientBy(RHS: GCDMinElts),
1179 ScalarTy: OrigTy.getElementType());
1180 }
1181 unsigned LCM = std::lcm(m: OrigTy.getSizeInBits().getKnownMinValue(),
1182 n: TargetTy.getSizeInBits().getKnownMinValue());
1183 return LLT::vector(
1184 EC: ElementCount::get(MinVal: LCM / OrigElt.getSizeInBits(), Scalable: OrigTy.isScalable()),
1185 ScalarTy: OrigElt);
1186 }
1187
1188 // One type is scalar, one type is vector
1189 if (OrigTy.isVector() || TargetTy.isVector()) {
1190 LLT VecTy = OrigTy.isVector() ? OrigTy : TargetTy;
1191 LLT ScalarTy = OrigTy.isVector() ? TargetTy : OrigTy;
1192 LLT EltTy = VecTy.getElementType();
1193 LLT OrigEltTy = OrigTy.isVector() ? OrigTy.getElementType() : OrigTy;
1194
1195 // Prefer scalar type from OrigTy.
1196 if (EltTy.getSizeInBits() == ScalarTy.getSizeInBits())
1197 return LLT::vector(EC: VecTy.getElementCount(), ScalarTy: OrigEltTy);
1198
1199 // Different size scalars. Create vector with the same total size.
1200 // LCM will take fixed/scalable from VecTy.
1201 unsigned LCM = std::lcm(m: EltTy.getSizeInBits().getFixedValue() *
1202 VecTy.getElementCount().getKnownMinValue(),
1203 n: ScalarTy.getSizeInBits().getFixedValue());
1204 // Prefer type from OrigTy
1205 return LLT::vector(EC: ElementCount::get(MinVal: LCM / OrigEltTy.getSizeInBits(),
1206 Scalable: VecTy.getElementCount().isScalable()),
1207 ScalarTy: OrigEltTy);
1208 }
1209
1210 // At this point, both types are scalars of different size
1211 unsigned LCM = std::lcm(m: OrigTy.getSizeInBits().getFixedValue(),
1212 n: TargetTy.getSizeInBits().getFixedValue());
1213 // Preserve pointer types.
1214 if (LCM == OrigTy.getSizeInBits())
1215 return OrigTy;
1216 if (LCM == TargetTy.getSizeInBits())
1217 return TargetTy;
1218 return LLT::scalar(SizeInBits: LCM);
1219}
1220
1221LLT llvm::getCoverTy(LLT OrigTy, LLT TargetTy) {
1222
1223 if ((OrigTy.isScalableVector() && TargetTy.isFixedVector()) ||
1224 (OrigTy.isFixedVector() && TargetTy.isScalableVector()))
1225 llvm_unreachable(
1226 "getCoverTy not implemented between fixed and scalable vectors.");
1227
1228 if (!OrigTy.isVector() || !TargetTy.isVector() || OrigTy == TargetTy ||
1229 (OrigTy.getScalarSizeInBits() != TargetTy.getScalarSizeInBits()))
1230 return getLCMType(OrigTy, TargetTy);
1231
1232 unsigned OrigTyNumElts = OrigTy.getElementCount().getKnownMinValue();
1233 unsigned TargetTyNumElts = TargetTy.getElementCount().getKnownMinValue();
1234 if (OrigTyNumElts % TargetTyNumElts == 0)
1235 return OrigTy;
1236
1237 unsigned NumElts = alignTo(Value: OrigTyNumElts, Align: TargetTyNumElts);
1238 return LLT::scalarOrVector(EC: ElementCount::getFixed(MinVal: NumElts),
1239 ScalarTy: OrigTy.getElementType());
1240}
1241
1242LLT llvm::getGCDType(LLT OrigTy, LLT TargetTy) {
1243 if (OrigTy.getSizeInBits() == TargetTy.getSizeInBits())
1244 return OrigTy;
1245
1246 if (OrigTy.isVector() && TargetTy.isVector()) {
1247 LLT OrigElt = OrigTy.getElementType();
1248
1249 // TODO: The docstring for this function says the intention is to use this
1250 // function to build MERGE/UNMERGE instructions. It won't be the case that
1251 // we generate a MERGE/UNMERGE between fixed and scalable vector types. We
1252 // could implement getGCDType between the two in the future if there was a
1253 // need, but it is not worth it now as this function should not be used in
1254 // that way.
1255 assert(((OrigTy.isScalableVector() && !TargetTy.isFixedVector()) ||
1256 (OrigTy.isFixedVector() && !TargetTy.isScalableVector())) &&
1257 "getGCDType not implemented between fixed and scalable vectors.");
1258
1259 unsigned GCD = std::gcd(m: OrigTy.getSizeInBits().getKnownMinValue(),
1260 n: TargetTy.getSizeInBits().getKnownMinValue());
1261 if (GCD == OrigElt.getSizeInBits())
1262 return LLT::scalarOrVector(EC: ElementCount::get(MinVal: 1, Scalable: OrigTy.isScalable()),
1263 ScalarTy: OrigElt);
1264
1265 // Cannot produce original element type, but both have vscale in common.
1266 if (GCD < OrigElt.getSizeInBits())
1267 return LLT::scalarOrVector(EC: ElementCount::get(MinVal: 1, Scalable: OrigTy.isScalable()),
1268 ScalarSize: GCD);
1269
1270 return LLT::vector(
1271 EC: ElementCount::get(MinVal: GCD / OrigElt.getSizeInBits().getFixedValue(),
1272 Scalable: OrigTy.isScalable()),
1273 ScalarTy: OrigElt);
1274 }
1275
1276 // If one type is vector and the element size matches the scalar size, then
1277 // the gcd is the scalar type.
1278 if (OrigTy.isVector() &&
1279 OrigTy.getElementType().getSizeInBits() == TargetTy.getSizeInBits())
1280 return OrigTy.getElementType();
1281 if (TargetTy.isVector() &&
1282 TargetTy.getElementType().getSizeInBits() == OrigTy.getSizeInBits())
1283 return OrigTy;
1284
1285 // At this point, both types are either scalars of different type or one is a
1286 // vector and one is a scalar. If both types are scalars, the GCD type is the
1287 // GCD between the two scalar sizes. If one is vector and one is scalar, then
1288 // the GCD type is the GCD between the scalar and the vector element size.
1289 LLT OrigScalar = OrigTy.getScalarType();
1290 LLT TargetScalar = TargetTy.getScalarType();
1291 unsigned GCD = std::gcd(m: OrigScalar.getSizeInBits().getFixedValue(),
1292 n: TargetScalar.getSizeInBits().getFixedValue());
1293 return LLT::integer(SizeInBits: GCD);
1294}
1295
1296std::optional<int> llvm::getSplatIndex(MachineInstr &MI) {
1297 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
1298 "Only G_SHUFFLE_VECTOR can have a splat index!");
1299 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
1300 auto FirstDefinedIdx = find_if(Range&: Mask, P: [](int Elt) { return Elt >= 0; });
1301
1302 // If all elements are undefined, this shuffle can be considered a splat.
1303 // Return 0 for better potential for callers to simplify.
1304 if (FirstDefinedIdx == Mask.end())
1305 return 0;
1306
1307 // Make sure all remaining elements are either undef or the same
1308 // as the first non-undef value.
1309 int SplatValue = *FirstDefinedIdx;
1310 if (any_of(Range: make_range(x: std::next(x: FirstDefinedIdx), y: Mask.end()),
1311 P: [&SplatValue](int Elt) { return Elt >= 0 && Elt != SplatValue; }))
1312 return std::nullopt;
1313
1314 return SplatValue;
1315}
1316
1317static bool isBuildVectorOp(unsigned Opcode) {
1318 return Opcode == TargetOpcode::G_BUILD_VECTOR ||
1319 Opcode == TargetOpcode::G_BUILD_VECTOR_TRUNC;
1320}
1321
1322namespace {
1323
1324std::optional<ValueAndVReg> getAnyConstantSplat(Register VReg,
1325 const MachineRegisterInfo &MRI,
1326 bool AllowUndef) {
1327 MachineInstr *MI = getDefIgnoringCopies(Reg: VReg, MRI);
1328 if (!MI)
1329 return std::nullopt;
1330
1331 bool isConcatVectorsOp = MI->getOpcode() == TargetOpcode::G_CONCAT_VECTORS;
1332 if (!isBuildVectorOp(Opcode: MI->getOpcode()) && !isConcatVectorsOp)
1333 return std::nullopt;
1334
1335 std::optional<ValueAndVReg> SplatValAndReg;
1336 for (MachineOperand &Op : MI->uses()) {
1337 Register Element = Op.getReg();
1338 // If we have a G_CONCAT_VECTOR, we recursively look into the
1339 // vectors that we're concatenating to see if they're splats.
1340 auto ElementValAndReg =
1341 isConcatVectorsOp
1342 ? getAnyConstantSplat(VReg: Element, MRI, AllowUndef)
1343 : getAnyConstantVRegValWithLookThrough(VReg: Element, MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
1344
1345 // If AllowUndef, treat undef as value that will result in a constant splat.
1346 if (!ElementValAndReg) {
1347 if (AllowUndef && isa<GImplicitDef>(Val: MRI.getVRegDef(Reg: Element)))
1348 continue;
1349 return std::nullopt;
1350 }
1351
1352 // Record splat value
1353 if (!SplatValAndReg)
1354 SplatValAndReg = ElementValAndReg;
1355
1356 // Different constant than the one already recorded, not a constant splat.
1357 if (SplatValAndReg->Value != ElementValAndReg->Value)
1358 return std::nullopt;
1359 }
1360
1361 return SplatValAndReg;
1362}
1363
1364} // end anonymous namespace
1365
1366bool llvm::isBuildVectorConstantSplat(const Register Reg,
1367 const MachineRegisterInfo &MRI,
1368 int64_t SplatValue, bool AllowUndef) {
1369 if (auto SplatValAndReg = getAnyConstantSplat(VReg: Reg, MRI, AllowUndef))
1370 return SplatValAndReg->Value.getSExtValue() == SplatValue;
1371
1372 return false;
1373}
1374
1375bool llvm::isBuildVectorConstantSplat(const Register Reg,
1376 const MachineRegisterInfo &MRI,
1377 const APInt &SplatValue,
1378 bool AllowUndef) {
1379 if (auto SplatValAndReg = getAnyConstantSplat(VReg: Reg, MRI, AllowUndef)) {
1380 if (SplatValAndReg->Value.getBitWidth() < SplatValue.getBitWidth())
1381 return APInt::isSameValue(
1382 I1: SplatValAndReg->Value.sext(width: SplatValue.getBitWidth()), I2: SplatValue);
1383 return APInt::isSameValue(
1384 I1: SplatValAndReg->Value,
1385 I2: SplatValue.sext(width: SplatValAndReg->Value.getBitWidth()));
1386 }
1387
1388 return false;
1389}
1390
1391bool llvm::isBuildVectorConstantSplat(const MachineInstr &MI,
1392 const MachineRegisterInfo &MRI,
1393 int64_t SplatValue, bool AllowUndef) {
1394 return isBuildVectorConstantSplat(Reg: MI.getOperand(i: 0).getReg(), MRI, SplatValue,
1395 AllowUndef);
1396}
1397
1398bool llvm::isBuildVectorConstantSplat(const MachineInstr &MI,
1399 const MachineRegisterInfo &MRI,
1400 const APInt &SplatValue,
1401 bool AllowUndef) {
1402 return isBuildVectorConstantSplat(Reg: MI.getOperand(i: 0).getReg(), MRI, SplatValue,
1403 AllowUndef);
1404}
1405
1406std::optional<APInt>
1407llvm::getIConstantSplatVal(const Register Reg, const MachineRegisterInfo &MRI) {
1408 if (auto SplatValAndReg =
1409 getAnyConstantSplat(VReg: Reg, MRI, /* AllowUndef */ false)) {
1410 if (std::optional<ValueAndVReg> ValAndVReg =
1411 getIConstantVRegValWithLookThrough(VReg: SplatValAndReg->VReg, MRI))
1412 return ValAndVReg->Value;
1413 }
1414
1415 return std::nullopt;
1416}
1417
1418std::optional<APInt>
1419llvm::getIConstantSplatVal(const MachineInstr &MI,
1420 const MachineRegisterInfo &MRI) {
1421 return getIConstantSplatVal(Reg: MI.getOperand(i: 0).getReg(), MRI);
1422}
1423
1424std::optional<int64_t>
1425llvm::getIConstantSplatSExtVal(const Register Reg,
1426 const MachineRegisterInfo &MRI) {
1427 if (auto SplatValAndReg =
1428 getAnyConstantSplat(VReg: Reg, MRI, /* AllowUndef */ false))
1429 return getIConstantVRegSExtVal(VReg: SplatValAndReg->VReg, MRI);
1430 return std::nullopt;
1431}
1432
1433std::optional<int64_t>
1434llvm::getIConstantSplatSExtVal(const MachineInstr &MI,
1435 const MachineRegisterInfo &MRI) {
1436 return getIConstantSplatSExtVal(Reg: MI.getOperand(i: 0).getReg(), MRI);
1437}
1438
1439std::optional<FPValueAndVReg>
1440llvm::getFConstantSplat(Register VReg, const MachineRegisterInfo &MRI,
1441 bool AllowUndef) {
1442 if (auto SplatValAndReg = getAnyConstantSplat(VReg, MRI, AllowUndef))
1443 return getFConstantVRegValWithLookThrough(VReg: SplatValAndReg->VReg, MRI);
1444 return std::nullopt;
1445}
1446
1447bool llvm::isBuildVectorAllZeros(const MachineInstr &MI,
1448 const MachineRegisterInfo &MRI,
1449 bool AllowUndef) {
1450 return isBuildVectorConstantSplat(MI, MRI, SplatValue: 0, AllowUndef);
1451}
1452
1453bool llvm::isBuildVectorAllOnes(const MachineInstr &MI,
1454 const MachineRegisterInfo &MRI,
1455 bool AllowUndef) {
1456 return isBuildVectorConstantSplat(MI, MRI, SplatValue: -1, AllowUndef);
1457}
1458
1459std::optional<RegOrConstant>
1460llvm::getVectorSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI) {
1461 unsigned Opc = MI.getOpcode();
1462 if (!isBuildVectorOp(Opcode: Opc))
1463 return std::nullopt;
1464 if (auto Splat = getIConstantSplatSExtVal(MI, MRI))
1465 return RegOrConstant(*Splat);
1466 auto Reg = MI.getOperand(i: 1).getReg();
1467 if (any_of(Range: drop_begin(RangeOrContainer: MI.operands(), N: 2),
1468 P: [&Reg](const MachineOperand &Op) { return Op.getReg() != Reg; }))
1469 return std::nullopt;
1470 return RegOrConstant(Reg);
1471}
1472
1473static bool isConstantScalar(const MachineInstr &MI,
1474 const MachineRegisterInfo &MRI,
1475 bool AllowFP = true,
1476 bool AllowOpaqueConstants = true) {
1477 switch (MI.getOpcode()) {
1478 case TargetOpcode::G_CONSTANT:
1479 case TargetOpcode::G_IMPLICIT_DEF:
1480 return true;
1481 case TargetOpcode::G_FCONSTANT:
1482 return AllowFP;
1483 case TargetOpcode::G_GLOBAL_VALUE:
1484 case TargetOpcode::G_FRAME_INDEX:
1485 case TargetOpcode::G_BLOCK_ADDR:
1486 case TargetOpcode::G_JUMP_TABLE:
1487 return AllowOpaqueConstants;
1488 default:
1489 return false;
1490 }
1491}
1492
1493bool llvm::isConstantOrConstantVector(MachineInstr &MI,
1494 const MachineRegisterInfo &MRI) {
1495 Register Def = MI.getOperand(i: 0).getReg();
1496 if (auto C = getIConstantVRegValWithLookThrough(VReg: Def, MRI))
1497 return true;
1498 GBuildVector *BV = dyn_cast<GBuildVector>(Val: &MI);
1499 if (!BV)
1500 return false;
1501 for (unsigned SrcIdx = 0; SrcIdx < BV->getNumSources(); ++SrcIdx) {
1502 if (getIConstantVRegValWithLookThrough(VReg: BV->getSourceReg(I: SrcIdx), MRI) ||
1503 getOpcodeDef<GImplicitDef>(Reg: BV->getSourceReg(I: SrcIdx), MRI))
1504 continue;
1505 return false;
1506 }
1507 return true;
1508}
1509
1510bool llvm::isConstantOrConstantVector(const MachineInstr &MI,
1511 const MachineRegisterInfo &MRI,
1512 bool AllowFP, bool AllowOpaqueConstants) {
1513 if (isConstantScalar(MI, MRI, AllowFP, AllowOpaqueConstants))
1514 return true;
1515
1516 if (!isBuildVectorOp(Opcode: MI.getOpcode()))
1517 return false;
1518
1519 const unsigned NumOps = MI.getNumOperands();
1520 for (unsigned I = 1; I != NumOps; ++I) {
1521 const MachineInstr *ElementDef = MRI.getVRegDef(Reg: MI.getOperand(i: I).getReg());
1522 if (!isConstantScalar(MI: *ElementDef, MRI, AllowFP, AllowOpaqueConstants))
1523 return false;
1524 }
1525
1526 return true;
1527}
1528
1529std::optional<APInt>
1530llvm::isConstantOrConstantSplatVector(MachineInstr &MI,
1531 const MachineRegisterInfo &MRI) {
1532 Register Def = MI.getOperand(i: 0).getReg();
1533 if (auto C = getIConstantVRegValWithLookThrough(VReg: Def, MRI))
1534 return C->Value;
1535 auto MaybeCst = getIConstantSplatSExtVal(MI, MRI);
1536 if (!MaybeCst)
1537 return std::nullopt;
1538 const unsigned ScalarSize = MRI.getType(Reg: Def).getScalarSizeInBits();
1539 return APInt(ScalarSize, *MaybeCst, true);
1540}
1541
1542std::optional<APFloat>
1543llvm::isConstantOrConstantSplatVectorFP(MachineInstr &MI,
1544 const MachineRegisterInfo &MRI) {
1545 Register Def = MI.getOperand(i: 0).getReg();
1546 if (auto FpConst = getFConstantVRegValWithLookThrough(VReg: Def, MRI))
1547 return FpConst->Value;
1548 auto MaybeCstFP = getFConstantSplat(VReg: Def, MRI, /*allowUndef=*/AllowUndef: false);
1549 if (!MaybeCstFP)
1550 return std::nullopt;
1551 return MaybeCstFP->Value;
1552}
1553
1554bool llvm::isNullOrNullSplat(const MachineInstr &MI,
1555 const MachineRegisterInfo &MRI, bool AllowUndefs) {
1556 switch (MI.getOpcode()) {
1557 case TargetOpcode::G_IMPLICIT_DEF:
1558 return AllowUndefs;
1559 case TargetOpcode::G_CONSTANT:
1560 return MI.getOperand(i: 1).getCImm()->isNullValue();
1561 case TargetOpcode::G_FCONSTANT: {
1562 const ConstantFP *FPImm = MI.getOperand(i: 1).getFPImm();
1563 return FPImm->isZero() && !FPImm->isNegative();
1564 }
1565 default:
1566 if (!AllowUndefs) // TODO: isBuildVectorAllZeros assumes undef is OK already
1567 return false;
1568 return isBuildVectorAllZeros(MI, MRI);
1569 }
1570}
1571
1572bool llvm::isAllOnesOrAllOnesSplat(const MachineInstr &MI,
1573 const MachineRegisterInfo &MRI,
1574 bool AllowUndefs) {
1575 switch (MI.getOpcode()) {
1576 case TargetOpcode::G_IMPLICIT_DEF:
1577 return AllowUndefs;
1578 case TargetOpcode::G_CONSTANT:
1579 return MI.getOperand(i: 1).getCImm()->isAllOnesValue();
1580 default:
1581 if (!AllowUndefs) // TODO: isBuildVectorAllOnes assumes undef is OK already
1582 return false;
1583 return isBuildVectorAllOnes(MI, MRI);
1584 }
1585}
1586
1587bool llvm::matchUnaryPredicate(
1588 const MachineRegisterInfo &MRI, Register Reg,
1589 std::function<bool(const Constant *ConstVal)> Match, bool AllowUndefs) {
1590
1591 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);
1592 if (AllowUndefs && Def->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1593 return Match(nullptr);
1594
1595 // TODO: Also handle fconstant
1596 if (Def->getOpcode() == TargetOpcode::G_CONSTANT)
1597 return Match(Def->getOperand(i: 1).getCImm());
1598
1599 if (Def->getOpcode() != TargetOpcode::G_BUILD_VECTOR)
1600 return false;
1601
1602 for (unsigned I = 1, E = Def->getNumOperands(); I != E; ++I) {
1603 Register SrcElt = Def->getOperand(i: I).getReg();
1604 const MachineInstr *SrcDef = getDefIgnoringCopies(Reg: SrcElt, MRI);
1605 if (AllowUndefs && SrcDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) {
1606 if (!Match(nullptr))
1607 return false;
1608 continue;
1609 }
1610
1611 if (SrcDef->getOpcode() != TargetOpcode::G_CONSTANT ||
1612 !Match(SrcDef->getOperand(i: 1).getCImm()))
1613 return false;
1614 }
1615
1616 return true;
1617}
1618
1619bool llvm::isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector,
1620 bool IsFP) {
1621 switch (TLI.getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
1622 case TargetLowering::UndefinedBooleanContent:
1623 return Val & 0x1;
1624 case TargetLowering::ZeroOrOneBooleanContent:
1625 return Val == 1;
1626 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1627 return Val == -1;
1628 }
1629 llvm_unreachable("Invalid boolean contents");
1630}
1631
1632bool llvm::isConstFalseVal(const TargetLowering &TLI, int64_t Val,
1633 bool IsVector, bool IsFP) {
1634 switch (TLI.getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
1635 case TargetLowering::UndefinedBooleanContent:
1636 return ~Val & 0x1;
1637 case TargetLowering::ZeroOrOneBooleanContent:
1638 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1639 return Val == 0;
1640 }
1641 llvm_unreachable("Invalid boolean contents");
1642}
1643
1644int64_t llvm::getICmpTrueVal(const TargetLowering &TLI, bool IsVector,
1645 bool IsFP) {
1646 switch (TLI.getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
1647 case TargetLowering::UndefinedBooleanContent:
1648 case TargetLowering::ZeroOrOneBooleanContent:
1649 return 1;
1650 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1651 return -1;
1652 }
1653 llvm_unreachable("Invalid boolean contents");
1654}
1655
1656void llvm::saveUsesAndErase(MachineInstr &MI, MachineRegisterInfo &MRI,
1657 LostDebugLocObserver *LocObserver,
1658 SmallInstListTy &DeadInstChain) {
1659 for (MachineOperand &Op : MI.uses()) {
1660 if (Op.isReg() && Op.getReg().isVirtual())
1661 DeadInstChain.insert(I: MRI.getVRegDef(Reg: Op.getReg()));
1662 }
1663 LLVM_DEBUG(dbgs() << MI << "Is dead; erasing.\n");
1664 DeadInstChain.remove(I: &MI);
1665 MI.eraseFromParent();
1666 if (LocObserver)
1667 LocObserver->checkpoint(CheckDebugLocs: false);
1668}
1669
1670void llvm::eraseInstrs(ArrayRef<MachineInstr *> DeadInstrs,
1671 MachineRegisterInfo &MRI,
1672 LostDebugLocObserver *LocObserver) {
1673 SmallInstListTy DeadInstChain;
1674 for (MachineInstr *MI : DeadInstrs)
1675 saveUsesAndErase(MI&: *MI, MRI, LocObserver, DeadInstChain);
1676
1677 while (!DeadInstChain.empty()) {
1678 MachineInstr *Inst = DeadInstChain.pop_back_val();
1679 if (!isTriviallyDead(MI: *Inst, MRI))
1680 continue;
1681 saveUsesAndErase(MI&: *Inst, MRI, LocObserver, DeadInstChain);
1682 }
1683}
1684
1685void llvm::eraseInstr(MachineInstr &MI, MachineRegisterInfo &MRI,
1686 LostDebugLocObserver *LocObserver) {
1687 return eraseInstrs(DeadInstrs: {&MI}, MRI, LocObserver);
1688}
1689
1690void llvm::salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI) {
1691 for (auto &Def : MI.defs()) {
1692 assert(Def.isReg() && "Must be a reg");
1693
1694 SmallVector<MachineOperand *, 16> DbgUsers;
1695 for (auto &MOUse : MRI.use_operands(Reg: Def.getReg())) {
1696 MachineInstr *DbgValue = MOUse.getParent();
1697 // Ignore partially formed DBG_VALUEs.
1698 if (DbgValue->isNonListDebugValue() && DbgValue->getNumOperands() == 4) {
1699 DbgUsers.push_back(Elt: &MOUse);
1700 }
1701 }
1702
1703 if (!DbgUsers.empty()) {
1704 salvageDebugInfoForDbgValue(MRI, MI, DbgUsers);
1705 }
1706 }
1707}
1708
1709bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) {
1710 switch (Opc) {
1711 case TargetOpcode::G_FABS:
1712 case TargetOpcode::G_FADD:
1713 case TargetOpcode::G_FCANONICALIZE:
1714 case TargetOpcode::G_FCEIL:
1715 case TargetOpcode::G_FCONSTANT:
1716 case TargetOpcode::G_FCOPYSIGN:
1717 case TargetOpcode::G_FCOS:
1718 case TargetOpcode::G_FDIV:
1719 case TargetOpcode::G_FEXP2:
1720 case TargetOpcode::G_FEXP:
1721 case TargetOpcode::G_FFLOOR:
1722 case TargetOpcode::G_FLOG10:
1723 case TargetOpcode::G_FLOG2:
1724 case TargetOpcode::G_FLOG:
1725 case TargetOpcode::G_FMA:
1726 case TargetOpcode::G_FMAD:
1727 case TargetOpcode::G_FMAXIMUM:
1728 case TargetOpcode::G_FMAXIMUMNUM:
1729 case TargetOpcode::G_FMAXNUM:
1730 case TargetOpcode::G_FMAXNUM_IEEE:
1731 case TargetOpcode::G_FMINIMUM:
1732 case TargetOpcode::G_FMINIMUMNUM:
1733 case TargetOpcode::G_FMINNUM:
1734 case TargetOpcode::G_FMINNUM_IEEE:
1735 case TargetOpcode::G_FMUL:
1736 case TargetOpcode::G_FNEARBYINT:
1737 case TargetOpcode::G_FNEG:
1738 case TargetOpcode::G_FPEXT:
1739 case TargetOpcode::G_FPEXTLOAD:
1740 case TargetOpcode::G_FPOW:
1741 case TargetOpcode::G_FPTRUNC:
1742 case TargetOpcode::G_FPTRUNCSTORE:
1743 case TargetOpcode::G_FREM:
1744 case TargetOpcode::G_FRINT:
1745 case TargetOpcode::G_FSIN:
1746 case TargetOpcode::G_FTAN:
1747 case TargetOpcode::G_FACOS:
1748 case TargetOpcode::G_FASIN:
1749 case TargetOpcode::G_FATAN:
1750 case TargetOpcode::G_FATAN2:
1751 case TargetOpcode::G_FCOSH:
1752 case TargetOpcode::G_FSINH:
1753 case TargetOpcode::G_FTANH:
1754 case TargetOpcode::G_FSQRT:
1755 case TargetOpcode::G_FSUB:
1756 case TargetOpcode::G_INTRINSIC_ROUND:
1757 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
1758 case TargetOpcode::G_INTRINSIC_TRUNC:
1759 return true;
1760 default:
1761 return false;
1762 }
1763}
1764
1765/// Shifts return poison if shiftwidth is larger than the bitwidth.
1766static bool shiftAmountKnownInRange(Register ShiftAmount,
1767 const MachineRegisterInfo &MRI) {
1768 LLT Ty = MRI.getType(Reg: ShiftAmount);
1769
1770 if (Ty.isScalableVector())
1771 return false; // Can't tell, just return false to be safe
1772
1773 if (Ty.isScalar()) {
1774 std::optional<ValueAndVReg> Val =
1775 getIConstantVRegValWithLookThrough(VReg: ShiftAmount, MRI);
1776 if (!Val)
1777 return false;
1778 return Val->Value.ult(RHS: Ty.getScalarSizeInBits());
1779 }
1780
1781 GBuildVector *BV = getOpcodeDef<GBuildVector>(Reg: ShiftAmount, MRI);
1782 if (!BV)
1783 return false;
1784
1785 unsigned Sources = BV->getNumSources();
1786 for (unsigned I = 0; I < Sources; ++I) {
1787 std::optional<ValueAndVReg> Val =
1788 getIConstantVRegValWithLookThrough(VReg: BV->getSourceReg(I), MRI);
1789 if (!Val)
1790 return false;
1791 if (!Val->Value.ult(RHS: Ty.getScalarSizeInBits()))
1792 return false;
1793 }
1794
1795 return true;
1796}
1797
1798static bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
1799 bool ConsiderFlagsAndMetadata,
1800 UndefPoisonKind Kind) {
1801 MachineInstr *RegDef = MRI.getVRegDef(Reg);
1802
1803 if (ConsiderFlagsAndMetadata && includesPoison(Kind))
1804 if (auto *GMI = dyn_cast<GenericMachineInstr>(Val: RegDef))
1805 if (GMI->hasPoisonGeneratingFlags())
1806 return true;
1807
1808 // Check whether opcode is a poison/undef-generating operation.
1809 switch (RegDef->getOpcode()) {
1810 case TargetOpcode::G_BUILD_VECTOR:
1811 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
1812 return false;
1813 case TargetOpcode::G_SHL:
1814 case TargetOpcode::G_ASHR:
1815 case TargetOpcode::G_LSHR:
1816 return includesPoison(Kind) &&
1817 !shiftAmountKnownInRange(ShiftAmount: RegDef->getOperand(i: 2).getReg(), MRI);
1818 case TargetOpcode::G_FPTOSI:
1819 case TargetOpcode::G_FPTOUI:
1820 // fptosi/ui yields poison if the resulting value does not fit in the
1821 // destination type.
1822 return true;
1823 case TargetOpcode::G_CTLZ:
1824 case TargetOpcode::G_CTTZ:
1825 case TargetOpcode::G_CTLS:
1826 case TargetOpcode::G_ABS:
1827 case TargetOpcode::G_CTPOP:
1828 case TargetOpcode::G_BSWAP:
1829 case TargetOpcode::G_BITREVERSE:
1830 case TargetOpcode::G_FSHL:
1831 case TargetOpcode::G_FSHR:
1832 case TargetOpcode::G_SMAX:
1833 case TargetOpcode::G_SMIN:
1834 case TargetOpcode::G_SCMP:
1835 case TargetOpcode::G_UMAX:
1836 case TargetOpcode::G_UMIN:
1837 case TargetOpcode::G_UCMP:
1838 case TargetOpcode::G_PTRMASK:
1839 case TargetOpcode::G_SADDO:
1840 case TargetOpcode::G_SSUBO:
1841 case TargetOpcode::G_UADDO:
1842 case TargetOpcode::G_USUBO:
1843 case TargetOpcode::G_SMULO:
1844 case TargetOpcode::G_UMULO:
1845 case TargetOpcode::G_SADDSAT:
1846 case TargetOpcode::G_UADDSAT:
1847 case TargetOpcode::G_SSUBSAT:
1848 case TargetOpcode::G_USUBSAT:
1849 case TargetOpcode::G_SBFX:
1850 case TargetOpcode::G_UBFX:
1851 return false;
1852 case TargetOpcode::G_SSHLSAT:
1853 case TargetOpcode::G_USHLSAT:
1854 return includesPoison(Kind) &&
1855 !shiftAmountKnownInRange(ShiftAmount: RegDef->getOperand(i: 2).getReg(), MRI);
1856 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1857 GInsertVectorElement *Insert = cast<GInsertVectorElement>(Val: RegDef);
1858 if (includesPoison(Kind)) {
1859 std::optional<ValueAndVReg> Index =
1860 getIConstantVRegValWithLookThrough(VReg: Insert->getIndexReg(), MRI);
1861 if (!Index)
1862 return true;
1863 LLT VecTy = MRI.getType(Reg: Insert->getVectorReg());
1864 return Index->Value.uge(RHS: VecTy.getElementCount().getKnownMinValue());
1865 }
1866 return false;
1867 }
1868 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1869 GExtractVectorElement *Extract = cast<GExtractVectorElement>(Val: RegDef);
1870 if (includesPoison(Kind)) {
1871 std::optional<ValueAndVReg> Index =
1872 getIConstantVRegValWithLookThrough(VReg: Extract->getIndexReg(), MRI);
1873 if (!Index)
1874 return true;
1875 LLT VecTy = MRI.getType(Reg: Extract->getVectorReg());
1876 return Index->Value.uge(RHS: VecTy.getElementCount().getKnownMinValue());
1877 }
1878 return false;
1879 }
1880 case TargetOpcode::G_SHUFFLE_VECTOR: {
1881 GShuffleVector *Shuffle = cast<GShuffleVector>(Val: RegDef);
1882 ArrayRef<int> Mask = Shuffle->getMask();
1883 return includesPoison(Kind) && is_contained(Range&: Mask, Element: -1);
1884 }
1885 case TargetOpcode::G_FNEG:
1886 case TargetOpcode::G_PHI:
1887 case TargetOpcode::G_SELECT:
1888 case TargetOpcode::G_UREM:
1889 case TargetOpcode::G_SREM:
1890 case TargetOpcode::G_FREEZE:
1891 case TargetOpcode::G_ICMP:
1892 case TargetOpcode::G_FCMP:
1893 case TargetOpcode::G_FADD:
1894 case TargetOpcode::G_FSUB:
1895 case TargetOpcode::G_FMUL:
1896 case TargetOpcode::G_FDIV:
1897 case TargetOpcode::G_FREM:
1898 case TargetOpcode::G_PTR_ADD:
1899 return false;
1900 default:
1901 return !isa<GCastOp>(Val: RegDef) && !isa<GBinOp>(Val: RegDef);
1902 }
1903}
1904
1905static bool isGuaranteedNotToBeUndefOrPoison(Register Reg,
1906 const MachineRegisterInfo &MRI,
1907 unsigned Depth,
1908 UndefPoisonKind Kind) {
1909 if (Depth >= MaxAnalysisRecursionDepth)
1910 return false;
1911
1912 MachineInstr *RegDef = MRI.getVRegDef(Reg);
1913
1914 switch (RegDef->getOpcode()) {
1915 case TargetOpcode::G_FREEZE:
1916 return true;
1917 case TargetOpcode::G_IMPLICIT_DEF:
1918 return !includesUndef(Kind);
1919 case TargetOpcode::G_CONSTANT:
1920 case TargetOpcode::G_FCONSTANT:
1921 return true;
1922 case TargetOpcode::G_BUILD_VECTOR: {
1923 GBuildVector *BV = cast<GBuildVector>(Val: RegDef);
1924 unsigned NumSources = BV->getNumSources();
1925 for (unsigned I = 0; I < NumSources; ++I)
1926 if (!::isGuaranteedNotToBeUndefOrPoison(Reg: BV->getSourceReg(I), MRI,
1927 Depth: Depth + 1, Kind))
1928 return false;
1929 return true;
1930 }
1931 case TargetOpcode::G_PHI: {
1932 GPhi *Phi = cast<GPhi>(Val: RegDef);
1933 unsigned NumIncoming = Phi->getNumIncomingValues();
1934 for (unsigned I = 0; I < NumIncoming; ++I)
1935 if (!::isGuaranteedNotToBeUndefOrPoison(Reg: Phi->getIncomingValue(I), MRI,
1936 Depth: Depth + 1, Kind))
1937 return false;
1938 return true;
1939 }
1940 default: {
1941 auto MOCheck = [&](const MachineOperand &MO) {
1942 if (!MO.isReg())
1943 return true;
1944 return ::isGuaranteedNotToBeUndefOrPoison(Reg: MO.getReg(), MRI, Depth: Depth + 1,
1945 Kind);
1946 };
1947 return !::canCreateUndefOrPoison(Reg, MRI,
1948 /*ConsiderFlagsAndMetadata=*/true, Kind) &&
1949 all_of(Range: RegDef->uses(), P: MOCheck);
1950 }
1951 }
1952}
1953
1954bool llvm::canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
1955 bool ConsiderFlagsAndMetadata) {
1956 return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata,
1957 Kind: UndefPoisonKind::UndefOrPoison);
1958}
1959
1960bool canCreatePoison(Register Reg, const MachineRegisterInfo &MRI,
1961 bool ConsiderFlagsAndMetadata = true) {
1962 return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata,
1963 Kind: UndefPoisonKind::PoisonOnly);
1964}
1965
1966bool llvm::isGuaranteedNotToBeUndefOrPoison(Register Reg,
1967 const MachineRegisterInfo &MRI,
1968 unsigned Depth) {
1969 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
1970 Kind: UndefPoisonKind::UndefOrPoison);
1971}
1972
1973bool llvm::isGuaranteedNotToBePoison(Register Reg,
1974 const MachineRegisterInfo &MRI,
1975 unsigned Depth) {
1976 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
1977 Kind: UndefPoisonKind::PoisonOnly);
1978}
1979
1980bool llvm::isGuaranteedNotToBeUndef(Register Reg,
1981 const MachineRegisterInfo &MRI,
1982 unsigned Depth) {
1983 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
1984 Kind: UndefPoisonKind::UndefOnly);
1985}
1986
1987Type *llvm::getTypeForLLT(LLT Ty, LLVMContext &C) {
1988 if (Ty.isVector())
1989 return VectorType::get(ElementType: IntegerType::get(C, NumBits: Ty.getScalarSizeInBits()),
1990 EC: Ty.getElementCount());
1991 return IntegerType::get(C, NumBits: Ty.getSizeInBits());
1992}
1993
1994bool llvm::isAssertMI(const MachineInstr &MI) {
1995 switch (MI.getOpcode()) {
1996 default:
1997 return false;
1998 case TargetOpcode::G_ASSERT_ALIGN:
1999 case TargetOpcode::G_ASSERT_SEXT:
2000 case TargetOpcode::G_ASSERT_ZEXT:
2001 return true;
2002 }
2003}
2004
2005APInt llvm::GIConstant::getScalarValue() const {
2006 assert(Kind == GIConstantKind::Scalar && "Expected scalar constant");
2007
2008 return Value;
2009}
2010
2011std::optional<GIConstant>
2012llvm::GIConstant::getConstant(Register Const, const MachineRegisterInfo &MRI) {
2013 MachineInstr *Constant = getDefIgnoringCopies(Reg: Const, MRI);
2014
2015 if (GSplatVector *Splat = dyn_cast<GSplatVector>(Val: Constant)) {
2016 std::optional<ValueAndVReg> MayBeConstant =
2017 getIConstantVRegValWithLookThrough(VReg: Splat->getScalarReg(), MRI);
2018 if (!MayBeConstant)
2019 return std::nullopt;
2020 return GIConstant(MayBeConstant->Value, GIConstantKind::ScalableVector);
2021 }
2022
2023 if (GBuildVector *Build = dyn_cast<GBuildVector>(Val: Constant)) {
2024 SmallVector<APInt> Values;
2025 unsigned NumSources = Build->getNumSources();
2026 for (unsigned I = 0; I < NumSources; ++I) {
2027 Register SrcReg = Build->getSourceReg(I);
2028 std::optional<ValueAndVReg> MayBeConstant =
2029 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI);
2030 if (!MayBeConstant)
2031 return std::nullopt;
2032 Values.push_back(Elt: MayBeConstant->Value);
2033 }
2034 return GIConstant(Values);
2035 }
2036
2037 std::optional<ValueAndVReg> MayBeConstant =
2038 getIConstantVRegValWithLookThrough(VReg: Const, MRI);
2039 if (!MayBeConstant)
2040 return std::nullopt;
2041
2042 return GIConstant(MayBeConstant->Value, GIConstantKind::Scalar);
2043}
2044
2045APFloat llvm::GFConstant::getScalarValue() const {
2046 assert(Kind == GFConstantKind::Scalar && "Expected scalar constant");
2047
2048 return Values[0];
2049}
2050
2051std::optional<GFConstant>
2052llvm::GFConstant::getConstant(Register Const, const MachineRegisterInfo &MRI) {
2053 MachineInstr *Constant = getDefIgnoringCopies(Reg: Const, MRI);
2054
2055 if (GSplatVector *Splat = dyn_cast<GSplatVector>(Val: Constant)) {
2056 std::optional<FPValueAndVReg> MayBeConstant =
2057 getFConstantVRegValWithLookThrough(VReg: Splat->getScalarReg(), MRI);
2058 if (!MayBeConstant)
2059 return std::nullopt;
2060 return GFConstant(MayBeConstant->Value, GFConstantKind::ScalableVector);
2061 }
2062
2063 if (GBuildVector *Build = dyn_cast<GBuildVector>(Val: Constant)) {
2064 SmallVector<APFloat> Values;
2065 unsigned NumSources = Build->getNumSources();
2066 for (unsigned I = 0; I < NumSources; ++I) {
2067 Register SrcReg = Build->getSourceReg(I);
2068 std::optional<FPValueAndVReg> MayBeConstant =
2069 getFConstantVRegValWithLookThrough(VReg: SrcReg, MRI);
2070 if (!MayBeConstant)
2071 return std::nullopt;
2072 Values.push_back(Elt: MayBeConstant->Value);
2073 }
2074 return GFConstant(Values);
2075 }
2076
2077 std::optional<FPValueAndVReg> MayBeConstant =
2078 getFConstantVRegValWithLookThrough(VReg: Const, MRI);
2079 if (!MayBeConstant)
2080 return std::nullopt;
2081
2082 return GFConstant(MayBeConstant->Value, GFConstantKind::Scalar);
2083}
2084
2085// Returns a list of types to use for memory op lowering in MemOps. A partial
2086// port of findOptimalMemOpLowering in TargetLowering.
2087static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
2088 unsigned Limit, const MemOp &Op,
2089 unsigned DstAS, unsigned SrcAS,
2090 const AttributeList &FuncAttributes,
2091 const TargetLowering &TLI) {
2092 if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
2093 return false;
2094
2095 LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes);
2096
2097 if (Ty == LLT()) {
2098 // Use the largest scalar type whose alignment constraints are satisfied.
2099 // We only need to check DstAlign here as SrcAlign is always greater or
2100 // equal to DstAlign (or zero).
2101 Ty = LLT::integer(SizeInBits: 64);
2102 if (Op.isFixedDstAlign())
2103 while (Op.getDstAlign() < Ty.getSizeInBytes() &&
2104 !TLI.allowsMisalignedMemoryAccesses(Ty, AddrSpace: DstAS, Alignment: Op.getDstAlign()))
2105 Ty = LLT::integer(SizeInBits: Ty.getSizeInBytes());
2106 assert(Ty.getSizeInBits() > 0 && "Could not find valid type");
2107 // FIXME: check for the largest legal type we can load/store to.
2108 }
2109
2110 unsigned NumMemOps = 0;
2111 uint64_t Size = Op.size();
2112 while (Size) {
2113 unsigned TySize = Ty.getSizeInBytes();
2114 while (TySize > Size) {
2115 // For now, only use non-vector load / store's for the left-over pieces.
2116 LLT NewTy = Ty;
2117 // FIXME: check for mem op safety and legality of the types. Not all of
2118 // SDAGisms map cleanly to GISel concepts.
2119 if (NewTy.isVector())
2120 NewTy =
2121 NewTy.getSizeInBits() > 64 ? LLT::integer(SizeInBits: 64) : LLT::integer(SizeInBits: 32);
2122 NewTy = LLT::integer(SizeInBits: llvm::bit_floor(Value: NewTy.getSizeInBits() - 1));
2123 unsigned NewTySize = NewTy.getSizeInBytes();
2124 assert(NewTySize > 0 && "Could not find appropriate type");
2125
2126 // If the new LLT cannot cover all of the remaining bits, then consider
2127 // issuing a (or a pair of) unaligned and overlapping load / store.
2128 unsigned Fast;
2129 // Need to get a VT equivalent for allowMisalignedMemoryAccesses().
2130 MVT VT = getMVTForLLT(Ty);
2131 if (NumMemOps && Op.allowOverlap() && NewTySize < Size &&
2132 TLI.allowsMisalignedMemoryAccesses(
2133 VT, AddrSpace: DstAS, Alignment: Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
2134 Flags: MachineMemOperand::MONone, &Fast) &&
2135 Fast)
2136 TySize = Size;
2137 else {
2138 Ty = NewTy;
2139 TySize = NewTySize;
2140 }
2141 }
2142
2143 if (++NumMemOps > Limit)
2144 return false;
2145
2146 MemOps.push_back(x: Ty);
2147 Size -= TySize;
2148 }
2149
2150 return true;
2151}
2152
2153bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
2154 const MachineRegisterInfo &MRI, unsigned MaxLen,
2155 Register &Dst, Register &Src,
2156 uint64_t &KnownLen, Align &Alignment,
2157 bool &DstAlignCanChange,
2158 std::vector<LLT> &MemOps) {
2159 const unsigned Opc = MI.getOpcode();
2160 assert((Opc == TargetOpcode::G_MEMCPY ||
2161 Opc == TargetOpcode::G_MEMCPY_INLINE ||
2162 Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET ||
2163 Opc == TargetOpcode::G_MEMSET_INLINE) &&
2164 "Expected memcpy like instruction");
2165
2166 auto MMOIt = MI.memoperands_begin();
2167 const MachineMemOperand *MemOp = *MMOIt;
2168
2169 Align DstAlign = MemOp->getBaseAlign();
2170 Align SrcAlign;
2171 Alignment = DstAlign;
2172 Register Len;
2173 std::tie(args&: Dst, args&: Src, args&: Len) = MI.getFirst3Regs();
2174
2175 if (Opc != TargetOpcode::G_MEMSET && Opc != TargetOpcode::G_MEMSET_INLINE) {
2176 assert(MMOIt != MI.memoperands_end() && "Expected a second MMO on MI");
2177 MemOp = *(++MMOIt);
2178 SrcAlign = MemOp->getBaseAlign();
2179 Alignment = std::min(a: DstAlign, b: SrcAlign);
2180 }
2181
2182 // See if this is a constant length copy.
2183 auto LenVRegAndVal = getIConstantVRegValWithLookThrough(VReg: Len, MRI);
2184 if (!LenVRegAndVal) {
2185 // FIXME: support dynamically sized G_MEMCPY_INLINE and G_MEMSET_INLINE
2186 assert(Opc != TargetOpcode::G_MEMCPY_INLINE &&
2187 Opc != TargetOpcode::G_MEMSET_INLINE &&
2188 "inline memcpy and memset with dynamic size are not yet supported");
2189 return false;
2190 }
2191
2192 KnownLen = LenVRegAndVal->Value.getZExtValue();
2193 DstAlignCanChange = false;
2194
2195 if (KnownLen == 0)
2196 return true;
2197
2198 if (Opc != TargetOpcode::G_MEMCPY_INLINE &&
2199 Opc != TargetOpcode::G_MEMSET_INLINE && MaxLen && KnownLen > MaxLen)
2200 return false;
2201
2202 bool IsVolatile = MemOp->isVolatile();
2203 const MachineFunction &MF = *MI.getParent()->getParent();
2204 const auto &TLI = *MF.getSubtarget().getTargetLowering();
2205 // On Darwin, -Os means optimize for size without hurting performance, so
2206 // only really optimize for size when -Oz (MinSize) is used.
2207 bool OptSize = MF.getTarget().getTargetTriple().isOSDarwin()
2208 ? MF.getFunction().hasMinSize()
2209 : MF.getFunction().hasOptSize();
2210
2211 const MachineFrameInfo &MFI = MF.getFrameInfo();
2212 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
2213 if (FIDef && !MFI.isFixedObjectIndex(ObjectIdx: FIDef->getOperand(i: 1).getIndex()))
2214 DstAlignCanChange = true;
2215
2216 const auto &DstMMO = **MI.memoperands_begin();
2217 MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo();
2218
2219 switch (Opc) {
2220 case TargetOpcode::G_MEMCPY_INLINE:
2221 case TargetOpcode::G_MEMCPY: {
2222 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
2223 MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
2224 uint64_t Limit = Opc == TargetOpcode::G_MEMCPY_INLINE
2225 ? std::numeric_limits<uint64_t>::max()
2226 : TLI.getMaxStoresPerMemcpy(OptSize);
2227 return findGISelOptimalMemOpLowering(
2228 MemOps, Limit,
2229 Op: MemOp::Copy(Size: KnownLen, DstAlignCanChange, DstAlign: std::min(a: DstAlign, b: SrcAlign),
2230 SrcAlign, IsVolatile),
2231 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
2232 FuncAttributes: MF.getFunction().getAttributes(), TLI);
2233 }
2234 case TargetOpcode::G_MEMMOVE: {
2235 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
2236 MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
2237 unsigned Limit = TLI.getMaxStoresPerMemmove(OptSize);
2238 // FIXME: SelectionDAG always passes false for 'AllowOverlap', apparently
2239 // due to a bug in it's findOptimalMemOpLowering implementation. For now do
2240 // the same thing here.
2241 return findGISelOptimalMemOpLowering(
2242 MemOps, Limit,
2243 Op: MemOp::Copy(Size: KnownLen, DstAlignCanChange, DstAlign: std::min(a: DstAlign, b: SrcAlign),
2244 SrcAlign, /*IsVolatile=*/true),
2245 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
2246 FuncAttributes: MF.getFunction().getAttributes(), TLI);
2247 }
2248 case TargetOpcode::G_MEMSET:
2249 case TargetOpcode::G_MEMSET_INLINE: {
2250 unsigned Limit = Opc == TargetOpcode::G_MEMSET_INLINE
2251 ? std::numeric_limits<unsigned>::max()
2252 : TLI.getMaxStoresPerMemset(OptSize);
2253 auto ValVRegAndVal = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
2254 bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0;
2255 return findGISelOptimalMemOpLowering(
2256 MemOps, Limit,
2257 Op: MemOp::Set(Size: KnownLen, DstAlignCanChange, DstAlign,
2258 /*IsZeroMemset=*/IsZeroVal,
2259 /*IsVolatile=*/IsVolatile),
2260 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes(), TLI);
2261 }
2262 default:
2263 llvm_unreachable("Unexpected memcpy-family opcode");
2264 }
2265}
2266