1//===-- lib/CodeGen/GlobalISel/GICombinerHelper.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
9#include "llvm/ADT/APFloat.h"
10#include "llvm/ADT/FloatingPointMode.h"
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/ADT/SetVector.h"
13#include "llvm/ADT/SmallBitVector.h"
14#include "llvm/Analysis/CmpInstAnalysis.h"
15#include "llvm/Analysis/ValueTracking.h"
16#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
17#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
18#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
19#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
20#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
21#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
22#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
23#include "llvm/CodeGen/GlobalISel/Utils.h"
24#include "llvm/CodeGen/LowLevelTypeUtils.h"
25#include "llvm/CodeGen/MachineBasicBlock.h"
26#include "llvm/CodeGen/MachineDominators.h"
27#include "llvm/CodeGen/MachineFunction.h"
28#include "llvm/CodeGen/MachineInstr.h"
29#include "llvm/CodeGen/MachineMemOperand.h"
30#include "llvm/CodeGen/MachineRegisterInfo.h"
31#include "llvm/CodeGen/Register.h"
32#include "llvm/CodeGen/RegisterBankInfo.h"
33#include "llvm/CodeGen/TargetInstrInfo.h"
34#include "llvm/CodeGen/TargetLowering.h"
35#include "llvm/CodeGen/TargetOpcodes.h"
36#include "llvm/IR/ConstantRange.h"
37#include "llvm/IR/DataLayout.h"
38#include "llvm/IR/InstrTypes.h"
39#include "llvm/Support/Casting.h"
40#include "llvm/Support/DivisionByConstantInfo.h"
41#include "llvm/Support/ErrorHandling.h"
42#include "llvm/Support/KnownBits.h"
43#include "llvm/Support/MathExtras.h"
44#include "llvm/Target/TargetMachine.h"
45#include <cmath>
46#include <optional>
47#include <tuple>
48
49#define DEBUG_TYPE "gi-combiner"
50
51using namespace llvm;
52using namespace MIPatternMatch;
53
54// Option to allow testing of the combiner while no targets know about indexed
55// addressing.
56static cl::opt<bool>
57 ForceLegalIndexing("force-legal-indexing", cl::Hidden, cl::init(Val: false),
58 cl::desc("Force all indexed operations to be "
59 "legal for the GlobalISel combiner"));
60
61CombinerHelper::CombinerHelper(GISelChangeObserver &Observer,
62 MachineIRBuilder &B, bool IsPreLegalize,
63 GISelValueTracking *VT,
64 MachineDominatorTree *MDT,
65 const LegalizerInfo *LI)
66 : Builder(B), MRI(Builder.getMF().getRegInfo()), Observer(Observer), VT(VT),
67 MDT(MDT), IsPreLegalize(IsPreLegalize), LI(LI),
68 TII(Builder.getMF().getSubtarget().getInstrInfo()),
69 RBI(Builder.getMF().getSubtarget().getRegBankInfo()),
70 TRI(Builder.getMF().getSubtarget().getRegisterInfo()) {
71 (void)this->VT;
72}
73
74const TargetLowering &CombinerHelper::getTargetLowering() const {
75 return *Builder.getMF().getSubtarget().getTargetLowering();
76}
77
78const MachineFunction &CombinerHelper::getMachineFunction() const {
79 return Builder.getMF();
80}
81
82LLVMContext &CombinerHelper::getContext() const { return Builder.getContext(); }
83
84/// \returns The little endian in-memory byte position of byte \p I in a
85/// \p ByteWidth bytes wide type.
86///
87/// E.g. Given a 4-byte type x, x[0] -> byte 0
88static unsigned littleEndianByteAt(const unsigned ByteWidth, const unsigned I) {
89 assert(I < ByteWidth && "I must be in [0, ByteWidth)");
90 return I;
91}
92
93/// Determines the LogBase2 value for a non-null input value using the
94/// transform: LogBase2(V) = (EltBits - 1) - ctlz(V).
95static Register buildLogBase2(Register V, MachineIRBuilder &MIB) {
96 auto &MRI = *MIB.getMRI();
97 LLT Ty = MRI.getType(Reg: V);
98 auto Ctlz = MIB.buildCTLZ(Dst: Ty, Src0: V);
99 auto Base = MIB.buildConstant(Res: Ty, Val: Ty.getScalarSizeInBits() - 1);
100 return MIB.buildSub(Dst: Ty, Src0: Base, Src1: Ctlz).getReg(Idx: 0);
101}
102
103/// \returns The big endian in-memory byte position of byte \p I in a
104/// \p ByteWidth bytes wide type.
105///
106/// E.g. Given a 4-byte type x, x[0] -> byte 3
107static unsigned bigEndianByteAt(const unsigned ByteWidth, const unsigned I) {
108 assert(I < ByteWidth && "I must be in [0, ByteWidth)");
109 return ByteWidth - I - 1;
110}
111
112/// Given a map from byte offsets in memory to indices in a load/store,
113/// determine if that map corresponds to a little or big endian byte pattern.
114///
115/// \param MemOffset2Idx maps memory offsets to address offsets.
116/// \param LowestIdx is the lowest index in \p MemOffset2Idx.
117///
118/// \returns true if the map corresponds to a big endian byte pattern, false if
119/// it corresponds to a little endian byte pattern, and std::nullopt otherwise.
120///
121/// E.g. given a 32-bit type x, and x[AddrOffset], the in-memory byte patterns
122/// are as follows:
123///
124/// AddrOffset Little endian Big endian
125/// 0 0 3
126/// 1 1 2
127/// 2 2 1
128/// 3 3 0
129static std::optional<bool>
130isBigEndian(const SmallDenseMap<int64_t, int64_t, 8> &MemOffset2Idx,
131 int64_t LowestIdx) {
132 // Need at least two byte positions to decide on endianness.
133 unsigned Width = MemOffset2Idx.size();
134 if (Width < 2)
135 return std::nullopt;
136 bool BigEndian = true, LittleEndian = true;
137 for (unsigned MemOffset = 0; MemOffset < Width; ++ MemOffset) {
138 auto MemOffsetAndIdx = MemOffset2Idx.find(Val: MemOffset);
139 if (MemOffsetAndIdx == MemOffset2Idx.end())
140 return std::nullopt;
141 const int64_t Idx = MemOffsetAndIdx->second - LowestIdx;
142 assert(Idx >= 0 && "Expected non-negative byte offset?");
143 LittleEndian &= Idx == littleEndianByteAt(ByteWidth: Width, I: MemOffset);
144 BigEndian &= Idx == bigEndianByteAt(ByteWidth: Width, I: MemOffset);
145 if (!BigEndian && !LittleEndian)
146 return std::nullopt;
147 }
148
149 assert((BigEndian != LittleEndian) &&
150 "Pattern cannot be both big and little endian!");
151 return BigEndian;
152}
153
154bool CombinerHelper::isPreLegalize() const { return IsPreLegalize; }
155
156bool CombinerHelper::isLegal(const LegalityQuery &Query) const {
157 assert(LI && "Must have LegalizerInfo to query isLegal!");
158 return LI->getAction(Query).Action == LegalizeActions::Legal;
159}
160
161bool CombinerHelper::isLegalOrBeforeLegalizer(
162 const LegalityQuery &Query) const {
163 return isPreLegalize() || isLegal(Query);
164}
165
166bool CombinerHelper::isLegalOrHasWidenScalar(const LegalityQuery &Query) const {
167 return isLegal(Query) ||
168 LI->getAction(Query).Action == LegalizeActions::WidenScalar;
169}
170
171bool CombinerHelper::isLegalOrHasFewerElements(
172 const LegalityQuery &Query) const {
173 LegalizeAction Action = LI->getAction(Query).Action;
174 return Action == LegalizeActions::Legal ||
175 Action == LegalizeActions::FewerElements;
176}
177
178bool CombinerHelper::isImmConstantLegalOrBeforeLegalizer(unsigned Opc,
179 const LLT Ty) const {
180 if (!Ty.isVector())
181 return isLegalOrBeforeLegalizer(Query: {Opc, {Ty}});
182 if (isPreLegalize())
183 return true;
184 // Scalable vectors cannot be constant folded here.
185 if (!Ty.isFixedVector())
186 return false;
187 LLT EltTy = Ty.getElementType();
188 return isLegal(Query: {TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}}) &&
189 isLegal(Query: {Opc, {EltTy}});
190}
191
192bool CombinerHelper::isConstantLegalOrBeforeLegalizer(const LLT Ty) const {
193 return isImmConstantLegalOrBeforeLegalizer(Opc: TargetOpcode::G_CONSTANT, Ty);
194}
195
196bool CombinerHelper::isFPConstantLegalOrBeforeLegalizer(const LLT Ty) const {
197 return isImmConstantLegalOrBeforeLegalizer(Opc: TargetOpcode::G_FCONSTANT, Ty);
198}
199
200void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg,
201 Register ToReg) const {
202 Observer.changingAllUsesOfReg(MRI, Reg: FromReg);
203
204 if (MRI.constrainRegAttrs(Reg: ToReg, ConstrainingReg: FromReg))
205 MRI.replaceRegWith(FromReg, ToReg);
206 else
207 Builder.buildCopy(Res: FromReg, Op: ToReg);
208
209 Observer.finishedChangingAllUsesOfReg();
210}
211
212void CombinerHelper::replaceRegOpWith(MachineRegisterInfo &MRI,
213 MachineOperand &FromRegOp,
214 Register ToReg) const {
215 assert(FromRegOp.getParent() && "Expected an operand in an MI");
216 Observer.changingInstr(MI&: *FromRegOp.getParent());
217
218 FromRegOp.setReg(ToReg);
219
220 Observer.changedInstr(MI&: *FromRegOp.getParent());
221}
222
223void CombinerHelper::replaceOpcodeWith(MachineInstr &FromMI,
224 unsigned ToOpcode) const {
225 Observer.changingInstr(MI&: FromMI);
226
227 FromMI.setDesc(Builder.getTII().get(Opcode: ToOpcode));
228
229 Observer.changedInstr(MI&: FromMI);
230}
231
232const RegisterBank *CombinerHelper::getRegBank(Register Reg) const {
233 return RBI->getRegBank(Reg, MRI, TRI: *TRI);
234}
235
236void CombinerHelper::setRegBank(Register Reg,
237 const RegisterBank *RegBank) const {
238 if (RegBank)
239 MRI.setRegBank(Reg, RegBank: *RegBank);
240}
241
242bool CombinerHelper::matchCombineCopy(MachineInstr &MI) const {
243 if (MI.getOpcode() != TargetOpcode::COPY)
244 return false;
245 Register DstReg = MI.getOperand(i: 0).getReg();
246 Register SrcReg = MI.getOperand(i: 1).getReg();
247 return canReplaceReg(DstReg, SrcReg, MRI);
248}
249void CombinerHelper::applyCombineCopy(MachineInstr &MI) const {
250 Register DstReg = MI.getOperand(i: 0).getReg();
251 Register SrcReg = MI.getOperand(i: 1).getReg();
252 replaceRegWith(MRI, FromReg: DstReg, ToReg: SrcReg);
253 MI.eraseFromParent();
254}
255
256bool CombinerHelper::matchFreezeOfSingleMaybePoisonOperand(
257 MachineInstr &MI, BuildFnTy &MatchInfo) const {
258 assert(MI.getOpcode() == TargetOpcode::G_FREEZE && "Invalid instruction");
259
260 // Ported from InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating.
261 Register DstOp = MI.getOperand(i: 0).getReg();
262 Register OrigOp = MI.getOperand(i: 1).getReg();
263
264 if (!MRI.hasOneNonDBGUse(RegNo: OrigOp))
265 return false;
266
267 MachineInstr *OrigDef;
268 if (!mi_match(R: OrigOp, MRI, P: m_MInstr(MI&: OrigDef)))
269 return false;
270 // Even if only a single operand of the PHI is not guaranteed non-poison,
271 // moving freeze() backwards across a PHI can cause optimization issues for
272 // other users of that operand.
273 //
274 // Moving freeze() from one of the output registers of a G_UNMERGE_VALUES to
275 // the source register is unprofitable because it makes the freeze() more
276 // strict than is necessary (it would affect the whole register instead of
277 // just the subreg being frozen).
278 if (OrigDef->isPHI() || isa<GUnmerge>(Val: OrigDef))
279 return false;
280
281 if (canCreateUndefOrPoison(Reg: OrigOp, MRI,
282 /*ConsiderFlagsAndMetadata=*/false))
283 return false;
284
285 std::optional<MachineOperand> MaybePoisonOperand;
286 for (MachineOperand &Operand : OrigDef->uses()) {
287 if (!Operand.isReg())
288 return false;
289
290 if (isGuaranteedNotToBeUndefOrPoison(Reg: Operand.getReg(), MRI))
291 continue;
292
293 if (!MaybePoisonOperand)
294 MaybePoisonOperand = Operand;
295 else {
296 // We have more than one maybe-poison operand. Moving the freeze is
297 // unsafe.
298 return false;
299 }
300 }
301
302 // Eliminate freeze if all operands are guaranteed non-poison.
303 if (!MaybePoisonOperand) {
304 MatchInfo = [=](MachineIRBuilder &B) {
305 Observer.changingInstr(MI&: *OrigDef);
306 cast<GenericMachineInstr>(Val: OrigDef)->dropPoisonGeneratingFlags();
307 Observer.changedInstr(MI&: *OrigDef);
308 B.buildCopy(Res: DstOp, Op: OrigOp);
309 };
310 return true;
311 }
312
313 Register MaybePoisonOperandReg = MaybePoisonOperand->getReg();
314 LLT MaybePoisonOperandRegTy = MRI.getType(Reg: MaybePoisonOperandReg);
315
316 if (!isLegalOrBeforeLegalizer(
317 Query: {TargetOpcode::G_FREEZE, {MaybePoisonOperandRegTy}}))
318 return false;
319
320 MatchInfo = [=](MachineIRBuilder &B) mutable {
321 Observer.changingInstr(MI&: *OrigDef);
322 cast<GenericMachineInstr>(Val: OrigDef)->dropPoisonGeneratingFlags();
323 Observer.changedInstr(MI&: *OrigDef);
324 B.setInsertPt(MBB&: *OrigDef->getParent(), II: OrigDef->getIterator());
325 auto Freeze = B.buildFreeze(Dst: MaybePoisonOperandRegTy, Src: MaybePoisonOperandReg);
326 replaceRegOpWith(
327 MRI, FromRegOp&: *OrigDef->findRegisterUseOperand(Reg: MaybePoisonOperandReg, TRI),
328 ToReg: Freeze.getReg(Idx: 0));
329 replaceRegWith(MRI, FromReg: DstOp, ToReg: OrigOp);
330 };
331 return true;
332}
333
334bool CombinerHelper::matchCombineConcatVectors(
335 MachineInstr &MI, SmallVector<Register> &Ops) const {
336 assert(MI.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
337 "Invalid instruction");
338 bool IsUndef = true;
339 MachineInstr *Undef = nullptr;
340
341 // Walk over all the operands of concat vectors and check if they are
342 // build_vector themselves or undef.
343 // Then collect their operands in Ops.
344 for (const MachineOperand &MO : MI.uses()) {
345 Register Reg = MO.getReg();
346 MachineInstr *Def;
347 if (!mi_match(R: Reg, MRI, P: m_MInstr(MI&: Def)))
348 return false;
349 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
350 return false;
351 switch (Def->getOpcode()) {
352 case TargetOpcode::G_BUILD_VECTOR:
353 IsUndef = false;
354 // Remember the operands of the build_vector to fold
355 // them into the yet-to-build flattened concat vectors.
356 for (const MachineOperand &BuildVecMO : Def->uses())
357 Ops.push_back(Elt: BuildVecMO.getReg());
358 break;
359 case TargetOpcode::G_IMPLICIT_DEF: {
360 LLT OpType = MRI.getType(Reg);
361 // Keep one undef value for all the undef operands.
362 if (!Undef) {
363 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
364 Undef = Builder.buildUndef(Res: OpType.getScalarType());
365 }
366 assert(MRI.getType(Undef->getOperand(0).getReg()) ==
367 OpType.getScalarType() &&
368 "All undefs should have the same type");
369 // Break the undef vector in as many scalar elements as needed
370 // for the flattening.
371 for (unsigned EltIdx = 0, EltEnd = OpType.getNumElements();
372 EltIdx != EltEnd; ++EltIdx)
373 Ops.push_back(Elt: Undef->getOperand(i: 0).getReg());
374 break;
375 }
376 default:
377 return false;
378 }
379 }
380
381 // Check if the combine is illegal
382 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
383 if (!isLegalOrBeforeLegalizer(
384 Query: {TargetOpcode::G_BUILD_VECTOR, {DstTy, MRI.getType(Reg: Ops[0])}})) {
385 return false;
386 }
387
388 if (IsUndef)
389 Ops.clear();
390
391 return true;
392}
393void CombinerHelper::applyCombineConcatVectors(
394 MachineInstr &MI, SmallVector<Register> &Ops) const {
395 // We determined that the concat_vectors can be flatten.
396 // Generate the flattened build_vector.
397 Register DstReg = MI.getOperand(i: 0).getReg();
398 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
399 Register NewDstReg = MRI.cloneVirtualRegister(VReg: DstReg);
400
401 // Note: IsUndef is sort of redundant. We could have determine it by
402 // checking that at all Ops are undef. Alternatively, we could have
403 // generate a build_vector of undefs and rely on another combine to
404 // clean that up. For now, given we already gather this information
405 // in matchCombineConcatVectors, just save compile time and issue the
406 // right thing.
407 if (Ops.empty())
408 Builder.buildUndef(Res: NewDstReg);
409 else
410 Builder.buildBuildVector(Res: NewDstReg, Ops);
411 replaceRegWith(MRI, FromReg: DstReg, ToReg: NewDstReg);
412 MI.eraseFromParent();
413}
414
415bool CombinerHelper::matchCombineBuildVectorOfBitcast(
416 MachineInstr &MI, SmallVector<Register> &Ops) const {
417 auto &BV = cast<GBuildVector>(Val&: MI);
418
419 // Look at the first operand for a unmerge(bitcast) from a scalar type.
420 GUnmerge *Unmerge = getOpcodeDef<GUnmerge>(Reg: BV.getSourceReg(I: 0), MRI);
421 if (!Unmerge || Unmerge->getReg(Idx: 0) != BV.getSourceReg(I: 0))
422 return false;
423 Register BCSrc;
424 if (!mi_match(R: Unmerge->getSourceReg(), MRI, P: m_GBitcast(Src: m_Reg(R&: BCSrc))))
425 return false;
426 LLT InputTy = MRI.getType(Reg: BCSrc);
427 unsigned Factor = Unmerge->getNumDefs();
428 if (!InputTy.isScalar() || BV.getNumSources() % Factor != 0)
429 return false;
430
431 // Check if the build_vector is legal
432 LLT BVDstTy = LLT::fixed_vector(NumElements: BV.getNumSources() / Factor, ScalarTy: InputTy);
433 if (!isLegal(Query: {TargetOpcode::G_BUILD_VECTOR, {BVDstTy, InputTy}}))
434 return false;
435
436 // Check all other operands are bitcasts or undef.
437 for (unsigned Idx = 0; Idx < BV.getNumSources(); Idx += Factor) {
438 GUnmerge *Unmerge = getOpcodeDef<GUnmerge>(Reg: BV.getSourceReg(I: Idx), MRI);
439 if (!all_of(Range: iota_range<unsigned>(0, Factor, false), P: [&](unsigned J) {
440 if (mi_match(R: BV.getSourceReg(I: Idx + J), MRI, P: m_GImplicitDef()))
441 return true;
442 return Unmerge && BV.getSourceReg(I: Idx + J) == Unmerge->getReg(Idx: J);
443 }))
444 return false;
445 if (!Unmerge)
446 Ops.push_back(Elt: 0);
447 else {
448 Register BCSrc;
449 if (!mi_match(
450 R: Unmerge->getSourceReg(), MRI,
451 P: m_GBitcast(Src: m_all_of(preds: m_Reg(R&: BCSrc), preds: m_SpecificType(Ty: InputTy)))))
452 return false;
453 Ops.push_back(Elt: BCSrc);
454 }
455 }
456
457 return true;
458}
459
460void CombinerHelper::applyCombineBuildVectorOfBitcast(
461 MachineInstr &MI, SmallVector<Register> &Ops) const {
462 LLT SrcTy = MRI.getType(Reg: Ops[0]);
463 // Build undef if any operations require it.
464 Register Undef = 0;
465 for (Register &Op : Ops) {
466 if (!Op) {
467 if (!Undef)
468 Undef = Builder.buildUndef(Res: SrcTy).getReg(Idx: 0);
469 Op = Undef;
470 }
471 }
472
473 LLT BVDstTy = LLT::fixed_vector(NumElements: Ops.size(), ScalarTy: SrcTy);
474 auto BV = Builder.buildBuildVector(Res: BVDstTy, Ops);
475 Builder.buildBitcast(Dst: MI.getOperand(i: 0).getReg(), Src: BV);
476 MI.eraseFromParent();
477}
478
479void CombinerHelper::applyCombineShuffleToBuildVector(MachineInstr &MI) const {
480 auto &Shuffle = cast<GShuffleVector>(Val&: MI);
481
482 Register SrcVec1 = Shuffle.getSrc1Reg();
483 Register SrcVec2 = Shuffle.getSrc2Reg();
484 LLT EltTy = MRI.getType(Reg: SrcVec1).getElementType();
485 int Width = MRI.getType(Reg: SrcVec1).getNumElements();
486
487 auto Unmerge1 = Builder.buildUnmerge(Res: EltTy, Op: SrcVec1);
488 auto Unmerge2 = Builder.buildUnmerge(Res: EltTy, Op: SrcVec2);
489
490 SmallVector<Register> Extracts;
491 // Select only applicable elements from unmerged values.
492 for (int Val : Shuffle.getMask()) {
493 if (Val == -1)
494 Extracts.push_back(Elt: Builder.buildUndef(Res: EltTy).getReg(Idx: 0));
495 else if (Val < Width)
496 Extracts.push_back(Elt: Unmerge1.getReg(Idx: Val));
497 else
498 Extracts.push_back(Elt: Unmerge2.getReg(Idx: Val - Width));
499 }
500 assert(Extracts.size() > 0 && "Expected at least one element in the shuffle");
501 if (Extracts.size() == 1)
502 Builder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: Extracts[0]);
503 else
504 Builder.buildBuildVector(Res: MI.getOperand(i: 0).getReg(), Ops: Extracts);
505 MI.eraseFromParent();
506}
507
508void CombinerHelper::applyCombineShuffleOfBuildVector(MachineInstr &MI) const {
509 auto &Shuffle = cast<GShuffleVector>(Val&: MI);
510 Register SrcVec1 = Shuffle.getSrc1Reg();
511 Register SrcVec2 = Shuffle.getSrc2Reg();
512 auto *BuildMI1 = getOpcodeDef<GBuildVector>(Reg: SrcVec1, MRI);
513 auto *BuildMI2 = getOpcodeDef<GBuildVector>(Reg: SrcVec2, MRI);
514 assert((BuildMI1 && (BuildMI2 || MRI.getVRegDef(SrcVec2)->getOpcode() ==
515 TargetOpcode::G_IMPLICIT_DEF)) &&
516 "Unexpected input to applyCombineShuffleOfBuildVector");
517 LLT EltTy = MRI.getType(Reg: SrcVec1).getElementType();
518 int NumElts = MRI.getType(Reg: SrcVec1).getNumElements();
519
520 SmallVector<Register> Extracts;
521 for (int Val : Shuffle.getMask()) {
522 if (Val == -1)
523 Extracts.push_back(Elt: Builder.buildUndef(Res: EltTy).getReg(Idx: 0));
524 else if (Val < NumElts)
525 Extracts.push_back(Elt: BuildMI1->getSourceReg(I: Val));
526 else if (BuildMI2)
527 Extracts.push_back(Elt: BuildMI2->getSourceReg(I: Val - NumElts));
528 else
529 Extracts.push_back(Elt: Builder.buildUndef(Res: EltTy).getReg(Idx: 0));
530 }
531
532 if (Extracts.size() == 1)
533 Builder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: Extracts[0]);
534 else
535 Builder.buildBuildVector(Res: MI.getOperand(i: 0).getReg(), Ops: Extracts);
536 MI.eraseFromParent();
537}
538
539bool CombinerHelper::matchCombineShuffleConcat(
540 MachineInstr &MI, SmallVector<Register> &Ops) const {
541 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
542 GConcatVectors *ConcatMI1, *ConcatMI2;
543 if (!mi_match(R: MI.getOperand(i: 1).getReg(), MRI, P: m_GConcatVectors(Inst&: ConcatMI1)) ||
544 !mi_match(R: MI.getOperand(i: 2).getReg(), MRI, P: m_GConcatVectors(Inst&: ConcatMI2)))
545 return false;
546
547 // Check that the sources of the Concat instructions have the same type
548 if (MRI.getType(Reg: ConcatMI1->getSourceReg(I: 0)) !=
549 MRI.getType(Reg: ConcatMI2->getSourceReg(I: 0)))
550 return false;
551
552 LLT ConcatSrcTy = MRI.getType(Reg: ConcatMI1->getReg(Idx: 1));
553 LLT ShuffleSrcTy1 = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
554 unsigned ConcatSrcNumElt = ConcatSrcTy.getNumElements();
555 for (unsigned i = 0; i < Mask.size(); i += ConcatSrcNumElt) {
556 // Check if the index takes a whole source register from G_CONCAT_VECTORS
557 // Assumes that all Sources of G_CONCAT_VECTORS are the same type
558 if (Mask[i] == -1) {
559 for (unsigned j = 1; j < ConcatSrcNumElt; j++) {
560 if (i + j >= Mask.size())
561 return false;
562 if (Mask[i + j] != -1)
563 return false;
564 }
565 if (!isLegalOrBeforeLegalizer(
566 Query: {TargetOpcode::G_IMPLICIT_DEF, {ConcatSrcTy}}))
567 return false;
568 Ops.push_back(Elt: 0);
569 } else if (Mask[i] % ConcatSrcNumElt == 0) {
570 for (unsigned j = 1; j < ConcatSrcNumElt; j++) {
571 if (i + j >= Mask.size())
572 return false;
573 if (Mask[i + j] != Mask[i] + static_cast<int>(j))
574 return false;
575 }
576 // Retrieve the source register from its respective G_CONCAT_VECTORS
577 // instruction
578 if (Mask[i] < ShuffleSrcTy1.getNumElements()) {
579 Ops.push_back(Elt: ConcatMI1->getSourceReg(I: Mask[i] / ConcatSrcNumElt));
580 } else {
581 Ops.push_back(Elt: ConcatMI2->getSourceReg(I: Mask[i] / ConcatSrcNumElt -
582 ConcatMI1->getNumSources()));
583 }
584 } else {
585 return false;
586 }
587 }
588
589 if (!isLegalOrBeforeLegalizer(
590 Query: {TargetOpcode::G_CONCAT_VECTORS,
591 {MRI.getType(Reg: MI.getOperand(i: 0).getReg()), ConcatSrcTy}}))
592 return false;
593
594 return !Ops.empty();
595}
596
597void CombinerHelper::applyCombineShuffleConcat(
598 MachineInstr &MI, SmallVector<Register> &Ops) const {
599 LLT SrcTy;
600 for (Register &Reg : Ops) {
601 if (Reg != 0)
602 SrcTy = MRI.getType(Reg);
603 }
604 assert(SrcTy.isValid() && "Unexpected full undef vector in concat combine");
605
606 Register UndefReg = 0;
607
608 for (Register &Reg : Ops) {
609 if (Reg == 0) {
610 if (UndefReg == 0)
611 UndefReg = Builder.buildUndef(Res: SrcTy).getReg(Idx: 0);
612 Reg = UndefReg;
613 }
614 }
615
616 if (Ops.size() > 1)
617 Builder.buildConcatVectors(Res: MI.getOperand(i: 0).getReg(), Ops);
618 else
619 Builder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: Ops[0]);
620 MI.eraseFromParent();
621}
622
623bool CombinerHelper::matchCombineShuffleVector(
624 MachineInstr &MI, SmallVectorImpl<Register> &Ops) const {
625 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
626 "Invalid instruction kind");
627 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
628 Register Src1 = MI.getOperand(i: 1).getReg();
629 LLT SrcType = MRI.getType(Reg: Src1);
630
631 unsigned DstNumElts = DstType.getNumElements();
632 unsigned SrcNumElts = SrcType.getNumElements();
633
634 // If the resulting vector is smaller than the size of the source
635 // vectors being concatenated, we won't be able to replace the
636 // shuffle vector into a concat_vectors.
637 //
638 // Note: We may still be able to produce a concat_vectors fed by
639 // extract_vector_elt and so on. It is less clear that would
640 // be better though, so don't bother for now.
641 //
642 // If the destination is a scalar, the size of the sources doesn't
643 // matter. we will lower the shuffle to a plain copy. This will
644 // work only if the source and destination have the same size. But
645 // that's covered by the next condition.
646 //
647 // TODO: If the size between the source and destination don't match
648 // we could still emit an extract vector element in that case.
649 if (DstNumElts < 2 * SrcNumElts)
650 return false;
651
652 // Check that the shuffle mask can be broken evenly between the
653 // different sources.
654 if (DstNumElts % SrcNumElts != 0)
655 return false;
656
657 // Mask length is a multiple of the source vector length.
658 // Check if the shuffle is some kind of concatenation of the input
659 // vectors.
660 unsigned NumConcat = DstNumElts / SrcNumElts;
661 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
662 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
663 for (unsigned i = 0; i != DstNumElts; ++i) {
664 int Idx = Mask[i];
665 // Undef value.
666 if (Idx < 0)
667 continue;
668 // Ensure the indices in each SrcType sized piece are sequential and that
669 // the same source is used for the whole piece.
670 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
671 (ConcatSrcs[i / SrcNumElts] >= 0 &&
672 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts)))
673 return false;
674 // Remember which source this index came from.
675 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
676 }
677
678 // The shuffle is concatenating multiple vectors together.
679 // Collect the different operands for that.
680 Register UndefReg;
681 Register Src2 = MI.getOperand(i: 2).getReg();
682 for (auto Src : ConcatSrcs) {
683 if (Src < 0) {
684 if (!UndefReg) {
685 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
686 UndefReg = Builder.buildUndef(Res: SrcType).getReg(Idx: 0);
687 }
688 Ops.push_back(Elt: UndefReg);
689 } else if (Src == 0)
690 Ops.push_back(Elt: Src1);
691 else
692 Ops.push_back(Elt: Src2);
693 }
694 return true;
695}
696
697void CombinerHelper::applyCombineShuffleVector(MachineInstr &MI,
698 ArrayRef<Register> Ops) const {
699 Register DstReg = MI.getOperand(i: 0).getReg();
700 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
701 Register NewDstReg = MRI.cloneVirtualRegister(VReg: DstReg);
702
703 if (Ops.size() == 1)
704 Builder.buildCopy(Res: NewDstReg, Op: Ops[0]);
705 else
706 Builder.buildMergeLikeInstr(Res: NewDstReg, Ops);
707
708 replaceRegWith(MRI, FromReg: DstReg, ToReg: NewDstReg);
709 MI.eraseFromParent();
710}
711
712namespace {
713
714/// Select a preference between two uses. CurrentUse is the current preference
715/// while *ForCandidate is attributes of the candidate under consideration.
716PreferredTuple ChoosePreferredUse(MachineInstr &LoadMI,
717 PreferredTuple &CurrentUse,
718 const LLT TyForCandidate,
719 unsigned OpcodeForCandidate,
720 MachineInstr *MIForCandidate) {
721 if (!CurrentUse.Ty.isValid()) {
722 if (CurrentUse.ExtendOpcode == OpcodeForCandidate ||
723 CurrentUse.ExtendOpcode == TargetOpcode::G_ANYEXT)
724 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
725 return CurrentUse;
726 }
727
728 // We permit the extend to hoist through basic blocks but this is only
729 // sensible if the target has extending loads. If you end up lowering back
730 // into a load and extend during the legalizer then the end result is
731 // hoisting the extend up to the load.
732
733 // Prefer defined extensions to undefined extensions as these are more
734 // likely to reduce the number of instructions.
735 if (OpcodeForCandidate == TargetOpcode::G_ANYEXT &&
736 CurrentUse.ExtendOpcode != TargetOpcode::G_ANYEXT)
737 return CurrentUse;
738 else if (CurrentUse.ExtendOpcode == TargetOpcode::G_ANYEXT &&
739 OpcodeForCandidate != TargetOpcode::G_ANYEXT)
740 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
741
742 // Prefer sign extensions to zero extensions as sign-extensions tend to be
743 // more expensive. Don't do this if the load is already a zero-extend load
744 // though, otherwise we'll rewrite a zero-extend load into a sign-extend
745 // later.
746 if (!isa<GZExtLoad>(Val: LoadMI) && CurrentUse.Ty == TyForCandidate) {
747 if (CurrentUse.ExtendOpcode == TargetOpcode::G_SEXT &&
748 OpcodeForCandidate == TargetOpcode::G_ZEXT)
749 return CurrentUse;
750 else if (CurrentUse.ExtendOpcode == TargetOpcode::G_ZEXT &&
751 OpcodeForCandidate == TargetOpcode::G_SEXT)
752 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
753 }
754
755 // This is potentially target specific. We've chosen the largest type
756 // because G_TRUNC is usually free. One potential catch with this is that
757 // some targets have a reduced number of larger registers than smaller
758 // registers and this choice potentially increases the live-range for the
759 // larger value.
760 if (TyForCandidate.getSizeInBits() > CurrentUse.Ty.getSizeInBits()) {
761 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
762 }
763 return CurrentUse;
764}
765
766/// Find a suitable place to insert some instructions and insert them. This
767/// function accounts for special cases like inserting before a PHI node.
768/// The current strategy for inserting before PHI's is to duplicate the
769/// instructions for each predecessor. However, while that's ok for G_TRUNC
770/// on most targets since it generally requires no code, other targets/cases may
771/// want to try harder to find a dominating block.
772static void InsertInsnsWithoutSideEffectsBeforeUse(
773 MachineIRBuilder &Builder, MachineInstr &DefMI, MachineOperand &UseMO,
774 std::function<void(MachineBasicBlock *, MachineBasicBlock::iterator,
775 MachineOperand &UseMO)>
776 Inserter) {
777 MachineInstr &UseMI = *UseMO.getParent();
778
779 MachineBasicBlock *InsertBB = UseMI.getParent();
780
781 // If the use is a PHI then we want the predecessor block instead.
782 if (UseMI.isPHI()) {
783 MachineOperand *PredBB = std::next(x: &UseMO);
784 InsertBB = PredBB->getMBB();
785 }
786
787 // If the block is the same block as the def then we want to insert just after
788 // the def instead of at the start of the block.
789 if (InsertBB == DefMI.getParent()) {
790 MachineBasicBlock::iterator InsertPt = &DefMI;
791 Inserter(InsertBB, std::next(x: InsertPt), UseMO);
792 return;
793 }
794
795 // Otherwise we want the start of the BB
796 Inserter(InsertBB, InsertBB->getFirstNonPHI(), UseMO);
797}
798} // end anonymous namespace
799
800bool CombinerHelper::tryCombineExtendingLoads(MachineInstr &MI) const {
801 PreferredTuple Preferred;
802 if (matchCombineExtendingLoads(MI, MatchInfo&: Preferred)) {
803 applyCombineExtendingLoads(MI, MatchInfo&: Preferred);
804 return true;
805 }
806 return false;
807}
808
809static unsigned getExtLoadOpcForExtend(unsigned ExtOpc) {
810 unsigned CandidateLoadOpc;
811 switch (ExtOpc) {
812 case TargetOpcode::G_ANYEXT:
813 CandidateLoadOpc = TargetOpcode::G_LOAD;
814 break;
815 case TargetOpcode::G_SEXT:
816 CandidateLoadOpc = TargetOpcode::G_SEXTLOAD;
817 break;
818 case TargetOpcode::G_ZEXT:
819 CandidateLoadOpc = TargetOpcode::G_ZEXTLOAD;
820 break;
821 default:
822 llvm_unreachable("Unexpected extend opc");
823 }
824 return CandidateLoadOpc;
825}
826
827bool CombinerHelper::matchCombineExtendingLoads(
828 MachineInstr &MI, PreferredTuple &Preferred) const {
829 // We match the loads and follow the uses to the extend instead of matching
830 // the extends and following the def to the load. This is because the load
831 // must remain in the same position for correctness (unless we also add code
832 // to find a safe place to sink it) whereas the extend is freely movable.
833 // It also prevents us from duplicating the load for the volatile case or just
834 // for performance.
835 GAnyLoad *LoadMI = dyn_cast<GAnyLoad>(Val: &MI);
836 if (!LoadMI)
837 return false;
838
839 Register LoadReg = LoadMI->getDstReg();
840
841 LLT LoadValueTy = MRI.getType(Reg: LoadReg);
842 if (!LoadValueTy.isScalar())
843 return false;
844
845 // Most architectures are going to legalize <s8 loads into at least a 1 byte
846 // load, and the MMOs can only describe memory accesses in multiples of bytes.
847 // If we try to perform extload combining on those, we can end up with
848 // %a(s8) = extload %ptr (load 1 byte from %ptr)
849 // ... which is an illegal extload instruction.
850 if (LoadValueTy.getSizeInBits() < 8)
851 return false;
852
853 // For non power-of-2 types, they will very likely be legalized into multiple
854 // loads. Don't bother trying to match them into extending loads.
855 if (!llvm::has_single_bit<uint32_t>(Value: LoadValueTy.getSizeInBits()))
856 return false;
857
858 // Find the preferred type aside from the any-extends (unless it's the only
859 // one) and non-extending ops. We'll emit an extending load to that type and
860 // and emit a variant of (extend (trunc X)) for the others according to the
861 // relative type sizes. At the same time, pick an extend to use based on the
862 // extend involved in the chosen type.
863 unsigned PreferredOpcode =
864 isa<GLoad>(Val: &MI)
865 ? TargetOpcode::G_ANYEXT
866 : isa<GSExtLoad>(Val: &MI) ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
867 Preferred = {.Ty: LLT(), .ExtendOpcode: PreferredOpcode, .MI: nullptr};
868 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: LoadReg)) {
869 if (UseMI.getOpcode() == TargetOpcode::G_SEXT ||
870 UseMI.getOpcode() == TargetOpcode::G_ZEXT ||
871 (UseMI.getOpcode() == TargetOpcode::G_ANYEXT)) {
872 const auto &MMO = LoadMI->getMMO();
873 // Don't do anything for atomics.
874 if (MMO.isAtomic())
875 continue;
876 // Check for legality.
877 if (!isPreLegalize()) {
878 LegalityQuery::MemDesc MMDesc(MMO);
879 unsigned CandidateLoadOpc = getExtLoadOpcForExtend(ExtOpc: UseMI.getOpcode());
880 LLT UseTy = MRI.getType(Reg: UseMI.getOperand(i: 0).getReg());
881 LLT SrcTy = MRI.getType(Reg: LoadMI->getPointerReg());
882 if (LI->getAction(Query: {CandidateLoadOpc, {UseTy, SrcTy}, {MMDesc}})
883 .Action != LegalizeActions::Legal)
884 continue;
885 }
886 Preferred = ChoosePreferredUse(LoadMI&: MI, CurrentUse&: Preferred,
887 TyForCandidate: MRI.getType(Reg: UseMI.getOperand(i: 0).getReg()),
888 OpcodeForCandidate: UseMI.getOpcode(), MIForCandidate: &UseMI);
889 }
890 }
891
892 // There were no extends
893 if (!Preferred.MI)
894 return false;
895 // It should be impossible to chose an extend without selecting a different
896 // type since by definition the result of an extend is larger.
897 assert(Preferred.Ty != LoadValueTy && "Extending to same type?");
898
899 LLVM_DEBUG(dbgs() << "Preferred use is: " << *Preferred.MI);
900 return true;
901}
902
903void CombinerHelper::applyCombineExtendingLoads(
904 MachineInstr &MI, PreferredTuple &Preferred) const {
905 // Rewrite the load to the chosen extending load.
906 Register ChosenDstReg = Preferred.MI->getOperand(i: 0).getReg();
907
908 // Inserter to insert a truncate back to the original type at a given point
909 // with some basic CSE to limit truncate duplication to one per BB.
910 DenseMap<MachineBasicBlock *, MachineInstr *> EmittedInsns;
911 auto InsertTruncAt = [&](MachineBasicBlock *InsertIntoBB,
912 MachineBasicBlock::iterator InsertBefore,
913 MachineOperand &UseMO) {
914 MachineInstr *PreviouslyEmitted = EmittedInsns.lookup(Val: InsertIntoBB);
915 if (PreviouslyEmitted) {
916 Observer.changingInstr(MI&: *UseMO.getParent());
917 UseMO.setReg(PreviouslyEmitted->getOperand(i: 0).getReg());
918 Observer.changedInstr(MI&: *UseMO.getParent());
919 return;
920 }
921
922 Builder.setInsertPt(MBB&: *InsertIntoBB, II: InsertBefore);
923 Register NewDstReg = MRI.cloneVirtualRegister(VReg: MI.getOperand(i: 0).getReg());
924 MachineInstr *NewMI = Builder.buildTrunc(Res: NewDstReg, Op: ChosenDstReg);
925 EmittedInsns[InsertIntoBB] = NewMI;
926 replaceRegOpWith(MRI, FromRegOp&: UseMO, ToReg: NewDstReg);
927 };
928
929 Observer.changingInstr(MI);
930 unsigned LoadOpc = getExtLoadOpcForExtend(ExtOpc: Preferred.ExtendOpcode);
931 MI.setDesc(Builder.getTII().get(Opcode: LoadOpc));
932
933 // Rewrite all the uses to fix up the types.
934 auto &LoadValue = MI.getOperand(i: 0);
935 SmallVector<MachineOperand *, 4> Uses(
936 llvm::make_pointer_range(Range: MRI.use_operands(Reg: LoadValue.getReg())));
937
938 for (auto *UseMO : Uses) {
939 MachineInstr *UseMI = UseMO->getParent();
940
941 // If the extend is compatible with the preferred extend then we should fix
942 // up the type and extend so that it uses the preferred use.
943 if (UseMI->getOpcode() == Preferred.ExtendOpcode ||
944 UseMI->getOpcode() == TargetOpcode::G_ANYEXT) {
945 Register UseDstReg = UseMI->getOperand(i: 0).getReg();
946 MachineOperand &UseSrcMO = UseMI->getOperand(i: 1);
947 const LLT UseDstTy = MRI.getType(Reg: UseDstReg);
948 if (UseDstReg != ChosenDstReg) {
949 if (Preferred.Ty == UseDstTy) {
950 // If the use has the same type as the preferred use, then merge
951 // the vregs and erase the extend. For example:
952 // %1:_(s8) = G_LOAD ...
953 // %2:_(s32) = G_SEXT %1(s8)
954 // %3:_(s32) = G_ANYEXT %1(s8)
955 // ... = ... %3(s32)
956 // rewrites to:
957 // %2:_(s32) = G_SEXTLOAD ...
958 // ... = ... %2(s32)
959 replaceRegWith(MRI, FromReg: UseDstReg, ToReg: ChosenDstReg);
960 Observer.erasingInstr(MI&: *UseMO->getParent());
961 UseMO->getParent()->eraseFromParent();
962 } else if (Preferred.Ty.getSizeInBits() < UseDstTy.getSizeInBits()) {
963 // If the preferred size is smaller, then keep the extend but extend
964 // from the result of the extending load. For example:
965 // %1:_(s8) = G_LOAD ...
966 // %2:_(s32) = G_SEXT %1(s8)
967 // %3:_(s64) = G_ANYEXT %1(s8)
968 // ... = ... %3(s64)
969 /// rewrites to:
970 // %2:_(s32) = G_SEXTLOAD ...
971 // %3:_(s64) = G_ANYEXT %2:_(s32)
972 // ... = ... %3(s64)
973 replaceRegOpWith(MRI, FromRegOp&: UseSrcMO, ToReg: ChosenDstReg);
974 } else {
975 // If the preferred size is large, then insert a truncate. For
976 // example:
977 // %1:_(s8) = G_LOAD ...
978 // %2:_(s64) = G_SEXT %1(s8)
979 // %3:_(s32) = G_ZEXT %1(s8)
980 // ... = ... %3(s32)
981 /// rewrites to:
982 // %2:_(s64) = G_SEXTLOAD ...
983 // %4:_(s8) = G_TRUNC %2:_(s32)
984 // %3:_(s64) = G_ZEXT %2:_(s8)
985 // ... = ... %3(s64)
986 InsertInsnsWithoutSideEffectsBeforeUse(Builder, DefMI&: MI, UseMO&: *UseMO,
987 Inserter: InsertTruncAt);
988 }
989 continue;
990 }
991 // The use is (one of) the uses of the preferred use we chose earlier.
992 // We're going to update the load to def this value later so just erase
993 // the old extend.
994 Observer.erasingInstr(MI&: *UseMO->getParent());
995 UseMO->getParent()->eraseFromParent();
996 continue;
997 }
998
999 // The use isn't an extend. Truncate back to the type we originally loaded.
1000 // This is free on many targets.
1001 InsertInsnsWithoutSideEffectsBeforeUse(Builder, DefMI&: MI, UseMO&: *UseMO, Inserter: InsertTruncAt);
1002 }
1003
1004 MI.getOperand(i: 0).setReg(ChosenDstReg);
1005 Observer.changedInstr(MI);
1006}
1007
1008bool CombinerHelper::matchCombineLoadWithAndMask(MachineInstr &MI,
1009 BuildFnTy &MatchInfo) const {
1010 assert(MI.getOpcode() == TargetOpcode::G_AND);
1011
1012 // If we have the following code:
1013 // %mask = G_CONSTANT 255
1014 // %ld = G_LOAD %ptr, (load s16)
1015 // %and = G_AND %ld, %mask
1016 //
1017 // Try to fold it into
1018 // %ld = G_ZEXTLOAD %ptr, (load s8)
1019
1020 Register Dst = MI.getOperand(i: 0).getReg();
1021 if (MRI.getType(Reg: Dst).isVector())
1022 return false;
1023
1024 auto MaybeMask =
1025 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
1026 if (!MaybeMask)
1027 return false;
1028
1029 APInt MaskVal = MaybeMask->Value;
1030
1031 if (!MaskVal.isMask())
1032 return false;
1033
1034 Register SrcReg = MI.getOperand(i: 1).getReg();
1035 // Don't use getOpcodeDef() here since intermediate instructions may have
1036 // multiple users.
1037 GAnyLoad *LoadMI;
1038 Register PtrReg;
1039 const MachineMemOperand *MMO;
1040 if (!mi_match(R: SrcReg, MRI, P: m_GAnyLoad(Inst&: LoadMI, Ptr: m_Reg(R&: PtrReg), MMO: m_MMO(MMO))))
1041 return false;
1042
1043 Register LoadReg = LoadMI->getDstReg();
1044 LLT RegTy = MRI.getType(Reg: LoadReg);
1045 unsigned RegSize = RegTy.getSizeInBits();
1046 unsigned LoadSizeBits = MMO->getSizeInBits().getValue();
1047 unsigned MaskSizeBits = MaskVal.countr_one();
1048
1049 if ((isa<GSExtLoad>(Val: LoadMI) || MaskSizeBits < LoadSizeBits) &&
1050 !MRI.hasOneNonDBGUse(RegNo: LoadReg))
1051 return false;
1052
1053 // The mask may not be larger than the in-memory type, as it might cover sign
1054 // extended bits
1055 if (MaskSizeBits > LoadSizeBits)
1056 return false;
1057
1058 // If the mask covers the whole destination register, there's nothing to
1059 // extend
1060 if (MaskSizeBits >= RegSize)
1061 return false;
1062
1063 // Most targets cannot deal with loads of size < 8 and need to re-legalize to
1064 // at least byte loads. Avoid creating such loads here
1065 if (MaskSizeBits < 8 || !isPowerOf2_32(Value: MaskSizeBits))
1066 return false;
1067
1068 LegalityQuery::MemDesc MemDesc(*MMO);
1069
1070 // Don't modify the memory access size if this is atomic/volatile, but we can
1071 // still adjust the opcode to indicate the high bit behavior.
1072 if (!MMO->isAtomic() && !MMO->isVolatile())
1073 MemDesc.MemoryTy = LLT::scalar(SizeInBits: MaskSizeBits);
1074 else if (LoadSizeBits > MaskSizeBits || LoadSizeBits == RegSize)
1075 return false;
1076
1077 // TODO: Could check if it's legal with the reduced or original memory size.
1078 if (!isLegalOrBeforeLegalizer(
1079 Query: {TargetOpcode::G_ZEXTLOAD, {RegTy, MRI.getType(Reg: PtrReg)}, {MemDesc}}))
1080 return false;
1081
1082 MatchInfo = [=](MachineIRBuilder &B) {
1083 B.setInstrAndDebugLoc(*LoadMI);
1084 auto &MF = B.getMF();
1085 auto PtrInfo = MMO->getPointerInfo();
1086 auto *NewMMO = MF.getMachineMemOperand(MMO, PtrInfo, Ty: MemDesc.MemoryTy);
1087 B.buildLoadInstr(Opcode: TargetOpcode::G_ZEXTLOAD, Res: Dst, Addr: PtrReg, MMO&: *NewMMO);
1088 replaceRegWith(MRI, FromReg: LoadReg, ToReg: Dst);
1089 LoadMI->eraseFromParent();
1090 };
1091 return true;
1092}
1093
1094bool CombinerHelper::isPredecessor(const MachineInstr &DefMI,
1095 const MachineInstr &UseMI) const {
1096 assert(!DefMI.isDebugInstr() && !UseMI.isDebugInstr() &&
1097 "shouldn't consider debug uses");
1098 assert(DefMI.getParent() == UseMI.getParent());
1099 if (&DefMI == &UseMI)
1100 return true;
1101 const MachineBasicBlock &MBB = *DefMI.getParent();
1102 auto DefOrUse = find_if(Range: MBB, P: [&DefMI, &UseMI](const MachineInstr &MI) {
1103 return &MI == &DefMI || &MI == &UseMI;
1104 });
1105 if (DefOrUse == MBB.end())
1106 llvm_unreachable("Block must contain both DefMI and UseMI!");
1107 return &*DefOrUse == &DefMI;
1108}
1109
1110bool CombinerHelper::dominates(const MachineInstr &DefMI,
1111 const MachineInstr &UseMI) const {
1112 assert(!DefMI.isDebugInstr() && !UseMI.isDebugInstr() &&
1113 "shouldn't consider debug uses");
1114 if (MDT)
1115 return MDT->dominates(A: &DefMI, B: &UseMI);
1116 else if (DefMI.getParent() != UseMI.getParent())
1117 return false;
1118
1119 return isPredecessor(DefMI, UseMI);
1120}
1121
1122bool CombinerHelper::matchSextTruncSextLoad(MachineInstr &MI) const {
1123 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1124 Register SrcReg = MI.getOperand(i: 1).getReg();
1125 Register LoadUser = SrcReg;
1126
1127 if (MRI.getType(Reg: SrcReg).isVector())
1128 return false;
1129
1130 Register TruncSrc;
1131 if (mi_match(R: SrcReg, MRI, P: m_GTrunc(Src: m_Reg(R&: TruncSrc))))
1132 LoadUser = TruncSrc;
1133
1134 uint64_t SizeInBits = MI.getOperand(i: 2).getImm();
1135 // If the source is a G_SEXTLOAD from the same bit width, then we don't
1136 // need any extend at all, just a truncate.
1137 if (auto *LoadMI = getOpcodeDef<GSExtLoad>(Reg: LoadUser, MRI)) {
1138 // If truncating more than the original extended value, abort.
1139 auto LoadSizeBits = LoadMI->getMemSizeInBits();
1140 if (TruncSrc &&
1141 MRI.getType(Reg: TruncSrc).getSizeInBits() < LoadSizeBits.getValue())
1142 return false;
1143 if (LoadSizeBits == SizeInBits)
1144 return true;
1145 }
1146 return false;
1147}
1148
1149bool CombinerHelper::matchSextInRegOfLoad(
1150 MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) const {
1151 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1152
1153 Register DstReg = MI.getOperand(i: 0).getReg();
1154 LLT RegTy = MRI.getType(Reg: DstReg);
1155
1156 // Only supports scalars for now.
1157 if (RegTy.isVector())
1158 return false;
1159
1160 Register SrcReg = MI.getOperand(i: 1).getReg();
1161 Register PtrReg;
1162 const MachineMemOperand *MMO;
1163 if (!mi_match(R: SrcReg, MRI, P: m_GLoad(Ptr: m_Reg(R&: PtrReg), MMO: m_MMO(MMO))))
1164 return false;
1165
1166 uint64_t MemBits = MMO->getSizeInBits().getValue();
1167 uint64_t ExtFrom = MI.getOperand(i: 2).getImm();
1168
1169 if (MemBits > ExtFrom && !MRI.hasOneNonDBGUse(RegNo: SrcReg))
1170 return false;
1171
1172 // If the sign extend extends from a narrower width than the load's width,
1173 // then we can narrow the load width when we combine to a G_SEXTLOAD.
1174 // Avoid widening the load at all.
1175 unsigned NewSizeBits = std::min(a: ExtFrom, b: MemBits);
1176
1177 // Don't generate G_SEXTLOADs with a < 1 byte width.
1178 if (NewSizeBits < 8)
1179 return false;
1180 // Don't bother creating a non-power-2 sextload, it will likely be broken up
1181 // anyway for most targets.
1182 if (!isPowerOf2_32(Value: NewSizeBits))
1183 return false;
1184
1185 LegalityQuery::MemDesc MMDesc(*MMO);
1186
1187 // Don't modify the memory access size if this is atomic/volatile, but we can
1188 // still adjust the opcode to indicate the high bit behavior.
1189 if (!MMO->isAtomic() && !MMO->isVolatile())
1190 MMDesc.MemoryTy = LLT::scalar(SizeInBits: NewSizeBits);
1191 else if (MemBits > NewSizeBits || MemBits == RegTy.getSizeInBits())
1192 return false;
1193
1194 // TODO: Could check if it's legal with the reduced or original memory size.
1195 if (!isLegalOrBeforeLegalizer(
1196 Query: {TargetOpcode::G_SEXTLOAD, {RegTy, MRI.getType(Reg: PtrReg)}, {MMDesc}}))
1197 return false;
1198
1199 MatchInfo = std::make_tuple(args&: SrcReg, args&: NewSizeBits);
1200 return true;
1201}
1202
1203void CombinerHelper::applySextInRegOfLoad(
1204 MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) const {
1205 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1206 Register LoadReg;
1207 unsigned ScalarSizeBits;
1208 std::tie(args&: LoadReg, args&: ScalarSizeBits) = MatchInfo;
1209 GLoad *LoadDef = cast<GLoad>(Val: MRI.getVRegDef(Reg: LoadReg));
1210
1211 // If we have the following:
1212 // %ld = G_LOAD %ptr, (load 2)
1213 // %ext = G_SEXT_INREG %ld, 8
1214 // ==>
1215 // %ld = G_SEXTLOAD %ptr (load 1)
1216
1217 auto &MMO = LoadDef->getMMO();
1218 Builder.setInstrAndDebugLoc(*LoadDef);
1219 auto &MF = Builder.getMF();
1220 auto PtrInfo = MMO.getPointerInfo();
1221 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Size: ScalarSizeBits / 8);
1222 Builder.buildLoadInstr(Opcode: TargetOpcode::G_SEXTLOAD, Res: MI.getOperand(i: 0).getReg(),
1223 Addr: LoadDef->getPointerReg(), MMO&: *NewMMO);
1224 replaceRegWith(MRI, FromReg: LoadReg, ToReg: MI.getOperand(i: 0).getReg());
1225 MI.eraseFromParent();
1226
1227 // Not all loads can be deleted, so make sure the old one is removed.
1228 LoadDef->eraseFromParent();
1229}
1230
1231/// Return true if 'MI' is a load or a store that may be fold it's address
1232/// operand into the load / store addressing mode.
1233static bool canFoldInAddressingMode(GLoadStore *MI, const TargetLowering &TLI,
1234 MachineRegisterInfo &MRI) {
1235 TargetLowering::AddrMode AM;
1236 auto *MF = MI->getMF();
1237 auto *Addr = getOpcodeDef<GPtrAdd>(Reg: MI->getPointerReg(), MRI);
1238 if (!Addr)
1239 return false;
1240
1241 AM.HasBaseReg = true;
1242 if (auto CstOff = getIConstantVRegVal(VReg: Addr->getOffsetReg(), MRI))
1243 AM.BaseOffs = CstOff->getSExtValue(); // [reg +/- imm]
1244 else
1245 AM.Scale = 1; // [reg +/- reg]
1246
1247 return TLI.isLegalAddressingMode(
1248 DL: MF->getDataLayout(), AM,
1249 Ty: getTypeForLLT(Ty: MI->getMMO().getMemoryType(),
1250 C&: MF->getFunction().getContext()),
1251 AddrSpace: MI->getMMO().getAddrSpace());
1252}
1253
1254static unsigned getIndexedOpc(unsigned LdStOpc) {
1255 switch (LdStOpc) {
1256 case TargetOpcode::G_LOAD:
1257 return TargetOpcode::G_INDEXED_LOAD;
1258 case TargetOpcode::G_STORE:
1259 return TargetOpcode::G_INDEXED_STORE;
1260 case TargetOpcode::G_ZEXTLOAD:
1261 return TargetOpcode::G_INDEXED_ZEXTLOAD;
1262 case TargetOpcode::G_SEXTLOAD:
1263 return TargetOpcode::G_INDEXED_SEXTLOAD;
1264 default:
1265 llvm_unreachable("Unexpected opcode");
1266 }
1267}
1268
1269bool CombinerHelper::isIndexedLoadStoreLegal(GLoadStore &LdSt) const {
1270 // Check for legality.
1271 LLT PtrTy = MRI.getType(Reg: LdSt.getPointerReg());
1272 LLT Ty = MRI.getType(Reg: LdSt.getReg(Idx: 0));
1273 LLT MemTy = LdSt.getMMO().getMemoryType();
1274 SmallVector<LegalityQuery::MemDesc, 2> MemDescrs(
1275 {{MemTy, MemTy.getSizeInBits().getKnownMinValue(),
1276 AtomicOrdering::NotAtomic, AtomicOrdering::NotAtomic}});
1277 unsigned IndexedOpc = getIndexedOpc(LdStOpc: LdSt.getOpcode());
1278 SmallVector<LLT> OpTys;
1279 if (IndexedOpc == TargetOpcode::G_INDEXED_STORE)
1280 OpTys = {PtrTy, Ty, Ty};
1281 else
1282 OpTys = {Ty, PtrTy}; // For G_INDEXED_LOAD, G_INDEXED_[SZ]EXTLOAD
1283
1284 LegalityQuery Q(IndexedOpc, OpTys, MemDescrs);
1285 return isLegal(Query: Q);
1286}
1287
1288static cl::opt<unsigned> PostIndexUseThreshold(
1289 "post-index-use-threshold", cl::Hidden, cl::init(Val: 32),
1290 cl::desc("Number of uses of a base pointer to check before it is no longer "
1291 "considered for post-indexing."));
1292
1293bool CombinerHelper::findPostIndexCandidate(GLoadStore &LdSt, Register &Addr,
1294 Register &Base, Register &Offset,
1295 bool &RematOffset) const {
1296 // We're looking for the following pattern, for either load or store:
1297 // %baseptr:_(p0) = ...
1298 // G_STORE %val(s64), %baseptr(p0)
1299 // %offset:_(s64) = G_CONSTANT i64 -256
1300 // %new_addr:_(p0) = G_PTR_ADD %baseptr, %offset(s64)
1301 const auto &TLI = getTargetLowering();
1302
1303 Register Ptr = LdSt.getPointerReg();
1304 // If the store is the only use, don't bother.
1305 if (MRI.hasOneNonDBGUse(RegNo: Ptr))
1306 return false;
1307
1308 if (!isIndexedLoadStoreLegal(LdSt))
1309 return false;
1310
1311 if (getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Ptr, MRI))
1312 return false;
1313
1314 MachineInstr *StoredValDef = getDefIgnoringCopies(Reg: LdSt.getReg(Idx: 0), MRI);
1315 MachineInstr *PtrDef;
1316 if (!mi_match(R: Ptr, MRI, P: m_MInstr(MI&: PtrDef)))
1317 return false;
1318
1319 unsigned NumUsesChecked = 0;
1320 for (auto &Use : MRI.use_nodbg_instructions(Reg: Ptr)) {
1321 if (++NumUsesChecked > PostIndexUseThreshold)
1322 return false; // Try to avoid exploding compile time.
1323
1324 auto *PtrAdd = dyn_cast<GPtrAdd>(Val: &Use);
1325 // The use itself might be dead. This can happen during combines if DCE
1326 // hasn't had a chance to run yet. Don't allow it to form an indexed op.
1327 if (!PtrAdd || MRI.use_nodbg_empty(RegNo: PtrAdd->getReg(Idx: 0)))
1328 continue;
1329
1330 // Check the user of this isn't the store, otherwise we'd be generate a
1331 // indexed store defining its own use.
1332 if (StoredValDef == &Use)
1333 continue;
1334
1335 Offset = PtrAdd->getOffsetReg();
1336 if (!ForceLegalIndexing &&
1337 !TLI.isIndexingLegal(MI&: LdSt, Base: PtrAdd->getBaseReg(), Offset,
1338 /*IsPre*/ false, MRI))
1339 continue;
1340
1341 // Make sure the offset calculation is before the potentially indexed op.
1342 MachineInstr *OffsetDef;
1343 if (!mi_match(R: Offset, MRI, P: m_MInstr(MI&: OffsetDef)))
1344 continue;
1345 RematOffset = false;
1346 if (!dominates(DefMI: *OffsetDef, UseMI: LdSt)) {
1347 // If the offset however is just a G_CONSTANT, we can always just
1348 // rematerialize it where we need it.
1349 if (OffsetDef->getOpcode() != TargetOpcode::G_CONSTANT)
1350 continue;
1351 RematOffset = true;
1352 }
1353
1354 for (auto &BasePtrUse : MRI.use_nodbg_instructions(Reg: PtrAdd->getBaseReg())) {
1355 if (&BasePtrUse == PtrDef)
1356 continue;
1357
1358 // If the user is a later load/store that can be post-indexed, then don't
1359 // combine this one.
1360 auto *BasePtrLdSt = dyn_cast<GLoadStore>(Val: &BasePtrUse);
1361 if (BasePtrLdSt && BasePtrLdSt != &LdSt &&
1362 dominates(DefMI: LdSt, UseMI: *BasePtrLdSt) &&
1363 isIndexedLoadStoreLegal(LdSt&: *BasePtrLdSt))
1364 return false;
1365
1366 // Now we're looking for the key G_PTR_ADD instruction, which contains
1367 // the offset add that we want to fold.
1368 if (auto *BasePtrUseDef = dyn_cast<GPtrAdd>(Val: &BasePtrUse)) {
1369 Register PtrAddDefReg = BasePtrUseDef->getReg(Idx: 0);
1370 for (auto &BaseUseUse : MRI.use_nodbg_instructions(Reg: PtrAddDefReg)) {
1371 // If the use is in a different block, then we may produce worse code
1372 // due to the extra register pressure.
1373 if (BaseUseUse.getParent() != LdSt.getParent())
1374 return false;
1375
1376 if (auto *UseUseLdSt = dyn_cast<GLoadStore>(Val: &BaseUseUse))
1377 if (canFoldInAddressingMode(MI: UseUseLdSt, TLI, MRI))
1378 return false;
1379 }
1380 if (!dominates(DefMI: LdSt, UseMI: BasePtrUse))
1381 return false; // All use must be dominated by the load/store.
1382 }
1383 }
1384
1385 Addr = PtrAdd->getReg(Idx: 0);
1386 Base = PtrAdd->getBaseReg();
1387 return true;
1388 }
1389
1390 return false;
1391}
1392
1393bool CombinerHelper::findPreIndexCandidate(GLoadStore &LdSt, Register &Addr,
1394 Register &Base,
1395 Register &Offset) const {
1396 auto &MF = *LdSt.getParent()->getParent();
1397 const auto &TLI = *MF.getSubtarget().getTargetLowering();
1398
1399 Addr = LdSt.getPointerReg();
1400 if (!mi_match(R: Addr, MRI, P: m_GPtrAdd(L: m_Reg(R&: Base), R: m_Reg(R&: Offset))) ||
1401 MRI.hasOneNonDBGUse(RegNo: Addr))
1402 return false;
1403
1404 if (!ForceLegalIndexing &&
1405 !TLI.isIndexingLegal(MI&: LdSt, Base, Offset, /*IsPre*/ true, MRI))
1406 return false;
1407
1408 if (!isIndexedLoadStoreLegal(LdSt))
1409 return false;
1410
1411 MachineInstr *BaseDef = getDefIgnoringCopies(Reg: Base, MRI);
1412 if (BaseDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
1413 return false;
1414
1415 if (auto *St = dyn_cast<GStore>(Val: &LdSt)) {
1416 // Would require a copy.
1417 if (Base == St->getValueReg())
1418 return false;
1419
1420 // We're expecting one use of Addr in MI, but it could also be the
1421 // value stored, which isn't actually dominated by the instruction.
1422 if (St->getValueReg() == Addr)
1423 return false;
1424 }
1425
1426 // Avoid increasing cross-block register pressure.
1427 for (auto &AddrUse : MRI.use_nodbg_instructions(Reg: Addr))
1428 if (AddrUse.getParent() != LdSt.getParent())
1429 return false;
1430
1431 // FIXME: check whether all uses of the base pointer are constant PtrAdds.
1432 // That might allow us to end base's liveness here by adjusting the constant.
1433 bool RealUse = false;
1434 for (auto &AddrUse : MRI.use_nodbg_instructions(Reg: Addr)) {
1435 if (!dominates(DefMI: LdSt, UseMI: AddrUse))
1436 return false; // All use must be dominated by the load/store.
1437
1438 // If Ptr may be folded in addressing mode of other use, then it's
1439 // not profitable to do this transformation.
1440 if (auto *UseLdSt = dyn_cast<GLoadStore>(Val: &AddrUse)) {
1441 if (!canFoldInAddressingMode(MI: UseLdSt, TLI, MRI))
1442 RealUse = true;
1443 } else {
1444 RealUse = true;
1445 }
1446 }
1447 return RealUse;
1448}
1449
1450bool CombinerHelper::matchCombineExtractedVectorLoad(
1451 MachineInstr &MI, BuildFnTy &MatchInfo) const {
1452 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
1453
1454 // Check if there is a load that defines the vector being extracted from.
1455 auto *LoadMI = getOpcodeDef<GLoad>(Reg: MI.getOperand(i: 1).getReg(), MRI);
1456 if (!LoadMI)
1457 return false;
1458
1459 Register Vector = MI.getOperand(i: 1).getReg();
1460 LLT VecEltTy = MRI.getType(Reg: Vector).getElementType();
1461
1462 assert(MRI.getType(MI.getOperand(0).getReg()) == VecEltTy);
1463
1464 // Checking whether we should reduce the load width.
1465 if (!MRI.hasOneNonDBGUse(RegNo: Vector))
1466 return false;
1467
1468 // Check if the defining load is simple.
1469 if (!LoadMI->isSimple())
1470 return false;
1471
1472 // If the vector element type is not a multiple of a byte then we are unable
1473 // to correctly compute an address to load only the extracted element as a
1474 // scalar.
1475 if (!VecEltTy.isByteSized())
1476 return false;
1477
1478 // Check for load fold barriers between the extraction and the load.
1479 if (MI.getParent() != LoadMI->getParent())
1480 return false;
1481 const unsigned MaxIter = 20;
1482 unsigned Iter = 0;
1483 for (const auto &Instr :
1484 instructionsWithoutDebug(It: LoadMI->getIterator(), End: MI.getIterator())) {
1485 if (Instr.isLoadFoldBarrier())
1486 return false;
1487 if (Iter++ == MaxIter)
1488 return false;
1489 }
1490
1491 // Check if the new load that we are going to create is legal
1492 // if we are in the post-legalization phase.
1493 MachineMemOperand MMO = LoadMI->getMMO();
1494 Align Alignment = MMO.getAlign();
1495 MachinePointerInfo PtrInfo;
1496 uint64_t Offset;
1497
1498 // Finding the appropriate PtrInfo if offset is a known constant.
1499 // This is required to create the memory operand for the narrowed load.
1500 // This machine memory operand object helps us infer about legality
1501 // before we proceed to combine the instruction.
1502 Register Idx = MI.getOperand(i: 2).getReg();
1503 if (auto CVal = getIConstantVRegVal(VReg: Idx, MRI)) {
1504 int Elt = CVal->getZExtValue();
1505 Offset = VecEltTy.getSizeInBits() * Elt / 8;
1506 PtrInfo = MMO.getPointerInfo().getWithOffset(O: Offset);
1507 } else {
1508 // Discard the pointer info except the address space because the memory
1509 // operand can't represent this new access since the offset is variable.
1510 Offset = VecEltTy.getSizeInBits() / 8;
1511 PtrInfo = MachinePointerInfo(MMO.getPointerInfo().getAddrSpace());
1512 }
1513
1514 Alignment = commonAlignment(A: Alignment, Offset);
1515
1516 Register VecPtr = LoadMI->getPointerReg();
1517 LLT PtrTy = MRI.getType(Reg: VecPtr);
1518
1519 MachineFunction &MF = *MI.getMF();
1520 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Ty: VecEltTy);
1521
1522 LegalityQuery::MemDesc MMDesc(*NewMMO);
1523
1524 if (!isLegalOrBeforeLegalizer(
1525 Query: {TargetOpcode::G_LOAD, {VecEltTy, PtrTy}, {MMDesc}}))
1526 return false;
1527
1528 // Load must be allowed and fast on the target.
1529 LLVMContext &C = MF.getFunction().getContext();
1530 auto &DL = MF.getDataLayout();
1531 unsigned Fast = 0;
1532 if (!getTargetLowering().allowsMemoryAccess(Context&: C, DL, Ty: VecEltTy, MMO: *NewMMO,
1533 Fast: &Fast) ||
1534 !Fast)
1535 return false;
1536
1537 Register Result = MI.getOperand(i: 0).getReg();
1538 Register Index = MI.getOperand(i: 2).getReg();
1539
1540 MatchInfo = [=](MachineIRBuilder &B) {
1541 GISelObserverWrapper DummyObserver;
1542 LegalizerHelper Helper(B.getMF(), DummyObserver, B);
1543 //// Get pointer to the vector element.
1544 Register finalPtr = Helper.getVectorElementPointer(
1545 VecPtr: LoadMI->getPointerReg(), VecTy: MRI.getType(Reg: LoadMI->getOperand(i: 0).getReg()),
1546 Index);
1547 // New G_LOAD instruction.
1548 B.buildLoad(Res: Result, Addr: finalPtr, PtrInfo, Alignment);
1549 // Remove original GLOAD instruction.
1550 LoadMI->eraseFromParent();
1551 };
1552
1553 return true;
1554}
1555
1556bool CombinerHelper::matchCombineIndexedLoadStore(
1557 MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) const {
1558 auto &LdSt = cast<GLoadStore>(Val&: MI);
1559
1560 if (LdSt.isAtomic())
1561 return false;
1562
1563 MatchInfo.IsPre = findPreIndexCandidate(LdSt, Addr&: MatchInfo.Addr, Base&: MatchInfo.Base,
1564 Offset&: MatchInfo.Offset);
1565 if (!MatchInfo.IsPre &&
1566 !findPostIndexCandidate(LdSt, Addr&: MatchInfo.Addr, Base&: MatchInfo.Base,
1567 Offset&: MatchInfo.Offset, RematOffset&: MatchInfo.RematOffset))
1568 return false;
1569
1570 return true;
1571}
1572
1573void CombinerHelper::applyCombineIndexedLoadStore(
1574 MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) const {
1575 MachineInstr &AddrDef = *MRI.getVRegDef(Reg: MatchInfo.Addr);
1576 unsigned Opcode = MI.getOpcode();
1577 bool IsStore = Opcode == TargetOpcode::G_STORE;
1578 unsigned NewOpcode = getIndexedOpc(LdStOpc: Opcode);
1579
1580 // If the offset constant didn't happen to dominate the load/store, we can
1581 // just clone it as needed.
1582 if (MatchInfo.RematOffset) {
1583 auto *OldCst = MRI.getVRegDef(Reg: MatchInfo.Offset);
1584 auto NewCst = Builder.buildConstant(Res: MRI.getType(Reg: MatchInfo.Offset),
1585 Val: *OldCst->getOperand(i: 1).getCImm());
1586 MatchInfo.Offset = NewCst.getReg(Idx: 0);
1587 }
1588
1589 auto MIB = Builder.buildInstr(Opcode: NewOpcode);
1590 if (IsStore) {
1591 MIB.addDef(RegNo: MatchInfo.Addr);
1592 MIB.addUse(RegNo: MI.getOperand(i: 0).getReg());
1593 } else {
1594 MIB.addDef(RegNo: MI.getOperand(i: 0).getReg());
1595 MIB.addDef(RegNo: MatchInfo.Addr);
1596 }
1597
1598 MIB.addUse(RegNo: MatchInfo.Base);
1599 MIB.addUse(RegNo: MatchInfo.Offset);
1600 MIB.addImm(Val: MatchInfo.IsPre);
1601 MIB->cloneMemRefs(MF&: *MI.getMF(), MI);
1602 MI.eraseFromParent();
1603 AddrDef.eraseFromParent();
1604
1605 LLVM_DEBUG(dbgs() << " Combinined to indexed operation");
1606}
1607
1608bool CombinerHelper::matchCombineDivRem(MachineInstr &MI,
1609 MachineInstr *&OtherMI) const {
1610 unsigned Opcode = MI.getOpcode();
1611 bool IsDiv, IsSigned;
1612
1613 switch (Opcode) {
1614 default:
1615 llvm_unreachable("Unexpected opcode!");
1616 case TargetOpcode::G_SDIV:
1617 case TargetOpcode::G_UDIV: {
1618 IsDiv = true;
1619 IsSigned = Opcode == TargetOpcode::G_SDIV;
1620 break;
1621 }
1622 case TargetOpcode::G_SREM:
1623 case TargetOpcode::G_UREM: {
1624 IsDiv = false;
1625 IsSigned = Opcode == TargetOpcode::G_SREM;
1626 break;
1627 }
1628 }
1629
1630 Register Src1 = MI.getOperand(i: 1).getReg();
1631 unsigned DivOpcode, RemOpcode, DivremOpcode;
1632 if (IsSigned) {
1633 DivOpcode = TargetOpcode::G_SDIV;
1634 RemOpcode = TargetOpcode::G_SREM;
1635 DivremOpcode = TargetOpcode::G_SDIVREM;
1636 } else {
1637 DivOpcode = TargetOpcode::G_UDIV;
1638 RemOpcode = TargetOpcode::G_UREM;
1639 DivremOpcode = TargetOpcode::G_UDIVREM;
1640 }
1641
1642 if (!isLegalOrBeforeLegalizer(Query: {DivremOpcode, {MRI.getType(Reg: Src1)}}))
1643 return false;
1644
1645 // Combine:
1646 // %div:_ = G_[SU]DIV %src1:_, %src2:_
1647 // %rem:_ = G_[SU]REM %src1:_, %src2:_
1648 // into:
1649 // %div:_, %rem:_ = G_[SU]DIVREM %src1:_, %src2:_
1650
1651 // Combine:
1652 // %rem:_ = G_[SU]REM %src1:_, %src2:_
1653 // %div:_ = G_[SU]DIV %src1:_, %src2:_
1654 // into:
1655 // %div:_, %rem:_ = G_[SU]DIVREM %src1:_, %src2:_
1656
1657 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: Src1)) {
1658 if (MI.getParent() == UseMI.getParent() &&
1659 ((IsDiv && UseMI.getOpcode() == RemOpcode) ||
1660 (!IsDiv && UseMI.getOpcode() == DivOpcode)) &&
1661 matchEqualDefs(MOP1: MI.getOperand(i: 2), MOP2: UseMI.getOperand(i: 2)) &&
1662 matchEqualDefs(MOP1: MI.getOperand(i: 1), MOP2: UseMI.getOperand(i: 1))) {
1663 OtherMI = &UseMI;
1664 return true;
1665 }
1666 }
1667
1668 return false;
1669}
1670
1671void CombinerHelper::applyCombineDivRem(MachineInstr &MI,
1672 MachineInstr *&OtherMI) const {
1673 unsigned Opcode = MI.getOpcode();
1674 assert(OtherMI && "OtherMI shouldn't be empty.");
1675
1676 Register DestDivReg, DestRemReg;
1677 if (Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_UDIV) {
1678 DestDivReg = MI.getOperand(i: 0).getReg();
1679 DestRemReg = OtherMI->getOperand(i: 0).getReg();
1680 } else {
1681 DestDivReg = OtherMI->getOperand(i: 0).getReg();
1682 DestRemReg = MI.getOperand(i: 0).getReg();
1683 }
1684
1685 bool IsSigned =
1686 Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM;
1687
1688 // Check which instruction is first in the block so we don't break def-use
1689 // deps by "moving" the instruction incorrectly. Also keep track of which
1690 // instruction is first so we pick it's operands, avoiding use-before-def
1691 // bugs.
1692 MachineInstr *FirstInst = dominates(DefMI: MI, UseMI: *OtherMI) ? &MI : OtherMI;
1693 Builder.setInstrAndDebugLoc(*FirstInst);
1694
1695 Builder.buildInstr(Opc: IsSigned ? TargetOpcode::G_SDIVREM
1696 : TargetOpcode::G_UDIVREM,
1697 DstOps: {DestDivReg, DestRemReg},
1698 SrcOps: { FirstInst->getOperand(i: 1), FirstInst->getOperand(i: 2) });
1699 MI.eraseFromParent();
1700 OtherMI->eraseFromParent();
1701}
1702
1703bool CombinerHelper::matchOptBrCondByInvertingCond(
1704 MachineInstr &MI, MachineInstr *&BrCond) const {
1705 assert(MI.getOpcode() == TargetOpcode::G_BR);
1706
1707 // Try to match the following:
1708 // bb1:
1709 // G_BRCOND %c1, %bb2
1710 // G_BR %bb3
1711 // bb2:
1712 // ...
1713 // bb3:
1714
1715 // The above pattern does not have a fall through to the successor bb2, always
1716 // resulting in a branch no matter which path is taken. Here we try to find
1717 // and replace that pattern with conditional branch to bb3 and otherwise
1718 // fallthrough to bb2. This is generally better for branch predictors.
1719
1720 MachineBasicBlock *MBB = MI.getParent();
1721 MachineBasicBlock::iterator BrIt(MI);
1722 if (BrIt == MBB->begin())
1723 return false;
1724 assert(std::next(BrIt) == MBB->end() && "expected G_BR to be a terminator");
1725
1726 BrCond = &*std::prev(x: BrIt);
1727 if (BrCond->getOpcode() != TargetOpcode::G_BRCOND)
1728 return false;
1729
1730 // Check that the next block is the conditional branch target. Also make sure
1731 // that it isn't the same as the G_BR's target (otherwise, this will loop.)
1732 MachineBasicBlock *BrCondTarget = BrCond->getOperand(i: 1).getMBB();
1733 return BrCondTarget != MI.getOperand(i: 0).getMBB() &&
1734 MBB->isLayoutSuccessor(MBB: BrCondTarget);
1735}
1736
1737void CombinerHelper::applyOptBrCondByInvertingCond(
1738 MachineInstr &MI, MachineInstr *&BrCond) const {
1739 MachineBasicBlock *BrTarget = MI.getOperand(i: 0).getMBB();
1740 Builder.setInstrAndDebugLoc(*BrCond);
1741 LLT Ty = MRI.getType(Reg: BrCond->getOperand(i: 0).getReg());
1742 // FIXME: Does int/fp matter for this? If so, we might need to restrict
1743 // this to i1 only since we might not know for sure what kind of
1744 // compare generated the condition value.
1745 auto True = Builder.buildConstant(
1746 Res: Ty, Val: getICmpTrueVal(TLI: getTargetLowering(), IsVector: false, IsFP: false));
1747 auto Xor = Builder.buildXor(Dst: Ty, Src0: BrCond->getOperand(i: 0), Src1: True);
1748
1749 auto *FallthroughBB = BrCond->getOperand(i: 1).getMBB();
1750 Observer.changingInstr(MI);
1751 MI.getOperand(i: 0).setMBB(FallthroughBB);
1752 Observer.changedInstr(MI);
1753
1754 // Change the conditional branch to use the inverted condition and
1755 // new target block.
1756 Observer.changingInstr(MI&: *BrCond);
1757 BrCond->getOperand(i: 0).setReg(Xor.getReg(Idx: 0));
1758 BrCond->getOperand(i: 1).setMBB(BrTarget);
1759 Observer.changedInstr(MI&: *BrCond);
1760}
1761
1762bool CombinerHelper::matchCombineMemCpyFamily(
1763 MachineInstr &MI, MemCpyFamilyLoweringInfo &MatchInfo,
1764 unsigned MaxLen) const {
1765 auto &[Dst, Src, KnownLen, Alignment, DstAlignCanChange, MemOps] = MatchInfo;
1766 return canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, Alignment,
1767 DstAlignCanChange, MemOps);
1768}
1769
1770void CombinerHelper::applyCombineMemCpyFamily(
1771 MachineInstr &MI, MemCpyFamilyLoweringInfo &MatchInfo) const {
1772 auto &[Dst, Src, KnownLen, Alignment, DstAlignCanChange, MemOps] = MatchInfo;
1773 MachineIRBuilder HelperBuilder(MI);
1774 GISelObserverWrapper DummyObserver;
1775 LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);
1776 bool Changed = Helper.lowerMemCpyFamily(MI, Dst, Src, KnownLen, Alignment,
1777 DstAlignCanChange, MemOps) ==
1778 LegalizerHelper::LegalizeResult::Legalized;
1779 assert(Changed && "expected memcpy-family instruction to lower");
1780 (void)Changed;
1781}
1782
1783bool CombinerHelper::tryCombineMemCpyFamily(MachineInstr &MI,
1784 unsigned MaxLen) const {
1785 MachineIRBuilder HelperBuilder(MI);
1786 GISelObserverWrapper DummyObserver;
1787 LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);
1788 return Helper.lowerMemCpyFamily(MI, MaxLen) ==
1789 LegalizerHelper::LegalizeResult::Legalized;
1790}
1791
1792bool CombinerHelper::matchPtrAddImmedChain(MachineInstr &MI,
1793 PtrAddChain &MatchInfo) const {
1794 // We're trying to match the following pattern:
1795 // %t1 = G_PTR_ADD %base, G_CONSTANT imm1
1796 // %root = G_PTR_ADD %t1, G_CONSTANT imm2
1797 // -->
1798 // %root = G_PTR_ADD %base, G_CONSTANT (imm1 + imm2)
1799
1800 if (MI.getOpcode() != TargetOpcode::G_PTR_ADD)
1801 return false;
1802
1803 Register Add2 = MI.getOperand(i: 1).getReg();
1804 Register Imm1 = MI.getOperand(i: 2).getReg();
1805 auto MaybeImmVal = getIConstantVRegValWithLookThrough(VReg: Imm1, MRI);
1806 if (!MaybeImmVal)
1807 return false;
1808
1809 Register Base, Imm2;
1810 uint32_t LHSPtrAddFlags;
1811 if (!mi_match(R: Add2, MRI,
1812 P: m_GPtrAdd(L: m_Reg(R&: Base), R: m_Reg(R&: Imm2), Flags: m_MIFlags(Flags&: LHSPtrAddFlags))))
1813 return false;
1814
1815 auto MaybeImm2Val = getIConstantVRegValWithLookThrough(VReg: Imm2, MRI);
1816 if (!MaybeImm2Val)
1817 return false;
1818
1819 // Check if the new combined immediate forms an illegal addressing mode.
1820 // Do not combine if it was legal before but would get illegal.
1821 // To do so, we need to find a load/store user of the pointer to get
1822 // the access type.
1823 Type *AccessTy = nullptr;
1824 auto &MF = *MI.getMF();
1825 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg())) {
1826 if (auto *LdSt = dyn_cast<GLoadStore>(Val: &UseMI)) {
1827 AccessTy = getTypeForLLT(Ty: MRI.getType(Reg: LdSt->getReg(Idx: 0)),
1828 C&: MF.getFunction().getContext());
1829 break;
1830 }
1831 }
1832 TargetLoweringBase::AddrMode AMNew;
1833 APInt CombinedImm = MaybeImmVal->Value + MaybeImm2Val->Value;
1834 AMNew.BaseOffs = CombinedImm.getSExtValue();
1835 if (AccessTy) {
1836 AMNew.HasBaseReg = true;
1837 TargetLoweringBase::AddrMode AMOld;
1838 AMOld.BaseOffs = MaybeImmVal->Value.getSExtValue();
1839 AMOld.HasBaseReg = true;
1840 unsigned AS = MRI.getType(Reg: Add2).getAddressSpace();
1841 const auto &TLI = *MF.getSubtarget().getTargetLowering();
1842 if (TLI.isLegalAddressingMode(DL: MF.getDataLayout(), AM: AMOld, Ty: AccessTy, AddrSpace: AS) &&
1843 !TLI.isLegalAddressingMode(DL: MF.getDataLayout(), AM: AMNew, Ty: AccessTy, AddrSpace: AS))
1844 return false;
1845 }
1846
1847 // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
1848 // inbounds, reaching the same result in one G_PTR_ADD is also inbounds.
1849 // The nusw constraints are satisfied because imm1+imm2 cannot exceed the
1850 // largest signed integer that fits into the index type, which is the maximum
1851 // size of allocated objects according to the IR Language Reference.
1852 unsigned PtrAddFlags = MI.getFlags();
1853 bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
1854 bool IsInBounds =
1855 PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::InBounds;
1856 unsigned Flags = 0;
1857 if (IsNoUWrap)
1858 Flags |= MachineInstr::MIFlag::NoUWrap;
1859 if (IsInBounds) {
1860 Flags |= MachineInstr::MIFlag::InBounds;
1861 Flags |= MachineInstr::MIFlag::NoUSWrap;
1862 }
1863
1864 // Pass the combined immediate to the apply function.
1865 MatchInfo.Imm = AMNew.BaseOffs;
1866 MatchInfo.Base = Base;
1867 MatchInfo.Bank = getRegBank(Reg: Imm2);
1868 MatchInfo.Flags = Flags;
1869 return true;
1870}
1871
1872void CombinerHelper::applyPtrAddImmedChain(MachineInstr &MI,
1873 PtrAddChain &MatchInfo) const {
1874 assert(MI.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD");
1875 MachineIRBuilder MIB(MI);
1876 LLT OffsetTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
1877 auto NewOffset = MIB.buildConstant(Res: OffsetTy, Val: MatchInfo.Imm);
1878 setRegBank(Reg: NewOffset.getReg(Idx: 0), RegBank: MatchInfo.Bank);
1879 Observer.changingInstr(MI);
1880 MI.getOperand(i: 1).setReg(MatchInfo.Base);
1881 MI.getOperand(i: 2).setReg(NewOffset.getReg(Idx: 0));
1882 MI.setFlags(MatchInfo.Flags);
1883 Observer.changedInstr(MI);
1884}
1885
1886bool CombinerHelper::matchShiftImmedChain(MachineInstr &MI,
1887 RegisterImmPair &MatchInfo) const {
1888 // We're trying to match the following pattern with any of
1889 // G_SHL/G_ASHR/G_LSHR/G_SSHLSAT/G_USHLSAT shift instructions:
1890 // %t1 = SHIFT %base, G_CONSTANT imm1
1891 // %root = SHIFT %t1, G_CONSTANT imm2
1892 // -->
1893 // %root = SHIFT %base, G_CONSTANT (imm1 + imm2)
1894
1895 unsigned Opcode = MI.getOpcode();
1896 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1897 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1898 Opcode == TargetOpcode::G_USHLSAT) &&
1899 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1900
1901 Register Shl2 = MI.getOperand(i: 1).getReg();
1902 Register Imm1 = MI.getOperand(i: 2).getReg();
1903 auto MaybeImmVal = getIConstantVRegValWithLookThrough(VReg: Imm1, MRI);
1904 if (!MaybeImmVal)
1905 return false;
1906
1907 MachineInstr *Shl2Def;
1908 if (!mi_match(R: Shl2, MRI, P: m_MInstr(MI&: Shl2Def)) || Shl2Def->getOpcode() != Opcode)
1909 return false;
1910
1911 Register Base = Shl2Def->getOperand(i: 1).getReg();
1912 Register Imm2 = Shl2Def->getOperand(i: 2).getReg();
1913 auto MaybeImm2Val = getIConstantVRegValWithLookThrough(VReg: Imm2, MRI);
1914 if (!MaybeImm2Val)
1915 return false;
1916
1917 // Pass the combined immediate to the apply function.
1918 MatchInfo.Imm =
1919 (MaybeImmVal->Value.getZExtValue() + MaybeImm2Val->Value).getZExtValue();
1920 MatchInfo.Reg = Base;
1921
1922 // There is no simple replacement for a saturating unsigned left shift that
1923 // exceeds the scalar size.
1924 if (Opcode == TargetOpcode::G_USHLSAT &&
1925 MatchInfo.Imm >= MRI.getType(Reg: Shl2).getScalarSizeInBits())
1926 return false;
1927
1928 return true;
1929}
1930
1931void CombinerHelper::applyShiftImmedChain(MachineInstr &MI,
1932 RegisterImmPair &MatchInfo) const {
1933 unsigned Opcode = MI.getOpcode();
1934 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1935 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1936 Opcode == TargetOpcode::G_USHLSAT) &&
1937 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1938
1939 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1940 unsigned const ScalarSizeInBits = Ty.getScalarSizeInBits();
1941 auto Imm = MatchInfo.Imm;
1942
1943 if (Imm >= ScalarSizeInBits) {
1944 // Any logical shift that exceeds scalar size will produce zero.
1945 if (Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR) {
1946 Builder.buildConstant(Res: MI.getOperand(i: 0), Val: 0);
1947 MI.eraseFromParent();
1948 return;
1949 }
1950 // Arithmetic shift and saturating signed left shift have no effect beyond
1951 // scalar size.
1952 Imm = ScalarSizeInBits - 1;
1953 }
1954
1955 LLT ImmTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
1956 Register NewImm = Builder.buildConstant(Res: ImmTy, Val: Imm).getReg(Idx: 0);
1957 Observer.changingInstr(MI);
1958 MI.getOperand(i: 1).setReg(MatchInfo.Reg);
1959 MI.getOperand(i: 2).setReg(NewImm);
1960 Observer.changedInstr(MI);
1961}
1962
1963bool CombinerHelper::matchShiftOfShiftedLogic(
1964 MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo) const {
1965 // We're trying to match the following pattern with any of
1966 // G_SHL/G_ASHR/G_LSHR shift instructions in combination
1967 // with any of G_AND/G_OR/G_XOR logic instructions.
1968 // %t1 = SHIFT %X, G_CONSTANT C0
1969 // %t2 = LOGIC %t1, %Y
1970 // %root = SHIFT %t2, G_CONSTANT C1
1971 // -->
1972 // %t3 = SHIFT %X, G_CONSTANT (C0+C1)
1973 // %t4 = SHIFT %Y, G_CONSTANT C1
1974 // %root = LOGIC %t3, %t4
1975 unsigned ShiftOpcode = MI.getOpcode();
1976 assert((ShiftOpcode == TargetOpcode::G_SHL ||
1977 ShiftOpcode == TargetOpcode::G_ASHR ||
1978 ShiftOpcode == TargetOpcode::G_LSHR) &&
1979 "Expected G_SHL, G_ASHR and G_LSHR");
1980
1981 // Match a one-use bitwise logic op.
1982 Register LogicDest = MI.getOperand(i: 1).getReg();
1983 if (!MRI.hasOneNonDBGUse(RegNo: LogicDest))
1984 return false;
1985
1986 MachineInstr *LogicMI;
1987 if (!mi_match(R: LogicDest, MRI, P: m_MInstr(MI&: LogicMI)))
1988 return false;
1989 unsigned LogicOpcode = LogicMI->getOpcode();
1990 if (LogicOpcode != TargetOpcode::G_AND && LogicOpcode != TargetOpcode::G_OR &&
1991 LogicOpcode != TargetOpcode::G_XOR)
1992 return false;
1993
1994 // Find a matching one-use shift by constant.
1995 const Register C1 = MI.getOperand(i: 2).getReg();
1996 auto MaybeImmVal = getIConstantVRegValWithLookThrough(VReg: C1, MRI);
1997 if (!MaybeImmVal || MaybeImmVal->Value == 0)
1998 return false;
1999
2000 const uint64_t C1Val = MaybeImmVal->Value.getZExtValue();
2001
2002 auto matchFirstShift = [&](const MachineInstr *MI, uint64_t &ShiftVal) {
2003 // Shift should match previous one and should be a one-use.
2004 if (MI->getOpcode() != ShiftOpcode ||
2005 !MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
2006 return false;
2007
2008 // Must be a constant.
2009 auto MaybeImmVal =
2010 getIConstantVRegValWithLookThrough(VReg: MI->getOperand(i: 2).getReg(), MRI);
2011 if (!MaybeImmVal)
2012 return false;
2013
2014 ShiftVal = MaybeImmVal->Value.getSExtValue();
2015 return true;
2016 };
2017
2018 // Logic ops are commutative, so check each operand for a match.
2019 Register LogicMIReg1 = LogicMI->getOperand(i: 1).getReg();
2020 MachineInstr *LogicMIOp1;
2021 Register LogicMIReg2 = LogicMI->getOperand(i: 2).getReg();
2022 MachineInstr *LogicMIOp2;
2023 if (!mi_match(R: LogicMIReg1, MRI, P: m_MInstr(MI&: LogicMIOp1)) ||
2024 !mi_match(R: LogicMIReg2, MRI, P: m_MInstr(MI&: LogicMIOp2)))
2025 return false;
2026 uint64_t C0Val;
2027
2028 if (matchFirstShift(LogicMIOp1, C0Val)) {
2029 MatchInfo.LogicNonShiftReg = LogicMIReg2;
2030 MatchInfo.Shift2 = LogicMIOp1;
2031 } else if (matchFirstShift(LogicMIOp2, C0Val)) {
2032 MatchInfo.LogicNonShiftReg = LogicMIReg1;
2033 MatchInfo.Shift2 = LogicMIOp2;
2034 } else
2035 return false;
2036
2037 MatchInfo.ValSum = C0Val + C1Val;
2038
2039 // The fold is not valid if the sum of the shift values exceeds bitwidth.
2040 if (MatchInfo.ValSum >= MRI.getType(Reg: LogicDest).getScalarSizeInBits())
2041 return false;
2042
2043 MatchInfo.Logic = LogicMI;
2044 return true;
2045}
2046
2047void CombinerHelper::applyShiftOfShiftedLogic(
2048 MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo) const {
2049 unsigned Opcode = MI.getOpcode();
2050 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
2051 Opcode == TargetOpcode::G_LSHR) &&
2052 "Expected G_SHL, G_ASHR and G_LSHR");
2053
2054 LLT ShlType = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
2055 LLT DestType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2056
2057 Register Const = Builder.buildConstant(Res: ShlType, Val: MatchInfo.ValSum).getReg(Idx: 0);
2058
2059 Register Shift1Base = MatchInfo.Shift2->getOperand(i: 1).getReg();
2060 Register Shift1 =
2061 Builder.buildInstr(Opc: Opcode, DstOps: {DestType}, SrcOps: {Shift1Base, Const}).getReg(Idx: 0);
2062
2063 // If LogicNonShiftReg is the same to Shift1Base, and shift1 const is the same
2064 // to MatchInfo.Shift2 const, CSEMIRBuilder will reuse the old shift1 when
2065 // build shift2. So, if we erase MatchInfo.Shift2 at the end, actually we
2066 // remove old shift1. And it will cause crash later. So erase it earlier to
2067 // avoid the crash.
2068 MatchInfo.Shift2->eraseFromParent();
2069
2070 Register Shift2Const = MI.getOperand(i: 2).getReg();
2071 Register Shift2 = Builder
2072 .buildInstr(Opc: Opcode, DstOps: {DestType},
2073 SrcOps: {MatchInfo.LogicNonShiftReg, Shift2Const})
2074 .getReg(Idx: 0);
2075
2076 Register Dest = MI.getOperand(i: 0).getReg();
2077 Builder.buildInstr(Opc: MatchInfo.Logic->getOpcode(), DstOps: {Dest}, SrcOps: {Shift1, Shift2});
2078
2079 // This was one use so it's safe to remove it.
2080 MatchInfo.Logic->eraseFromParent();
2081
2082 MI.eraseFromParent();
2083}
2084
2085bool CombinerHelper::isDesirableToCommuteWithShift(
2086 const MachineInstr &MI) const {
2087 return getTargetLowering().isDesirableToCommuteWithShift(MI,
2088 IsAfterLegal: !isPreLegalize());
2089}
2090
2091bool CombinerHelper::matchLshrOfTruncOfLshr(MachineInstr &MI,
2092 LshrOfTruncOfLshr &MatchInfo,
2093 MachineInstr &ShiftMI) const {
2094 assert(MI.getOpcode() == TargetOpcode::G_LSHR && "Expected a G_LSHR");
2095
2096 Register N0 = MI.getOperand(i: 1).getReg();
2097 Register N1 = MI.getOperand(i: 2).getReg();
2098 unsigned OpSizeInBits = MRI.getType(Reg: N0).getScalarSizeInBits();
2099
2100 APInt N1C, N001C;
2101 if (!mi_match(R: N1, MRI, P: m_ICstOrSplat(Cst&: N1C)))
2102 return false;
2103 auto N001 = ShiftMI.getOperand(i: 2).getReg();
2104 if (!mi_match(R: N001, MRI, P: m_ICstOrSplat(Cst&: N001C)))
2105 return false;
2106
2107 if (N001C.getBitWidth() > N1C.getBitWidth())
2108 N1C = N1C.zext(width: N001C.getBitWidth());
2109 else
2110 N001C = N001C.zext(width: N1C.getBitWidth());
2111
2112 Register InnerShift = ShiftMI.getOperand(i: 0).getReg();
2113 LLT InnerShiftTy = MRI.getType(Reg: InnerShift);
2114 uint64_t InnerShiftSize = InnerShiftTy.getScalarSizeInBits();
2115 if ((N1C + N001C).ult(RHS: InnerShiftSize)) {
2116 MatchInfo.Src = ShiftMI.getOperand(i: 1).getReg();
2117 MatchInfo.ShiftAmt = N1C + N001C;
2118 MatchInfo.ShiftAmtTy = MRI.getType(Reg: N001);
2119 MatchInfo.InnerShiftTy = InnerShiftTy;
2120
2121 if ((N001C + OpSizeInBits) == InnerShiftSize)
2122 return true;
2123 if (MRI.hasOneNonDBGUse(RegNo: N0) && MRI.hasOneNonDBGUse(RegNo: InnerShift)) {
2124 MatchInfo.Mask = true;
2125 MatchInfo.MaskVal = APInt(N1C.getBitWidth(), OpSizeInBits) - N1C;
2126 return true;
2127 }
2128 }
2129 return false;
2130}
2131
2132void CombinerHelper::applyLshrOfTruncOfLshr(
2133 MachineInstr &MI, LshrOfTruncOfLshr &MatchInfo) const {
2134 assert(MI.getOpcode() == TargetOpcode::G_LSHR && "Expected a G_LSHR");
2135
2136 Register Dst = MI.getOperand(i: 0).getReg();
2137 auto ShiftAmt =
2138 Builder.buildConstant(Res: MatchInfo.ShiftAmtTy, Val: MatchInfo.ShiftAmt);
2139 auto Shift =
2140 Builder.buildLShr(Dst: MatchInfo.InnerShiftTy, Src0: MatchInfo.Src, Src1: ShiftAmt);
2141 if (MatchInfo.Mask == true) {
2142 APInt MaskVal =
2143 APInt::getLowBitsSet(numBits: MatchInfo.InnerShiftTy.getScalarSizeInBits(),
2144 loBitsSet: MatchInfo.MaskVal.getZExtValue());
2145 auto Mask = Builder.buildConstant(Res: MatchInfo.InnerShiftTy, Val: MaskVal);
2146 auto And = Builder.buildAnd(Dst: MatchInfo.InnerShiftTy, Src0: Shift, Src1: Mask);
2147 Builder.buildTrunc(Res: Dst, Op: And);
2148 } else
2149 Builder.buildTrunc(Res: Dst, Op: Shift);
2150 MI.eraseFromParent();
2151}
2152
2153bool CombinerHelper::matchCombineMulToShl(MachineInstr &MI,
2154 unsigned &ShiftVal) const {
2155 assert(MI.getOpcode() == TargetOpcode::G_MUL && "Expected a G_MUL");
2156 auto MaybeImmVal =
2157 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
2158 if (!MaybeImmVal)
2159 return false;
2160
2161 ShiftVal = MaybeImmVal->Value.exactLogBase2();
2162 return (static_cast<int32_t>(ShiftVal) != -1);
2163}
2164
2165void CombinerHelper::applyCombineMulToShl(MachineInstr &MI,
2166 unsigned &ShiftVal) const {
2167 assert(MI.getOpcode() == TargetOpcode::G_MUL && "Expected a G_MUL");
2168 MachineIRBuilder MIB(MI);
2169 LLT ShiftTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2170 auto ShiftCst = MIB.buildConstant(Res: ShiftTy, Val: ShiftVal);
2171 Observer.changingInstr(MI);
2172 MI.setDesc(MIB.getTII().get(Opcode: TargetOpcode::G_SHL));
2173 MI.getOperand(i: 2).setReg(ShiftCst.getReg(Idx: 0));
2174 if (ShiftVal == ShiftTy.getScalarSizeInBits() - 1)
2175 MI.clearFlag(Flag: MachineInstr::MIFlag::NoSWrap);
2176 Observer.changedInstr(MI);
2177}
2178
2179bool CombinerHelper::matchCombineSubToAdd(MachineInstr &MI,
2180 BuildFnTy &MatchInfo) const {
2181 GSub &Sub = cast<GSub>(Val&: MI);
2182
2183 LLT Ty = MRI.getType(Reg: Sub.getReg(Idx: 0));
2184
2185 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ADD, {Ty}}))
2186 return false;
2187
2188 if (!isConstantLegalOrBeforeLegalizer(Ty))
2189 return false;
2190
2191 APInt Imm = getIConstantFromReg(VReg: Sub.getRHSReg(), MRI);
2192
2193 MatchInfo = [=, &MI](MachineIRBuilder &B) {
2194 auto NegCst = B.buildConstant(Res: Ty, Val: -Imm);
2195 Observer.changingInstr(MI);
2196 MI.setDesc(B.getTII().get(Opcode: TargetOpcode::G_ADD));
2197 MI.getOperand(i: 2).setReg(NegCst.getReg(Idx: 0));
2198 MI.clearFlag(Flag: MachineInstr::MIFlag::NoUWrap);
2199 if (Imm.isMinSignedValue())
2200 MI.clearFlags(flags: MachineInstr::MIFlag::NoSWrap);
2201 Observer.changedInstr(MI);
2202 };
2203 return true;
2204}
2205
2206// shl ([sza]ext x), y => zext (shl x, y), if shift does not overflow source
2207bool CombinerHelper::matchCombineShlOfExtend(MachineInstr &MI,
2208 RegisterImmPair &MatchData) const {
2209 assert(MI.getOpcode() == TargetOpcode::G_SHL && VT);
2210 if (!getTargetLowering().isDesirableToPullExtFromShl(MI))
2211 return false;
2212
2213 Register LHS = MI.getOperand(i: 1).getReg();
2214
2215 Register ExtSrc;
2216 if (!mi_match(R: LHS, MRI, P: m_GAnyExt(Src: m_Reg(R&: ExtSrc))) &&
2217 !mi_match(R: LHS, MRI, P: m_GZExt(Src: m_Reg(R&: ExtSrc))) &&
2218 !mi_match(R: LHS, MRI, P: m_GSExt(Src: m_Reg(R&: ExtSrc))))
2219 return false;
2220
2221 Register RHS = MI.getOperand(i: 2).getReg();
2222 auto MaybeShiftAmtVal = isConstantOrConstantSplatVector(Def: RHS, MRI);
2223 if (!MaybeShiftAmtVal)
2224 return false;
2225
2226 if (LI) {
2227 LLT SrcTy = MRI.getType(Reg: ExtSrc);
2228
2229 // We only really care about the legality with the shifted value. We can
2230 // pick any type the constant shift amount, so ask the target what to
2231 // use. Otherwise we would have to guess and hope it is reported as legal.
2232 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: SrcTy);
2233 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SHL, {SrcTy, ShiftAmtTy}}))
2234 return false;
2235 }
2236
2237 int64_t ShiftAmt = MaybeShiftAmtVal->getSExtValue();
2238 MatchData.Reg = ExtSrc;
2239 MatchData.Imm = ShiftAmt;
2240
2241 unsigned MinLeadingZeros = VT->getKnownZeroes(R: ExtSrc).countl_one();
2242 unsigned SrcTySize = MRI.getType(Reg: ExtSrc).getScalarSizeInBits();
2243 return MinLeadingZeros >= ShiftAmt && ShiftAmt < SrcTySize;
2244}
2245
2246void CombinerHelper::applyCombineShlOfExtend(
2247 MachineInstr &MI, const RegisterImmPair &MatchData) const {
2248 Register ExtSrcReg = MatchData.Reg;
2249 int64_t ShiftAmtVal = MatchData.Imm;
2250
2251 LLT ExtSrcTy = MRI.getType(Reg: ExtSrcReg);
2252 auto ShiftAmt = Builder.buildConstant(Res: ExtSrcTy, Val: ShiftAmtVal);
2253 auto NarrowShift =
2254 Builder.buildShl(Dst: ExtSrcTy, Src0: ExtSrcReg, Src1: ShiftAmt, Flags: MI.getFlags());
2255 Builder.buildZExt(Res: MI.getOperand(i: 0), Op: NarrowShift);
2256 MI.eraseFromParent();
2257}
2258
2259static Register peekThroughBitcast(Register Reg,
2260 const MachineRegisterInfo &MRI) {
2261 while (mi_match(R: Reg, MRI, P: m_GBitcast(Src: m_Reg(R&: Reg))))
2262 ;
2263
2264 return Reg;
2265}
2266
2267bool CombinerHelper::matchCombineUnmergeMergeToPlainValues(
2268 MachineInstr &MI, SmallVectorImpl<Register> &Operands) const {
2269 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2270 "Expected an unmerge");
2271 auto &Unmerge = cast<GUnmerge>(Val&: MI);
2272 Register SrcReg = peekThroughBitcast(Reg: Unmerge.getSourceReg(), MRI);
2273
2274 auto *SrcInstr = getOpcodeDef<GMergeLikeInstr>(Reg: SrcReg, MRI);
2275 if (!SrcInstr)
2276 return false;
2277
2278 // Check the source type of the merge.
2279 LLT SrcMergeTy = MRI.getType(Reg: SrcInstr->getSourceReg(I: 0));
2280 LLT Dst0Ty = MRI.getType(Reg: Unmerge.getReg(Idx: 0));
2281 bool SameSize = Dst0Ty.getSizeInBits() == SrcMergeTy.getSizeInBits();
2282 if (SrcMergeTy != Dst0Ty && !SameSize)
2283 return false;
2284 // They are the same now (modulo a bitcast).
2285 // We can collect all the src registers.
2286 for (unsigned Idx = 0; Idx < SrcInstr->getNumSources(); ++Idx)
2287 Operands.push_back(Elt: SrcInstr->getSourceReg(I: Idx));
2288 return true;
2289}
2290
2291void CombinerHelper::applyCombineUnmergeMergeToPlainValues(
2292 MachineInstr &MI, SmallVectorImpl<Register> &Operands) const {
2293 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2294 "Expected an unmerge");
2295 assert((MI.getNumOperands() - 1 == Operands.size()) &&
2296 "Not enough operands to replace all defs");
2297 unsigned NumElems = MI.getNumOperands() - 1;
2298
2299 LLT SrcTy = MRI.getType(Reg: Operands[0]);
2300 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2301 bool CanReuseInputDirectly = DstTy == SrcTy;
2302 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2303 Register DstReg = MI.getOperand(i: Idx).getReg();
2304 Register SrcReg = Operands[Idx];
2305
2306 // This combine may run after RegBankSelect, so we need to be aware of
2307 // register banks.
2308 const auto &DstCB = MRI.getRegClassOrRegBank(Reg: DstReg);
2309 if (!DstCB.isNull() && DstCB != MRI.getRegClassOrRegBank(Reg: SrcReg)) {
2310 SrcReg = Builder.buildCopy(Res: MRI.getType(Reg: SrcReg), Op: SrcReg).getReg(Idx: 0);
2311 MRI.setRegClassOrRegBank(Reg: SrcReg, RCOrRB: DstCB);
2312 }
2313
2314 if (CanReuseInputDirectly)
2315 replaceRegWith(MRI, FromReg: DstReg, ToReg: SrcReg);
2316 else
2317 Builder.buildCast(Dst: DstReg, Src: SrcReg);
2318 }
2319 MI.eraseFromParent();
2320}
2321
2322bool CombinerHelper::matchCombineUnmergeConstant(
2323 MachineInstr &MI, SmallVectorImpl<APInt> &Csts) const {
2324 unsigned SrcIdx = MI.getNumOperands() - 1;
2325 Register SrcReg = MI.getOperand(i: SrcIdx).getReg();
2326 // Break down the big constant in smaller ones.
2327 APInt Val;
2328 if (!mi_match(R: SrcReg, MRI, P: m_GConstantOrFConstantBits(Bits&: Val)))
2329 return false;
2330
2331 LLT Dst0Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2332 unsigned ShiftAmt = Dst0Ty.getSizeInBits();
2333 // Unmerge a constant.
2334 for (unsigned Idx = 0; Idx != SrcIdx; ++Idx) {
2335 Csts.emplace_back(Args: Val.trunc(width: ShiftAmt));
2336 Val = Val.lshr(shiftAmt: ShiftAmt);
2337 }
2338
2339 return true;
2340}
2341
2342void CombinerHelper::applyCombineUnmergeConstant(
2343 MachineInstr &MI, SmallVectorImpl<APInt> &Csts) const {
2344 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2345 "Expected an unmerge");
2346 assert((MI.getNumOperands() - 1 == Csts.size()) &&
2347 "Not enough operands to replace all defs");
2348 unsigned NumElems = MI.getNumOperands() - 1;
2349 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2350 Register DstReg = MI.getOperand(i: Idx).getReg();
2351 Builder.buildConstant(Res: DstReg, Val: Csts[Idx]);
2352 }
2353
2354 MI.eraseFromParent();
2355}
2356
2357bool CombinerHelper::matchCombineUnmergeUndef(
2358 MachineInstr &MI,
2359 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
2360 unsigned SrcIdx = MI.getNumOperands() - 1;
2361 Register SrcReg = MI.getOperand(i: SrcIdx).getReg();
2362 MatchInfo = [&MI](MachineIRBuilder &B) {
2363 unsigned NumElems = MI.getNumOperands() - 1;
2364 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2365 Register DstReg = MI.getOperand(i: Idx).getReg();
2366 B.buildUndef(Res: DstReg);
2367 }
2368 };
2369 return mi_match(R: SrcReg, MRI, P: m_GImplicitDef());
2370}
2371
2372bool CombinerHelper::matchCombineUnmergeWithDeadLanesToTrunc(
2373 MachineInstr &MI) const {
2374 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2375 "Expected an unmerge");
2376 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isScalar() ||
2377 !MRI.getType(Reg: MI.getOperand(i: MI.getNumDefs()).getReg()).isScalar())
2378 return false;
2379 // Check that all the lanes are dead except the first one.
2380 for (unsigned Idx = 1, EndIdx = MI.getNumDefs(); Idx != EndIdx; ++Idx) {
2381 if (!MRI.use_nodbg_empty(RegNo: MI.getOperand(i: Idx).getReg()))
2382 return false;
2383 }
2384 return true;
2385}
2386
2387void CombinerHelper::applyCombineUnmergeWithDeadLanesToTrunc(
2388 MachineInstr &MI) const {
2389 Register SrcReg = MI.getOperand(i: MI.getNumDefs()).getReg();
2390 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2391 Builder.buildTrunc(Res: Dst0Reg, Op: SrcReg);
2392 MI.eraseFromParent();
2393}
2394
2395bool CombinerHelper::matchCombineUnmergeZExtToZExt(MachineInstr &MI) const {
2396 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2397 "Expected an unmerge");
2398 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2399 LLT Dst0Ty = MRI.getType(Reg: Dst0Reg);
2400 // G_ZEXT on vector applies to each lane, so it will
2401 // affect all destinations. Therefore we won't be able
2402 // to simplify the unmerge to just the first definition.
2403 if (Dst0Ty.isVector())
2404 return false;
2405 Register SrcReg = MI.getOperand(i: MI.getNumDefs()).getReg();
2406 LLT SrcTy = MRI.getType(Reg: SrcReg);
2407 if (SrcTy.isVector())
2408 return false;
2409
2410 Register ZExtSrcReg;
2411 if (!mi_match(R: SrcReg, MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg))))
2412 return false;
2413
2414 // Finally we can replace the first definition with
2415 // a zext of the source if the definition is big enough to hold
2416 // all of ZExtSrc bits.
2417 LLT ZExtSrcTy = MRI.getType(Reg: ZExtSrcReg);
2418 return ZExtSrcTy.getSizeInBits() <= Dst0Ty.getSizeInBits();
2419}
2420
2421void CombinerHelper::applyCombineUnmergeZExtToZExt(MachineInstr &MI) const {
2422 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2423 "Expected an unmerge");
2424
2425 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2426
2427 GZext *ZExtInstr =
2428 cast<GZext>(Val: MRI.getVRegDef(Reg: MI.getOperand(i: MI.getNumDefs()).getReg()));
2429 Register ZExtSrcReg = ZExtInstr->getSrcReg();
2430 LLT Dst0Ty = MRI.getType(Reg: Dst0Reg);
2431 LLT ZExtSrcTy = MRI.getType(Reg: ZExtSrcReg);
2432
2433 if (Dst0Ty.getSizeInBits() > ZExtSrcTy.getSizeInBits()) {
2434 Builder.buildZExt(Res: Dst0Reg, Op: ZExtSrcReg);
2435 } else {
2436 assert(Dst0Ty.getSizeInBits() == ZExtSrcTy.getSizeInBits() &&
2437 "ZExt src doesn't fit in destination");
2438 replaceRegWith(MRI, FromReg: Dst0Reg, ToReg: ZExtSrcReg);
2439 }
2440
2441 Register ZeroReg;
2442 for (unsigned Idx = 1, EndIdx = MI.getNumDefs(); Idx != EndIdx; ++Idx) {
2443 if (!ZeroReg)
2444 ZeroReg = Builder.buildConstant(Res: Dst0Ty, Val: 0).getReg(Idx: 0);
2445 replaceRegWith(MRI, FromReg: MI.getOperand(i: Idx).getReg(), ToReg: ZeroReg);
2446 }
2447 MI.eraseFromParent();
2448}
2449
2450bool CombinerHelper::matchCombineShiftToUnmerge(MachineInstr &MI,
2451 unsigned TargetShiftSize,
2452 unsigned &ShiftVal) const {
2453 assert((MI.getOpcode() == TargetOpcode::G_SHL ||
2454 MI.getOpcode() == TargetOpcode::G_LSHR ||
2455 MI.getOpcode() == TargetOpcode::G_ASHR) && "Expected a shift");
2456
2457 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2458 if (Ty.isVector()) // TODO:
2459 return false;
2460
2461 // Don't narrow further than the requested size.
2462 unsigned Size = Ty.getSizeInBits();
2463 if (Size <= TargetShiftSize)
2464 return false;
2465
2466 auto MaybeImmVal =
2467 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
2468 if (!MaybeImmVal)
2469 return false;
2470
2471 ShiftVal = MaybeImmVal->Value.getSExtValue();
2472 return ShiftVal >= Size / 2 && ShiftVal < Size;
2473}
2474
2475void CombinerHelper::applyCombineShiftToUnmerge(
2476 MachineInstr &MI, const unsigned &ShiftVal) const {
2477 Register DstReg = MI.getOperand(i: 0).getReg();
2478 Register SrcReg = MI.getOperand(i: 1).getReg();
2479 LLT Ty = MRI.getType(Reg: SrcReg);
2480 unsigned Size = Ty.getSizeInBits();
2481 unsigned HalfSize = Size / 2;
2482 assert(ShiftVal >= HalfSize);
2483
2484 LLT HalfTy = Ty.changeElementSize(NewEltSize: HalfSize);
2485
2486 auto Unmerge = Builder.buildUnmerge(Res: HalfTy, Op: SrcReg);
2487 unsigned NarrowShiftAmt = ShiftVal - HalfSize;
2488
2489 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
2490 Register Narrowed = Unmerge.getReg(Idx: 1);
2491
2492 // dst = G_LSHR s64:x, C for C >= 32
2493 // =>
2494 // lo, hi = G_UNMERGE_VALUES x
2495 // dst = G_MERGE_VALUES (G_LSHR hi, C - 32), 0
2496
2497 if (NarrowShiftAmt != 0) {
2498 Narrowed = Builder.buildLShr(Dst: HalfTy, Src0: Narrowed,
2499 Src1: Builder.buildConstant(Res: HalfTy, Val: NarrowShiftAmt)).getReg(Idx: 0);
2500 }
2501
2502 auto Zero = Builder.buildConstant(Res: HalfTy, Val: 0);
2503 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Narrowed, Zero});
2504 } else if (MI.getOpcode() == TargetOpcode::G_SHL) {
2505 Register Narrowed = Unmerge.getReg(Idx: 0);
2506 // dst = G_SHL s64:x, C for C >= 32
2507 // =>
2508 // lo, hi = G_UNMERGE_VALUES x
2509 // dst = G_MERGE_VALUES 0, (G_SHL hi, C - 32)
2510 if (NarrowShiftAmt != 0) {
2511 Narrowed = Builder.buildShl(Dst: HalfTy, Src0: Narrowed,
2512 Src1: Builder.buildConstant(Res: HalfTy, Val: NarrowShiftAmt)).getReg(Idx: 0);
2513 }
2514
2515 auto Zero = Builder.buildConstant(Res: HalfTy, Val: 0);
2516 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Zero, Narrowed});
2517 } else {
2518 assert(MI.getOpcode() == TargetOpcode::G_ASHR);
2519 auto Hi = Builder.buildAShr(
2520 Dst: HalfTy, Src0: Unmerge.getReg(Idx: 1),
2521 Src1: Builder.buildConstant(Res: HalfTy, Val: HalfSize - 1));
2522
2523 if (ShiftVal == HalfSize) {
2524 // (G_ASHR i64:x, 32) ->
2525 // G_MERGE_VALUES hi_32(x), (G_ASHR hi_32(x), 31)
2526 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Unmerge.getReg(Idx: 1), Hi});
2527 } else if (ShiftVal == Size - 1) {
2528 // Don't need a second shift.
2529 // (G_ASHR i64:x, 63) ->
2530 // %narrowed = (G_ASHR hi_32(x), 31)
2531 // G_MERGE_VALUES %narrowed, %narrowed
2532 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Hi, Hi});
2533 } else {
2534 auto Lo = Builder.buildAShr(
2535 Dst: HalfTy, Src0: Unmerge.getReg(Idx: 1),
2536 Src1: Builder.buildConstant(Res: HalfTy, Val: ShiftVal - HalfSize));
2537
2538 // (G_ASHR i64:x, C) ->, for C >= 32
2539 // G_MERGE_VALUES (G_ASHR hi_32(x), C - 32), (G_ASHR hi_32(x), 31)
2540 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Lo, Hi});
2541 }
2542 }
2543
2544 MI.eraseFromParent();
2545}
2546
2547bool CombinerHelper::tryCombineShiftToUnmerge(
2548 MachineInstr &MI, unsigned TargetShiftAmount) const {
2549 unsigned ShiftAmt;
2550 if (matchCombineShiftToUnmerge(MI, TargetShiftSize: TargetShiftAmount, ShiftVal&: ShiftAmt)) {
2551 applyCombineShiftToUnmerge(MI, ShiftVal: ShiftAmt);
2552 return true;
2553 }
2554
2555 return false;
2556}
2557
2558bool CombinerHelper::matchCombineAnyExtTrunc(MachineInstr &MI,
2559 Register &Reg) const {
2560 assert(MI.getOpcode() == TargetOpcode::G_ANYEXT && "Expected a G_ANYEXT");
2561 Register DstReg = MI.getOperand(i: 0).getReg();
2562 Register SrcReg = MI.getOperand(i: 1).getReg();
2563 Register OriginalSrcReg = getSrcRegIgnoringCopies(Reg: SrcReg, MRI);
2564 if (OriginalSrcReg.isValid())
2565 SrcReg = OriginalSrcReg;
2566 LLT DstTy = MRI.getType(Reg: DstReg);
2567 return mi_match(R: SrcReg, MRI,
2568 P: m_GTrunc(Src: m_all_of(preds: m_Reg(R&: Reg), preds: m_SpecificType(Ty: DstTy)))) &&
2569 canReplaceReg(DstReg, SrcReg: Reg, MRI);
2570}
2571
2572bool CombinerHelper::matchCombineZextTrunc(MachineInstr &MI,
2573 Register &Reg) const {
2574 assert(MI.getOpcode() == TargetOpcode::G_ZEXT && "Expected a G_ZEXT");
2575 Register DstReg = MI.getOperand(i: 0).getReg();
2576 Register SrcReg = MI.getOperand(i: 1).getReg();
2577 LLT DstTy = MRI.getType(Reg: DstReg);
2578 if (mi_match(R: SrcReg, MRI,
2579 P: m_GTrunc(Src: m_all_of(preds: m_Reg(R&: Reg), preds: m_SpecificType(Ty: DstTy)))) &&
2580 canReplaceReg(DstReg, SrcReg: Reg, MRI)) {
2581 unsigned DstSize = DstTy.getScalarSizeInBits();
2582 unsigned SrcSize = MRI.getType(Reg: SrcReg).getScalarSizeInBits();
2583 return VT->getKnownBits(R: Reg).countMinLeadingZeros() >= DstSize - SrcSize;
2584 }
2585 return false;
2586}
2587
2588static LLT getMidVTForTruncRightShiftCombine(LLT ShiftTy, LLT TruncTy) {
2589 const unsigned ShiftSize = ShiftTy.getScalarSizeInBits();
2590 const unsigned TruncSize = TruncTy.getScalarSizeInBits();
2591
2592 // ShiftTy > 32 > TruncTy -> 32
2593 if (ShiftSize > 32 && TruncSize < 32)
2594 return ShiftTy.changeElementSize(NewEltSize: 32);
2595
2596 // TODO: We could also reduce to 16 bits, but that's more target-dependent.
2597 // Some targets like it, some don't, some only like it under certain
2598 // conditions/processor versions, etc.
2599 // A TL hook might be needed for this.
2600
2601 // Don't combine
2602 return ShiftTy;
2603}
2604
2605bool CombinerHelper::matchCombineTruncOfShift(
2606 MachineInstr &MI, std::pair<MachineInstr *, LLT> &MatchInfo) const {
2607 assert(MI.getOpcode() == TargetOpcode::G_TRUNC && "Expected a G_TRUNC");
2608 Register DstReg = MI.getOperand(i: 0).getReg();
2609 Register SrcReg = MI.getOperand(i: 1).getReg();
2610
2611 if (!MRI.hasOneNonDBGUse(RegNo: SrcReg))
2612 return false;
2613
2614 LLT SrcTy = MRI.getType(Reg: SrcReg);
2615 LLT DstTy = MRI.getType(Reg: DstReg);
2616
2617 MachineInstr *SrcMI = getDefIgnoringCopies(Reg: SrcReg, MRI);
2618 const auto &TL = getTargetLowering();
2619
2620 LLT NewShiftTy;
2621 switch (SrcMI->getOpcode()) {
2622 default:
2623 return false;
2624 case TargetOpcode::G_SHL: {
2625 NewShiftTy = DstTy;
2626
2627 // Make sure new shift amount is legal.
2628 KnownBits Known = VT->getKnownBits(R: SrcMI->getOperand(i: 2).getReg());
2629 if (Known.getMaxValue().uge(RHS: NewShiftTy.getScalarSizeInBits()))
2630 return false;
2631 break;
2632 }
2633 case TargetOpcode::G_LSHR:
2634 case TargetOpcode::G_ASHR: {
2635 // For right shifts, we conservatively do not do the transform if the TRUNC
2636 // has any STORE users. The reason is that if we change the type of the
2637 // shift, we may break the truncstore combine.
2638 //
2639 // TODO: Fix truncstore combine to handle (trunc(lshr (trunc x), k)).
2640 for (auto &User : MRI.use_instructions(Reg: DstReg))
2641 if (User.getOpcode() == TargetOpcode::G_STORE)
2642 return false;
2643
2644 NewShiftTy = getMidVTForTruncRightShiftCombine(ShiftTy: SrcTy, TruncTy: DstTy);
2645 if (NewShiftTy == SrcTy)
2646 return false;
2647
2648 // Make sure we won't lose information by truncating the high bits.
2649 KnownBits Known = VT->getKnownBits(R: SrcMI->getOperand(i: 2).getReg());
2650 if (Known.getMaxValue().ugt(RHS: NewShiftTy.getScalarSizeInBits() -
2651 DstTy.getScalarSizeInBits()))
2652 return false;
2653 break;
2654 }
2655 }
2656
2657 if (!isLegalOrBeforeLegalizer(
2658 Query: {SrcMI->getOpcode(),
2659 {NewShiftTy, TL.getPreferredShiftAmountTy(ShiftValueTy: NewShiftTy)}}))
2660 return false;
2661
2662 MatchInfo = std::make_pair(x&: SrcMI, y&: NewShiftTy);
2663 return true;
2664}
2665
2666void CombinerHelper::applyCombineTruncOfShift(
2667 MachineInstr &MI, std::pair<MachineInstr *, LLT> &MatchInfo) const {
2668 MachineInstr *ShiftMI = MatchInfo.first;
2669 LLT NewShiftTy = MatchInfo.second;
2670
2671 Register Dst = MI.getOperand(i: 0).getReg();
2672 LLT DstTy = MRI.getType(Reg: Dst);
2673
2674 Register ShiftAmt = ShiftMI->getOperand(i: 2).getReg();
2675 Register ShiftSrc = ShiftMI->getOperand(i: 1).getReg();
2676 ShiftSrc = Builder.buildTrunc(Res: NewShiftTy, Op: ShiftSrc).getReg(Idx: 0);
2677
2678 const auto &TL = getTargetLowering();
2679 LLT PrefShiftTy = TL.getPreferredShiftAmountTy(ShiftValueTy: NewShiftTy);
2680 if (MRI.getType(Reg: ShiftAmt) != PrefShiftTy)
2681 ShiftAmt = Builder.buildZExtOrTrunc(Res: PrefShiftTy, Op: ShiftAmt).getReg(Idx: 0);
2682
2683 Register NewShift =
2684 Builder
2685 .buildInstr(Opc: ShiftMI->getOpcode(), DstOps: {NewShiftTy}, SrcOps: {ShiftSrc, ShiftAmt})
2686 .getReg(Idx: 0);
2687
2688 if (NewShiftTy == DstTy)
2689 replaceRegWith(MRI, FromReg: Dst, ToReg: NewShift);
2690 else
2691 Builder.buildTrunc(Res: Dst, Op: NewShift);
2692
2693 eraseInst(MI);
2694}
2695
2696bool CombinerHelper::matchAllExplicitUsesAreUndef(MachineInstr &MI) const {
2697 return all_of(Range: MI.explicit_uses(), P: [this](const MachineOperand &MO) {
2698 return !MO.isReg() ||
2699 getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: MO.getReg(), MRI);
2700 });
2701}
2702
2703bool CombinerHelper::matchUndefShuffleVectorMask(MachineInstr &MI) const {
2704 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
2705 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
2706 return all_of(Range&: Mask, P: [](int Elt) { return Elt < 0; });
2707}
2708
2709bool CombinerHelper::matchInsertExtractVecEltOutOfBounds(
2710 MachineInstr &MI) const {
2711 assert((MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT ||
2712 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT) &&
2713 "Expected an insert/extract element op");
2714 LLT VecTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
2715 if (VecTy.isScalableVector())
2716 return false;
2717
2718 unsigned IdxIdx =
2719 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
2720 auto Idx = getIConstantVRegVal(VReg: MI.getOperand(i: IdxIdx).getReg(), MRI);
2721 if (!Idx)
2722 return false;
2723 return Idx->getZExtValue() >= VecTy.getNumElements();
2724}
2725
2726void CombinerHelper::eraseInst(MachineInstr &MI) const { MI.eraseFromParent(); }
2727
2728bool CombinerHelper::matchEqualDefs(const MachineOperand &MOP1,
2729 const MachineOperand &MOP2) const {
2730 if (!MOP1.isReg() || !MOP2.isReg())
2731 return false;
2732 auto InstAndDef1 = getDefSrcRegIgnoringCopies(Reg: MOP1.getReg(), MRI);
2733 if (!InstAndDef1)
2734 return false;
2735 auto InstAndDef2 = getDefSrcRegIgnoringCopies(Reg: MOP2.getReg(), MRI);
2736 if (!InstAndDef2)
2737 return false;
2738 MachineInstr *I1 = InstAndDef1->MI;
2739 MachineInstr *I2 = InstAndDef2->MI;
2740
2741 // Handle a case like this:
2742 //
2743 // %0:_(s64), %1:_(s64) = G_UNMERGE_VALUES %2:_(<2 x s64>)
2744 //
2745 // Even though %0 and %1 are produced by the same instruction they are not
2746 // the same values.
2747 if (I1 == I2)
2748 return MOP1.getReg() == MOP2.getReg();
2749
2750 // If we have an instruction which loads or stores, we can't guarantee that
2751 // it is identical.
2752 //
2753 // For example, we may have
2754 //
2755 // %x1 = G_LOAD %addr (load N from @somewhere)
2756 // ...
2757 // call @foo
2758 // ...
2759 // %x2 = G_LOAD %addr (load N from @somewhere)
2760 // ...
2761 // %or = G_OR %x1, %x2
2762 //
2763 // It's possible that @foo will modify whatever lives at the address we're
2764 // loading from. To be safe, let's just assume that all loads and stores
2765 // are different (unless we have something which is guaranteed to not
2766 // change.)
2767 if (I1->mayLoadOrStore() && !I1->isDereferenceableInvariantLoad())
2768 return false;
2769
2770 // If both instructions are loads or stores, they are equal only if both
2771 // are dereferenceable invariant loads with the same number of bits.
2772 if (I1->mayLoadOrStore() && I2->mayLoadOrStore()) {
2773 GLoadStore *LS1 = dyn_cast<GLoadStore>(Val: I1);
2774 GLoadStore *LS2 = dyn_cast<GLoadStore>(Val: I2);
2775 if (!LS1 || !LS2)
2776 return false;
2777
2778 if (!I2->isDereferenceableInvariantLoad() ||
2779 (LS1->getMemSizeInBits() != LS2->getMemSizeInBits()))
2780 return false;
2781 }
2782
2783 // Check for physical registers on the instructions first to avoid cases
2784 // like this:
2785 //
2786 // %a = COPY $physreg
2787 // ...
2788 // SOMETHING implicit-def $physreg
2789 // ...
2790 // %b = COPY $physreg
2791 //
2792 // These copies are not equivalent.
2793 if (any_of(Range: I1->uses(), P: [](const MachineOperand &MO) {
2794 return MO.isReg() && MO.getReg().isPhysical();
2795 })) {
2796 // Check if we have a case like this:
2797 //
2798 // %a = COPY $physreg
2799 // %b = COPY %a
2800 //
2801 // In this case, I1 and I2 will both be equal to %a = COPY $physreg.
2802 // From that, we know that they must have the same value, since they must
2803 // have come from the same COPY.
2804 return I1->isIdenticalTo(Other: *I2);
2805 }
2806
2807 // We don't have any physical registers, so we don't necessarily need the
2808 // same vreg defs.
2809 //
2810 // On the off-chance that there's some target instruction feeding into the
2811 // instruction, let's use produceSameValue instead of isIdenticalTo.
2812 if (Builder.getTII().produceSameValue(MI0: *I1, MI1: *I2, MRI: &MRI)) {
2813 // Handle instructions with multiple defs that produce same values. Values
2814 // are same for operands with same index.
2815 // %0:_(s8), %1:_(s8), %2:_(s8), %3:_(s8) = G_UNMERGE_VALUES %4:_(<4 x s8>)
2816 // %5:_(s8), %6:_(s8), %7:_(s8), %8:_(s8) = G_UNMERGE_VALUES %4:_(<4 x s8>)
2817 // I1 and I2 are different instructions but produce same values,
2818 // %1 and %6 are same, %1 and %7 are not the same value.
2819 return I1->findRegisterDefOperandIdx(Reg: InstAndDef1->Reg, /*TRI=*/nullptr) ==
2820 I2->findRegisterDefOperandIdx(Reg: InstAndDef2->Reg, /*TRI=*/nullptr);
2821 }
2822 return false;
2823}
2824
2825bool CombinerHelper::matchConstantOp(const MachineOperand &MOP,
2826 int64_t C) const {
2827 if (!MOP.isReg())
2828 return false;
2829 auto MaybeCst = isConstantOrConstantSplatVector(Def: MOP.getReg(), MRI);
2830 // Compare as APInt so constants wider than 64 bits (e.g. i128 2) still match.
2831 return MaybeCst &&
2832 APInt::isSameValue(I1: *MaybeCst, I2: APInt(64, C, /*isSigned=*/true),
2833 /*SignedCompare=*/true);
2834}
2835
2836bool CombinerHelper::matchConstantFPOp(const MachineOperand &MOP,
2837 double C) const {
2838 if (!MOP.isReg())
2839 return false;
2840 std::optional<FPValueAndVReg> MaybeCst;
2841 if (!mi_match(R: MOP.getReg(), MRI, P: m_GFCstOrSplat(FPValReg&: MaybeCst)))
2842 return false;
2843
2844 return MaybeCst->Value.isExactlyValue(V: C);
2845}
2846
2847void CombinerHelper::replaceSingleDefInstWithOperand(MachineInstr &MI,
2848 unsigned OpIdx) const {
2849 assert(MI.getNumExplicitDefs() == 1 && "Expected one explicit def?");
2850 Register OldReg = MI.getOperand(i: 0).getReg();
2851 Register Replacement = MI.getOperand(i: OpIdx).getReg();
2852 assert(canReplaceReg(OldReg, Replacement, MRI) && "Cannot replace register?");
2853 replaceRegWith(MRI, FromReg: OldReg, ToReg: Replacement);
2854 MI.eraseFromParent();
2855}
2856
2857void CombinerHelper::replaceSingleDefInstWithReg(MachineInstr &MI,
2858 Register Replacement) const {
2859 assert(MI.getNumExplicitDefs() == 1 && "Expected one explicit def?");
2860 Register OldReg = MI.getOperand(i: 0).getReg();
2861 assert(canReplaceReg(OldReg, Replacement, MRI) && "Cannot replace register?");
2862 replaceRegWith(MRI, FromReg: OldReg, ToReg: Replacement);
2863 MI.eraseFromParent();
2864}
2865
2866bool CombinerHelper::matchConstantLargerBitWidth(MachineInstr &MI,
2867 unsigned ConstIdx) const {
2868 Register ConstReg = MI.getOperand(i: ConstIdx).getReg();
2869 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2870
2871 // Get the shift amount
2872 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstReg, MRI);
2873 if (!VRegAndVal)
2874 return false;
2875
2876 // Return true of shift amount >= Bitwidth
2877 return (VRegAndVal->Value.uge(RHS: DstTy.getSizeInBits()));
2878}
2879
2880void CombinerHelper::applyFunnelShiftConstantModulo(MachineInstr &MI) const {
2881 assert((MI.getOpcode() == TargetOpcode::G_FSHL ||
2882 MI.getOpcode() == TargetOpcode::G_FSHR) &&
2883 "This is not a funnel shift operation");
2884
2885 Register ConstReg = MI.getOperand(i: 3).getReg();
2886 LLT ConstTy = MRI.getType(Reg: ConstReg);
2887 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2888
2889 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstReg, MRI);
2890 assert((VRegAndVal) && "Value is not a constant");
2891
2892 // Calculate the new Shift Amount = Old Shift Amount % BitWidth
2893 APInt NewConst = VRegAndVal->Value.urem(
2894 RHS: APInt(ConstTy.getSizeInBits(), DstTy.getScalarSizeInBits()));
2895
2896 auto NewConstInstr = Builder.buildConstant(Res: ConstTy, Val: NewConst.getZExtValue());
2897 Builder.buildInstr(
2898 Opc: MI.getOpcode(), DstOps: {MI.getOperand(i: 0)},
2899 SrcOps: {MI.getOperand(i: 1), MI.getOperand(i: 2), NewConstInstr.getReg(Idx: 0)});
2900
2901 MI.eraseFromParent();
2902}
2903
2904bool CombinerHelper::matchSelectSameVal(MachineInstr &MI) const {
2905 assert(MI.getOpcode() == TargetOpcode::G_SELECT);
2906 // Match (cond ? x : x)
2907 return matchEqualDefs(MOP1: MI.getOperand(i: 2), MOP2: MI.getOperand(i: 3)) &&
2908 canReplaceReg(DstReg: MI.getOperand(i: 0).getReg(), SrcReg: MI.getOperand(i: 2).getReg(),
2909 MRI);
2910}
2911
2912bool CombinerHelper::matchOperandIsKnownToBeAPowerOfTwo(
2913 const MachineOperand &MO, bool OrNegative, bool OrZero) const {
2914 return isKnownToBeAPowerOfTwo(Val: MO.getReg(), MRI, ValueTracking: VT, OrNegative, OrZero);
2915}
2916
2917void CombinerHelper::replaceInstWithFConstant(MachineInstr &MI,
2918 double C) const {
2919 assert(MI.getNumDefs() == 1 && "Expected only one def?");
2920 Builder.buildFConstant(Res: MI.getOperand(i: 0), Val: C);
2921 MI.eraseFromParent();
2922}
2923
2924void CombinerHelper::replaceInstWithConstant(MachineInstr &MI,
2925 int64_t C) const {
2926 assert(MI.getNumDefs() == 1 && "Expected only one def?");
2927 Builder.buildConstant(Res: MI.getOperand(i: 0), Val: C);
2928 MI.eraseFromParent();
2929}
2930
2931void CombinerHelper::replaceInstWithConstant(MachineInstr &MI, APInt C) const {
2932 assert(MI.getNumDefs() == 1 && "Expected only one def?");
2933 Builder.buildConstant(Res: MI.getOperand(i: 0), Val: C);
2934 MI.eraseFromParent();
2935}
2936
2937void CombinerHelper::replaceInstWithFConstant(MachineInstr &MI,
2938 ConstantFP *CFP) const {
2939 assert(MI.getNumDefs() == 1 && "Expected only one def?");
2940 Builder.buildFConstant(Res: MI.getOperand(i: 0), Val: CFP->getValueAPF());
2941 MI.eraseFromParent();
2942}
2943
2944void CombinerHelper::replaceInstWithUndef(MachineInstr &MI) const {
2945 assert(MI.getNumDefs() == 1 && "Expected only one def?");
2946 Builder.buildUndef(Res: MI.getOperand(i: 0));
2947 MI.eraseFromParent();
2948}
2949
2950bool CombinerHelper::matchCombineInsertVecElts(
2951 MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo) const {
2952 assert(MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT &&
2953 "Invalid opcode");
2954 Register DstReg = MI.getOperand(i: 0).getReg();
2955 LLT DstTy = MRI.getType(Reg: DstReg);
2956 assert(DstTy.isVector() && "Invalid G_INSERT_VECTOR_ELT?");
2957
2958 if (DstTy.isScalableVector())
2959 return false;
2960
2961 unsigned NumElts = DstTy.getNumElements();
2962 // If this MI is part of a sequence of insert_vec_elts, then
2963 // don't do the combine in the middle of the sequence.
2964 if (MRI.hasOneNonDBGUse(RegNo: DstReg) &&
2965 MRI.use_nodbg_instructions(Reg: DstReg).begin()->getOpcode() ==
2966 TargetOpcode::G_INSERT_VECTOR_ELT)
2967 return false;
2968 MachineInstr *CurrInst = &MI;
2969 MachineInstr *TmpInst;
2970 int64_t IntImm;
2971 Register TmpReg;
2972 MatchInfo.resize(N: NumElts);
2973 while (mi_match(
2974 MI&: *CurrInst, MRI,
2975 P: m_GInsertVecElt(Src0: m_MInstr(MI&: TmpInst), Src1: m_Reg(R&: TmpReg), Src2: m_ICst(Cst&: IntImm)))) {
2976 if (IntImm >= NumElts || IntImm < 0)
2977 return false;
2978 if (!MatchInfo[IntImm])
2979 MatchInfo[IntImm] = TmpReg;
2980 CurrInst = TmpInst;
2981 }
2982 // Variable index.
2983 if (CurrInst->getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
2984 return false;
2985 if (TmpInst->getOpcode() == TargetOpcode::G_BUILD_VECTOR) {
2986 for (unsigned I = 1; I < TmpInst->getNumOperands(); ++I) {
2987 if (!MatchInfo[I - 1].isValid())
2988 MatchInfo[I - 1] = TmpInst->getOperand(i: I).getReg();
2989 }
2990 return true;
2991 }
2992 // If we didn't end in a G_IMPLICIT_DEF and the source is not fully
2993 // overwritten, bail out.
2994 return TmpInst->getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
2995 all_of(Range&: MatchInfo, P: [](Register Reg) { return !!Reg; });
2996}
2997
2998void CombinerHelper::applyCombineInsertVecElts(
2999 MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo) const {
3000 Register UndefReg;
3001 auto GetUndef = [&]() {
3002 if (UndefReg)
3003 return UndefReg;
3004 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3005 UndefReg = Builder.buildUndef(Res: DstTy.getScalarType()).getReg(Idx: 0);
3006 return UndefReg;
3007 };
3008 for (Register &Reg : MatchInfo) {
3009 if (!Reg)
3010 Reg = GetUndef();
3011 }
3012 Builder.buildBuildVector(Res: MI.getOperand(i: 0).getReg(), Ops: MatchInfo);
3013 MI.eraseFromParent();
3014}
3015
3016bool CombinerHelper::matchBinopWithNegInner(Register MInner, Register Other,
3017 unsigned RootOpc, Register Dst,
3018 LLT Ty,
3019 BuildFnTy &MatchInfo) const {
3020 /// Helper function for matchBinopWithNeg: tries to match one commuted form
3021 /// of `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`.
3022 MachineInstr *InnerDef;
3023 if (!mi_match(R: MInner, MRI, P: m_MInstr(MI&: InnerDef)))
3024 return false;
3025
3026 unsigned InnerOpc = InnerDef->getOpcode();
3027 if (InnerOpc != TargetOpcode::G_ADD && InnerOpc != TargetOpcode::G_SUB)
3028 return false;
3029
3030 if (!MRI.hasOneNonDBGUse(RegNo: MInner))
3031 return false;
3032
3033 Register InnerLHS = InnerDef->getOperand(i: 1).getReg();
3034 Register InnerRHS = InnerDef->getOperand(i: 2).getReg();
3035 Register NotSrc;
3036 Register B, C;
3037
3038 // Check if either operand is ~b
3039 auto TryMatch = [&](Register MaybeNot, Register Other) {
3040 if (mi_match(R: MaybeNot, MRI, P: m_Not(Src: m_Reg(R&: NotSrc)))) {
3041 if (!MRI.hasOneNonDBGUse(RegNo: MaybeNot))
3042 return false;
3043 B = NotSrc;
3044 C = Other;
3045 return true;
3046 }
3047 return false;
3048 };
3049
3050 // For SUB, the not must be the LHS. For ADD, it can be either operand.
3051 if (!TryMatch(InnerLHS, InnerRHS) &&
3052 !(InnerOpc == TargetOpcode::G_ADD && TryMatch(InnerRHS, InnerLHS)))
3053 return false;
3054
3055 // Flip add/sub
3056 unsigned FlippedOpc = (InnerOpc == TargetOpcode::G_ADD) ? TargetOpcode::G_SUB
3057 : TargetOpcode::G_ADD;
3058
3059 Register A = Other;
3060 MatchInfo = [=](MachineIRBuilder &Builder) {
3061 auto NewInner = Builder.buildInstr(Opc: FlippedOpc, DstOps: {Ty}, SrcOps: {B, C});
3062 auto NewNot = Builder.buildNot(Dst: Ty, Src0: NewInner);
3063 Builder.buildInstr(Opc: RootOpc, DstOps: {Dst}, SrcOps: {A, NewNot});
3064 };
3065 return true;
3066}
3067
3068bool CombinerHelper::matchBinopWithNeg(MachineInstr &MI,
3069 BuildFnTy &MatchInfo) const {
3070 // Fold `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`
3071 // Root MI is one of G_AND, G_OR, G_XOR.
3072 // We also look for commuted forms of operations. Pattern shouldn't apply
3073 // if there are multiple reasons of inner operations.
3074
3075 unsigned RootOpc = MI.getOpcode();
3076 Register Dst = MI.getOperand(i: 0).getReg();
3077 LLT Ty = MRI.getType(Reg: Dst);
3078
3079 Register LHS = MI.getOperand(i: 1).getReg();
3080 Register RHS = MI.getOperand(i: 2).getReg();
3081 // Check the commuted and uncommuted forms of the operation.
3082 return matchBinopWithNegInner(MInner: LHS, Other: RHS, RootOpc, Dst, Ty, MatchInfo) ||
3083 matchBinopWithNegInner(MInner: RHS, Other: LHS, RootOpc, Dst, Ty, MatchInfo);
3084}
3085
3086bool CombinerHelper::matchHoistLogicOpWithSameOpcodeHands(
3087 MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) const {
3088 // Matches: logic (hand x, ...), (hand y, ...) -> hand (logic x, y), ...
3089 //
3090 // Creates the new hand + logic instruction (but does not insert them.)
3091 //
3092 // On success, MatchInfo is populated with the new instructions. These are
3093 // inserted in applyHoistLogicOpWithSameOpcodeHands.
3094 unsigned LogicOpcode = MI.getOpcode();
3095 assert(LogicOpcode == TargetOpcode::G_AND ||
3096 LogicOpcode == TargetOpcode::G_OR ||
3097 LogicOpcode == TargetOpcode::G_XOR);
3098 Register Dst = MI.getOperand(i: 0).getReg();
3099 Register LHSReg = MI.getOperand(i: 1).getReg();
3100 Register RHSReg = MI.getOperand(i: 2).getReg();
3101
3102 // Don't recompute anything.
3103 if (!MRI.hasOneNonDBGUse(RegNo: LHSReg) || !MRI.hasOneNonDBGUse(RegNo: RHSReg))
3104 return false;
3105
3106 // Make sure we have (hand x, ...), (hand y, ...)
3107 MachineInstr *LeftHandInst = getDefIgnoringCopies(Reg: LHSReg, MRI);
3108 MachineInstr *RightHandInst = getDefIgnoringCopies(Reg: RHSReg, MRI);
3109 if (!LeftHandInst || !RightHandInst)
3110 return false;
3111 unsigned HandOpcode = LeftHandInst->getOpcode();
3112 if (HandOpcode != RightHandInst->getOpcode())
3113 return false;
3114 if (LeftHandInst->getNumOperands() < 2 ||
3115 !LeftHandInst->getOperand(i: 1).isReg() ||
3116 RightHandInst->getNumOperands() < 2 ||
3117 !RightHandInst->getOperand(i: 1).isReg())
3118 return false;
3119
3120 // Make sure the types match up, and if we're doing this post-legalization,
3121 // we end up with legal types.
3122 Register X = LeftHandInst->getOperand(i: 1).getReg();
3123 Register Y = RightHandInst->getOperand(i: 1).getReg();
3124 LLT XTy = MRI.getType(Reg: X);
3125 LLT YTy = MRI.getType(Reg: Y);
3126 if (!XTy.isValid() || XTy != YTy)
3127 return false;
3128
3129 // Optional extra source register.
3130 Register ExtraHandOpSrcReg;
3131 switch (HandOpcode) {
3132 default:
3133 return false;
3134 case TargetOpcode::G_ANYEXT:
3135 case TargetOpcode::G_SEXT:
3136 case TargetOpcode::G_ZEXT: {
3137 // Match: logic (ext X), (ext Y) --> ext (logic X, Y)
3138 break;
3139 }
3140 case TargetOpcode::G_TRUNC: {
3141 // Match: logic (trunc X), (trunc Y) -> trunc (logic X, Y)
3142 const MachineFunction *MF = MI.getMF();
3143 LLVMContext &Ctx = MF->getFunction().getContext();
3144
3145 LLT DstTy = MRI.getType(Reg: Dst);
3146 const TargetLowering &TLI = getTargetLowering();
3147
3148 // Be extra careful sinking truncate. If it's free, there's no benefit in
3149 // widening a binop.
3150 if (TLI.isZExtFree(FromTy: DstTy, ToTy: XTy, Ctx) && TLI.isTruncateFree(FromTy: XTy, ToTy: DstTy, Ctx))
3151 return false;
3152 break;
3153 }
3154 case TargetOpcode::G_AND:
3155 case TargetOpcode::G_ASHR:
3156 case TargetOpcode::G_LSHR:
3157 case TargetOpcode::G_SHL: {
3158 // Match: logic (binop x, z), (binop y, z) -> binop (logic x, y), z
3159 MachineOperand &ZOp = LeftHandInst->getOperand(i: 2);
3160 if (!matchEqualDefs(MOP1: ZOp, MOP2: RightHandInst->getOperand(i: 2)))
3161 return false;
3162 ExtraHandOpSrcReg = ZOp.getReg();
3163 break;
3164 }
3165 }
3166
3167 if (!isLegalOrBeforeLegalizer(Query: {LogicOpcode, {XTy, YTy}}))
3168 return false;
3169
3170 // Record the steps to build the new instructions.
3171 //
3172 // Steps to build (logic x, y)
3173 auto NewLogicDst = MRI.createGenericVirtualRegister(Ty: XTy);
3174 OperandBuildSteps LogicBuildSteps = {
3175 [=](MachineInstrBuilder &MIB) { MIB.addDef(RegNo: NewLogicDst); },
3176 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: X); },
3177 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Y); }};
3178 InstructionBuildSteps LogicSteps(LogicOpcode, LogicBuildSteps);
3179
3180 // Steps to build hand (logic x, y), ...z
3181 OperandBuildSteps HandBuildSteps = {
3182 [=](MachineInstrBuilder &MIB) { MIB.addDef(RegNo: Dst); },
3183 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: NewLogicDst); }};
3184 if (ExtraHandOpSrcReg.isValid())
3185 HandBuildSteps.push_back(
3186 Elt: [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ExtraHandOpSrcReg); });
3187 InstructionBuildSteps HandSteps(HandOpcode, HandBuildSteps);
3188
3189 MatchInfo = InstructionStepsMatchInfo({LogicSteps, HandSteps});
3190 return true;
3191}
3192
3193void CombinerHelper::applyBuildInstructionSteps(
3194 MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) const {
3195 assert(MatchInfo.InstrsToBuild.size() &&
3196 "Expected at least one instr to build?");
3197 for (auto &InstrToBuild : MatchInfo.InstrsToBuild) {
3198 assert(InstrToBuild.Opcode && "Expected a valid opcode?");
3199 assert(InstrToBuild.OperandFns.size() && "Expected at least one operand?");
3200 MachineInstrBuilder Instr = Builder.buildInstr(Opcode: InstrToBuild.Opcode);
3201 for (auto &OperandFn : InstrToBuild.OperandFns)
3202 OperandFn(Instr);
3203 }
3204 MI.eraseFromParent();
3205}
3206
3207bool CombinerHelper::matchAshrShlToSextInreg(
3208 MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {
3209 assert(MI.getOpcode() == TargetOpcode::G_ASHR);
3210 int64_t ShlCst, AshrCst;
3211 Register Src;
3212 if (!mi_match(R: MI.getOperand(i: 0).getReg(), MRI,
3213 P: m_GAShr(L: m_GShl(L: m_Reg(R&: Src), R: m_ICstOrSplat(Cst&: ShlCst)),
3214 R: m_ICstOrSplat(Cst&: AshrCst))))
3215 return false;
3216 if (ShlCst != AshrCst)
3217 return false;
3218 if (!isLegalOrBeforeLegalizer(
3219 Query: {TargetOpcode::G_SEXT_INREG,
3220 {MRI.getType(Reg: Src)},
3221 {},
3222 {MRI.getType(Reg: Src).getScalarSizeInBits() - ShlCst}}))
3223 return false;
3224 MatchInfo = std::make_tuple(args&: Src, args&: ShlCst);
3225 return true;
3226}
3227
3228void CombinerHelper::applyAshShlToSextInreg(
3229 MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {
3230 assert(MI.getOpcode() == TargetOpcode::G_ASHR);
3231 Register Src;
3232 int64_t ShiftAmt;
3233 std::tie(args&: Src, args&: ShiftAmt) = MatchInfo;
3234 unsigned Size = MRI.getType(Reg: Src).getScalarSizeInBits();
3235 Builder.buildSExtInReg(Res: MI.getOperand(i: 0).getReg(), Op: Src, ImmOp: Size - ShiftAmt);
3236 MI.eraseFromParent();
3237}
3238
3239/// and(and(x, C1), C2) -> C1&C2 ? and(x, C1&C2) : 0
3240bool CombinerHelper::matchOverlappingAnd(
3241 MachineInstr &MI,
3242 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
3243 assert(MI.getOpcode() == TargetOpcode::G_AND);
3244
3245 Register Dst = MI.getOperand(i: 0).getReg();
3246 LLT Ty = MRI.getType(Reg: Dst);
3247
3248 Register R;
3249 int64_t C1;
3250 int64_t C2;
3251 if (!mi_match(
3252 R: Dst, MRI,
3253 P: m_GAnd(L: m_GAnd(L: m_Reg(R), R: m_ICst(Cst&: C1)), R: m_ICst(Cst&: C2))))
3254 return false;
3255
3256 MatchInfo = [=](MachineIRBuilder &B) {
3257 if (C1 & C2) {
3258 B.buildAnd(Dst, Src0: R, Src1: B.buildConstant(Res: Ty, Val: C1 & C2));
3259 return;
3260 }
3261 auto Zero = B.buildConstant(Res: Ty, Val: 0);
3262 replaceRegWith(MRI, FromReg: Dst, ToReg: Zero->getOperand(i: 0).getReg());
3263 };
3264 return true;
3265}
3266
3267bool CombinerHelper::matchRedundantAnd(MachineInstr &MI,
3268 Register &Replacement) const {
3269 // Given
3270 //
3271 // %y:_(sN) = G_SOMETHING
3272 // %x:_(sN) = G_SOMETHING
3273 // %res:_(sN) = G_AND %x, %y
3274 //
3275 // Eliminate the G_AND when it is known that x & y == x or x & y == y.
3276 //
3277 // Patterns like this can appear as a result of legalization. E.g.
3278 //
3279 // %cmp:_(s32) = G_ICMP intpred(pred), %x(s32), %y
3280 // %one:_(s32) = G_CONSTANT i32 1
3281 // %and:_(s32) = G_AND %cmp, %one
3282 //
3283 // In this case, G_ICMP only produces a single bit, so x & 1 == x.
3284 assert(MI.getOpcode() == TargetOpcode::G_AND);
3285 if (!VT)
3286 return false;
3287
3288 Register AndDst = MI.getOperand(i: 0).getReg();
3289 Register LHS = MI.getOperand(i: 1).getReg();
3290 Register RHS = MI.getOperand(i: 2).getReg();
3291
3292 // Check the RHS (maybe a constant) first, and if we have no KnownBits there,
3293 // we can't do anything. If we do, then it depends on whether we have
3294 // KnownBits on the LHS.
3295 KnownBits RHSBits = VT->getKnownBits(R: RHS);
3296 if (RHSBits.isUnknown())
3297 return false;
3298
3299 KnownBits LHSBits = VT->getKnownBits(R: LHS);
3300
3301 // Check that x & Mask == x.
3302 // x & 1 == x, always
3303 // x & 0 == x, only if x is also 0
3304 // Meaning Mask has no effect if every bit is either one in Mask or zero in x.
3305 //
3306 // Check if we can replace AndDst with the LHS of the G_AND
3307 if (canReplaceReg(DstReg: AndDst, SrcReg: LHS, MRI) &&
3308 (LHSBits.Zero | RHSBits.One).isAllOnes()) {
3309 Replacement = LHS;
3310 return true;
3311 }
3312
3313 // Check if we can replace AndDst with the RHS of the G_AND
3314 if (canReplaceReg(DstReg: AndDst, SrcReg: RHS, MRI) &&
3315 (LHSBits.One | RHSBits.Zero).isAllOnes()) {
3316 Replacement = RHS;
3317 return true;
3318 }
3319
3320 return false;
3321}
3322
3323bool CombinerHelper::matchRedundantOr(MachineInstr &MI,
3324 Register &Replacement) const {
3325 // Given
3326 //
3327 // %y:_(sN) = G_SOMETHING
3328 // %x:_(sN) = G_SOMETHING
3329 // %res:_(sN) = G_OR %x, %y
3330 //
3331 // Eliminate the G_OR when it is known that x | y == x or x | y == y.
3332 assert(MI.getOpcode() == TargetOpcode::G_OR);
3333 if (!VT)
3334 return false;
3335
3336 Register OrDst = MI.getOperand(i: 0).getReg();
3337 Register LHS = MI.getOperand(i: 1).getReg();
3338 Register RHS = MI.getOperand(i: 2).getReg();
3339
3340 KnownBits LHSBits = VT->getKnownBits(R: LHS);
3341 KnownBits RHSBits = VT->getKnownBits(R: RHS);
3342
3343 // Check that x | Mask == x.
3344 // x | 0 == x, always
3345 // x | 1 == x, only if x is also 1
3346 // Meaning Mask has no effect if every bit is either zero in Mask or one in x.
3347 //
3348 // Check if we can replace OrDst with the LHS of the G_OR
3349 if (canReplaceReg(DstReg: OrDst, SrcReg: LHS, MRI) &&
3350 (LHSBits.One | RHSBits.Zero).isAllOnes()) {
3351 Replacement = LHS;
3352 return true;
3353 }
3354
3355 // Check if we can replace OrDst with the RHS of the G_OR
3356 if (canReplaceReg(DstReg: OrDst, SrcReg: RHS, MRI) &&
3357 (LHSBits.Zero | RHSBits.One).isAllOnes()) {
3358 Replacement = RHS;
3359 return true;
3360 }
3361
3362 return false;
3363}
3364
3365bool CombinerHelper::matchRedundantSExtInReg(MachineInstr &MI) const {
3366 // If the input is already sign extended, just drop the extension.
3367 Register Src = MI.getOperand(i: 1).getReg();
3368 unsigned ExtBits = MI.getOperand(i: 2).getImm();
3369 unsigned TypeSize = MRI.getType(Reg: Src).getScalarSizeInBits();
3370 return VT->computeNumSignBits(R: Src) >= (TypeSize - ExtBits + 1);
3371}
3372
3373static bool isConstValidTrue(const TargetLowering &TLI, unsigned ScalarSizeBits,
3374 int64_t Cst, bool IsVector, bool IsFP) {
3375 // For i1, Cst will always be -1 regardless of boolean contents.
3376 return (ScalarSizeBits == 1 && Cst == -1) ||
3377 isConstTrueVal(TLI, Val: Cst, IsVector, IsFP);
3378}
3379
3380// This pattern aims to match the following shape to avoid extra mov
3381// instructions
3382// G_BUILD_VECTOR(
3383// G_UNMERGE_VALUES(src, 0)
3384// G_UNMERGE_VALUES(src, 1)
3385// G_IMPLICIT_DEF
3386// G_IMPLICIT_DEF
3387// )
3388// ->
3389// G_CONCAT_VECTORS(
3390// src,
3391// undef
3392// )
3393bool CombinerHelper::matchCombineBuildUnmerge(MachineInstr &MI,
3394 MachineRegisterInfo &MRI,
3395 Register &UnmergeSrc) const {
3396 auto &BV = cast<GBuildVector>(Val&: MI);
3397
3398 unsigned BuildUseCount = BV.getNumSources();
3399 if (BuildUseCount % 2 != 0)
3400 return false;
3401
3402 unsigned NumUnmerge = BuildUseCount / 2;
3403
3404 auto *Unmerge = getOpcodeDef<GUnmerge>(Reg: BV.getSourceReg(I: 0), MRI);
3405
3406 // Check the first operand is an unmerge and has the correct number of
3407 // operands
3408 if (!Unmerge || Unmerge->getNumDefs() != NumUnmerge)
3409 return false;
3410
3411 UnmergeSrc = Unmerge->getSourceReg();
3412
3413 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3414 LLT UnmergeSrcTy = MRI.getType(Reg: UnmergeSrc);
3415
3416 if (!UnmergeSrcTy.isVector())
3417 return false;
3418
3419 // Ensure we only generate legal instructions post-legalizer
3420 if (!IsPreLegalize &&
3421 !isLegal(Query: {TargetOpcode::G_CONCAT_VECTORS, {DstTy, UnmergeSrcTy}}))
3422 return false;
3423
3424 // Check that all of the operands before the midpoint come from the same
3425 // unmerge and are in the same order as they are used in the build_vector
3426 for (unsigned I = 0; I < NumUnmerge; ++I) {
3427 auto MaybeUnmergeReg = BV.getSourceReg(I);
3428 auto *LoopUnmerge = getOpcodeDef<GUnmerge>(Reg: MaybeUnmergeReg, MRI);
3429
3430 if (!LoopUnmerge || LoopUnmerge != Unmerge)
3431 return false;
3432
3433 if (LoopUnmerge->getOperand(i: I).getReg() != MaybeUnmergeReg)
3434 return false;
3435 }
3436
3437 // Check that all of the unmerged values are used
3438 if (Unmerge->getNumDefs() != NumUnmerge)
3439 return false;
3440
3441 // Check that all of the operands after the mid point are undefs.
3442 for (unsigned I = NumUnmerge; I < BuildUseCount; ++I) {
3443 auto *Undef = getDefIgnoringCopies(Reg: BV.getSourceReg(I), MRI);
3444
3445 if (Undef->getOpcode() != TargetOpcode::G_IMPLICIT_DEF)
3446 return false;
3447 }
3448
3449 return true;
3450}
3451
3452void CombinerHelper::applyCombineBuildUnmerge(MachineInstr &MI,
3453 MachineRegisterInfo &MRI,
3454 MachineIRBuilder &B,
3455 Register &UnmergeSrc) const {
3456 assert(UnmergeSrc && "Expected there to be one matching G_UNMERGE_VALUES");
3457 B.setInstrAndDebugLoc(MI);
3458
3459 Register UndefVec = B.buildUndef(Res: MRI.getType(Reg: UnmergeSrc)).getReg(Idx: 0);
3460 B.buildConcatVectors(Res: MI.getOperand(i: 0), Ops: {UnmergeSrc, UndefVec});
3461
3462 MI.eraseFromParent();
3463}
3464
3465// This combine tries to reduce the number of scalarised G_TRUNC instructions by
3466// using vector truncates instead
3467//
3468// EXAMPLE:
3469// %a(i32), %b(i32) = G_UNMERGE_VALUES %src(<2 x i32>)
3470// %T_a(i16) = G_TRUNC %a(i32)
3471// %T_b(i16) = G_TRUNC %b(i32)
3472// %Undef(i16) = G_IMPLICIT_DEF(i16)
3473// %dst(v4i16) = G_BUILD_VECTORS %T_a(i16), %T_b(i16), %Undef(i16), %Undef(i16)
3474//
3475// ===>
3476// %Undef(<2 x i32>) = G_IMPLICIT_DEF(<2 x i32>)
3477// %Mid(<4 x s32>) = G_CONCAT_VECTORS %src(<2 x i32>), %Undef(<2 x i32>)
3478// %dst(<4 x s16>) = G_TRUNC %Mid(<4 x s32>)
3479//
3480// Only matches sources made up of G_TRUNCs followed by G_IMPLICIT_DEFs
3481bool CombinerHelper::matchUseVectorTruncate(MachineInstr &MI,
3482 Register &MatchInfo) const {
3483 auto BuildMI = cast<GBuildVector>(Val: &MI);
3484 unsigned NumOperands = BuildMI->getNumSources();
3485 LLT DstTy = MRI.getType(Reg: BuildMI->getReg(Idx: 0));
3486
3487 // Check the G_BUILD_VECTOR sources
3488 unsigned I;
3489 GUnmerge *UnmergeMI = nullptr;
3490
3491 // Check all source TRUNCs come from the same UNMERGE instruction
3492 // and that the element order matches (BUILD_VECTOR position I
3493 // corresponds to UNMERGE result I)
3494 for (I = 0; I < NumOperands; ++I) {
3495 // Check if the G_TRUNC instructions all come from the same MI
3496 Register TruncSrcReg;
3497 if (!mi_match(R: BuildMI->getSourceReg(I), MRI, P: m_GTrunc(Src: m_Reg(R&: TruncSrcReg))))
3498 break;
3499
3500 if (!UnmergeMI) {
3501 if (!mi_match(R: TruncSrcReg, MRI, P: m_GUnmerge(Inst&: UnmergeMI)))
3502 return false;
3503 } else {
3504 MachineInstr *UnmergeSrcMI;
3505 if (!mi_match(R: TruncSrcReg, MRI, P: m_MInstr(MI&: UnmergeSrcMI)) ||
3506 UnmergeMI != UnmergeSrcMI)
3507 return false;
3508 }
3509 // Element order must match: position I must use UNMERGE result I.
3510 if (UnmergeMI->getOperand(i: I).getReg() != TruncSrcReg)
3511 return false;
3512 }
3513 if (I < 2)
3514 return false;
3515
3516 // Check the remaining source elements are only G_IMPLICIT_DEF
3517 for (; I < NumOperands; ++I) {
3518 if (!mi_match(R: BuildMI->getSourceReg(I), MRI, P: m_GImplicitDef()))
3519 return false;
3520 }
3521
3522 // Check the size of unmerge source
3523 MatchInfo = UnmergeMI->getSourceReg();
3524 LLT UnmergeSrcTy = MRI.getType(Reg: MatchInfo);
3525 if (!UnmergeSrcTy.isVector())
3526 return false;
3527
3528 if (!DstTy.getElementCount().isKnownMultipleOf(RHS: UnmergeSrcTy.getNumElements()))
3529 return false;
3530
3531 // Check the unmerge source and destination element types match
3532 LLT UnmergeSrcEltTy = UnmergeSrcTy.getElementType();
3533 Register UnmergeDstReg = UnmergeMI->getOperand(i: 0).getReg();
3534 LLT UnmergeDstEltTy = MRI.getType(Reg: UnmergeDstReg);
3535 if (UnmergeSrcEltTy != UnmergeDstEltTy)
3536 return false;
3537
3538 // Only generate legal instructions post-legalizer
3539 if (!IsPreLegalize) {
3540 LLT MidTy = DstTy.changeElementType(NewEltTy: UnmergeSrcTy.getScalarType());
3541
3542 if (DstTy.getElementCount() != UnmergeSrcTy.getElementCount() &&
3543 !isLegal(Query: {TargetOpcode::G_CONCAT_VECTORS, {MidTy, UnmergeSrcTy}}))
3544 return false;
3545
3546 if (!isLegal(Query: {TargetOpcode::G_TRUNC, {DstTy, MidTy}}))
3547 return false;
3548 }
3549
3550 return true;
3551}
3552
3553void CombinerHelper::applyUseVectorTruncate(MachineInstr &MI,
3554 Register &MatchInfo) const {
3555 Register MidReg;
3556 auto BuildMI = cast<GBuildVector>(Val: &MI);
3557 Register DstReg = BuildMI->getReg(Idx: 0);
3558 LLT DstTy = MRI.getType(Reg: DstReg);
3559 LLT UnmergeSrcTy = MRI.getType(Reg: MatchInfo);
3560 unsigned DstTyNumElt = DstTy.getNumElements();
3561 unsigned UnmergeSrcTyNumElt = UnmergeSrcTy.getNumElements();
3562
3563 // No need to pad vector if only G_TRUNC is needed
3564 if (DstTyNumElt / UnmergeSrcTyNumElt == 1) {
3565 MidReg = MatchInfo;
3566 } else {
3567 Register UndefReg = Builder.buildUndef(Res: UnmergeSrcTy).getReg(Idx: 0);
3568 SmallVector<Register> ConcatRegs = {MatchInfo};
3569 for (unsigned I = 1; I < DstTyNumElt / UnmergeSrcTyNumElt; ++I)
3570 ConcatRegs.push_back(Elt: UndefReg);
3571
3572 auto MidTy = DstTy.changeElementType(NewEltTy: UnmergeSrcTy.getScalarType());
3573 MidReg = Builder.buildConcatVectors(Res: MidTy, Ops: ConcatRegs).getReg(Idx: 0);
3574 }
3575
3576 Builder.buildTrunc(Res: DstReg, Op: MidReg);
3577 MI.eraseFromParent();
3578}
3579
3580bool CombinerHelper::matchNotCmp(
3581 MachineInstr &MI, SmallVectorImpl<Register> &RegsToNegate) const {
3582 assert(MI.getOpcode() == TargetOpcode::G_XOR);
3583 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3584 const auto &TLI = *Builder.getMF().getSubtarget().getTargetLowering();
3585 Register XorSrc;
3586 Register CstReg;
3587 // We match xor(src, true) here.
3588 if (!mi_match(R: MI.getOperand(i: 0).getReg(), MRI,
3589 P: m_GXor(L: m_Reg(R&: XorSrc), R: m_Reg(R&: CstReg))))
3590 return false;
3591
3592 if (!MRI.hasOneNonDBGUse(RegNo: XorSrc))
3593 return false;
3594
3595 // Check that XorSrc is the root of a tree of comparisons combined with ANDs
3596 // and ORs. The suffix of RegsToNegate starting from index I is used a work
3597 // list of tree nodes to visit.
3598 RegsToNegate.push_back(Elt: XorSrc);
3599 // Remember whether the comparisons are all integer or all floating point.
3600 bool IsInt = false;
3601 bool IsFP = false;
3602 for (unsigned I = 0; I < RegsToNegate.size(); ++I) {
3603 Register Reg = RegsToNegate[I];
3604 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
3605 return false;
3606 MachineInstr *Def;
3607 if (!mi_match(R: Reg, MRI, P: m_MInstr(MI&: Def)))
3608 return false;
3609 switch (Def->getOpcode()) {
3610 default:
3611 // Don't match if the tree contains anything other than ANDs, ORs and
3612 // comparisons.
3613 return false;
3614 case TargetOpcode::G_ICMP:
3615 if (IsFP)
3616 return false;
3617 IsInt = true;
3618 // When we apply the combine we will invert the predicate.
3619 break;
3620 case TargetOpcode::G_FCMP:
3621 if (IsInt)
3622 return false;
3623 IsFP = true;
3624 // When we apply the combine we will invert the predicate.
3625 break;
3626 case TargetOpcode::G_AND:
3627 case TargetOpcode::G_OR:
3628 // Implement De Morgan's laws:
3629 // ~(x & y) -> ~x | ~y
3630 // ~(x | y) -> ~x & ~y
3631 // When we apply the combine we will change the opcode and recursively
3632 // negate the operands.
3633 RegsToNegate.push_back(Elt: Def->getOperand(i: 1).getReg());
3634 RegsToNegate.push_back(Elt: Def->getOperand(i: 2).getReg());
3635 break;
3636 }
3637 }
3638
3639 // Now we know whether the comparisons are integer or floating point, check
3640 // the constant in the xor.
3641 int64_t Cst;
3642 if (Ty.isVector()) {
3643 int64_t SplatCst;
3644 if (!mi_match(R: CstReg, MRI, P: m_ICstOrSplat(Cst&: SplatCst)))
3645 return false;
3646 if (!isConstValidTrue(TLI, ScalarSizeBits: Ty.getScalarSizeInBits(), Cst: SplatCst, IsVector: true, IsFP))
3647 return false;
3648 } else {
3649 if (!mi_match(R: CstReg, MRI, P: m_ICst(Cst)))
3650 return false;
3651 if (!isConstValidTrue(TLI, ScalarSizeBits: Ty.getSizeInBits(), Cst, IsVector: false, IsFP))
3652 return false;
3653 }
3654
3655 return true;
3656}
3657
3658void CombinerHelper::applyNotCmp(
3659 MachineInstr &MI, SmallVectorImpl<Register> &RegsToNegate) const {
3660 for (Register Reg : RegsToNegate) {
3661 MachineInstr *Def = MRI.getVRegDef(Reg);
3662 Observer.changingInstr(MI&: *Def);
3663 // For each comparison, invert the opcode. For each AND and OR, change the
3664 // opcode.
3665 switch (Def->getOpcode()) {
3666 default:
3667 llvm_unreachable("Unexpected opcode");
3668 case TargetOpcode::G_ICMP:
3669 case TargetOpcode::G_FCMP: {
3670 MachineOperand &PredOp = Def->getOperand(i: 1);
3671 CmpInst::Predicate NewP = CmpInst::getInversePredicate(
3672 pred: (CmpInst::Predicate)PredOp.getPredicate());
3673 PredOp.setPredicate(NewP);
3674 break;
3675 }
3676 case TargetOpcode::G_AND:
3677 Def->setDesc(Builder.getTII().get(Opcode: TargetOpcode::G_OR));
3678 break;
3679 case TargetOpcode::G_OR:
3680 Def->setDesc(Builder.getTII().get(Opcode: TargetOpcode::G_AND));
3681 break;
3682 }
3683 Observer.changedInstr(MI&: *Def);
3684 }
3685
3686 replaceRegWith(MRI, FromReg: MI.getOperand(i: 0).getReg(), ToReg: MI.getOperand(i: 1).getReg());
3687 MI.eraseFromParent();
3688}
3689
3690bool CombinerHelper::matchXorOfAndWithSameReg(
3691 MachineInstr &MI, std::pair<Register, Register> &MatchInfo) const {
3692 // Match (xor (and x, y), y) (or any of its commuted cases)
3693 assert(MI.getOpcode() == TargetOpcode::G_XOR);
3694 Register &X = MatchInfo.first;
3695 Register &Y = MatchInfo.second;
3696 Register AndReg = MI.getOperand(i: 1).getReg();
3697 Register SharedReg = MI.getOperand(i: 2).getReg();
3698
3699 // Find a G_AND on either side of the G_XOR.
3700 // Look for one of
3701 //
3702 // (xor (and x, y), SharedReg)
3703 // (xor SharedReg, (and x, y))
3704 if (!mi_match(R: AndReg, MRI, P: m_GAnd(L: m_Reg(R&: X), R: m_Reg(R&: Y)))) {
3705 std::swap(a&: AndReg, b&: SharedReg);
3706 if (!mi_match(R: AndReg, MRI, P: m_GAnd(L: m_Reg(R&: X), R: m_Reg(R&: Y))))
3707 return false;
3708 }
3709
3710 // Only do this if we'll eliminate the G_AND.
3711 if (!MRI.hasOneNonDBGUse(RegNo: AndReg))
3712 return false;
3713
3714 // We can combine if SharedReg is the same as either the LHS or RHS of the
3715 // G_AND.
3716 if (Y != SharedReg)
3717 std::swap(a&: X, b&: Y);
3718 return Y == SharedReg;
3719}
3720
3721void CombinerHelper::applyXorOfAndWithSameReg(
3722 MachineInstr &MI, std::pair<Register, Register> &MatchInfo) const {
3723 // Fold (xor (and x, y), y) -> (and (not x), y)
3724 Register X, Y;
3725 std::tie(args&: X, args&: Y) = MatchInfo;
3726 auto Not = Builder.buildNot(Dst: MRI.getType(Reg: X), Src0: X);
3727 Observer.changingInstr(MI);
3728 MI.setDesc(Builder.getTII().get(Opcode: TargetOpcode::G_AND));
3729 MI.getOperand(i: 1).setReg(Not->getOperand(i: 0).getReg());
3730 MI.getOperand(i: 2).setReg(Y);
3731 Observer.changedInstr(MI);
3732}
3733
3734bool CombinerHelper::matchPtrAddZeroVector(MachineInstr &MI) const {
3735 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
3736 Register DstReg = PtrAdd.getReg(Idx: 0);
3737 LLT Ty = MRI.getType(Reg: DstReg);
3738 const DataLayout &DL = Builder.getMF().getDataLayout();
3739
3740 if (DL.isNonIntegralAddressSpace(AddrSpace: Ty.getScalarType().getAddressSpace()))
3741 return false;
3742
3743 if (!Ty.isVector())
3744 return false;
3745 const MachineInstr *VecMI;
3746 if (!mi_match(R: PtrAdd.getBaseReg(), MRI, P: m_MInstr(MI&: VecMI)))
3747 return false;
3748 return isBuildVectorAllZeros(MI: *VecMI, MRI);
3749}
3750
3751/// The second source operand is known to be a power of 2.
3752void CombinerHelper::applySimplifyURemByPow2(MachineInstr &MI) const {
3753 Register DstReg = MI.getOperand(i: 0).getReg();
3754 Register Src0 = MI.getOperand(i: 1).getReg();
3755 Register Pow2Src1 = MI.getOperand(i: 2).getReg();
3756 LLT Ty = MRI.getType(Reg: DstReg);
3757
3758 // Fold (urem x, pow2) -> (and x, pow2-1)
3759 auto NegOne = Builder.buildConstant(Res: Ty, Val: -1);
3760 auto Add = Builder.buildAdd(Dst: Ty, Src0: Pow2Src1, Src1: NegOne);
3761 Builder.buildAnd(Dst: DstReg, Src0, Src1: Add);
3762 MI.eraseFromParent();
3763}
3764
3765bool CombinerHelper::matchFoldBinOpIntoSelect(MachineInstr &MI,
3766 unsigned &SelectOpNo) const {
3767 Register LHS = MI.getOperand(i: 1).getReg();
3768 Register RHS = MI.getOperand(i: 2).getReg();
3769
3770 Register OtherOperandReg = RHS;
3771 SelectOpNo = 1;
3772 Register SelectTrue, SelectFalse;
3773
3774 // Don't do this unless the old select is going away. We want to eliminate the
3775 // binary operator, not replace a binop with a select.
3776 if (!mi_match(R: LHS, MRI,
3777 P: m_GISelect(Src0: m_Reg(), Src1: m_Reg(R&: SelectTrue), Src2: m_Reg(R&: SelectFalse))) ||
3778 !MRI.hasOneNonDBGUse(RegNo: LHS)) {
3779 OtherOperandReg = LHS;
3780 SelectOpNo = 2;
3781 if (!mi_match(R: RHS, MRI,
3782 P: m_GISelect(Src0: m_Reg(), Src1: m_Reg(R&: SelectTrue), Src2: m_Reg(R&: SelectFalse))) ||
3783 !MRI.hasOneNonDBGUse(RegNo: RHS))
3784 return false;
3785 }
3786
3787 MachineInstr *SelectLHS, *SelectRHS;
3788 if (!mi_match(R: SelectTrue, MRI, P: m_MInstr(MI&: SelectLHS)) ||
3789 !mi_match(R: SelectFalse, MRI, P: m_MInstr(MI&: SelectRHS)))
3790 return false;
3791
3792 if (!isConstantOrConstantVector(MI: *SelectLHS, MRI,
3793 /*AllowFP*/ true,
3794 /*AllowOpaqueConstants*/ false))
3795 return false;
3796 if (!isConstantOrConstantVector(MI: *SelectRHS, MRI,
3797 /*AllowFP*/ true,
3798 /*AllowOpaqueConstants*/ false))
3799 return false;
3800
3801 unsigned BinOpcode = MI.getOpcode();
3802
3803 // We know that one of the operands is a select of constants. Now verify that
3804 // the other binary operator operand is either a constant, or we can handle a
3805 // variable.
3806 bool CanFoldNonConst =
3807 (BinOpcode == TargetOpcode::G_AND || BinOpcode == TargetOpcode::G_OR) &&
3808 (isNullOrNullSplat(MI: *SelectLHS, MRI) ||
3809 isAllOnesOrAllOnesSplat(MI: *SelectLHS, MRI)) &&
3810 (isNullOrNullSplat(MI: *SelectRHS, MRI) ||
3811 isAllOnesOrAllOnesSplat(MI: *SelectRHS, MRI));
3812 if (CanFoldNonConst)
3813 return true;
3814
3815 MachineInstr *OtherOperandDef;
3816 if (!mi_match(R: OtherOperandReg, MRI, P: m_MInstr(MI&: OtherOperandDef)))
3817 return false;
3818 return isConstantOrConstantVector(MI: *OtherOperandDef, MRI,
3819 /*AllowFP*/ true,
3820 /*AllowOpaqueConstants*/ false);
3821}
3822
3823/// \p SelectOperand is the operand in binary operator \p MI that is the select
3824/// to fold.
3825void CombinerHelper::applyFoldBinOpIntoSelect(
3826 MachineInstr &MI, const unsigned &SelectOperand) const {
3827 Register Dst = MI.getOperand(i: 0).getReg();
3828 Register LHS = MI.getOperand(i: 1).getReg();
3829 Register RHS = MI.getOperand(i: 2).getReg();
3830 GSelect *Select =
3831 cast<GSelect>(Val: MRI.getVRegDef(Reg: MI.getOperand(i: SelectOperand).getReg()));
3832
3833 Register SelectCond = Select->getCondReg();
3834 Register SelectTrue = Select->getTrueReg();
3835 Register SelectFalse = Select->getFalseReg();
3836
3837 LLT Ty = MRI.getType(Reg: Dst);
3838 unsigned BinOpcode = MI.getOpcode();
3839
3840 Register FoldTrue, FoldFalse;
3841
3842 // We have a select-of-constants followed by a binary operator with a
3843 // constant. Eliminate the binop by pulling the constant math into the select.
3844 // Example: add (select Cond, CT, CF), CBO --> select Cond, CT + CBO, CF + CBO
3845 if (SelectOperand == 1) {
3846 // TODO: SelectionDAG verifies this actually constant folds before
3847 // committing to the combine.
3848
3849 FoldTrue = Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {SelectTrue, RHS}).getReg(Idx: 0);
3850 FoldFalse =
3851 Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {SelectFalse, RHS}).getReg(Idx: 0);
3852 } else {
3853 FoldTrue = Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {LHS, SelectTrue}).getReg(Idx: 0);
3854 FoldFalse =
3855 Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {LHS, SelectFalse}).getReg(Idx: 0);
3856 }
3857
3858 Builder.buildSelect(Res: Dst, Tst: SelectCond, Op0: FoldTrue, Op1: FoldFalse, Flags: MI.getFlags());
3859 MI.eraseFromParent();
3860}
3861
3862std::optional<SmallVector<Register, 8>>
3863CombinerHelper::findCandidatesForLoadOrCombine(const MachineInstr *Root) const {
3864 assert(Root->getOpcode() == TargetOpcode::G_OR && "Expected G_OR only!");
3865 // We want to detect if Root is part of a tree which represents a bunch
3866 // of loads being merged into a larger load. We'll try to recognize patterns
3867 // like, for example:
3868 //
3869 // Reg Reg
3870 // \ /
3871 // OR_1 Reg
3872 // \ /
3873 // OR_2
3874 // \ Reg
3875 // .. /
3876 // Root
3877 //
3878 // Reg Reg Reg Reg
3879 // \ / \ /
3880 // OR_1 OR_2
3881 // \ /
3882 // \ /
3883 // ...
3884 // Root
3885 //
3886 // Each "Reg" may have been produced by a load + some arithmetic. This
3887 // function will save each of them.
3888 SmallVector<Register, 8> RegsToVisit;
3889 SmallVector<const MachineInstr *, 7> Ors = {Root};
3890
3891 // In the "worst" case, we're dealing with a load for each byte. So, there
3892 // are at most #bytes - 1 ORs.
3893 const unsigned MaxIter =
3894 MRI.getType(Reg: Root->getOperand(i: 0).getReg()).getSizeInBytes() - 1;
3895 for (unsigned Iter = 0; Iter < MaxIter; ++Iter) {
3896 if (Ors.empty())
3897 break;
3898 const MachineInstr *Curr = Ors.pop_back_val();
3899 Register OrLHS = Curr->getOperand(i: 1).getReg();
3900 Register OrRHS = Curr->getOperand(i: 2).getReg();
3901
3902 // In the combine, we want to elimate the entire tree.
3903 if (!MRI.hasOneNonDBGUse(RegNo: OrLHS) || !MRI.hasOneNonDBGUse(RegNo: OrRHS))
3904 return std::nullopt;
3905
3906 // If it's a G_OR, save it and continue to walk. If it's not, then it's
3907 // something that may be a load + arithmetic.
3908 if (const MachineInstr *Or = getOpcodeDef(Opcode: TargetOpcode::G_OR, Reg: OrLHS, MRI))
3909 Ors.push_back(Elt: Or);
3910 else
3911 RegsToVisit.push_back(Elt: OrLHS);
3912 if (const MachineInstr *Or = getOpcodeDef(Opcode: TargetOpcode::G_OR, Reg: OrRHS, MRI))
3913 Ors.push_back(Elt: Or);
3914 else
3915 RegsToVisit.push_back(Elt: OrRHS);
3916 }
3917
3918 // We're going to try and merge each register into a wider power-of-2 type,
3919 // so we ought to have an even number of registers.
3920 if (RegsToVisit.empty() || RegsToVisit.size() % 2 != 0)
3921 return std::nullopt;
3922 return RegsToVisit;
3923}
3924
3925/// Helper function for findLoadOffsetsForLoadOrCombine.
3926///
3927/// Check if \p Reg is the result of loading a \p MemSizeInBits wide value,
3928/// and then moving that value into a specific byte offset.
3929///
3930/// e.g. x[i] << 24
3931///
3932/// \returns The load instruction and the byte offset it is moved into.
3933static std::optional<std::pair<GZExtLoad *, int64_t>>
3934matchLoadAndBytePosition(Register Reg, unsigned MemSizeInBits,
3935 const MachineRegisterInfo &MRI) {
3936 assert(MRI.hasOneNonDBGUse(Reg) &&
3937 "Expected Reg to only have one non-debug use?");
3938 Register MaybeLoad;
3939 int64_t Shift;
3940 if (!mi_match(R: Reg, MRI, P: m_OneUse(SP: m_GShl(L: m_Reg(R&: MaybeLoad), R: m_ICst(Cst&: Shift))))) {
3941 Shift = 0;
3942 MaybeLoad = Reg;
3943 }
3944
3945 if (Shift % MemSizeInBits != 0)
3946 return std::nullopt;
3947
3948 // TODO: Handle other types of loads.
3949 auto *Load = getOpcodeDef<GZExtLoad>(Reg: MaybeLoad, MRI);
3950 if (!Load)
3951 return std::nullopt;
3952
3953 if (!Load->isUnordered() || Load->getMemSizeInBits() != MemSizeInBits)
3954 return std::nullopt;
3955
3956 return std::make_pair(x&: Load, y: Shift / MemSizeInBits);
3957}
3958
3959std::optional<std::tuple<GZExtLoad *, int64_t, GZExtLoad *>>
3960CombinerHelper::findLoadOffsetsForLoadOrCombine(
3961 SmallDenseMap<int64_t, int64_t, 8> &MemOffset2Idx,
3962 const SmallVector<Register, 8> &RegsToVisit,
3963 const unsigned MemSizeInBits) const {
3964
3965 // Each load found for the pattern. There should be one for each RegsToVisit.
3966 SmallSetVector<const MachineInstr *, 8> Loads;
3967
3968 // The lowest index used in any load. (The lowest "i" for each x[i].)
3969 int64_t LowestIdx = INT64_MAX;
3970
3971 // The load which uses the lowest index.
3972 GZExtLoad *LowestIdxLoad = nullptr;
3973
3974 // Keeps track of the load indices we see. We shouldn't see any indices twice.
3975 SmallSet<int64_t, 8> SeenIdx;
3976
3977 // Ensure each load is in the same MBB.
3978 // TODO: Support multiple MachineBasicBlocks.
3979 MachineBasicBlock *MBB = nullptr;
3980 const MachineMemOperand *MMO = nullptr;
3981
3982 // Earliest instruction-order load in the pattern.
3983 GZExtLoad *EarliestLoad = nullptr;
3984
3985 // Latest instruction-order load in the pattern.
3986 GZExtLoad *LatestLoad = nullptr;
3987
3988 // Base pointer which every load should share.
3989 Register BasePtr;
3990
3991 // We want to find a load for each register. Each load should have some
3992 // appropriate bit twiddling arithmetic. During this loop, we will also keep
3993 // track of the load which uses the lowest index. Later, we will check if we
3994 // can use its pointer in the final, combined load.
3995 for (auto Reg : RegsToVisit) {
3996 // Find the load, and find the position that it will end up in (e.g. a
3997 // shifted) value.
3998 auto LoadAndPos = matchLoadAndBytePosition(Reg, MemSizeInBits, MRI);
3999 if (!LoadAndPos)
4000 return std::nullopt;
4001 GZExtLoad *Load;
4002 int64_t DstPos;
4003 std::tie(args&: Load, args&: DstPos) = *LoadAndPos;
4004
4005 // TODO: Handle multiple MachineBasicBlocks. Currently not handled because
4006 // it is difficult to check for stores/calls/etc between loads.
4007 MachineBasicBlock *LoadMBB = Load->getParent();
4008 if (!MBB)
4009 MBB = LoadMBB;
4010 if (LoadMBB != MBB)
4011 return std::nullopt;
4012
4013 // Make sure that the MachineMemOperands of every seen load are compatible.
4014 auto &LoadMMO = Load->getMMO();
4015 if (!MMO)
4016 MMO = &LoadMMO;
4017 if (MMO->getAddrSpace() != LoadMMO.getAddrSpace())
4018 return std::nullopt;
4019
4020 // Find out what the base pointer and index for the load is.
4021 Register LoadPtr;
4022 int64_t Idx;
4023 if (!mi_match(R: Load->getOperand(i: 1).getReg(), MRI,
4024 P: m_GPtrAdd(L: m_Reg(R&: LoadPtr), R: m_ICst(Cst&: Idx)))) {
4025 LoadPtr = Load->getOperand(i: 1).getReg();
4026 Idx = 0;
4027 }
4028
4029 // Don't combine things like a[i], a[i] -> a bigger load.
4030 if (!SeenIdx.insert(V: Idx).second)
4031 return std::nullopt;
4032
4033 // Every load must share the same base pointer; don't combine things like:
4034 //
4035 // a[i], b[i + 1] -> a bigger load.
4036 if (!BasePtr.isValid())
4037 BasePtr = LoadPtr;
4038 if (BasePtr != LoadPtr)
4039 return std::nullopt;
4040
4041 if (Idx < LowestIdx) {
4042 LowestIdx = Idx;
4043 LowestIdxLoad = Load;
4044 }
4045
4046 // Keep track of the byte offset that this load ends up at. If we have seen
4047 // the byte offset, then stop here. We do not want to combine:
4048 //
4049 // a[i] << 16, a[i + k] << 16 -> a bigger load.
4050 if (!MemOffset2Idx.try_emplace(Key: DstPos, Args&: Idx).second)
4051 return std::nullopt;
4052 Loads.insert(X: Load);
4053
4054 // Keep track of the position of the earliest/latest loads in the pattern.
4055 // We will check that there are no load fold barriers between them later
4056 // on.
4057 //
4058 // FIXME: Is there a better way to check for load fold barriers?
4059 if (!EarliestLoad || dominates(DefMI: *Load, UseMI: *EarliestLoad))
4060 EarliestLoad = Load;
4061 if (!LatestLoad || dominates(DefMI: *LatestLoad, UseMI: *Load))
4062 LatestLoad = Load;
4063 }
4064
4065 // We found a load for each register. Let's check if each load satisfies the
4066 // pattern.
4067 assert(Loads.size() == RegsToVisit.size() &&
4068 "Expected to find a load for each register?");
4069 assert(EarliestLoad != LatestLoad && EarliestLoad &&
4070 LatestLoad && "Expected at least two loads?");
4071
4072 // Check if there are any stores, calls, etc. between any of the loads. If
4073 // there are, then we can't safely perform the combine.
4074 //
4075 // MaxIter is chosen based off the (worst case) number of iterations it
4076 // typically takes to succeed in the LLVM test suite plus some padding.
4077 //
4078 // FIXME: Is there a better way to check for load fold barriers?
4079 const unsigned MaxIter = 20;
4080 unsigned Iter = 0;
4081 for (const auto &MI : instructionsWithoutDebug(It: EarliestLoad->getIterator(),
4082 End: LatestLoad->getIterator())) {
4083 if (Loads.count(key: &MI))
4084 continue;
4085 if (MI.isLoadFoldBarrier())
4086 return std::nullopt;
4087 if (Iter++ == MaxIter)
4088 return std::nullopt;
4089 }
4090
4091 return std::make_tuple(args&: LowestIdxLoad, args&: LowestIdx, args&: LatestLoad);
4092}
4093
4094bool CombinerHelper::matchLoadOrCombine(
4095 MachineInstr &MI,
4096 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4097 assert(MI.getOpcode() == TargetOpcode::G_OR);
4098 MachineFunction &MF = *MI.getMF();
4099 // Assuming a little-endian target, transform:
4100 // s8 *a = ...
4101 // s32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
4102 // =>
4103 // s32 val = *((i32)a)
4104 //
4105 // s8 *a = ...
4106 // s32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
4107 // =>
4108 // s32 val = BSWAP(*((s32)a))
4109 Register Dst = MI.getOperand(i: 0).getReg();
4110 LLT Ty = MRI.getType(Reg: Dst);
4111 if (Ty.isVector())
4112 return false;
4113
4114 // We need to combine at least two loads into this type. Since the smallest
4115 // possible load is into a byte, we need at least a 16-bit wide type.
4116 const unsigned WideMemSizeInBits = Ty.getSizeInBits();
4117 if (WideMemSizeInBits < 16 || WideMemSizeInBits % 8 != 0)
4118 return false;
4119
4120 // Match a collection of non-OR instructions in the pattern.
4121 auto RegsToVisit = findCandidatesForLoadOrCombine(Root: &MI);
4122 if (!RegsToVisit)
4123 return false;
4124
4125 // We have a collection of non-OR instructions. Figure out how wide each of
4126 // the small loads should be based off of the number of potential loads we
4127 // found.
4128 const unsigned NarrowMemSizeInBits = WideMemSizeInBits / RegsToVisit->size();
4129 if (NarrowMemSizeInBits % 8 != 0)
4130 return false;
4131
4132 // Check if each register feeding into each OR is a load from the same
4133 // base pointer + some arithmetic.
4134 //
4135 // e.g. a[0], a[1] << 8, a[2] << 16, etc.
4136 //
4137 // Also verify that each of these ends up putting a[i] into the same memory
4138 // offset as a load into a wide type would.
4139 SmallDenseMap<int64_t, int64_t, 8> MemOffset2Idx;
4140 GZExtLoad *LowestIdxLoad, *LatestLoad;
4141 int64_t LowestIdx;
4142 auto MaybeLoadInfo = findLoadOffsetsForLoadOrCombine(
4143 MemOffset2Idx, RegsToVisit: *RegsToVisit, MemSizeInBits: NarrowMemSizeInBits);
4144 if (!MaybeLoadInfo)
4145 return false;
4146 std::tie(args&: LowestIdxLoad, args&: LowestIdx, args&: LatestLoad) = *MaybeLoadInfo;
4147
4148 // We have a bunch of loads being OR'd together. Using the addresses + offsets
4149 // we found before, check if this corresponds to a big or little endian byte
4150 // pattern. If it does, then we can represent it using a load + possibly a
4151 // BSWAP.
4152 bool IsBigEndianTarget = MF.getDataLayout().isBigEndian();
4153 std::optional<bool> IsBigEndian = isBigEndian(MemOffset2Idx, LowestIdx);
4154 if (!IsBigEndian)
4155 return false;
4156 bool NeedsBSwap = IsBigEndianTarget != *IsBigEndian;
4157 if (NeedsBSwap && !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_BSWAP, {Ty}}))
4158 return false;
4159
4160 // Make sure that the load from the lowest index produces offset 0 in the
4161 // final value.
4162 //
4163 // This ensures that we won't combine something like this:
4164 //
4165 // load x[i] -> byte 2
4166 // load x[i+1] -> byte 0 ---> wide_load x[i]
4167 // load x[i+2] -> byte 1
4168 const unsigned NumLoadsInTy = WideMemSizeInBits / NarrowMemSizeInBits;
4169 const unsigned ZeroByteOffset =
4170 *IsBigEndian
4171 ? bigEndianByteAt(ByteWidth: NumLoadsInTy, I: 0)
4172 : littleEndianByteAt(ByteWidth: NumLoadsInTy, I: 0);
4173 auto ZeroOffsetIdx = MemOffset2Idx.find(Val: ZeroByteOffset);
4174 if (ZeroOffsetIdx == MemOffset2Idx.end() ||
4175 ZeroOffsetIdx->second != LowestIdx)
4176 return false;
4177
4178 // We wil reuse the pointer from the load which ends up at byte offset 0. It
4179 // may not use index 0.
4180 Register Ptr = LowestIdxLoad->getPointerReg();
4181 const MachineMemOperand &MMO = LowestIdxLoad->getMMO();
4182 LegalityQuery::MemDesc MMDesc(MMO);
4183 MMDesc.MemoryTy = Ty;
4184 if (!isLegalOrBeforeLegalizer(
4185 Query: {TargetOpcode::G_LOAD, {Ty, MRI.getType(Reg: Ptr)}, {MMDesc}}))
4186 return false;
4187 auto PtrInfo = MMO.getPointerInfo();
4188 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Size: WideMemSizeInBits / 8);
4189
4190 // Load must be allowed and fast on the target.
4191 LLVMContext &C = MF.getFunction().getContext();
4192 auto &DL = MF.getDataLayout();
4193 unsigned Fast = 0;
4194 if (!getTargetLowering().allowsMemoryAccess(Context&: C, DL, Ty, MMO: *NewMMO, Fast: &Fast) ||
4195 !Fast)
4196 return false;
4197
4198 MatchInfo = [=](MachineIRBuilder &MIB) {
4199 MIB.setInstrAndDebugLoc(*LatestLoad);
4200 Register LoadDst = NeedsBSwap ? MRI.cloneVirtualRegister(VReg: Dst) : Dst;
4201 MIB.buildLoad(Res: LoadDst, Addr: Ptr, MMO&: *NewMMO);
4202 if (NeedsBSwap)
4203 MIB.buildBSwap(Dst, Src0: LoadDst);
4204 };
4205 return true;
4206}
4207
4208bool CombinerHelper::matchExtendThroughPhis(MachineInstr &MI,
4209 MachineInstr *&ExtMI) const {
4210 auto &PHI = cast<GPhi>(Val&: MI);
4211 Register DstReg = PHI.getReg(Idx: 0);
4212
4213 // TODO: Extending a vector may be expensive, don't do this until heuristics
4214 // are better.
4215 if (MRI.getType(Reg: DstReg).isVector())
4216 return false;
4217
4218 // Try to match a phi, whose only use is an extend.
4219 if (!MRI.hasOneNonDBGUse(RegNo: DstReg))
4220 return false;
4221 ExtMI = &*MRI.use_instr_nodbg_begin(RegNo: DstReg);
4222 switch (ExtMI->getOpcode()) {
4223 case TargetOpcode::G_ANYEXT:
4224 return true; // G_ANYEXT is usually free.
4225 case TargetOpcode::G_ZEXT:
4226 case TargetOpcode::G_SEXT:
4227 break;
4228 default:
4229 return false;
4230 }
4231
4232 // If the target is likely to fold this extend away, don't propagate.
4233 if (Builder.getTII().isExtendLikelyToBeFolded(ExtMI&: *ExtMI, MRI))
4234 return false;
4235
4236 // We don't want to propagate the extends unless there's a good chance that
4237 // they'll be optimized in some way.
4238 // Collect the unique incoming values.
4239 SmallPtrSet<MachineInstr *, 4> InSrcs;
4240 for (unsigned I = 0; I < PHI.getNumIncomingValues(); ++I) {
4241 auto *DefMI = getDefIgnoringCopies(Reg: PHI.getIncomingValue(I), MRI);
4242 switch (DefMI->getOpcode()) {
4243 case TargetOpcode::G_LOAD:
4244 case TargetOpcode::G_TRUNC:
4245 case TargetOpcode::G_SEXT:
4246 case TargetOpcode::G_ZEXT:
4247 case TargetOpcode::G_ANYEXT:
4248 case TargetOpcode::G_CONSTANT:
4249 InSrcs.insert(Ptr: DefMI);
4250 // Don't try to propagate if there are too many places to create new
4251 // extends, chances are it'll increase code size.
4252 if (InSrcs.size() > 2)
4253 return false;
4254 break;
4255 default:
4256 return false;
4257 }
4258 }
4259 return true;
4260}
4261
4262void CombinerHelper::applyExtendThroughPhis(MachineInstr &MI,
4263 MachineInstr *&ExtMI) const {
4264 auto &PHI = cast<GPhi>(Val&: MI);
4265 Register DstReg = ExtMI->getOperand(i: 0).getReg();
4266 LLT ExtTy = MRI.getType(Reg: DstReg);
4267
4268 // Propagate the extension into the block of each incoming reg's block.
4269 // Use a SetVector here because PHIs can have duplicate edges, and we want
4270 // deterministic iteration order.
4271 SmallSetVector<MachineInstr *, 8> SrcMIs;
4272 SmallDenseMap<MachineInstr *, MachineInstr *, 8> OldToNewSrcMap;
4273 for (unsigned I = 0; I < PHI.getNumIncomingValues(); ++I) {
4274 auto SrcReg = PHI.getIncomingValue(I);
4275 MachineInstr *SrcMI;
4276 if (!mi_match(R: SrcReg, MRI, P: m_MInstr(MI&: SrcMI)))
4277 continue;
4278 if (!SrcMIs.insert(X: SrcMI))
4279 continue;
4280
4281 // Build an extend after each src inst.
4282 auto *MBB = SrcMI->getParent();
4283 MachineBasicBlock::iterator InsertPt = ++SrcMI->getIterator();
4284 if (InsertPt != MBB->end() && InsertPt->isPHI())
4285 InsertPt = MBB->getFirstNonPHI();
4286
4287 Builder.setInsertPt(MBB&: *SrcMI->getParent(), II: InsertPt);
4288 Builder.setDebugLoc(MI.getDebugLoc());
4289 auto NewExt = Builder.buildExtOrTrunc(ExtOpc: ExtMI->getOpcode(), Res: ExtTy, Op: SrcReg);
4290 OldToNewSrcMap[SrcMI] = NewExt;
4291 }
4292
4293 // Create a new phi with the extended inputs.
4294 Builder.setInstrAndDebugLoc(MI);
4295 auto NewPhi = Builder.buildInstrNoInsert(Opcode: TargetOpcode::G_PHI);
4296 NewPhi.addDef(RegNo: DstReg);
4297 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
4298 if (!MO.isReg()) {
4299 NewPhi.addMBB(MBB: MO.getMBB());
4300 continue;
4301 }
4302 auto *NewSrc = OldToNewSrcMap[MRI.getVRegDef(Reg: MO.getReg())];
4303 NewPhi.addUse(RegNo: NewSrc->getOperand(i: 0).getReg());
4304 }
4305 Builder.insertInstr(MIB: NewPhi);
4306 ExtMI->eraseFromParent();
4307}
4308
4309bool CombinerHelper::matchExtractVecEltBuildVec(MachineInstr &MI,
4310 Register &Reg) const {
4311 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
4312 // If we have a constant index, look for a G_BUILD_VECTOR source
4313 // and find the source register that the index maps to.
4314 Register SrcVec = MI.getOperand(i: 1).getReg();
4315 LLT SrcTy = MRI.getType(Reg: SrcVec);
4316 if (SrcTy.isScalableVector())
4317 return false;
4318
4319 auto Cst = getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
4320 if (!Cst || Cst->Value.getZExtValue() >= SrcTy.getNumElements())
4321 return false;
4322
4323 unsigned VecIdx = Cst->Value.getZExtValue();
4324
4325 // Check if we have a build_vector or build_vector_trunc with an optional
4326 // trunc in front.
4327 MachineInstr *SrcVecMI;
4328 Register TruncSrc;
4329 if (mi_match(R: SrcVec, MRI, P: m_GTrunc(Src: m_Reg(R&: TruncSrc)))) {
4330 if (!mi_match(R: TruncSrc, MRI, P: m_MInstr(MI&: SrcVecMI)))
4331 return false;
4332 } else if (!mi_match(R: SrcVec, MRI, P: m_MInstr(MI&: SrcVecMI)))
4333 return false;
4334
4335 if (SrcVecMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR &&
4336 SrcVecMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR_TRUNC)
4337 return false;
4338
4339 EVT Ty(getMVTForLLT(Ty: SrcTy));
4340 if (!MRI.hasOneNonDBGUse(RegNo: SrcVec) &&
4341 !getTargetLowering().aggressivelyPreferBuildVectorSources(VecVT: Ty))
4342 return false;
4343
4344 Reg = SrcVecMI->getOperand(i: VecIdx + 1).getReg();
4345 return true;
4346}
4347
4348void CombinerHelper::applyExtractVecEltBuildVec(MachineInstr &MI,
4349 Register &Reg) const {
4350 // Check the type of the register, since it may have come from a
4351 // G_BUILD_VECTOR_TRUNC.
4352 LLT ScalarTy = MRI.getType(Reg);
4353 Register DstReg = MI.getOperand(i: 0).getReg();
4354 LLT DstTy = MRI.getType(Reg: DstReg);
4355
4356 if (ScalarTy != DstTy) {
4357 assert(ScalarTy.getSizeInBits() > DstTy.getSizeInBits());
4358 Builder.buildTrunc(Res: DstReg, Op: Reg);
4359 MI.eraseFromParent();
4360 return;
4361 }
4362 replaceSingleDefInstWithReg(MI, Replacement: Reg);
4363}
4364
4365bool CombinerHelper::matchExtractAllEltsFromBuildVector(
4366 MachineInstr &MI,
4367 SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs) const {
4368 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4369 // This combine tries to find build_vector's which have every source element
4370 // extracted using G_EXTRACT_VECTOR_ELT. This can happen when transforms like
4371 // the masked load scalarization is run late in the pipeline. There's already
4372 // a combine for a similar pattern starting from the extract, but that
4373 // doesn't attempt to do it if there are multiple uses of the build_vector,
4374 // which in this case is true. Starting the combine from the build_vector
4375 // feels more natural than trying to find sibling nodes of extracts.
4376 // E.g.
4377 // %vec(<4 x s32>) = G_BUILD_VECTOR %s1(s32), %s2, %s3, %s4
4378 // %ext1 = G_EXTRACT_VECTOR_ELT %vec, 0
4379 // %ext2 = G_EXTRACT_VECTOR_ELT %vec, 1
4380 // %ext3 = G_EXTRACT_VECTOR_ELT %vec, 2
4381 // %ext4 = G_EXTRACT_VECTOR_ELT %vec, 3
4382 // ==>
4383 // replace ext{1,2,3,4} with %s{1,2,3,4}
4384
4385 Register DstReg = MI.getOperand(i: 0).getReg();
4386 LLT DstTy = MRI.getType(Reg: DstReg);
4387 unsigned NumElts = DstTy.getNumElements();
4388
4389 SmallBitVector ExtractedElts(NumElts);
4390 for (MachineInstr &II : MRI.use_nodbg_instructions(Reg: DstReg)) {
4391 if (II.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)
4392 return false;
4393 auto Cst = getIConstantVRegVal(VReg: II.getOperand(i: 2).getReg(), MRI);
4394 if (!Cst)
4395 return false;
4396 unsigned Idx = Cst->getZExtValue();
4397 if (Idx >= NumElts)
4398 return false; // Out of range.
4399 ExtractedElts.set(Idx);
4400 SrcDstPairs.emplace_back(
4401 Args: std::make_pair(x: MI.getOperand(i: Idx + 1).getReg(), y: &II));
4402 }
4403 // Match if every element was extracted.
4404 return ExtractedElts.all();
4405}
4406
4407void CombinerHelper::applyExtractAllEltsFromBuildVector(
4408 MachineInstr &MI,
4409 SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs) const {
4410 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4411 for (auto &Pair : SrcDstPairs) {
4412 auto *ExtMI = Pair.second;
4413 replaceRegWith(MRI, FromReg: ExtMI->getOperand(i: 0).getReg(), ToReg: Pair.first);
4414 ExtMI->eraseFromParent();
4415 }
4416 MI.eraseFromParent();
4417}
4418
4419void CombinerHelper::applyBuildFn(
4420 MachineInstr &MI,
4421 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4422 applyBuildFnNoErase(MI, MatchInfo);
4423 MI.eraseFromParent();
4424}
4425
4426void CombinerHelper::applyBuildFnNoErase(
4427 MachineInstr &MI,
4428 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4429 MatchInfo(Builder);
4430}
4431
4432bool CombinerHelper::matchOrShiftToFunnelShift(MachineInstr &MI,
4433 bool AllowScalarConstants,
4434 BuildFnTy &MatchInfo) const {
4435 assert(MI.getOpcode() == TargetOpcode::G_OR);
4436
4437 Register Dst = MI.getOperand(i: 0).getReg();
4438 LLT Ty = MRI.getType(Reg: Dst);
4439 unsigned BitWidth = Ty.getScalarSizeInBits();
4440
4441 Register ShlSrc, ShlAmt, LShrSrc, LShrAmt, Amt;
4442 unsigned FshOpc = 0;
4443
4444 // Match (or (shl ...), (lshr ...)).
4445 if (!mi_match(R: Dst, MRI,
4446 // m_GOr() handles the commuted version as well.
4447 P: m_GOr(L: m_GShl(L: m_Reg(R&: ShlSrc), R: m_Reg(R&: ShlAmt)),
4448 R: m_GLShr(L: m_Reg(R&: LShrSrc), R: m_Reg(R&: LShrAmt)))))
4449 return false;
4450
4451 // Given constants C0 and C1 such that C0 + C1 is bit-width:
4452 // (or (shl x, C0), (lshr y, C1)) -> (fshl x, y, C0) or (fshr x, y, C1)
4453 int64_t CstShlAmt = 0, CstLShrAmt;
4454 if (mi_match(R: ShlAmt, MRI, P: m_ICstOrSplat(Cst&: CstShlAmt)) &&
4455 mi_match(R: LShrAmt, MRI, P: m_ICstOrSplat(Cst&: CstLShrAmt)) &&
4456 CstShlAmt + CstLShrAmt == BitWidth) {
4457 FshOpc = TargetOpcode::G_FSHR;
4458 Amt = LShrAmt;
4459 } else if (mi_match(R: LShrAmt, MRI,
4460 P: m_GSub(L: m_SpecificICstOrSplat(RequestedValue: BitWidth), R: m_Reg(R&: Amt))) &&
4461 ShlAmt == Amt) {
4462 // (or (shl x, amt), (lshr y, (sub bw, amt))) -> (fshl x, y, amt)
4463 FshOpc = TargetOpcode::G_FSHL;
4464 } else if (mi_match(R: ShlAmt, MRI,
4465 P: m_GSub(L: m_SpecificICstOrSplat(RequestedValue: BitWidth), R: m_Reg(R&: Amt))) &&
4466 LShrAmt == Amt) {
4467 // (or (shl x, (sub bw, amt)), (lshr y, amt)) -> (fshr x, y, amt)
4468 FshOpc = TargetOpcode::G_FSHR;
4469 } else {
4470 return false;
4471 }
4472
4473 LLT AmtTy = MRI.getType(Reg: Amt);
4474 if (!isLegalOrBeforeLegalizer(Query: {FshOpc, {Ty, AmtTy}}) &&
4475 (!AllowScalarConstants || CstShlAmt == 0 || !Ty.isScalar()))
4476 return false;
4477
4478 MatchInfo = [=](MachineIRBuilder &B) {
4479 B.buildInstr(Opc: FshOpc, DstOps: {Dst}, SrcOps: {ShlSrc, LShrSrc, Amt});
4480 };
4481 return true;
4482}
4483
4484// Fold (rot x, c) -> (rot x, c % BitSize)
4485bool CombinerHelper::matchRotateOutOfRange(MachineInstr &MI) const {
4486 assert(MI.getOpcode() == TargetOpcode::G_ROTL ||
4487 MI.getOpcode() == TargetOpcode::G_ROTR);
4488 unsigned Bitsize =
4489 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getScalarSizeInBits();
4490 Register AmtReg = MI.getOperand(i: 2).getReg();
4491 bool OutOfRange = false;
4492 auto MatchOutOfRange = [Bitsize, &OutOfRange](const Constant *C) {
4493 if (auto *CI = dyn_cast<ConstantInt>(Val: C))
4494 OutOfRange |= CI->getValue().uge(RHS: Bitsize);
4495 return true;
4496 };
4497 return matchUnaryPredicate(MRI, Reg: AmtReg, Match: MatchOutOfRange) && OutOfRange;
4498}
4499
4500void CombinerHelper::applyRotateOutOfRange(MachineInstr &MI) const {
4501 assert(MI.getOpcode() == TargetOpcode::G_ROTL ||
4502 MI.getOpcode() == TargetOpcode::G_ROTR);
4503 unsigned Bitsize =
4504 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getScalarSizeInBits();
4505 Register Amt = MI.getOperand(i: 2).getReg();
4506 LLT AmtTy = MRI.getType(Reg: Amt);
4507 auto Bits = Builder.buildConstant(Res: AmtTy, Val: Bitsize);
4508 Amt = Builder.buildURem(Dst: AmtTy, Src0: MI.getOperand(i: 2).getReg(), Src1: Bits).getReg(Idx: 0);
4509 Observer.changingInstr(MI);
4510 MI.getOperand(i: 2).setReg(Amt);
4511 Observer.changedInstr(MI);
4512}
4513
4514bool CombinerHelper::matchICmpToTrueFalseKnownBits(MachineInstr &MI,
4515 int64_t &MatchInfo) const {
4516 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
4517 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
4518
4519 // We want to avoid calling KnownBits on the LHS if possible, as this combine
4520 // has no filter and runs on every G_ICMP instruction. We can avoid calling
4521 // KnownBits on the LHS in two cases:
4522 //
4523 // - The RHS is unknown: Constants are always on RHS. If the RHS is unknown
4524 // we cannot do any transforms so we can safely bail out early.
4525 // - The RHS is zero: we don't need to know the LHS to do unsigned <0 and
4526 // >=0.
4527 auto KnownRHS = VT->getKnownBits(R: MI.getOperand(i: 3).getReg());
4528 if (KnownRHS.isUnknown())
4529 return false;
4530
4531 std::optional<bool> KnownVal;
4532 if (KnownRHS.isZero()) {
4533 // ? uge 0 -> always true
4534 // ? ult 0 -> always false
4535 if (Pred == CmpInst::ICMP_UGE)
4536 KnownVal = true;
4537 else if (Pred == CmpInst::ICMP_ULT)
4538 KnownVal = false;
4539 }
4540
4541 if (!KnownVal) {
4542 auto KnownLHS = VT->getKnownBits(R: MI.getOperand(i: 2).getReg());
4543 KnownVal = ICmpInst::compare(LHS: KnownLHS, RHS: KnownRHS, Pred);
4544 }
4545
4546 if (!KnownVal)
4547 return false;
4548 MatchInfo =
4549 *KnownVal
4550 ? getICmpTrueVal(TLI: getTargetLowering(),
4551 /*IsVector = */
4552 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector(),
4553 /* IsFP = */ false)
4554 : 0;
4555 return true;
4556}
4557
4558bool CombinerHelper::matchICmpToLHSKnownBits(
4559 MachineInstr &MI,
4560 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4561 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
4562 // Given:
4563 //
4564 // %x = G_WHATEVER (... x is known to be 0 or 1 ...)
4565 // %cmp = G_ICMP ne %x, 0
4566 //
4567 // Or:
4568 //
4569 // %x = G_WHATEVER (... x is known to be 0 or 1 ...)
4570 // %cmp = G_ICMP eq %x, 1
4571 //
4572 // We can replace %cmp with %x assuming true is 1 on the target.
4573 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
4574 if (!CmpInst::isEquality(pred: Pred))
4575 return false;
4576 Register Dst = MI.getOperand(i: 0).getReg();
4577 LLT DstTy = MRI.getType(Reg: Dst);
4578 if (getICmpTrueVal(TLI: getTargetLowering(), IsVector: DstTy.isVector(),
4579 /* IsFP = */ false) != 1)
4580 return false;
4581 int64_t OneOrZero = Pred == CmpInst::ICMP_EQ;
4582 if (!mi_match(R: MI.getOperand(i: 3).getReg(), MRI, P: m_SpecificICst(RequestedValue: OneOrZero)))
4583 return false;
4584 Register LHS = MI.getOperand(i: 2).getReg();
4585 auto KnownLHS = VT->getKnownBits(R: LHS);
4586 if (KnownLHS.getMinValue() != 0 || KnownLHS.getMaxValue() != 1)
4587 return false;
4588 // Make sure replacing Dst with the LHS is a legal operation.
4589 LLT LHSTy = MRI.getType(Reg: LHS);
4590 unsigned LHSSize = LHSTy.getSizeInBits();
4591 unsigned DstSize = DstTy.getSizeInBits();
4592 unsigned Op = TargetOpcode::COPY;
4593 if (DstSize != LHSSize)
4594 Op = DstSize < LHSSize ? TargetOpcode::G_TRUNC : TargetOpcode::G_ZEXT;
4595 if (!isLegalOrBeforeLegalizer(Query: {Op, {DstTy, LHSTy}}))
4596 return false;
4597 MatchInfo = [=](MachineIRBuilder &B) { B.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {LHS}); };
4598 return true;
4599}
4600
4601// Replace (and (or x, c1), c2) with (and x, c2) iff c1 & c2 == 0
4602bool CombinerHelper::matchAndOrDisjointMask(
4603 MachineInstr &MI,
4604 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4605 assert(MI.getOpcode() == TargetOpcode::G_AND);
4606
4607 // Ignore vector types to simplify matching the two constants.
4608 // TODO: do this for vectors and scalars via a demanded bits analysis.
4609 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4610 if (Ty.isVector())
4611 return false;
4612
4613 Register Src;
4614 Register AndMaskReg;
4615 int64_t AndMaskBits;
4616 int64_t OrMaskBits;
4617 if (!mi_match(MI, MRI,
4618 P: m_GAnd(L: m_GOr(L: m_Reg(R&: Src), R: m_ICst(Cst&: OrMaskBits)),
4619 R: m_all_of(preds: m_ICst(Cst&: AndMaskBits), preds: m_Reg(R&: AndMaskReg)))))
4620 return false;
4621
4622 // Check if OrMask could turn on any bits in Src.
4623 if (AndMaskBits & OrMaskBits)
4624 return false;
4625
4626 MatchInfo = [=, &MI](MachineIRBuilder &B) {
4627 Observer.changingInstr(MI);
4628 // Canonicalize the result to have the constant on the RHS.
4629 if (MI.getOperand(i: 1).getReg() == AndMaskReg)
4630 MI.getOperand(i: 2).setReg(AndMaskReg);
4631 MI.getOperand(i: 1).setReg(Src);
4632 Observer.changedInstr(MI);
4633 };
4634 return true;
4635}
4636
4637/// Form a G_SBFX from a G_SEXT_INREG fed by a right shift.
4638bool CombinerHelper::matchBitfieldExtractFromSExtInReg(
4639 MachineInstr &MI,
4640 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4641 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
4642 Register Dst = MI.getOperand(i: 0).getReg();
4643 Register Src = MI.getOperand(i: 1).getReg();
4644 LLT Ty = MRI.getType(Reg: Src);
4645 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
4646 if (!LI || !LI->isLegalOrCustom(Query: {TargetOpcode::G_SBFX, {Ty, ExtractTy}}))
4647 return false;
4648 int64_t Width = MI.getOperand(i: 2).getImm();
4649 Register ShiftSrc;
4650 int64_t ShiftImm;
4651 if (!mi_match(R: Src, MRI,
4652 P: m_OneUse(SP: m_any_of(preds: m_GAShr(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: ShiftImm)),
4653 preds: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: ShiftImm))))))
4654 return false;
4655 if (ShiftImm < 0 || ShiftImm + Width > Ty.getScalarSizeInBits())
4656 return false;
4657
4658 MatchInfo = [=](MachineIRBuilder &B) {
4659 auto Cst1 = B.buildConstant(Res: ExtractTy, Val: ShiftImm);
4660 auto Cst2 = B.buildConstant(Res: ExtractTy, Val: Width);
4661 B.buildSbfx(Dst, Src: ShiftSrc, LSB: Cst1, Width: Cst2);
4662 };
4663 return true;
4664}
4665
4666/// Form a G_UBFX from "(a srl b) & mask", where b and mask are constants.
4667bool CombinerHelper::matchBitfieldExtractFromAnd(MachineInstr &MI,
4668 BuildFnTy &MatchInfo) const {
4669 GAnd *And = cast<GAnd>(Val: &MI);
4670 Register Dst = And->getReg(Idx: 0);
4671 LLT Ty = MRI.getType(Reg: Dst);
4672 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
4673 // Note that isLegalOrBeforeLegalizer is stricter and does not take custom
4674 // into account.
4675 if (LI && !LI->isLegalOrCustom(Query: {TargetOpcode::G_UBFX, {Ty, ExtractTy}}))
4676 return false;
4677
4678 int64_t AndImm, LSBImm;
4679 Register ShiftSrc;
4680 const unsigned Size = Ty.getScalarSizeInBits();
4681 if (!mi_match(R: And->getReg(Idx: 0), MRI,
4682 P: m_GAnd(L: m_OneUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: LSBImm))),
4683 R: m_ICst(Cst&: AndImm))))
4684 return false;
4685
4686 // AndImm is sign-extended to 64 bits by m_ICst; restrict it to the operand
4687 // width so an all-ones mask (a redundant AND) is not misread as a wider mask.
4688 uint64_t MaybeMask = static_cast<uint64_t>(AndImm);
4689 if (Size < 64)
4690 MaybeMask &= maskTrailingOnes<uint64_t>(N: Size);
4691
4692 // The mask is a mask of the low bits iff imm & (imm+1) == 0.
4693 if (MaybeMask & (MaybeMask + 1))
4694 return false;
4695
4696 // LSB must fit within the register.
4697 if (static_cast<uint64_t>(LSBImm) >= Size)
4698 return false;
4699
4700 uint64_t Width = APInt(Size, MaybeMask).countr_one();
4701 // The extracted field [LSB, LSB+Width) must fit within the register.
4702 // Otherwise this is a redundant AND (e.g. an all-ones mask combined with a
4703 // non-zero shift) that is better handled by other combines, and would form
4704 // an out-of-range bitfield extract.
4705 if (static_cast<uint64_t>(LSBImm) + Width > Size)
4706 return false;
4707
4708 MatchInfo = [=](MachineIRBuilder &B) {
4709 auto WidthCst = B.buildConstant(Res: ExtractTy, Val: Width);
4710 auto LSBCst = B.buildConstant(Res: ExtractTy, Val: LSBImm);
4711 B.buildInstr(Opc: TargetOpcode::G_UBFX, DstOps: {Dst}, SrcOps: {ShiftSrc, LSBCst, WidthCst});
4712 };
4713 return true;
4714}
4715
4716bool CombinerHelper::matchBitfieldExtractFromShr(
4717 MachineInstr &MI,
4718 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4719 const unsigned Opcode = MI.getOpcode();
4720 assert(Opcode == TargetOpcode::G_ASHR || Opcode == TargetOpcode::G_LSHR);
4721
4722 const Register Dst = MI.getOperand(i: 0).getReg();
4723
4724 const unsigned ExtrOpcode = Opcode == TargetOpcode::G_ASHR
4725 ? TargetOpcode::G_SBFX
4726 : TargetOpcode::G_UBFX;
4727
4728 // Check if the type we would use for the extract is legal
4729 LLT Ty = MRI.getType(Reg: Dst);
4730 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
4731 if (!LI || !LI->isLegalOrCustom(Query: {ExtrOpcode, {Ty, ExtractTy}}))
4732 return false;
4733
4734 Register ShlSrc;
4735 int64_t ShrAmt;
4736 int64_t ShlAmt;
4737 const unsigned Size = Ty.getScalarSizeInBits();
4738
4739 // Try to match shr (shl x, c1), c2
4740 if (!mi_match(R: Dst, MRI,
4741 P: m_BinOp(Opcode, L: m_OneUse(SP: m_GShl(L: m_Reg(R&: ShlSrc), R: m_ICst(Cst&: ShlAmt))),
4742 R: m_ICst(Cst&: ShrAmt))))
4743 return false;
4744
4745 // Make sure that the shift sizes can fit a bitfield extract
4746 if (ShlAmt < 0 || ShlAmt > ShrAmt || ShrAmt >= Size)
4747 return false;
4748
4749 // Skip this combine if the G_SEXT_INREG combine could handle it
4750 if (Opcode == TargetOpcode::G_ASHR && ShlAmt == ShrAmt)
4751 return false;
4752
4753 // Calculate start position and width of the extract
4754 const int64_t Pos = ShrAmt - ShlAmt;
4755 const int64_t Width = Size - ShrAmt;
4756
4757 MatchInfo = [=](MachineIRBuilder &B) {
4758 auto WidthCst = B.buildConstant(Res: ExtractTy, Val: Width);
4759 auto PosCst = B.buildConstant(Res: ExtractTy, Val: Pos);
4760 B.buildInstr(Opc: ExtrOpcode, DstOps: {Dst}, SrcOps: {ShlSrc, PosCst, WidthCst});
4761 };
4762 return true;
4763}
4764
4765bool CombinerHelper::matchBitfieldExtractFromShrAnd(
4766 MachineInstr &MI,
4767 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4768 const unsigned Opcode = MI.getOpcode();
4769 assert(Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_ASHR);
4770
4771 const Register Dst = MI.getOperand(i: 0).getReg();
4772 LLT Ty = MRI.getType(Reg: Dst);
4773 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
4774 if (LI && !LI->isLegalOrCustom(Query: {TargetOpcode::G_UBFX, {Ty, ExtractTy}}))
4775 return false;
4776
4777 // Try to match shr (and x, c1), c2
4778 Register AndSrc;
4779 int64_t ShrAmt;
4780 int64_t SMask;
4781 if (!mi_match(R: Dst, MRI,
4782 P: m_BinOp(Opcode, L: m_OneUse(SP: m_GAnd(L: m_Reg(R&: AndSrc), R: m_ICst(Cst&: SMask))),
4783 R: m_ICst(Cst&: ShrAmt))))
4784 return false;
4785
4786 const unsigned Size = Ty.getScalarSizeInBits();
4787 if (ShrAmt < 0 || ShrAmt >= Size)
4788 return false;
4789
4790 // If the shift subsumes the mask, emit the 0 directly.
4791 if (0 == (SMask >> ShrAmt)) {
4792 MatchInfo = [=](MachineIRBuilder &B) {
4793 B.buildConstant(Res: Dst, Val: 0);
4794 };
4795 return true;
4796 }
4797
4798 // Check that ubfx can do the extraction, with no holes in the mask.
4799 uint64_t UMask = SMask;
4800 UMask |= maskTrailingOnes<uint64_t>(N: ShrAmt);
4801 UMask &= maskTrailingOnes<uint64_t>(N: Size);
4802 if (!isMask_64(Value: UMask))
4803 return false;
4804
4805 // Calculate start position and width of the extract.
4806 const int64_t Pos = ShrAmt;
4807 const int64_t Width = llvm::countr_one(Value: UMask) - ShrAmt;
4808
4809 // It's preferable to keep the shift, rather than form G_SBFX.
4810 // TODO: remove the G_AND via demanded bits analysis.
4811 if (Opcode == TargetOpcode::G_ASHR && Width + ShrAmt == Size)
4812 return false;
4813
4814 MatchInfo = [=](MachineIRBuilder &B) {
4815 auto WidthCst = B.buildConstant(Res: ExtractTy, Val: Width);
4816 auto PosCst = B.buildConstant(Res: ExtractTy, Val: Pos);
4817 B.buildInstr(Opc: TargetOpcode::G_UBFX, DstOps: {Dst}, SrcOps: {AndSrc, PosCst, WidthCst});
4818 };
4819 return true;
4820}
4821
4822bool CombinerHelper::reassociationCanBreakAddressingModePattern(
4823 MachineInstr &MI) const {
4824 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
4825
4826 Register Src1Reg = PtrAdd.getBaseReg();
4827 auto *Src1Def = getOpcodeDef<GPtrAdd>(Reg: Src1Reg, MRI);
4828 if (!Src1Def)
4829 return false;
4830
4831 Register Src2Reg = PtrAdd.getOffsetReg();
4832
4833 if (MRI.hasOneNonDBGUse(RegNo: Src1Reg))
4834 return false;
4835
4836 auto C1 = getIConstantVRegVal(VReg: Src1Def->getOffsetReg(), MRI);
4837 if (!C1)
4838 return false;
4839 auto C2 = getIConstantVRegVal(VReg: Src2Reg, MRI);
4840 if (!C2)
4841 return false;
4842
4843 const APInt &C1APIntVal = *C1;
4844 const APInt &C2APIntVal = *C2;
4845 const int64_t CombinedValue = (C1APIntVal + C2APIntVal).getSExtValue();
4846
4847 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: PtrAdd.getReg(Idx: 0))) {
4848 // This combine may end up running before ptrtoint/inttoptr combines
4849 // manage to eliminate redundant conversions, so try to look through them.
4850 MachineInstr *ConvUseMI = &UseMI;
4851 unsigned ConvUseOpc = ConvUseMI->getOpcode();
4852 while (ConvUseOpc == TargetOpcode::G_INTTOPTR ||
4853 ConvUseOpc == TargetOpcode::G_PTRTOINT) {
4854 Register DefReg = ConvUseMI->getOperand(i: 0).getReg();
4855 if (!MRI.hasOneNonDBGUse(RegNo: DefReg))
4856 break;
4857 ConvUseMI = &*MRI.use_instr_nodbg_begin(RegNo: DefReg);
4858 ConvUseOpc = ConvUseMI->getOpcode();
4859 }
4860 auto *LdStMI = dyn_cast<GLoadStore>(Val: ConvUseMI);
4861 if (!LdStMI)
4862 continue;
4863 // Is x[offset2] already not a legal addressing mode? If so then
4864 // reassociating the constants breaks nothing (we test offset2 because
4865 // that's the one we hope to fold into the load or store).
4866 TargetLoweringBase::AddrMode AM;
4867 AM.HasBaseReg = true;
4868 AM.BaseOffs = C2APIntVal.getSExtValue();
4869 unsigned AS = MRI.getType(Reg: LdStMI->getPointerReg()).getAddressSpace();
4870 Type *AccessTy = getTypeForLLT(Ty: LdStMI->getMMO().getMemoryType(),
4871 C&: PtrAdd.getMF()->getFunction().getContext());
4872 const auto &TLI = *PtrAdd.getMF()->getSubtarget().getTargetLowering();
4873 if (!TLI.isLegalAddressingMode(DL: PtrAdd.getMF()->getDataLayout(), AM,
4874 Ty: AccessTy, AddrSpace: AS))
4875 continue;
4876
4877 // Would x[offset1+offset2] still be a legal addressing mode?
4878 AM.BaseOffs = CombinedValue;
4879 if (!TLI.isLegalAddressingMode(DL: PtrAdd.getMF()->getDataLayout(), AM,
4880 Ty: AccessTy, AddrSpace: AS))
4881 return true;
4882 }
4883
4884 return false;
4885}
4886
4887bool CombinerHelper::matchReassocConstantInnerRHS(GPtrAdd &MI,
4888 MachineInstr *RHS,
4889 BuildFnTy &MatchInfo) const {
4890 // G_PTR_ADD(BASE, G_ADD(X, C)) -> G_PTR_ADD(G_PTR_ADD(BASE, X), C)
4891 Register Src1Reg = MI.getOperand(i: 1).getReg();
4892 if (RHS->getOpcode() != TargetOpcode::G_ADD)
4893 return false;
4894 auto C2 = getIConstantVRegVal(VReg: RHS->getOperand(i: 2).getReg(), MRI);
4895 if (!C2)
4896 return false;
4897
4898 // If both additions are nuw, the reassociated additions are also nuw.
4899 // If the original G_PTR_ADD is additionally nusw, X and C are both not
4900 // negative, so BASE+X is between BASE and BASE+(X+C). The new G_PTR_ADDs are
4901 // therefore also nusw.
4902 // If the original G_PTR_ADD is additionally inbounds (which implies nusw),
4903 // the new G_PTR_ADDs are then also inbounds.
4904 unsigned PtrAddFlags = MI.getFlags();
4905 unsigned AddFlags = RHS->getFlags();
4906 bool IsNoUWrap = PtrAddFlags & AddFlags & MachineInstr::MIFlag::NoUWrap;
4907 bool IsNoUSWrap = IsNoUWrap && (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap);
4908 bool IsInBounds = IsNoUWrap && (PtrAddFlags & MachineInstr::MIFlag::InBounds);
4909 unsigned Flags = 0;
4910 if (IsNoUWrap)
4911 Flags |= MachineInstr::MIFlag::NoUWrap;
4912 if (IsNoUSWrap)
4913 Flags |= MachineInstr::MIFlag::NoUSWrap;
4914 if (IsInBounds)
4915 Flags |= MachineInstr::MIFlag::InBounds;
4916
4917 MatchInfo = [=, &MI](MachineIRBuilder &B) {
4918 LLT PtrTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4919
4920 auto NewBase =
4921 Builder.buildPtrAdd(Res: PtrTy, Op0: Src1Reg, Op1: RHS->getOperand(i: 1).getReg(), Flags);
4922 Observer.changingInstr(MI);
4923 MI.getOperand(i: 1).setReg(NewBase.getReg(Idx: 0));
4924 MI.getOperand(i: 2).setReg(RHS->getOperand(i: 2).getReg());
4925 MI.setFlags(Flags);
4926 Observer.changedInstr(MI);
4927 };
4928 return !reassociationCanBreakAddressingModePattern(MI);
4929}
4930
4931bool CombinerHelper::matchReassocConstantInnerLHS(GPtrAdd &MI,
4932 MachineInstr *LHS,
4933 MachineInstr *RHS,
4934 BuildFnTy &MatchInfo) const {
4935 // G_PTR_ADD (G_PTR_ADD X, C), Y) -> (G_PTR_ADD (G_PTR_ADD(X, Y), C)
4936 // if and only if (G_PTR_ADD X, C) has one use.
4937 Register LHSBase;
4938 std::optional<ValueAndVReg> LHSCstOff;
4939 if (!mi_match(R: MI.getBaseReg(), MRI,
4940 P: m_OneUse(SP: m_GPtrAdd(L: m_Reg(R&: LHSBase), R: m_GCst(ValReg&: LHSCstOff)))))
4941 return false;
4942
4943 auto *LHSPtrAdd = cast<GPtrAdd>(Val: LHS);
4944
4945 // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
4946 // nuw and inbounds (which implies nusw), the offsets are both non-negative,
4947 // so the new G_PTR_ADDs are also inbounds.
4948 unsigned PtrAddFlags = MI.getFlags();
4949 unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();
4950 bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
4951 bool IsNoUSWrap = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &
4952 MachineInstr::MIFlag::NoUSWrap);
4953 bool IsInBounds = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &
4954 MachineInstr::MIFlag::InBounds);
4955 unsigned Flags = 0;
4956 if (IsNoUWrap)
4957 Flags |= MachineInstr::MIFlag::NoUWrap;
4958 if (IsNoUSWrap)
4959 Flags |= MachineInstr::MIFlag::NoUSWrap;
4960 if (IsInBounds)
4961 Flags |= MachineInstr::MIFlag::InBounds;
4962
4963 MatchInfo = [=, &MI](MachineIRBuilder &B) {
4964 // When we change LHSPtrAdd's offset register we might cause it to use a reg
4965 // before its def. Sink the instruction so the outer PTR_ADD to ensure this
4966 // doesn't happen.
4967 LHSPtrAdd->moveBefore(MovePos: &MI);
4968 Register RHSReg = MI.getOffsetReg();
4969 // set VReg will cause type mismatch if it comes from extend/trunc
4970 auto NewCst = B.buildConstant(Res: MRI.getType(Reg: RHSReg), Val: LHSCstOff->Value);
4971 Observer.changingInstr(MI);
4972 MI.getOperand(i: 2).setReg(NewCst.getReg(Idx: 0));
4973 MI.setFlags(Flags);
4974 Observer.changedInstr(MI);
4975 Observer.changingInstr(MI&: *LHSPtrAdd);
4976 LHSPtrAdd->getOperand(i: 2).setReg(RHSReg);
4977 LHSPtrAdd->setFlags(Flags);
4978 Observer.changedInstr(MI&: *LHSPtrAdd);
4979 };
4980 return !reassociationCanBreakAddressingModePattern(MI);
4981}
4982
4983bool CombinerHelper::matchReassocFoldConstantsInSubTree(
4984 GPtrAdd &MI, MachineInstr *LHS, MachineInstr *RHS,
4985 BuildFnTy &MatchInfo) const {
4986 // G_PTR_ADD(G_PTR_ADD(BASE, C1), C2) -> G_PTR_ADD(BASE, C1+C2)
4987 auto *LHSPtrAdd = dyn_cast<GPtrAdd>(Val: LHS);
4988 if (!LHSPtrAdd)
4989 return false;
4990
4991 Register Src2Reg = MI.getOperand(i: 2).getReg();
4992 Register LHSSrc1 = LHSPtrAdd->getBaseReg();
4993 Register LHSSrc2 = LHSPtrAdd->getOffsetReg();
4994 auto C1 = getIConstantVRegVal(VReg: LHSSrc2, MRI);
4995 if (!C1)
4996 return false;
4997 auto C2 = getIConstantVRegVal(VReg: Src2Reg, MRI);
4998 if (!C2)
4999 return false;
5000
5001 // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
5002 // inbounds, reaching the same result in one G_PTR_ADD is also inbounds.
5003 // The nusw constraints are satisfied because imm1+imm2 cannot exceed the
5004 // largest signed integer that fits into the index type, which is the maximum
5005 // size of allocated objects according to the IR Language Reference.
5006 unsigned PtrAddFlags = MI.getFlags();
5007 unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();
5008 bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
5009 bool IsInBounds =
5010 PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::InBounds;
5011 unsigned Flags = 0;
5012 if (IsNoUWrap)
5013 Flags |= MachineInstr::MIFlag::NoUWrap;
5014 if (IsInBounds) {
5015 Flags |= MachineInstr::MIFlag::InBounds;
5016 Flags |= MachineInstr::MIFlag::NoUSWrap;
5017 }
5018
5019 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5020 auto NewCst = B.buildConstant(Res: MRI.getType(Reg: Src2Reg), Val: *C1 + *C2);
5021 Observer.changingInstr(MI);
5022 MI.getOperand(i: 1).setReg(LHSSrc1);
5023 MI.getOperand(i: 2).setReg(NewCst.getReg(Idx: 0));
5024 MI.setFlags(Flags);
5025 Observer.changedInstr(MI);
5026 };
5027 return !reassociationCanBreakAddressingModePattern(MI);
5028}
5029
5030bool CombinerHelper::matchReassocPtrAdd(MachineInstr &MI,
5031 BuildFnTy &MatchInfo) const {
5032 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
5033 // We're trying to match a few pointer computation patterns here for
5034 // re-association opportunities.
5035 // 1) Isolating a constant operand to be on the RHS, e.g.:
5036 // G_PTR_ADD(BASE, G_ADD(X, C)) -> G_PTR_ADD(G_PTR_ADD(BASE, X), C)
5037 //
5038 // 2) Folding two constants in each sub-tree as long as such folding
5039 // doesn't break a legal addressing mode.
5040 // G_PTR_ADD(G_PTR_ADD(BASE, C1), C2) -> G_PTR_ADD(BASE, C1+C2)
5041 //
5042 // 3) Move a constant from the LHS of an inner op to the RHS of the outer.
5043 // G_PTR_ADD (G_PTR_ADD X, C), Y) -> G_PTR_ADD (G_PTR_ADD(X, Y), C)
5044 // iif (G_PTR_ADD X, C) has one use.
5045 MachineInstr *LHS, *RHS;
5046 if (!mi_match(R: PtrAdd.getBaseReg(), MRI, P: m_MInstr(MI&: LHS)) ||
5047 !mi_match(R: PtrAdd.getOffsetReg(), MRI, P: m_MInstr(MI&: RHS)))
5048 return false;
5049
5050 // Try to match example 2.
5051 if (matchReassocFoldConstantsInSubTree(MI&: PtrAdd, LHS, RHS, MatchInfo))
5052 return true;
5053
5054 // Try to match example 3.
5055 if (matchReassocConstantInnerLHS(MI&: PtrAdd, LHS, RHS, MatchInfo))
5056 return true;
5057
5058 // Try to match example 1.
5059 if (matchReassocConstantInnerRHS(MI&: PtrAdd, RHS, MatchInfo))
5060 return true;
5061
5062 return false;
5063}
5064bool CombinerHelper::tryReassocBinOp(unsigned Opc, Register DstReg,
5065 Register OpLHS, Register OpRHS,
5066 BuildFnTy &MatchInfo) const {
5067 LLT OpRHSTy = MRI.getType(Reg: OpRHS);
5068 MachineInstr *OpLHSDef;
5069 if (!mi_match(R: OpLHS, MRI, P: m_MInstr(MI&: OpLHSDef)) || OpLHSDef->getOpcode() != Opc)
5070 return false;
5071
5072 Register OpLHSLHS = OpLHSDef->getOperand(i: 1).getReg();
5073 Register OpLHSRHS = OpLHSDef->getOperand(i: 2).getReg();
5074
5075 // If the inner op is (X op C), pull the constant out so it can be folded with
5076 // other constants in the expression tree. Folding is not guaranteed so we
5077 // might have (C1 op C2). In that case do not pull a constant out because it
5078 // won't help and can lead to infinite loops.
5079 if (isConstantOrConstantSplatVector(Def: OpLHSRHS, MRI) &&
5080 !isConstantOrConstantSplatVector(Def: OpLHSLHS, MRI)) {
5081 if (isConstantOrConstantSplatVector(Def: OpRHS, MRI)) {
5082 // (Opc (Opc X, C1), C2) -> (Opc X, (Opc C1, C2))
5083 MatchInfo = [=](MachineIRBuilder &B) {
5084 auto NewCst = B.buildInstr(Opc, DstOps: {OpRHSTy}, SrcOps: {OpLHSRHS, OpRHS});
5085 B.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {OpLHSLHS, NewCst});
5086 };
5087 return true;
5088 }
5089 if (getTargetLowering().isReassocProfitable(MRI, N0: OpLHS, N1: OpRHS)) {
5090 // Reassociate: (op (op x, c1), y) -> (op (op x, y), c1)
5091 // iff (op x, c1) has one use
5092 MatchInfo = [=](MachineIRBuilder &B) {
5093 auto NewLHSLHS = B.buildInstr(Opc, DstOps: {OpRHSTy}, SrcOps: {OpLHSLHS, OpRHS});
5094 B.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {NewLHSLHS, OpLHSRHS});
5095 };
5096 return true;
5097 }
5098 }
5099
5100 return false;
5101}
5102
5103bool CombinerHelper::matchReassocCommBinOp(MachineInstr &MI,
5104 BuildFnTy &MatchInfo) const {
5105 // We don't check if the reassociation will break a legal addressing mode
5106 // here since pointer arithmetic is handled by G_PTR_ADD.
5107 unsigned Opc = MI.getOpcode();
5108 Register DstReg = MI.getOperand(i: 0).getReg();
5109 Register LHSReg = MI.getOperand(i: 1).getReg();
5110 Register RHSReg = MI.getOperand(i: 2).getReg();
5111
5112 if (tryReassocBinOp(Opc, DstReg, OpLHS: LHSReg, OpRHS: RHSReg, MatchInfo))
5113 return true;
5114 if (tryReassocBinOp(Opc, DstReg, OpLHS: RHSReg, OpRHS: LHSReg, MatchInfo))
5115 return true;
5116 return false;
5117}
5118
5119bool CombinerHelper::matchConstantFoldCastOp(MachineInstr &MI,
5120 BuildFnTy &MatchInfo) const {
5121 Register Dst = MI.getOperand(i: 0).getReg();
5122 Register SrcOp = MI.getOperand(i: 1).getReg();
5123 LLT DstTy = MRI.getType(Reg: Dst);
5124 unsigned Opc = MI.getOpcode();
5125
5126 if (!isConstantLegalOrBeforeLegalizer(Ty: DstTy))
5127 return false;
5128
5129 if (DstTy.isScalableVector())
5130 return false;
5131
5132 if (DstTy.isFixedVector()) {
5133 auto *BV = getOpcodeDef<GBuildVector>(Reg: SrcOp, MRI);
5134 if (!BV)
5135 return false;
5136 SmallVector<APInt> Csts;
5137 for (unsigned I = 0, E = BV->getNumSources(); I != E; ++I) {
5138 auto MaybeCst = ConstantFoldCastOp(Opcode: Opc, DstTy, Op0: BV->getSourceReg(I), MRI);
5139 if (!MaybeCst)
5140 return false;
5141 Csts.push_back(Elt: *MaybeCst);
5142 }
5143 MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
5144 B.buildBuildVectorConstant(Res: Dst, Ops: Csts);
5145 };
5146 return true;
5147 }
5148
5149 if (auto MaybeCst = ConstantFoldCastOp(Opcode: Opc, DstTy, Op0: SrcOp, MRI)) {
5150 MatchInfo = [Dst, Cst = *MaybeCst](MachineIRBuilder &B) {
5151 B.buildConstant(Res: Dst, Val: Cst);
5152 };
5153 return true;
5154 }
5155
5156 return false;
5157}
5158
5159bool CombinerHelper::matchConstantFoldUnaryIntOp(MachineInstr &MI,
5160 BuildFnTy &MatchInfo) const {
5161 Register Dst = MI.getOperand(i: 0).getReg();
5162 LLT DstTy = MRI.getType(Reg: Dst);
5163
5164 SmallVector<APInt> Csts = ConstantFoldUnaryIntOp(
5165 Opcode: MI.getOpcode(), DstTy, Src: MI.getOperand(i: 1).getReg(), MRI);
5166 if (Csts.empty())
5167 return false;
5168
5169 MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
5170 if (Csts.size() == 1)
5171 B.buildConstant(Res: Dst, Val: Csts[0]);
5172 else
5173 B.buildBuildVectorConstant(Res: Dst, Ops: Csts);
5174 };
5175 return true;
5176}
5177
5178bool CombinerHelper::matchConstantFoldBinOp(MachineInstr &MI,
5179 BuildFnTy &MatchInfo) const {
5180 Register Dst = MI.getOperand(i: 0).getReg();
5181 Register Op1 = MI.getOperand(i: 1).getReg();
5182 Register Op2 = MI.getOperand(i: 2).getReg();
5183 LLT DstTy = MRI.getType(Reg: Dst);
5184
5185 // Scalable and pointer-element vectors have no G_BUILD_VECTOR of
5186 // G_CONSTANT to fold to.
5187 if (DstTy.isScalableVector() ||
5188 (DstTy.isVector() && DstTy.getElementType().isPointer()))
5189 return false;
5190
5191 if (DstTy.isFixedVector()) {
5192 SmallVector<APInt> Csts =
5193 ConstantFoldVectorBinop(Opcode: MI.getOpcode(), Op1, Op2, MRI);
5194 if (Csts.empty())
5195 return false;
5196 MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
5197 B.buildBuildVectorConstant(Res: Dst, Ops: Csts);
5198 };
5199 return true;
5200 }
5201
5202 auto MaybeCst = ConstantFoldBinOp(Opcode: MI.getOpcode(), Op1, Op2, MRI);
5203 if (!MaybeCst)
5204 return false;
5205 MatchInfo = [Dst, Cst = *MaybeCst](MachineIRBuilder &B) {
5206 B.buildConstant(Res: Dst, Val: Cst);
5207 };
5208 return true;
5209}
5210
5211// Vector FP constant folding assumes IEEE denormal arithmetic. Flush-to-zero
5212// and dynamic modes can produce a different value, so skip those folds.
5213// Targets such as RISC-V represent FP values as untyped scalars
5214// (s16/s32/s64/s128). getFltSemanticForLLT maps those onto the IEEE formats.
5215//
5216// TODO: Honor non-IEEE and dynamic denormal modes instead of skipping the
5217// vector folds. Scalar folds have the same limitation and stay unchanged here.
5218static bool vectorFPFoldUsesIEEEDenormals(const MachineFunction &MF, LLT Ty) {
5219 LLT ScalarTy = Ty.getScalarType();
5220 if (!ScalarTy.isFloat() && !ScalarTy.isAnyScalar())
5221 return true;
5222 return MF.getDenormalMode(FPType: getFltSemanticForLLT(Ty: ScalarTy)) ==
5223 DenormalMode::getIEEE();
5224}
5225
5226bool CombinerHelper::matchConstantFoldFPBinOp(MachineInstr &MI,
5227 BuildFnTy &MatchInfo) const {
5228 Register Dst = MI.getOperand(i: 0).getReg();
5229 Register Op1 = MI.getOperand(i: 1).getReg();
5230 Register Op2 = MI.getOperand(i: 2).getReg();
5231 LLT DstTy = MRI.getType(Reg: Dst);
5232
5233 if (DstTy.isScalableVector())
5234 return false;
5235
5236 if (DstTy.isFixedVector()) {
5237 if (!vectorFPFoldUsesIEEEDenormals(MF: getMachineFunction(), Ty: DstTy))
5238 return false;
5239 SmallVector<APFloat> Csts =
5240 ConstantFoldVectorFPBinop(Opcode: MI.getOpcode(), Op1, Op2, MRI);
5241 if (Csts.empty())
5242 return false;
5243 MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
5244 B.buildBuildVectorFConstant(Res: Dst, Ops: Csts);
5245 };
5246 return true;
5247 }
5248
5249 auto MaybeCst = ConstantFoldFPBinOp(Opcode: MI.getOpcode(), Op1, Op2, MRI);
5250 if (!MaybeCst)
5251 return false;
5252 MatchInfo = [Dst, Cst = *MaybeCst](MachineIRBuilder &B) {
5253 B.buildFConstant(Res: Dst, Val: Cst);
5254 };
5255 return true;
5256}
5257
5258bool CombinerHelper::matchConstantFoldFPUnary(MachineInstr &MI,
5259 BuildFnTy &MatchInfo) const {
5260 Register Dst = MI.getOperand(i: 0).getReg();
5261 Register Src = MI.getOperand(i: 1).getReg();
5262 LLT DstTy = MRI.getType(Reg: Dst);
5263 LLT SrcTy = MRI.getType(Reg: Src);
5264
5265 if (!isFPConstantLegalOrBeforeLegalizer(Ty: DstTy))
5266 return false;
5267
5268 // Source and destination element types can differ (G_FPEXT/G_FPTRUNC) and
5269 // can have different denormal modes.
5270 if (SrcTy.isVector() &&
5271 (!vectorFPFoldUsesIEEEDenormals(MF: getMachineFunction(), Ty: SrcTy) ||
5272 !vectorFPFoldUsesIEEEDenormals(MF: getMachineFunction(), Ty: DstTy)))
5273 return false;
5274
5275 SmallVector<APFloat> Csts =
5276 ConstantFoldUnaryFPOp(Opcode: MI.getOpcode(), DstTy, Src, MRI);
5277 if (Csts.empty())
5278 return false;
5279 MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
5280 if (Csts.size() == 1)
5281 B.buildFConstant(Res: Dst, Val: Csts[0]);
5282 else
5283 B.buildBuildVectorFConstant(Res: Dst, Ops: Csts);
5284 };
5285 return true;
5286}
5287
5288bool CombinerHelper::matchConstantFoldFMA(MachineInstr &MI,
5289 BuildFnTy &MatchInfo) const {
5290 assert(MI.getOpcode() == TargetOpcode::G_FMA ||
5291 MI.getOpcode() == TargetOpcode::G_FMAD);
5292 Register Dst = MI.getOperand(i: 0).getReg();
5293 auto [_, Op1, Op2, Op3] = MI.getFirst4Regs();
5294 LLT DstTy = MRI.getType(Reg: Dst);
5295
5296 auto FoldFMA = [](const APFloat &A, const APFloat &B,
5297 const APFloat &C) -> APFloat {
5298 APFloat Res(A);
5299 Res.fusedMultiplyAdd(Multiplicand: B, Addend: C, RM: APFloat::rmNearestTiesToEven);
5300 return Res;
5301 };
5302
5303 if (DstTy.isScalableVector())
5304 return false;
5305
5306 if (DstTy.isFixedVector()) {
5307 if (!vectorFPFoldUsesIEEEDenormals(MF: getMachineFunction(), Ty: DstTy))
5308 return false;
5309 auto *BV1 = getOpcodeDef<GBuildVector>(Reg: Op1, MRI);
5310 auto *BV2 = getOpcodeDef<GBuildVector>(Reg: Op2, MRI);
5311 auto *BV3 = getOpcodeDef<GBuildVector>(Reg: Op3, MRI);
5312 if (!BV1 || !BV2 || !BV3)
5313 return false;
5314 unsigned NumElts = BV1->getNumSources();
5315 if (BV2->getNumSources() != NumElts || BV3->getNumSources() != NumElts)
5316 return false;
5317 SmallVector<APFloat> Csts;
5318 for (unsigned I = 0; I != NumElts; ++I) {
5319 const ConstantFP *C1 = getConstantFPVRegVal(VReg: BV1->getSourceReg(I), MRI);
5320 const ConstantFP *C2 = getConstantFPVRegVal(VReg: BV2->getSourceReg(I), MRI);
5321 const ConstantFP *C3 = getConstantFPVRegVal(VReg: BV3->getSourceReg(I), MRI);
5322 if (!C1 || !C2 || !C3)
5323 return false;
5324 Csts.push_back(
5325 Elt: FoldFMA(C1->getValueAPF(), C2->getValueAPF(), C3->getValueAPF()));
5326 }
5327 MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
5328 B.buildBuildVectorFConstant(Res: Dst, Ops: Csts);
5329 };
5330 return true;
5331 }
5332
5333 const ConstantFP *Op3Cst = getConstantFPVRegVal(VReg: Op3, MRI);
5334 const ConstantFP *Op2Cst = getConstantFPVRegVal(VReg: Op2, MRI);
5335 const ConstantFP *Op1Cst = getConstantFPVRegVal(VReg: Op1, MRI);
5336 if (!Op1Cst || !Op2Cst || !Op3Cst)
5337 return false;
5338
5339 APFloat Res = FoldFMA(Op1Cst->getValueAPF(), Op2Cst->getValueAPF(),
5340 Op3Cst->getValueAPF());
5341 MatchInfo = [Dst, Res](MachineIRBuilder &B) { B.buildFConstant(Res: Dst, Val: Res); };
5342 return true;
5343}
5344
5345bool CombinerHelper::matchNarrowBinopFeedingAnd(
5346 MachineInstr &MI,
5347 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
5348 // Look for a binop feeding into an AND with a mask:
5349 //
5350 // %add = G_ADD %lhs, %rhs
5351 // %and = G_AND %add, 000...11111111
5352 //
5353 // Check if it's possible to perform the binop at a narrower width and zext
5354 // back to the original width like so:
5355 //
5356 // %narrow_lhs = G_TRUNC %lhs
5357 // %narrow_rhs = G_TRUNC %rhs
5358 // %narrow_add = G_ADD %narrow_lhs, %narrow_rhs
5359 // %new_add = G_ZEXT %narrow_add
5360 // %and = G_AND %new_add, 000...11111111
5361 //
5362 // This can allow later combines to eliminate the G_AND if it turns out
5363 // that the mask is irrelevant.
5364 assert(MI.getOpcode() == TargetOpcode::G_AND);
5365 Register Dst = MI.getOperand(i: 0).getReg();
5366 Register AndLHS = MI.getOperand(i: 1).getReg();
5367 Register AndRHS = MI.getOperand(i: 2).getReg();
5368 LLT WideTy = MRI.getType(Reg: Dst);
5369
5370 // If the potential binop has more than one use, then it's possible that one
5371 // of those uses will need its full width.
5372 if (!WideTy.isScalar() || !MRI.hasOneNonDBGUse(RegNo: AndLHS))
5373 return false;
5374
5375 // Check if the LHS feeding the AND is impacted by the high bits that we're
5376 // masking out.
5377 //
5378 // e.g. for 64-bit x, y:
5379 //
5380 // add_64(x, y) & 65535 == zext(add_16(trunc(x), trunc(y))) & 65535
5381 MachineInstr *LHSInst = getDefIgnoringCopies(Reg: AndLHS, MRI);
5382 if (!LHSInst)
5383 return false;
5384 unsigned LHSOpc = LHSInst->getOpcode();
5385 switch (LHSOpc) {
5386 default:
5387 return false;
5388 case TargetOpcode::G_ADD:
5389 case TargetOpcode::G_SUB:
5390 case TargetOpcode::G_MUL:
5391 case TargetOpcode::G_AND:
5392 case TargetOpcode::G_OR:
5393 case TargetOpcode::G_XOR:
5394 break;
5395 }
5396
5397 // Find the mask on the RHS.
5398 auto Cst = getIConstantVRegValWithLookThrough(VReg: AndRHS, MRI);
5399 if (!Cst)
5400 return false;
5401 auto Mask = Cst->Value;
5402 if (!Mask.isMask())
5403 return false;
5404
5405 // No point in combining if there's nothing to truncate.
5406 unsigned NarrowWidth = Mask.countr_one();
5407 if (NarrowWidth == WideTy.getSizeInBits())
5408 return false;
5409 LLT NarrowTy = LLT::integer(SizeInBits: NarrowWidth);
5410
5411 // Check if adding the zext + truncates could be harmful.
5412 auto &MF = *MI.getMF();
5413 const auto &TLI = getTargetLowering();
5414 LLVMContext &Ctx = MF.getFunction().getContext();
5415 if (!TLI.isTruncateFree(FromTy: WideTy, ToTy: NarrowTy, Ctx) ||
5416 !TLI.isZExtFree(FromTy: NarrowTy, ToTy: WideTy, Ctx))
5417 return false;
5418 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_TRUNC, {NarrowTy, WideTy}}) ||
5419 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ZEXT, {WideTy, NarrowTy}}))
5420 return false;
5421 Register BinOpLHS = LHSInst->getOperand(i: 1).getReg();
5422 Register BinOpRHS = LHSInst->getOperand(i: 2).getReg();
5423 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5424 auto NarrowLHS = Builder.buildTrunc(Res: NarrowTy, Op: BinOpLHS);
5425 auto NarrowRHS = Builder.buildTrunc(Res: NarrowTy, Op: BinOpRHS);
5426 auto NarrowBinOp =
5427 Builder.buildInstr(Opc: LHSOpc, DstOps: {NarrowTy}, SrcOps: {NarrowLHS, NarrowRHS});
5428 auto Ext = Builder.buildZExt(Res: WideTy, Op: NarrowBinOp);
5429 Observer.changingInstr(MI);
5430 MI.getOperand(i: 1).setReg(Ext.getReg(Idx: 0));
5431 Observer.changedInstr(MI);
5432 };
5433 return true;
5434}
5435
5436bool CombinerHelper::matchMulOBy0(MachineInstr &MI,
5437 BuildFnTy &MatchInfo) const {
5438 // (G_*MULO x, 0) -> 0 + no carry out
5439 assert(MI.getOpcode() == TargetOpcode::G_UMULO ||
5440 MI.getOpcode() == TargetOpcode::G_SMULO);
5441 if (!mi_match(R: MI.getOperand(i: 3).getReg(), MRI, P: m_SpecificICstOrSplat(RequestedValue: 0)))
5442 return false;
5443 Register Dst = MI.getOperand(i: 0).getReg();
5444 Register Carry = MI.getOperand(i: 1).getReg();
5445 if (!isConstantLegalOrBeforeLegalizer(Ty: MRI.getType(Reg: Dst)) ||
5446 !isConstantLegalOrBeforeLegalizer(Ty: MRI.getType(Reg: Carry)))
5447 return false;
5448 MatchInfo = [=](MachineIRBuilder &B) {
5449 B.buildConstant(Res: Dst, Val: 0);
5450 B.buildConstant(Res: Carry, Val: 0);
5451 };
5452 return true;
5453}
5454
5455bool CombinerHelper::matchSameRegOrICstOrSplat(Register A, Register B) const {
5456 Register SrcA = getSrcRegIgnoringCopies(Reg: A, MRI);
5457 Register SrcB = getSrcRegIgnoringCopies(Reg: B, MRI);
5458 if (SrcA.isValid() && SrcA == SrcB)
5459 return true;
5460 int64_t Cst;
5461 return mi_match(R: A, MRI, P: m_ICstOrSplat(Cst)) &&
5462 mi_match(R: B, MRI, P: m_SpecificICstOrSplat(RequestedValue: Cst));
5463}
5464
5465MachineInstr *CombinerHelper::buildUDivOrURemUsingMul(MachineInstr &MI) const {
5466 unsigned Opcode = MI.getOpcode();
5467 assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
5468 auto &UDivorRem = cast<GenericMachineInstr>(Val&: MI);
5469 Register Dst = UDivorRem.getReg(Idx: 0);
5470 Register LHS = UDivorRem.getReg(Idx: 1);
5471 Register RHS = UDivorRem.getReg(Idx: 2);
5472 LLT Ty = MRI.getType(Reg: Dst);
5473 LLT ScalarTy = Ty.getScalarType();
5474 const unsigned EltBits = ScalarTy.getScalarSizeInBits();
5475 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5476 LLT ScalarShiftAmtTy = ShiftAmtTy.getScalarType();
5477
5478 auto &MIB = Builder;
5479
5480 bool UseSRL = false;
5481 SmallVector<Register, 16> Shifts, Factors;
5482 auto *RHSDefInstr = cast<GenericMachineInstr>(Val: getDefIgnoringCopies(Reg: RHS, MRI));
5483 bool IsSplat = getIConstantSplatVal(MI: *RHSDefInstr, MRI).has_value();
5484
5485 auto BuildExactUDIVPattern = [&](const Constant *C) {
5486 // Don't recompute inverses for each splat element.
5487 if (IsSplat && !Factors.empty()) {
5488 Shifts.push_back(Elt: Shifts[0]);
5489 Factors.push_back(Elt: Factors[0]);
5490 return true;
5491 }
5492
5493 auto *CI = cast<ConstantInt>(Val: C);
5494 APInt Divisor = CI->getValue();
5495 unsigned Shift = Divisor.countr_zero();
5496 if (Shift) {
5497 Divisor.lshrInPlace(ShiftAmt: Shift);
5498 UseSRL = true;
5499 }
5500
5501 // Calculate the multiplicative inverse modulo BW.
5502 APInt Factor = Divisor.multiplicativeInverse();
5503 Shifts.push_back(Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: Shift).getReg(Idx: 0));
5504 Factors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Factor).getReg(Idx: 0));
5505 return true;
5506 };
5507
5508 if (MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
5509 // Collect all magic values from the build vector.
5510 if (!matchUnaryPredicate(MRI, Reg: RHS, Match: BuildExactUDIVPattern))
5511 llvm_unreachable("Expected unary predicate match to succeed");
5512
5513 Register Shift, Factor;
5514 if (Ty.isVector()) {
5515 Shift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: Shifts).getReg(Idx: 0);
5516 Factor = MIB.buildBuildVector(Res: Ty, Ops: Factors).getReg(Idx: 0);
5517 } else {
5518 Shift = Shifts[0];
5519 Factor = Factors[0];
5520 }
5521
5522 Register Res = LHS;
5523
5524 if (UseSRL)
5525 Res = MIB.buildLShr(Dst: Ty, Src0: Res, Src1: Shift, Flags: MachineInstr::IsExact).getReg(Idx: 0);
5526
5527 return MIB.buildMul(Dst: Ty, Src0: Res, Src1: Factor);
5528 }
5529
5530 unsigned KnownLeadingZeros =
5531 VT ? VT->getKnownBits(R: LHS).countMinLeadingZeros() : 0;
5532
5533 bool UseNPQ = false;
5534 SmallVector<Register, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
5535 auto BuildUDIVPattern = [&](const Constant *C) {
5536 auto *CI = cast<ConstantInt>(Val: C);
5537 const APInt &Divisor = CI->getValue();
5538
5539 bool SelNPQ = false;
5540 APInt Magic(Divisor.getBitWidth(), 0);
5541 unsigned PreShift = 0, PostShift = 0;
5542
5543 // Magic algorithm doesn't work for division by 1. We need to emit a select
5544 // at the end.
5545 // TODO: Use undef values for divisor of 1.
5546 if (!Divisor.isOne()) {
5547
5548 // UnsignedDivisionByConstantInfo doesn't work correctly if leading zeros
5549 // in the dividend exceeds the leading zeros for the divisor.
5550 UnsignedDivisionByConstantInfo magics =
5551 UnsignedDivisionByConstantInfo::get(
5552 D: Divisor, LeadingZeros: std::min(a: KnownLeadingZeros, b: Divisor.countl_zero()));
5553
5554 Magic = std::move(magics.Magic);
5555
5556 assert(magics.PreShift < Divisor.getBitWidth() &&
5557 "We shouldn't generate an undefined shift!");
5558 assert(magics.PostShift < Divisor.getBitWidth() &&
5559 "We shouldn't generate an undefined shift!");
5560 assert((!magics.IsAdd || magics.PreShift == 0) && "Unexpected pre-shift");
5561 PreShift = magics.PreShift;
5562 PostShift = magics.PostShift;
5563 SelNPQ = magics.IsAdd;
5564 }
5565
5566 PreShifts.push_back(
5567 Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: PreShift).getReg(Idx: 0));
5568 MagicFactors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Magic).getReg(Idx: 0));
5569 NPQFactors.push_back(
5570 Elt: MIB.buildConstant(Res: ScalarTy,
5571 Val: SelNPQ ? APInt::getOneBitSet(numBits: EltBits, BitNo: EltBits - 1)
5572 : APInt::getZero(numBits: EltBits))
5573 .getReg(Idx: 0));
5574 PostShifts.push_back(
5575 Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: PostShift).getReg(Idx: 0));
5576 UseNPQ |= SelNPQ;
5577 return true;
5578 };
5579
5580 // Collect the shifts/magic values from each element.
5581 bool Matched = matchUnaryPredicate(MRI, Reg: RHS, Match: BuildUDIVPattern);
5582 (void)Matched;
5583 assert(Matched && "Expected unary predicate match to succeed");
5584
5585 Register PreShift, PostShift, MagicFactor, NPQFactor;
5586 auto *RHSDef = getOpcodeDef<GBuildVector>(Reg: RHS, MRI);
5587 if (RHSDef) {
5588 PreShift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: PreShifts).getReg(Idx: 0);
5589 MagicFactor = MIB.buildBuildVector(Res: Ty, Ops: MagicFactors).getReg(Idx: 0);
5590 NPQFactor = MIB.buildBuildVector(Res: Ty, Ops: NPQFactors).getReg(Idx: 0);
5591 PostShift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: PostShifts).getReg(Idx: 0);
5592 } else {
5593 assert(MRI.getType(RHS).isScalar() &&
5594 "Non-build_vector operation should have been a scalar");
5595 PreShift = PreShifts[0];
5596 MagicFactor = MagicFactors[0];
5597 PostShift = PostShifts[0];
5598 }
5599
5600 Register Q = LHS;
5601 Q = MIB.buildLShr(Dst: Ty, Src0: Q, Src1: PreShift).getReg(Idx: 0);
5602
5603 // Multiply the numerator (operand 0) by the magic value.
5604 Q = MIB.buildUMulH(Dst: Ty, Src0: Q, Src1: MagicFactor).getReg(Idx: 0);
5605
5606 if (UseNPQ) {
5607 Register NPQ = MIB.buildSub(Dst: Ty, Src0: LHS, Src1: Q).getReg(Idx: 0);
5608
5609 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
5610 // G_UMULH to act as a SRL-by-1 for NPQ, else multiply by zero.
5611 if (Ty.isVector())
5612 NPQ = MIB.buildUMulH(Dst: Ty, Src0: NPQ, Src1: NPQFactor).getReg(Idx: 0);
5613 else
5614 NPQ = MIB.buildLShr(Dst: Ty, Src0: NPQ, Src1: MIB.buildConstant(Res: ShiftAmtTy, Val: 1)).getReg(Idx: 0);
5615
5616 Q = MIB.buildAdd(Dst: Ty, Src0: NPQ, Src1: Q).getReg(Idx: 0);
5617 }
5618
5619 Q = MIB.buildLShr(Dst: Ty, Src0: Q, Src1: PostShift).getReg(Idx: 0);
5620 auto One = MIB.buildConstant(Res: Ty, Val: 1);
5621 auto IsOne = MIB.buildICmp(
5622 Pred: CmpInst::Predicate::ICMP_EQ,
5623 Res: Ty.isScalar() ? LLT::integer(SizeInBits: 1) : Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1)),
5624 Op0: RHS, Op1: One);
5625 auto ret = MIB.buildSelect(Res: Ty, Tst: IsOne, Op0: LHS, Op1: Q);
5626
5627 if (Opcode == TargetOpcode::G_UREM) {
5628 auto Prod = MIB.buildMul(Dst: Ty, Src0: ret, Src1: RHS);
5629 return MIB.buildSub(Dst: Ty, Src0: LHS, Src1: Prod);
5630 }
5631 return ret;
5632}
5633
5634bool CombinerHelper::matchUDivOrURemByConst(MachineInstr &MI) const {
5635 unsigned Opcode = MI.getOpcode();
5636 assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
5637 Register Dst = MI.getOperand(i: 0).getReg();
5638 Register RHS = MI.getOperand(i: 2).getReg();
5639 LLT DstTy = MRI.getType(Reg: Dst);
5640
5641 auto &MF = *MI.getMF();
5642 AttributeList Attr = MF.getFunction().getAttributes();
5643 const auto &TLI = getTargetLowering();
5644 LLVMContext &Ctx = MF.getFunction().getContext();
5645 if (DstTy.getScalarSizeInBits() == 1 ||
5646 TLI.isIntDivCheap(VT: getApproximateEVTForLLT(Ty: DstTy, Ctx), Attr))
5647 return false;
5648
5649 // Don't do this for minsize because the instruction sequence is usually
5650 // larger.
5651 if (MF.getFunction().hasMinSize())
5652 return false;
5653
5654 if (Opcode == TargetOpcode::G_UDIV &&
5655 MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
5656 return matchUnaryPredicate(
5657 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5658 }
5659
5660 MachineInstr *RHSDef;
5661 if (!mi_match(R: RHS, MRI, P: m_MInstr(MI&: RHSDef)) ||
5662 !isConstantOrConstantVector(MI&: *RHSDef, MRI))
5663 return false;
5664
5665 // Don't do this if the types are not going to be legal.
5666 if (LI) {
5667 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_MUL, {DstTy, DstTy}}))
5668 return false;
5669 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_UMULH, {DstTy}}))
5670 return false;
5671 if (!isLegalOrBeforeLegalizer(
5672 Query: {TargetOpcode::G_ICMP,
5673 {DstTy.isVector() ? DstTy.changeElementSize(NewEltSize: 1) : LLT::scalar(SizeInBits: 1),
5674 DstTy}}))
5675 return false;
5676 if (Opcode == TargetOpcode::G_UREM &&
5677 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SUB, {DstTy, DstTy}}))
5678 return false;
5679 }
5680
5681 return matchUnaryPredicate(
5682 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5683}
5684
5685void CombinerHelper::applyUDivOrURemByConst(MachineInstr &MI) const {
5686 auto *NewMI = buildUDivOrURemUsingMul(MI);
5687 replaceSingleDefInstWithReg(MI, Replacement: NewMI->getOperand(i: 0).getReg());
5688}
5689
5690bool CombinerHelper::matchSDivOrSRemByConst(MachineInstr &MI) const {
5691 unsigned Opcode = MI.getOpcode();
5692 assert(Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM);
5693 Register Dst = MI.getOperand(i: 0).getReg();
5694 Register RHS = MI.getOperand(i: 2).getReg();
5695 LLT DstTy = MRI.getType(Reg: Dst);
5696 auto SizeInBits = DstTy.getScalarSizeInBits();
5697 LLT WideTy = DstTy.changeElementSize(NewEltSize: SizeInBits * 2);
5698
5699 auto &MF = *MI.getMF();
5700 AttributeList Attr = MF.getFunction().getAttributes();
5701 const auto &TLI = getTargetLowering();
5702 LLVMContext &Ctx = MF.getFunction().getContext();
5703 if (DstTy.getScalarSizeInBits() < 3 ||
5704 TLI.isIntDivCheap(VT: getApproximateEVTForLLT(Ty: DstTy, Ctx), Attr))
5705 return false;
5706
5707 // Don't do this for minsize because the instruction sequence is usually
5708 // larger.
5709 if (MF.getFunction().hasMinSize())
5710 return false;
5711
5712 // If the sdiv has an 'exact' flag we can use a simpler lowering.
5713 if (Opcode == TargetOpcode::G_SDIV &&
5714 MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
5715 return matchUnaryPredicate(
5716 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5717 }
5718
5719 MachineInstr *RHSDef;
5720 if (!mi_match(R: RHS, MRI, P: m_MInstr(MI&: RHSDef)) ||
5721 !isConstantOrConstantVector(MI&: *RHSDef, MRI))
5722 return false;
5723
5724 // Don't do this if the types are not going to be legal.
5725 if (LI) {
5726 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_MUL, {DstTy, DstTy}}))
5727 return false;
5728 if (isPreLegalize()) {
5729 // The legalizer can still split a vector G_SMULH or lower it through a
5730 // wider G_MUL.
5731 if (!isLegalOrHasFewerElements(Query: {TargetOpcode::G_SMULH, {DstTy}}) &&
5732 !isLegalOrHasWidenScalar(Query: {TargetOpcode::G_MUL, {WideTy, WideTy}}))
5733 return false;
5734 } else if (!isLegal(Query: {TargetOpcode::G_SMULH, {DstTy}})) {
5735 // After legalization nothing will fix up an illegal G_SMULH.
5736 return false;
5737 }
5738 if (Opcode == TargetOpcode::G_SREM &&
5739 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SUB, {DstTy, DstTy}}))
5740 return false;
5741 }
5742
5743 return matchUnaryPredicate(
5744 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5745}
5746
5747void CombinerHelper::applySDivOrSRemByConst(MachineInstr &MI) const {
5748 auto *NewMI = buildSDivOrSRemUsingMul(MI);
5749 replaceSingleDefInstWithReg(MI, Replacement: NewMI->getOperand(i: 0).getReg());
5750}
5751
5752MachineInstr *CombinerHelper::buildSDivOrSRemUsingMul(MachineInstr &MI) const {
5753 unsigned Opcode = MI.getOpcode();
5754 assert(MI.getOpcode() == TargetOpcode::G_SDIV ||
5755 Opcode == TargetOpcode::G_SREM);
5756 auto &SDivorRem = cast<GenericMachineInstr>(Val&: MI);
5757 Register Dst = SDivorRem.getReg(Idx: 0);
5758 Register LHS = SDivorRem.getReg(Idx: 1);
5759 Register RHS = SDivorRem.getReg(Idx: 2);
5760 LLT Ty = MRI.getType(Reg: Dst);
5761 LLT ScalarTy = Ty.getScalarType();
5762 const unsigned EltBits = ScalarTy.getScalarSizeInBits();
5763 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5764 LLT ScalarShiftAmtTy = ShiftAmtTy.getScalarType();
5765 auto &MIB = Builder;
5766
5767 bool UseSRA = false;
5768 SmallVector<Register, 16> ExactShifts, ExactFactors;
5769
5770 auto *RHSDefInstr = cast<GenericMachineInstr>(Val: getDefIgnoringCopies(Reg: RHS, MRI));
5771 bool IsSplat = getIConstantSplatVal(MI: *RHSDefInstr, MRI).has_value();
5772
5773 auto BuildExactSDIVPattern = [&](const Constant *C) {
5774 // Don't recompute inverses for each splat element.
5775 if (IsSplat && !ExactFactors.empty()) {
5776 ExactShifts.push_back(Elt: ExactShifts[0]);
5777 ExactFactors.push_back(Elt: ExactFactors[0]);
5778 return true;
5779 }
5780
5781 auto *CI = cast<ConstantInt>(Val: C);
5782 APInt Divisor = CI->getValue();
5783 unsigned Shift = Divisor.countr_zero();
5784 if (Shift) {
5785 Divisor.ashrInPlace(ShiftAmt: Shift);
5786 UseSRA = true;
5787 }
5788
5789 // Calculate the multiplicative inverse modulo BW.
5790 // 2^W requires W + 1 bits, so we have to extend and then truncate.
5791 APInt Factor = Divisor.multiplicativeInverse();
5792 ExactShifts.push_back(Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: Shift).getReg(Idx: 0));
5793 ExactFactors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Factor).getReg(Idx: 0));
5794 return true;
5795 };
5796
5797 if (MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
5798 // Collect all magic values from the build vector.
5799 bool Matched = matchUnaryPredicate(MRI, Reg: RHS, Match: BuildExactSDIVPattern);
5800 (void)Matched;
5801 assert(Matched && "Expected unary predicate match to succeed");
5802
5803 Register Shift, Factor;
5804 if (Ty.isVector()) {
5805 Shift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: ExactShifts).getReg(Idx: 0);
5806 Factor = MIB.buildBuildVector(Res: Ty, Ops: ExactFactors).getReg(Idx: 0);
5807 } else {
5808 Shift = ExactShifts[0];
5809 Factor = ExactFactors[0];
5810 }
5811
5812 Register Res = LHS;
5813
5814 if (UseSRA)
5815 Res = MIB.buildAShr(Dst: Ty, Src0: Res, Src1: Shift, Flags: MachineInstr::IsExact).getReg(Idx: 0);
5816
5817 return MIB.buildMul(Dst: Ty, Src0: Res, Src1: Factor);
5818 }
5819
5820 SmallVector<Register, 16> MagicFactors, Factors, Shifts, ShiftMasks;
5821
5822 auto BuildSDIVPattern = [&](const Constant *C) {
5823 auto *CI = cast<ConstantInt>(Val: C);
5824 const APInt &Divisor = CI->getValue();
5825
5826 SignedDivisionByConstantInfo Magics =
5827 SignedDivisionByConstantInfo::get(D: Divisor);
5828 int NumeratorFactor = 0;
5829 int ShiftMask = -1;
5830
5831 if (Divisor.isOne() || Divisor.isAllOnes()) {
5832 // If d is +1/-1, we just multiply the numerator by +1/-1.
5833 NumeratorFactor = Divisor.getSExtValue();
5834 Magics.Magic = 0;
5835 Magics.ShiftAmount = 0;
5836 ShiftMask = 0;
5837 } else if (Divisor.isStrictlyPositive() && Magics.Magic.isNegative()) {
5838 // If d > 0 and m < 0, add the numerator.
5839 NumeratorFactor = 1;
5840 } else if (Divisor.isNegative() && Magics.Magic.isStrictlyPositive()) {
5841 // If d < 0 and m > 0, subtract the numerator.
5842 NumeratorFactor = -1;
5843 }
5844
5845 MagicFactors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Magics.Magic).getReg(Idx: 0));
5846 Factors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: NumeratorFactor).getReg(Idx: 0));
5847 Shifts.push_back(
5848 Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: Magics.ShiftAmount).getReg(Idx: 0));
5849 ShiftMasks.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: ShiftMask).getReg(Idx: 0));
5850
5851 return true;
5852 };
5853
5854 // Collect the shifts/magic values from each element.
5855 bool Matched = matchUnaryPredicate(MRI, Reg: RHS, Match: BuildSDIVPattern);
5856 (void)Matched;
5857 assert(Matched && "Expected unary predicate match to succeed");
5858
5859 Register MagicFactor, Factor, Shift, ShiftMask;
5860 auto *RHSDef = getOpcodeDef<GBuildVector>(Reg: RHS, MRI);
5861 if (RHSDef) {
5862 MagicFactor = MIB.buildBuildVector(Res: Ty, Ops: MagicFactors).getReg(Idx: 0);
5863 Factor = MIB.buildBuildVector(Res: Ty, Ops: Factors).getReg(Idx: 0);
5864 Shift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: Shifts).getReg(Idx: 0);
5865 ShiftMask = MIB.buildBuildVector(Res: Ty, Ops: ShiftMasks).getReg(Idx: 0);
5866 } else {
5867 assert(MRI.getType(RHS).isScalar() &&
5868 "Non-build_vector operation should have been a scalar");
5869 MagicFactor = MagicFactors[0];
5870 Factor = Factors[0];
5871 Shift = Shifts[0];
5872 ShiftMask = ShiftMasks[0];
5873 }
5874
5875 Register Q = LHS;
5876 Q = MIB.buildSMulH(Dst: Ty, Src0: LHS, Src1: MagicFactor).getReg(Idx: 0);
5877
5878 // (Optionally) Add/subtract the numerator using Factor.
5879 Factor = MIB.buildMul(Dst: Ty, Src0: LHS, Src1: Factor).getReg(Idx: 0);
5880 Q = MIB.buildAdd(Dst: Ty, Src0: Q, Src1: Factor).getReg(Idx: 0);
5881
5882 // Shift right algebraic by shift value.
5883 Q = MIB.buildAShr(Dst: Ty, Src0: Q, Src1: Shift).getReg(Idx: 0);
5884
5885 // Extract the sign bit, mask it and add it to the quotient.
5886 auto SignShift = MIB.buildConstant(Res: ShiftAmtTy, Val: EltBits - 1);
5887 auto T = MIB.buildLShr(Dst: Ty, Src0: Q, Src1: SignShift);
5888 T = MIB.buildAnd(Dst: Ty, Src0: T, Src1: ShiftMask);
5889 auto ret = MIB.buildAdd(Dst: Ty, Src0: Q, Src1: T);
5890
5891 if (Opcode == TargetOpcode::G_SREM) {
5892 auto Prod = MIB.buildMul(Dst: Ty, Src0: ret, Src1: RHS);
5893 return MIB.buildSub(Dst: Ty, Src0: LHS, Src1: Prod);
5894 }
5895 return ret;
5896}
5897
5898bool CombinerHelper::matchDivByPow2(MachineInstr &MI, bool IsSigned) const {
5899 assert((MI.getOpcode() == TargetOpcode::G_SDIV ||
5900 MI.getOpcode() == TargetOpcode::G_UDIV) &&
5901 "Expected SDIV or UDIV");
5902 auto &Div = cast<GenericMachineInstr>(Val&: MI);
5903 Register RHS = Div.getReg(Idx: 2);
5904 auto MatchPow2 = [&](const Constant *C) {
5905 auto *CI = dyn_cast<ConstantInt>(Val: C);
5906 return CI && (CI->getValue().isPowerOf2() ||
5907 (IsSigned && CI->getValue().isNegatedPowerOf2()));
5908 };
5909 return matchUnaryPredicate(MRI, Reg: RHS, Match: MatchPow2, /*AllowUndefs=*/false);
5910}
5911
5912void CombinerHelper::applySDivByPow2(MachineInstr &MI) const {
5913 assert(MI.getOpcode() == TargetOpcode::G_SDIV && "Expected SDIV");
5914 auto &SDiv = cast<GenericMachineInstr>(Val&: MI);
5915 Register Dst = SDiv.getReg(Idx: 0);
5916 Register LHS = SDiv.getReg(Idx: 1);
5917 Register RHS = SDiv.getReg(Idx: 2);
5918 LLT Ty = MRI.getType(Reg: Dst);
5919 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5920 LLT CCVT = Ty.isVector() ? LLT::vector(EC: Ty.getElementCount(), ScalarTy: LLT::integer(SizeInBits: 1))
5921 : LLT::integer(SizeInBits: 1);
5922
5923 // Effectively we want to lower G_SDIV %lhs, %rhs, where %rhs is a power of 2,
5924 // to the following version:
5925 //
5926 // %c1 = G_CTTZ %rhs
5927 // %inexact = G_SUB $bitwidth, %c1
5928 // %sign = %G_ASHR %lhs, $(bitwidth - 1)
5929 // %lshr = G_LSHR %sign, %inexact
5930 // %add = G_ADD %lhs, %lshr
5931 // %ashr = G_ASHR %add, %c1
5932 // %ashr = G_SELECT, %isoneorallones, %lhs, %ashr
5933 // %zero = G_CONSTANT $0
5934 // %neg = G_NEG %ashr
5935 // %isneg = G_ICMP SLT %rhs, %zero
5936 // %res = G_SELECT %isneg, %neg, %ashr
5937
5938 unsigned BitWidth = Ty.getScalarSizeInBits();
5939 auto Zero = Builder.buildConstant(Res: Ty, Val: 0);
5940
5941 auto Bits = Builder.buildConstant(Res: ShiftAmtTy, Val: BitWidth);
5942 auto C1 = Builder.buildCTTZ(Dst: ShiftAmtTy, Src0: RHS);
5943 auto Inexact = Builder.buildSub(Dst: ShiftAmtTy, Src0: Bits, Src1: C1);
5944 // Splat the sign bit into the register
5945 auto Sign = Builder.buildAShr(
5946 Dst: Ty, Src0: LHS, Src1: Builder.buildConstant(Res: ShiftAmtTy, Val: BitWidth - 1));
5947
5948 // Add (LHS < 0) ? abs2 - 1 : 0;
5949 auto LSrl = Builder.buildLShr(Dst: Ty, Src0: Sign, Src1: Inexact);
5950 auto Add = Builder.buildAdd(Dst: Ty, Src0: LHS, Src1: LSrl);
5951 auto AShr = Builder.buildAShr(Dst: Ty, Src0: Add, Src1: C1);
5952
5953 // Special case: (sdiv X, 1) -> X
5954 // Special Case: (sdiv X, -1) -> 0-X
5955 auto One = Builder.buildConstant(Res: Ty, Val: 1);
5956 auto MinusOne = Builder.buildConstant(Res: Ty, Val: -1);
5957 auto IsOne = Builder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: CCVT, Op0: RHS, Op1: One);
5958 auto IsMinusOne =
5959 Builder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: CCVT, Op0: RHS, Op1: MinusOne);
5960 auto IsOneOrMinusOne = Builder.buildOr(Dst: CCVT, Src0: IsOne, Src1: IsMinusOne);
5961 AShr = Builder.buildSelect(Res: Ty, Tst: IsOneOrMinusOne, Op0: LHS, Op1: AShr);
5962
5963 // If divided by a positive value, we're done. Otherwise, the result must be
5964 // negated.
5965 auto Neg = Builder.buildNeg(Dst: Ty, Src0: AShr);
5966 auto IsNeg = Builder.buildICmp(Pred: CmpInst::Predicate::ICMP_SLT, Res: CCVT, Op0: RHS, Op1: Zero);
5967 Builder.buildSelect(Res: MI.getOperand(i: 0).getReg(), Tst: IsNeg, Op0: Neg, Op1: AShr);
5968 MI.eraseFromParent();
5969}
5970
5971void CombinerHelper::applyUDivByPow2(MachineInstr &MI) const {
5972 assert(MI.getOpcode() == TargetOpcode::G_UDIV && "Expected UDIV");
5973 auto &UDiv = cast<GenericMachineInstr>(Val&: MI);
5974 Register Dst = UDiv.getReg(Idx: 0);
5975 Register LHS = UDiv.getReg(Idx: 1);
5976 Register RHS = UDiv.getReg(Idx: 2);
5977 LLT Ty = MRI.getType(Reg: Dst);
5978 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5979
5980 auto C1 = Builder.buildCTTZ(Dst: ShiftAmtTy, Src0: RHS);
5981 Builder.buildLShr(Dst: MI.getOperand(i: 0).getReg(), Src0: LHS, Src1: C1);
5982 MI.eraseFromParent();
5983}
5984
5985bool CombinerHelper::matchUDivByShiftedPow2(MachineInstr &MI,
5986 BuildFnTy &MatchInfo) const {
5987 Register Dst = MI.getOperand(i: 0).getReg();
5988 Register LHS = MI.getOperand(i: 1).getReg();
5989 Register RHS = MI.getOperand(i: 2).getReg();
5990 Register Base, Shift;
5991 if (!mi_match(R: RHS, MRI, P: m_GShl(L: m_Reg(R&: Base), R: m_Reg(R&: Shift))))
5992 return false;
5993
5994 if (!matchUnaryPredicate(MRI, Reg: Base, Match: [](const Constant *C) {
5995 auto *CI = dyn_cast_if_present<ConstantInt>(Val: C);
5996 return CI && CI->getValue().isPowerOf2();
5997 }))
5998 return false;
5999
6000 LLT Ty = MRI.getType(Reg: Dst);
6001 LLT ShiftTy = MRI.getType(Reg: Shift);
6002 // The shift type must be able to represent every valid shift amount.
6003 if (ShiftTy.getScalarSizeInBits() < Log2_64_Ceil(Value: Ty.getScalarSizeInBits()))
6004 return false;
6005 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_CTTZ, {ShiftTy, Ty}}) ||
6006 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ADD, {ShiftTy, ShiftTy}}) ||
6007 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_LSHR, {Ty, ShiftTy}}))
6008 return false;
6009
6010 unsigned Flags =
6011 MI.getFlag(Flag: MachineInstr::IsExact) ? MachineInstr::IsExact : 0;
6012 MatchInfo = [=](MachineIRBuilder &B) {
6013 auto Log2 = B.buildCTTZ(Dst: ShiftTy, Src0: Base);
6014 auto Amount = B.buildAdd(Dst: ShiftTy, Src0: Shift, Src1: Log2);
6015 B.buildLShr(Dst, Src0: LHS, Src1: Amount, Flags);
6016 };
6017 return true;
6018}
6019
6020bool CombinerHelper::matchSignedDivRemToUnsigned(
6021 MachineInstr &MI, unsigned UnsignedOpcode) const {
6022 Register Dst = MI.getOperand(i: 0).getReg();
6023 Register LHS = MI.getOperand(i: 1).getReg();
6024 Register RHS = MI.getOperand(i: 2).getReg();
6025 if (!VT || !VT->signBitIsZero(Op: LHS) || !VT->signBitIsZero(Op: RHS))
6026 return false;
6027
6028 LLT Ty = MRI.getType(Reg: Dst);
6029 return isLegalOrBeforeLegalizer(Query: {UnsignedOpcode, {Ty}});
6030}
6031
6032void CombinerHelper::applySimplifySRemByPow2(MachineInstr &MI) const {
6033 assert(MI.getOpcode() == TargetOpcode::G_SREM && "Expected SREM");
6034 auto &SRem = cast<GBinOp>(Val&: MI);
6035 Register Dst = SRem.getReg(Idx: 0);
6036 Register LHS = SRem.getLHSReg();
6037 Register RHS = SRem.getRHSReg();
6038 LLT Ty = MRI.getType(Reg: Dst);
6039 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
6040
6041 // Effectively we want to lower G_SREM %lhs, %rhs, where %rhs is +/- a power
6042 // of 2, to the following branch-free bias-and-mask version:
6043 //
6044 // %abs = G_ABS %rhs
6045 // %mask = G_SUB %abs, 1
6046 // %sign = G_ASHR %lhs, $(bitwidth - 1)
6047 // %bias = G_AND %sign, %mask
6048 // %biased = G_ADD %lhs, %bias
6049 // %masked = G_AND %biased, %mask
6050 // %res = G_SUB %masked, %bias
6051 //
6052 // The bias adds (|%rhs| - 1) for negative %lhs, correcting rounding towards
6053 // zero (instead of towards -inf that a plain mask would give). Constant
6054 // divisors collapse %mask to a single G_CONSTANT via the CSEMIRBuilder folds
6055 // for G_ABS and G_SUB.
6056
6057 unsigned BitWidth = Ty.getScalarSizeInBits();
6058 auto AbsRHS = Builder.buildAbs(Dst: Ty, Src: RHS);
6059 auto Mask = Builder.buildSub(Dst: Ty, Src0: AbsRHS, Src1: Builder.buildConstant(Res: Ty, Val: 1));
6060 auto BWMinusOne = Builder.buildConstant(Res: ShiftAmtTy, Val: BitWidth - 1);
6061 auto Sign = Builder.buildAShr(Dst: Ty, Src0: LHS, Src1: BWMinusOne);
6062 auto Bias = Builder.buildAnd(Dst: Ty, Src0: Sign, Src1: Mask);
6063 auto Biased = Builder.buildAdd(Dst: Ty, Src0: LHS, Src1: Bias);
6064 auto Masked = Builder.buildAnd(Dst: Ty, Src0: Biased, Src1: Mask);
6065 Builder.buildSub(Dst, Src0: Masked, Src1: Bias);
6066 MI.eraseFromParent();
6067}
6068
6069bool CombinerHelper::matchUMulHToLShr(MachineInstr &MI) const {
6070 assert(MI.getOpcode() == TargetOpcode::G_UMULH);
6071 Register RHS = MI.getOperand(i: 2).getReg();
6072 Register Dst = MI.getOperand(i: 0).getReg();
6073 LLT Ty = MRI.getType(Reg: Dst);
6074 LLT RHSTy = MRI.getType(Reg: RHS);
6075 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
6076 auto MatchPow2ExceptOne = [&](const Constant *C) {
6077 if (auto *CI = dyn_cast<ConstantInt>(Val: C))
6078 return CI->getValue().isPowerOf2() && !CI->getValue().isOne();
6079 return false;
6080 };
6081 if (!matchUnaryPredicate(MRI, Reg: RHS, Match: MatchPow2ExceptOne, AllowUndefs: false))
6082 return false;
6083 // We need to check both G_LSHR and G_CTLZ because the combine uses G_CTLZ to
6084 // get log base 2, and it is not always legal for on a target.
6085 return isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_LSHR, {Ty, ShiftAmtTy}}) &&
6086 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_CTLZ, {RHSTy, RHSTy}});
6087}
6088
6089void CombinerHelper::applyUMulHToLShr(MachineInstr &MI) const {
6090 Register LHS = MI.getOperand(i: 1).getReg();
6091 Register RHS = MI.getOperand(i: 2).getReg();
6092 Register Dst = MI.getOperand(i: 0).getReg();
6093 LLT Ty = MRI.getType(Reg: Dst);
6094 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
6095 unsigned NumEltBits = Ty.getScalarSizeInBits();
6096
6097 auto LogBase2 = buildLogBase2(V: RHS, MIB&: Builder);
6098 auto ShiftAmt =
6099 Builder.buildSub(Dst: Ty, Src0: Builder.buildConstant(Res: Ty, Val: NumEltBits), Src1: LogBase2);
6100 auto Trunc = Builder.buildZExtOrTrunc(Res: ShiftAmtTy, Op: ShiftAmt);
6101 Builder.buildLShr(Dst, Src0: LHS, Src1: Trunc);
6102 MI.eraseFromParent();
6103}
6104
6105bool CombinerHelper::matchTruncSSatS(MachineInstr &MI,
6106 Register &MatchInfo) const {
6107 Register Dst = MI.getOperand(i: 0).getReg();
6108 Register Src = MI.getOperand(i: 1).getReg();
6109 LLT DstTy = MRI.getType(Reg: Dst);
6110 LLT SrcTy = MRI.getType(Reg: Src);
6111 unsigned NumDstBits = DstTy.getScalarSizeInBits();
6112 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6113 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
6114
6115 if (!LI || !isLegalOrHasFewerElements(
6116 Query: {TargetOpcode::G_TRUNC_SSAT_S, {DstTy, SrcTy}}))
6117 return false;
6118
6119 APInt SignedMax = APInt::getSignedMaxValue(numBits: NumDstBits).sext(width: NumSrcBits);
6120 APInt SignedMin = APInt::getSignedMinValue(numBits: NumDstBits).sext(width: NumSrcBits);
6121 if (mi_match(
6122 R: Src, MRI,
6123 P: m_GSMin(L: m_GSMax(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: SignedMin)),
6124 R: m_SpecificICstOrSplat(RequestedValue: SignedMax))))
6125 return true;
6126 if (mi_match(
6127 R: Src, MRI,
6128 P: m_GSMax(L: m_GSMin(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: SignedMax)),
6129 R: m_SpecificICstOrSplat(RequestedValue: SignedMin))))
6130 return true;
6131
6132 // CVP in the midend will often transform trunc(smin(smax(..)) into
6133 // trunc nsw(smin(..)) as the smax against INT_MIN never saturates.
6134 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoSWrap) &&
6135 mi_match(R: Src, MRI,
6136 P: m_GSMin(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: SignedMax))))
6137 return true;
6138
6139 return false;
6140}
6141
6142void CombinerHelper::applyTruncSSatS(MachineInstr &MI,
6143 Register &MatchInfo) const {
6144 Register Dst = MI.getOperand(i: 0).getReg();
6145 Builder.buildTruncSSatS(Res: Dst, Op: MatchInfo);
6146 MI.eraseFromParent();
6147}
6148
6149bool CombinerHelper::matchTruncSSatU(MachineInstr &MI,
6150 Register &MatchInfo) const {
6151 Register Dst = MI.getOperand(i: 0).getReg();
6152 Register Src = MI.getOperand(i: 1).getReg();
6153 LLT DstTy = MRI.getType(Reg: Dst);
6154 LLT SrcTy = MRI.getType(Reg: Src);
6155 unsigned NumDstBits = DstTy.getScalarSizeInBits();
6156 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6157 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
6158
6159 if (!LI || !isLegalOrHasFewerElements(
6160 Query: {TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))
6161 return false;
6162 APInt UnsignedMax = APInt::getMaxValue(numBits: NumDstBits).zext(width: NumSrcBits);
6163 return mi_match(R: Src, MRI,
6164 P: m_GSMin(L: m_GSMax(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: 0)),
6165 R: m_SpecificICstOrSplat(RequestedValue: UnsignedMax))) ||
6166 mi_match(R: Src, MRI,
6167 P: m_GSMax(L: m_GSMin(L: m_Reg(R&: MatchInfo),
6168 R: m_SpecificICstOrSplat(RequestedValue: UnsignedMax)),
6169 R: m_SpecificICstOrSplat(RequestedValue: 0))) ||
6170 mi_match(R: Src, MRI,
6171 P: m_GUMin(L: m_GSMax(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: 0)),
6172 R: m_SpecificICstOrSplat(RequestedValue: UnsignedMax)));
6173}
6174
6175void CombinerHelper::applyTruncSSatU(MachineInstr &MI,
6176 Register &MatchInfo) const {
6177 Register Dst = MI.getOperand(i: 0).getReg();
6178 Builder.buildTruncSSatU(Res: Dst, Op: MatchInfo);
6179 MI.eraseFromParent();
6180}
6181
6182bool CombinerHelper::matchTruncUSatU(MachineInstr &MI,
6183 MachineInstr &MinMI) const {
6184 Register Min = MinMI.getOperand(i: 2).getReg();
6185 Register Val = MinMI.getOperand(i: 1).getReg();
6186 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6187 LLT SrcTy = MRI.getType(Reg: Val);
6188 unsigned NumDstBits = DstTy.getScalarSizeInBits();
6189 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6190 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
6191
6192 if (!LI || !isLegalOrHasFewerElements(
6193 Query: {TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))
6194 return false;
6195 APInt UnsignedMax = APInt::getMaxValue(numBits: NumDstBits).zext(width: NumSrcBits);
6196 return mi_match(R: Min, MRI, P: m_SpecificICstOrSplat(RequestedValue: UnsignedMax)) &&
6197 !mi_match(R: Val, MRI, P: m_GSMax(L: m_Reg(), R: m_Reg()));
6198}
6199
6200bool CombinerHelper::matchTruncUSatUToFPTOUISat(MachineInstr &MI,
6201 MachineInstr &SrcMI) const {
6202 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6203 LLT SrcTy = MRI.getType(Reg: SrcMI.getOperand(i: 1).getReg());
6204
6205 return LI &&
6206 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FPTOUI_SAT, {DstTy, SrcTy}});
6207}
6208
6209bool CombinerHelper::matchFsubToFneg(MachineInstr &MI) const {
6210 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6211
6212 Register LHS = MI.getOperand(i: 1).getReg();
6213 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6214
6215 const auto LHSCst = Ty.isVector()
6216 ? getFConstantSplat(VReg: LHS, MRI, /* allowUndef */ AllowUndef: true)
6217 : getFConstantVRegValWithLookThrough(VReg: LHS, MRI);
6218 if (!LHSCst)
6219 return false;
6220
6221 // Skip fsub_to_fneg when both LHS and RHS are scalar constants (vector FP
6222 // binop constant folding is not supported yet), so constant_fold_fp_binop
6223 // folds these cases.
6224 if (!Ty.isVector() && getConstantFPVRegVal(VReg: LHS, MRI) &&
6225 getConstantFPVRegVal(VReg: MI.getOperand(i: 2).getReg(), MRI))
6226 return false;
6227
6228 // -0.0 is always allowed
6229 if (LHSCst->Value.isNegZero())
6230 return true;
6231
6232 // +0.0 is only allowed if nsz is set.
6233 if (LHSCst->Value.isPosZero())
6234 return MI.getFlag(Flag: MachineInstr::FmNsz);
6235
6236 return false;
6237}
6238
6239/// Checks if \p MI is TargetOpcode::G_FMUL and contractable either
6240/// due to global flags or MachineInstr flags.
6241static bool isContractableFMul(MachineInstr &MI, bool AllowFusionGlobally) {
6242 if (MI.getOpcode() != TargetOpcode::G_FMUL)
6243 return false;
6244 return AllowFusionGlobally || MI.getFlag(Flag: MachineInstr::MIFlag::FmContract);
6245}
6246
6247static bool hasMoreUses(const MachineInstr &MI0, const MachineInstr &MI1,
6248 const MachineRegisterInfo &MRI) {
6249 return std::distance(first: MRI.use_instr_nodbg_begin(RegNo: MI0.getOperand(i: 0).getReg()),
6250 last: MRI.use_instr_nodbg_end()) >
6251 std::distance(first: MRI.use_instr_nodbg_begin(RegNo: MI1.getOperand(i: 0).getReg()),
6252 last: MRI.use_instr_nodbg_end());
6253}
6254
6255bool CombinerHelper::canCombineFMadOrFMA(MachineInstr &MI,
6256 bool &AllowFusionGlobally,
6257 bool &HasFMAD, bool &Aggressive,
6258 bool CanReassociate) const {
6259
6260 auto *MF = MI.getMF();
6261 const auto &TLI = *MF->getSubtarget().getTargetLowering();
6262 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6263
6264 if (CanReassociate && !MI.getFlag(Flag: MachineInstr::MIFlag::FmReassoc))
6265 return false;
6266
6267 // Floating-point multiply-add with intermediate rounding.
6268 HasFMAD = (!isPreLegalize() && TLI.isFMADLegal(MI, Ty: DstType));
6269 // Floating-point multiply-add without intermediate rounding.
6270 bool HasFMA = TLI.isFMAFasterThanFMulAndFAdd(MF: *MF, DstType) &&
6271 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FMA, {DstType}});
6272 // No valid opcode, do not combine.
6273 if (!HasFMAD && !HasFMA)
6274 return false;
6275
6276 // FMAD (with intermediate rounding) is always safe to form; FMA requires the
6277 // contract fast-math flag.
6278 AllowFusionGlobally = HasFMAD;
6279 // If the addition is not contractable, do not combine.
6280 if (!AllowFusionGlobally && !MI.getFlag(Flag: MachineInstr::MIFlag::FmContract))
6281 return false;
6282
6283 Aggressive = TLI.enableAggressiveFMAFusion(Ty: DstType);
6284 return true;
6285}
6286
6287bool CombinerHelper::matchCombineFAddFMulToFMadOrFMA(
6288 MachineInstr &MI,
6289 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6290 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6291
6292 bool AllowFusionGlobally, HasFMAD, Aggressive;
6293 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6294 return false;
6295
6296 Register Op1 = MI.getOperand(i: 1).getReg();
6297 Register Op2 = MI.getOperand(i: 2).getReg();
6298 MachineInstr *Op1Def, *Op2Def;
6299 if (!mi_match(R: Op1, MRI, P: m_MInstr(MI&: Op1Def)) ||
6300 !mi_match(R: Op2, MRI, P: m_MInstr(MI&: Op2Def)))
6301 return false;
6302 DefinitionAndSourceRegister LHS = {.MI: Op1Def, .Reg: Op1};
6303 DefinitionAndSourceRegister RHS = {.MI: Op2Def, .Reg: Op2};
6304 unsigned PreferredFusedOpcode =
6305 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6306
6307 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
6308 // prefer to fold the multiply with fewer uses.
6309 if (Aggressive && isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6310 isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally)) {
6311 if (hasMoreUses(MI0: *LHS.MI, MI1: *RHS.MI, MRI))
6312 std::swap(a&: LHS, b&: RHS);
6313 }
6314
6315 // fold (fadd (fmul x, y), z) -> (fma x, y, z)
6316 if (isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6317 (Aggressive || MRI.hasOneNonDBGUse(RegNo: LHS.Reg))) {
6318 unsigned Flags = MI.getFlags() & LHS.MI->getFlags();
6319 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6320 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6321 SrcOps: {LHS.MI->getOperand(i: 1).getReg(),
6322 LHS.MI->getOperand(i: 2).getReg(), RHS.Reg},
6323 Flags);
6324 };
6325 return true;
6326 }
6327
6328 // fold (fadd x, (fmul y, z)) -> (fma y, z, x)
6329 if (isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally) &&
6330 (Aggressive || MRI.hasOneNonDBGUse(RegNo: RHS.Reg))) {
6331 unsigned Flags = MI.getFlags() & RHS.MI->getFlags();
6332 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6333 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6334 SrcOps: {RHS.MI->getOperand(i: 1).getReg(),
6335 RHS.MI->getOperand(i: 2).getReg(), LHS.Reg},
6336 Flags);
6337 };
6338 return true;
6339 }
6340
6341 return false;
6342}
6343
6344bool CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMA(
6345 MachineInstr &MI,
6346 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6347 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6348
6349 bool AllowFusionGlobally, HasFMAD, Aggressive;
6350 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6351 return false;
6352
6353 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
6354 Register Op1 = MI.getOperand(i: 1).getReg();
6355 Register Op2 = MI.getOperand(i: 2).getReg();
6356 MachineInstr *Op1Def, *Op2Def;
6357 if (!mi_match(R: Op1, MRI, P: m_MInstr(MI&: Op1Def)) ||
6358 !mi_match(R: Op2, MRI, P: m_MInstr(MI&: Op2Def)))
6359 return false;
6360 DefinitionAndSourceRegister LHS = {.MI: Op1Def, .Reg: Op1};
6361 DefinitionAndSourceRegister RHS = {.MI: Op2Def, .Reg: Op2};
6362 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6363
6364 unsigned PreferredFusedOpcode =
6365 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6366
6367 MachineInstr *LHSFpExtSrc;
6368 bool LHSContractable =
6369 mi_match(R: LHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: LHSFpExtSrc))) &&
6370 isContractableFMul(MI&: *LHSFpExtSrc, AllowFusionGlobally) &&
6371 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6372 SrcTy: MRI.getType(Reg: LHSFpExtSrc->getOperand(i: 1).getReg()));
6373 MachineInstr *RHSFpExtSrc;
6374 bool RHSContractable =
6375 mi_match(R: RHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: RHSFpExtSrc))) &&
6376 isContractableFMul(MI&: *RHSFpExtSrc, AllowFusionGlobally) &&
6377 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6378 SrcTy: MRI.getType(Reg: RHSFpExtSrc->getOperand(i: 1).getReg()));
6379
6380 // fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
6381 if (LHSContractable || RHSContractable) {
6382 // Ensure that the contractable fmul with the fewest uses (if both are
6383 // contractable) is the LHS operand.
6384 if (!LHSContractable ||
6385 (RHSContractable && hasMoreUses(MI0: *LHSFpExtSrc, MI1: *RHSFpExtSrc, MRI))) {
6386 std::swap(a&: LHS, b&: RHS);
6387 LHSFpExtSrc = RHSFpExtSrc;
6388 }
6389
6390 unsigned Flags = MI.getFlags() & LHSFpExtSrc->getFlags();
6391 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6392 auto FpExtX = B.buildFPExt(Res: DstType, Op: LHSFpExtSrc->getOperand(i: 1).getReg());
6393 auto FpExtY = B.buildFPExt(Res: DstType, Op: LHSFpExtSrc->getOperand(i: 2).getReg());
6394 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6395 SrcOps: {FpExtX.getReg(Idx: 0), FpExtY.getReg(Idx: 0), RHS.Reg}, Flags);
6396 };
6397 return true;
6398 }
6399
6400 return false;
6401}
6402
6403bool CombinerHelper::matchCombineFAddFMAFMulToFMadOrFMA(
6404 MachineInstr &MI,
6405 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6406 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6407
6408 bool AllowFusionGlobally, HasFMAD, Aggressive;
6409 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive, CanReassociate: true))
6410 return false;
6411
6412 Register Op1 = MI.getOperand(i: 1).getReg();
6413 Register Op2 = MI.getOperand(i: 2).getReg();
6414 MachineInstr *Op1Def, *Op2Def;
6415 if (!mi_match(R: Op1, MRI, P: m_MInstr(MI&: Op1Def)) ||
6416 !mi_match(R: Op2, MRI, P: m_MInstr(MI&: Op2Def)))
6417 return false;
6418 DefinitionAndSourceRegister LHS = {.MI: Op1Def, .Reg: Op1};
6419 DefinitionAndSourceRegister RHS = {.MI: Op2Def, .Reg: Op2};
6420 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6421
6422 unsigned PreferredFusedOpcode =
6423 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6424
6425 MachineInstr *FMA = nullptr;
6426 Register Z;
6427 // fold (fadd (fma x, y, (fmul u, v)), z) -> (fma x, y, (fma u, v, z))
6428 if (LHS.MI->getOpcode() == PreferredFusedOpcode &&
6429 mi_match(R: LHS.MI->getOperand(i: 3).getReg(), MRI,
6430 P: m_GFMul(L: m_Reg(), R: m_Reg())) &&
6431 MRI.hasOneNonDBGUse(RegNo: LHS.MI->getOperand(i: 0).getReg()) &&
6432 MRI.hasOneNonDBGUse(RegNo: LHS.MI->getOperand(i: 3).getReg())) {
6433 FMA = LHS.MI;
6434 Z = RHS.Reg;
6435 }
6436 // fold (fadd z, (fma x, y, (fmul u, v))) -> (fma x, y, (fma u, v, z))
6437 else if (RHS.MI->getOpcode() == PreferredFusedOpcode &&
6438 mi_match(R: RHS.MI->getOperand(i: 3).getReg(), MRI,
6439 P: m_GFMul(L: m_Reg(), R: m_Reg())) &&
6440 MRI.hasOneNonDBGUse(RegNo: RHS.MI->getOperand(i: 0).getReg()) &&
6441 MRI.hasOneNonDBGUse(RegNo: RHS.MI->getOperand(i: 3).getReg())) {
6442 Z = LHS.Reg;
6443 FMA = RHS.MI;
6444 }
6445
6446 if (FMA) {
6447 MachineInstr *FMulMI;
6448 if (!mi_match(R: FMA->getOperand(i: 3).getReg(), MRI, P: m_MInstr(MI&: FMulMI)))
6449 return false;
6450 Register X = FMA->getOperand(i: 1).getReg();
6451 Register Y = FMA->getOperand(i: 2).getReg();
6452 Register U = FMulMI->getOperand(i: 1).getReg();
6453 Register V = FMulMI->getOperand(i: 2).getReg();
6454 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6455 unsigned OuterFlags = MI.getFlags() & FMA->getFlags();
6456
6457 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6458 Register InnerFMA = MRI.createGenericVirtualRegister(Ty: DstTy);
6459 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {InnerFMA}, SrcOps: {U, V, Z}, Flags: InnerFlags);
6460 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6461 SrcOps: {X, Y, InnerFMA}, Flags: OuterFlags);
6462 };
6463 return true;
6464 }
6465
6466 return false;
6467}
6468
6469bool CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMAAggressive(
6470 MachineInstr &MI,
6471 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6472 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6473
6474 bool AllowFusionGlobally, HasFMAD, Aggressive;
6475 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6476 return false;
6477
6478 if (!Aggressive)
6479 return false;
6480
6481 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
6482 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6483 Register Op1 = MI.getOperand(i: 1).getReg();
6484 Register Op2 = MI.getOperand(i: 2).getReg();
6485 MachineInstr *Op1Def, *Op2Def;
6486 if (!mi_match(R: Op1, MRI, P: m_MInstr(MI&: Op1Def)) ||
6487 !mi_match(R: Op2, MRI, P: m_MInstr(MI&: Op2Def)))
6488 return false;
6489 DefinitionAndSourceRegister LHS = {.MI: Op1Def, .Reg: Op1};
6490 DefinitionAndSourceRegister RHS = {.MI: Op2Def, .Reg: Op2};
6491
6492 unsigned PreferredFusedOpcode =
6493 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6494
6495 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
6496 // prefer to fold the multiply with fewer uses.
6497 if (Aggressive && isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6498 isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally)) {
6499 if (hasMoreUses(MI0: *LHS.MI, MI1: *RHS.MI, MRI))
6500 std::swap(a&: LHS, b&: RHS);
6501 }
6502
6503 // Builds: (fma x, y, (fma (fpext u), (fpext v), z))
6504 auto buildMatchInfo = [=, &MI](Register U, Register V, Register Z, Register X,
6505 Register Y, unsigned InnerFlags,
6506 unsigned OuterFlags, MachineIRBuilder &B) {
6507 Register FpExtU = B.buildFPExt(Res: DstType, Op: U).getReg(Idx: 0);
6508 Register FpExtV = B.buildFPExt(Res: DstType, Op: V).getReg(Idx: 0);
6509 Register InnerFMA = B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {DstType},
6510 SrcOps: {FpExtU, FpExtV, Z}, Flags: InnerFlags)
6511 .getReg(Idx: 0);
6512 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6513 SrcOps: {X, Y, InnerFMA}, Flags: OuterFlags);
6514 };
6515
6516 MachineInstr *FMulMI, *FMAMI;
6517 // fold (fadd (fma x, y, (fpext (fmul u, v))), z)
6518 // -> (fma x, y, (fma (fpext u), (fpext v), z))
6519 if (LHS.MI->getOpcode() == PreferredFusedOpcode &&
6520 mi_match(R: LHS.MI->getOperand(i: 3).getReg(), MRI,
6521 P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6522 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6523 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6524 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
6525 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6526 unsigned OuterFlags = MI.getFlags() & LHS.MI->getFlags();
6527 MatchInfo = [=](MachineIRBuilder &B) {
6528 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6529 FMulMI->getOperand(i: 2).getReg(), RHS.Reg,
6530 LHS.MI->getOperand(i: 1).getReg(),
6531 LHS.MI->getOperand(i: 2).getReg(), InnerFlags, OuterFlags, B);
6532 };
6533 return true;
6534 }
6535
6536 // fold (fadd (fpext (fma x, y, (fmul u, v))), z)
6537 // -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z))
6538 // FIXME: This turns two single-precision and one double-precision
6539 // operation into two double-precision operations, which might not be
6540 // interesting for all targets, especially GPUs.
6541 if (mi_match(R: LHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMAMI))) &&
6542 FMAMI->getOpcode() == PreferredFusedOpcode) {
6543 MachineInstr *FMulMI;
6544 if (!mi_match(R: FMAMI->getOperand(i: 3).getReg(), MRI, P: m_MInstr(MI&: FMulMI)))
6545 return false;
6546 if (isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6547 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6548 SrcTy: MRI.getType(Reg: FMAMI->getOperand(i: 0).getReg()))) {
6549 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6550 unsigned OuterFlags = MI.getFlags() & FMAMI->getFlags();
6551 MatchInfo = [=](MachineIRBuilder &B) {
6552 Register X = FMAMI->getOperand(i: 1).getReg();
6553 Register Y = FMAMI->getOperand(i: 2).getReg();
6554 X = B.buildFPExt(Res: DstType, Op: X).getReg(Idx: 0);
6555 Y = B.buildFPExt(Res: DstType, Op: Y).getReg(Idx: 0);
6556 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6557 FMulMI->getOperand(i: 2).getReg(), RHS.Reg, X, Y,
6558 InnerFlags, OuterFlags, B);
6559 };
6560
6561 return true;
6562 }
6563 }
6564
6565 // fold (fadd z, (fma x, y, (fpext (fmul u, v)))
6566 // -> (fma x, y, (fma (fpext u), (fpext v), z))
6567 if (RHS.MI->getOpcode() == PreferredFusedOpcode &&
6568 mi_match(R: RHS.MI->getOperand(i: 3).getReg(), MRI,
6569 P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6570 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6571 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6572 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
6573 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6574 unsigned OuterFlags = MI.getFlags() & RHS.MI->getFlags();
6575 MatchInfo = [=](MachineIRBuilder &B) {
6576 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6577 FMulMI->getOperand(i: 2).getReg(), LHS.Reg,
6578 RHS.MI->getOperand(i: 1).getReg(),
6579 RHS.MI->getOperand(i: 2).getReg(), InnerFlags, OuterFlags, B);
6580 };
6581 return true;
6582 }
6583
6584 // fold (fadd z, (fpext (fma x, y, (fmul u, v)))
6585 // -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z))
6586 // FIXME: This turns two single-precision and one double-precision
6587 // operation into two double-precision operations, which might not be
6588 // interesting for all targets, especially GPUs.
6589 if (mi_match(R: RHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMAMI))) &&
6590 FMAMI->getOpcode() == PreferredFusedOpcode) {
6591 MachineInstr *FMulMI;
6592 if (!mi_match(R: FMAMI->getOperand(i: 3).getReg(), MRI, P: m_MInstr(MI&: FMulMI)))
6593 return false;
6594 if (isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6595 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6596 SrcTy: MRI.getType(Reg: FMAMI->getOperand(i: 0).getReg()))) {
6597 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6598 unsigned OuterFlags = MI.getFlags() & FMAMI->getFlags();
6599 MatchInfo = [=](MachineIRBuilder &B) {
6600 Register X = FMAMI->getOperand(i: 1).getReg();
6601 Register Y = FMAMI->getOperand(i: 2).getReg();
6602 X = B.buildFPExt(Res: DstType, Op: X).getReg(Idx: 0);
6603 Y = B.buildFPExt(Res: DstType, Op: Y).getReg(Idx: 0);
6604 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6605 FMulMI->getOperand(i: 2).getReg(), LHS.Reg, X, Y,
6606 InnerFlags, OuterFlags, B);
6607 };
6608 return true;
6609 }
6610 }
6611
6612 return false;
6613}
6614
6615bool CombinerHelper::matchCombineFSubFMulToFMadOrFMA(
6616 MachineInstr &MI,
6617 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6618 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6619
6620 bool AllowFusionGlobally, HasFMAD, Aggressive;
6621 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6622 return false;
6623
6624 Register Op1 = MI.getOperand(i: 1).getReg();
6625 Register Op2 = MI.getOperand(i: 2).getReg();
6626 MachineInstr *Op1Def, *Op2Def;
6627 if (!mi_match(R: Op1, MRI, P: m_MInstr(MI&: Op1Def)) ||
6628 !mi_match(R: Op2, MRI, P: m_MInstr(MI&: Op2Def)))
6629 return false;
6630 DefinitionAndSourceRegister LHS = {.MI: Op1Def, .Reg: Op1};
6631 DefinitionAndSourceRegister RHS = {.MI: Op2Def, .Reg: Op2};
6632 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6633
6634 // If we have two choices trying to fold (fsub (fmul u, v), (fmul x, y)),
6635 // prefer to fold the multiply with fewer uses.
6636 int FirstMulHasFewerUses = true;
6637 if (isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6638 isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally) &&
6639 hasMoreUses(MI0: *LHS.MI, MI1: *RHS.MI, MRI))
6640 FirstMulHasFewerUses = false;
6641
6642 unsigned PreferredFusedOpcode =
6643 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6644
6645 // fold (fsub (fmul x, y), z) -> (fma x, y, -z)
6646 if (FirstMulHasFewerUses &&
6647 (isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6648 (Aggressive || MRI.hasOneNonDBGUse(RegNo: LHS.Reg)))) {
6649 unsigned Flags = MI.getFlags() & LHS.MI->getFlags();
6650 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6651 Register NegZ = B.buildFNeg(Dst: DstTy, Src0: RHS.Reg).getReg(Idx: 0);
6652 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6653 SrcOps: {LHS.MI->getOperand(i: 1).getReg(),
6654 LHS.MI->getOperand(i: 2).getReg(), NegZ},
6655 Flags);
6656 };
6657 return true;
6658 }
6659 // fold (fsub x, (fmul y, z)) -> (fma -y, z, x)
6660 else if ((isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally) &&
6661 (Aggressive || MRI.hasOneNonDBGUse(RegNo: RHS.Reg)))) {
6662 unsigned Flags = MI.getFlags() & RHS.MI->getFlags();
6663 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6664 Register NegY =
6665 B.buildFNeg(Dst: DstTy, Src0: RHS.MI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6666 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6667 SrcOps: {NegY, RHS.MI->getOperand(i: 2).getReg(), LHS.Reg}, Flags);
6668 };
6669 return true;
6670 }
6671
6672 return false;
6673}
6674
6675bool CombinerHelper::matchCombineFSubFNegFMulToFMadOrFMA(
6676 MachineInstr &MI,
6677 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6678 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6679
6680 bool AllowFusionGlobally, HasFMAD, Aggressive;
6681 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6682 return false;
6683
6684 Register LHSReg = MI.getOperand(i: 1).getReg();
6685 Register RHSReg = MI.getOperand(i: 2).getReg();
6686 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6687
6688 unsigned PreferredFusedOpcode =
6689 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6690
6691 MachineInstr *FMulMI;
6692 // fold (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
6693 if (mi_match(R: LHSReg, MRI, P: m_GFNeg(Src: m_MInstr(MI&: FMulMI))) &&
6694 (Aggressive || (MRI.hasOneNonDBGUse(RegNo: LHSReg) &&
6695 MRI.hasOneNonDBGUse(RegNo: FMulMI->getOperand(i: 0).getReg()))) &&
6696 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally)) {
6697 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6698 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6699 Register NegX =
6700 B.buildFNeg(Dst: DstTy, Src0: FMulMI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6701 Register NegZ = B.buildFNeg(Dst: DstTy, Src0: RHSReg).getReg(Idx: 0);
6702 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6703 SrcOps: {NegX, FMulMI->getOperand(i: 2).getReg(), NegZ}, Flags);
6704 };
6705 return true;
6706 }
6707
6708 // fold (fsub x, (fneg (fmul, y, z))) -> (fma y, z, x)
6709 if (mi_match(R: RHSReg, MRI, P: m_GFNeg(Src: m_MInstr(MI&: FMulMI))) &&
6710 (Aggressive || (MRI.hasOneNonDBGUse(RegNo: RHSReg) &&
6711 MRI.hasOneNonDBGUse(RegNo: FMulMI->getOperand(i: 0).getReg()))) &&
6712 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally)) {
6713 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6714 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6715 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6716 SrcOps: {FMulMI->getOperand(i: 1).getReg(),
6717 FMulMI->getOperand(i: 2).getReg(), LHSReg},
6718 Flags);
6719 };
6720 return true;
6721 }
6722
6723 return false;
6724}
6725
6726bool CombinerHelper::matchCombineFSubFpExtFMulToFMadOrFMA(
6727 MachineInstr &MI,
6728 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6729 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6730
6731 bool AllowFusionGlobally, HasFMAD, Aggressive;
6732 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6733 return false;
6734
6735 Register LHSReg = MI.getOperand(i: 1).getReg();
6736 Register RHSReg = MI.getOperand(i: 2).getReg();
6737 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6738
6739 unsigned PreferredFusedOpcode =
6740 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6741
6742 MachineInstr *FMulMI;
6743 // fold (fsub (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), (fneg z))
6744 if (mi_match(R: LHSReg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6745 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6746 (Aggressive || MRI.hasOneNonDBGUse(RegNo: LHSReg))) {
6747 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6748 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6749 Register FpExtX =
6750 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6751 Register FpExtY =
6752 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 2).getReg()).getReg(Idx: 0);
6753 Register NegZ = B.buildFNeg(Dst: DstTy, Src0: RHSReg).getReg(Idx: 0);
6754 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6755 SrcOps: {FpExtX, FpExtY, NegZ}, Flags);
6756 };
6757 return true;
6758 }
6759
6760 // fold (fsub x, (fpext (fmul y, z))) -> (fma (fneg (fpext y)), (fpext z), x)
6761 if (mi_match(R: RHSReg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6762 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6763 (Aggressive || MRI.hasOneNonDBGUse(RegNo: RHSReg))) {
6764 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6765 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6766 Register FpExtY =
6767 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6768 Register NegY = B.buildFNeg(Dst: DstTy, Src0: FpExtY).getReg(Idx: 0);
6769 Register FpExtZ =
6770 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 2).getReg()).getReg(Idx: 0);
6771 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6772 SrcOps: {NegY, FpExtZ, LHSReg}, Flags);
6773 };
6774 return true;
6775 }
6776
6777 return false;
6778}
6779
6780bool CombinerHelper::matchCombineFSubFpExtFNegFMulToFMadOrFMA(
6781 MachineInstr &MI,
6782 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6783 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6784
6785 bool AllowFusionGlobally, HasFMAD, Aggressive;
6786 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6787 return false;
6788
6789 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
6790 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6791 Register LHSReg = MI.getOperand(i: 1).getReg();
6792 Register RHSReg = MI.getOperand(i: 2).getReg();
6793
6794 unsigned PreferredFusedOpcode =
6795 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6796
6797 auto buildMatchInfo = [=](Register Dst, Register X, Register Y, Register Z,
6798 unsigned Flags, MachineIRBuilder &B) {
6799 Register FpExtX = B.buildFPExt(Res: DstTy, Op: X).getReg(Idx: 0);
6800 Register FpExtY = B.buildFPExt(Res: DstTy, Op: Y).getReg(Idx: 0);
6801 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {Dst}, SrcOps: {FpExtX, FpExtY, Z}, Flags);
6802 };
6803
6804 MachineInstr *FMulMI;
6805 // fold (fsub (fpext (fneg (fmul x, y))), z) ->
6806 // (fneg (fma (fpext x), (fpext y), z))
6807 // fold (fsub (fneg (fpext (fmul x, y))), z) ->
6808 // (fneg (fma (fpext x), (fpext y), z))
6809 if ((mi_match(R: LHSReg, MRI, P: m_GFPExt(Src: m_GFNeg(Src: m_MInstr(MI&: FMulMI)))) ||
6810 mi_match(R: LHSReg, MRI, P: m_GFNeg(Src: m_GFPExt(Src: m_MInstr(MI&: FMulMI))))) &&
6811 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6812 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstTy,
6813 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
6814 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6815 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6816 Register FMAReg = MRI.createGenericVirtualRegister(Ty: DstTy);
6817 buildMatchInfo(FMAReg, FMulMI->getOperand(i: 1).getReg(),
6818 FMulMI->getOperand(i: 2).getReg(), RHSReg, Flags, B);
6819 B.buildFNeg(Dst: MI.getOperand(i: 0).getReg(), Src0: FMAReg);
6820 };
6821 return true;
6822 }
6823
6824 // fold (fsub x, (fpext (fneg (fmul y, z)))) -> (fma (fpext y), (fpext z), x)
6825 // fold (fsub x, (fneg (fpext (fmul y, z)))) -> (fma (fpext y), (fpext z), x)
6826 if ((mi_match(R: RHSReg, MRI, P: m_GFPExt(Src: m_GFNeg(Src: m_MInstr(MI&: FMulMI)))) ||
6827 mi_match(R: RHSReg, MRI, P: m_GFNeg(Src: m_GFPExt(Src: m_MInstr(MI&: FMulMI))))) &&
6828 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6829 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstTy,
6830 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
6831 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6832 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6833 buildMatchInfo(MI.getOperand(i: 0).getReg(), FMulMI->getOperand(i: 1).getReg(),
6834 FMulMI->getOperand(i: 2).getReg(), LHSReg, Flags, B);
6835 };
6836 return true;
6837 }
6838
6839 return false;
6840}
6841
6842// Combine multiple FDIVs with the same divisor into multiple FMULs by the
6843// reciprocal.
6844// E.g., (a / Y; b / Y;) -> (recip = 1.0 / Y; a * recip; b * recip)
6845bool CombinerHelper::matchRepeatedFPDivisor(
6846 MachineInstr &MI, SmallVector<MachineInstr *> &MatchInfo) const {
6847 assert(MI.getOpcode() == TargetOpcode::G_FDIV);
6848
6849 Register X = MI.getOperand(i: 1).getReg();
6850 Register Y = MI.getOperand(i: 2).getReg();
6851
6852 if (!MI.getFlag(Flag: MachineInstr::MIFlag::FmArcp))
6853 return false;
6854
6855 auto IsOne = [this](Register X) {
6856 auto N0CFP = isConstantOrConstantSplatVectorFP(Def: X, MRI);
6857 return N0CFP && (N0CFP->isOne() || N0CFP->isMinusOne());
6858 };
6859
6860 // Skip if current node is a reciprocal/fneg-reciprocal.
6861 if (IsOne(X))
6862 return false;
6863
6864 // Exit early if the target does not want this transform or if there can't
6865 // possibly be enough uses of the divisor to make the transform worthwhile.
6866 unsigned MinUses = getTargetLowering().combineRepeatedFPDivisors();
6867 if (!MinUses)
6868 return false;
6869
6870 // Find all FDIV users of the same divisor. For the moment we limit all
6871 // instructions to a single BB and use the first Instr in MatchInfo as the
6872 // dominating position.
6873 MatchInfo.push_back(Elt: &MI);
6874 for (auto &U : MRI.use_nodbg_instructions(Reg: Y)) {
6875 if (&U == &MI || U.getParent() != MI.getParent())
6876 continue;
6877 if (U.getOpcode() == TargetOpcode::G_FDIV &&
6878 U.getOperand(i: 2).getReg() == Y && U.getOperand(i: 1).getReg() != Y &&
6879 !IsOne(U.getOperand(i: 1).getReg())) {
6880 // This division is eligible for optimization only if global unsafe math
6881 // is enabled or if this division allows reciprocal formation.
6882 if (U.getFlag(Flag: MachineInstr::MIFlag::FmArcp)) {
6883 MatchInfo.push_back(Elt: &U);
6884 if (dominates(DefMI: U, UseMI: *MatchInfo[0]))
6885 std::swap(a&: MatchInfo[0], b&: MatchInfo.back());
6886 }
6887 }
6888 }
6889
6890 // Now that we have the actual number of divisor uses, make sure it meets
6891 // the minimum threshold specified by the target.
6892 return MatchInfo.size() >= MinUses;
6893}
6894
6895void CombinerHelper::applyRepeatedFPDivisor(
6896 SmallVector<MachineInstr *> &MatchInfo) const {
6897 // Generate the new div at the position of the first instruction, that we have
6898 // ensured will dominate all other instructions.
6899 Builder.setInsertPt(MBB&: *MatchInfo[0]->getParent(), II: MatchInfo[0]);
6900 LLT Ty = MRI.getType(Reg: MatchInfo[0]->getOperand(i: 0).getReg());
6901 auto Div = Builder.buildFDiv(Dst: Ty, Src0: Builder.buildFConstant(Res: Ty, Val: 1.0),
6902 Src1: MatchInfo[0]->getOperand(i: 2).getReg(),
6903 Flags: MatchInfo[0]->getFlags());
6904
6905 // Replace all found div's with fmul instructions.
6906 for (MachineInstr *MI : MatchInfo) {
6907 Builder.setInsertPt(MBB&: *MI->getParent(), II: MI);
6908 Builder.buildFMul(Dst: MI->getOperand(i: 0).getReg(), Src0: MI->getOperand(i: 1).getReg(),
6909 Src1: Div->getOperand(i: 0).getReg(), Flags: MI->getFlags());
6910 MI->eraseFromParent();
6911 }
6912}
6913
6914bool CombinerHelper::matchBuildVectorIdentityFold(MachineInstr &MI,
6915 Register &MatchInfo) const {
6916 // This combine folds the following patterns:
6917 //
6918 // G_BUILD_VECTOR_TRUNC (G_BITCAST(x), G_LSHR(G_BITCAST(x), k))
6919 // G_BUILD_VECTOR(G_TRUNC(G_BITCAST(x)), G_TRUNC(G_LSHR(G_BITCAST(x), k)))
6920 // into
6921 // x
6922 // if
6923 // k == sizeof(VecEltTy)/2
6924 // type(x) == type(dst)
6925 //
6926 // G_BUILD_VECTOR(G_TRUNC(G_BITCAST(x)), undef)
6927 // into
6928 // x
6929 // if
6930 // type(x) == type(dst)
6931
6932 LLT DstVecTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6933 LLT DstEltTy = DstVecTy.getElementType();
6934
6935 Register Lo, Hi;
6936
6937 if (mi_match(
6938 MI, MRI,
6939 P: m_GBuildVector(L: m_GTrunc(Src: m_GBitcast(Src: m_Reg(R&: Lo))), R: m_GImplicitDef()))) {
6940 MatchInfo = Lo;
6941 return MRI.getType(Reg: MatchInfo) == DstVecTy;
6942 }
6943
6944 std::optional<ValueAndVReg> ShiftAmount;
6945 const auto LoPattern = m_GBitcast(Src: m_Reg(R&: Lo));
6946 const auto HiPattern = m_GLShr(L: m_GBitcast(Src: m_Reg(R&: Hi)), R: m_GCst(ValReg&: ShiftAmount));
6947 if (mi_match(
6948 MI, MRI,
6949 P: m_any_of(preds: m_GBuildVectorTrunc(L: LoPattern, R: HiPattern),
6950 preds: m_GBuildVector(L: m_GTrunc(Src: LoPattern), R: m_GTrunc(Src: HiPattern))))) {
6951 if (Lo == Hi && ShiftAmount->Value == DstEltTy.getSizeInBits()) {
6952 MatchInfo = Lo;
6953 return MRI.getType(Reg: MatchInfo) == DstVecTy;
6954 }
6955 }
6956
6957 return false;
6958}
6959
6960bool CombinerHelper::matchTruncBuildVectorFold(MachineInstr &MI,
6961 Register &MatchInfo) const {
6962 // Replace (G_TRUNC (G_BITCAST (G_BUILD_VECTOR x, y)) with just x
6963 // if type(x) == type(G_TRUNC)
6964 if (!mi_match(R: MI.getOperand(i: 1).getReg(), MRI,
6965 P: m_GBitcast(Src: m_GBuildVector(L: m_Reg(R&: MatchInfo), R: m_Reg()))))
6966 return false;
6967
6968 return MRI.getType(Reg: MatchInfo) == MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6969}
6970
6971bool CombinerHelper::matchTruncLshrBuildVectorFold(MachineInstr &MI,
6972 Register &MatchInfo) const {
6973 // Replace (G_TRUNC (G_LSHR (G_BITCAST (G_BUILD_VECTOR x, y)), K)) with
6974 // y if K == size of vector element type
6975 std::optional<ValueAndVReg> ShiftAmt;
6976 if (!mi_match(R: MI.getOperand(i: 1).getReg(), MRI,
6977 P: m_GLShr(L: m_GBitcast(Src: m_GBuildVector(L: m_Reg(), R: m_Reg(R&: MatchInfo))),
6978 R: m_GCst(ValReg&: ShiftAmt))))
6979 return false;
6980
6981 LLT MatchTy = MRI.getType(Reg: MatchInfo);
6982 return ShiftAmt->Value.getZExtValue() == MatchTy.getSizeInBits() &&
6983 MatchTy == MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6984}
6985
6986unsigned CombinerHelper::getFPMinMaxOpcForSelect(
6987 CmpInst::Predicate Pred, LLT DstTy,
6988 SelectPatternNaNBehaviour VsNaNRetVal) const {
6989 assert(VsNaNRetVal != SelectPatternNaNBehaviour::NOT_APPLICABLE &&
6990 "Expected a NaN behaviour?");
6991 // Choose an opcode based off of legality or the behaviour when one of the
6992 // LHS/RHS may be NaN.
6993 switch (Pred) {
6994 default:
6995 return 0;
6996 case CmpInst::FCMP_UGT:
6997 case CmpInst::FCMP_UGE:
6998 case CmpInst::FCMP_OGT:
6999 case CmpInst::FCMP_OGE:
7000 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
7001 return TargetOpcode::G_FMAXNUM;
7002 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
7003 return TargetOpcode::G_FMAXIMUM;
7004 if (isLegal(Query: {TargetOpcode::G_FMAXNUM, {DstTy}}))
7005 return TargetOpcode::G_FMAXNUM;
7006 if (isLegal(Query: {TargetOpcode::G_FMAXIMUM, {DstTy}}))
7007 return TargetOpcode::G_FMAXIMUM;
7008 return 0;
7009 case CmpInst::FCMP_ULT:
7010 case CmpInst::FCMP_ULE:
7011 case CmpInst::FCMP_OLT:
7012 case CmpInst::FCMP_OLE:
7013 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
7014 return TargetOpcode::G_FMINNUM;
7015 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
7016 return TargetOpcode::G_FMINIMUM;
7017 if (isLegal(Query: {TargetOpcode::G_FMINNUM, {DstTy}}))
7018 return TargetOpcode::G_FMINNUM;
7019 if (!isLegal(Query: {TargetOpcode::G_FMINIMUM, {DstTy}}))
7020 return 0;
7021 return TargetOpcode::G_FMINIMUM;
7022 }
7023}
7024
7025CombinerHelper::SelectPatternNaNBehaviour
7026CombinerHelper::computeRetValAgainstNaN(Register LHS, Register RHS,
7027 bool IsOrderedComparison) const {
7028 bool LHSSafe = VT->isKnownNeverNaN(Val: LHS);
7029 bool RHSSafe = VT->isKnownNeverNaN(Val: RHS);
7030 // Completely unsafe.
7031 if (!LHSSafe && !RHSSafe)
7032 return SelectPatternNaNBehaviour::NOT_APPLICABLE;
7033 if (LHSSafe && RHSSafe)
7034 return SelectPatternNaNBehaviour::RETURNS_ANY;
7035 // An ordered comparison will return false when given a NaN, so it
7036 // returns the RHS.
7037 if (IsOrderedComparison)
7038 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_NAN
7039 : SelectPatternNaNBehaviour::RETURNS_OTHER;
7040 // An unordered comparison will return true when given a NaN, so it
7041 // returns the LHS.
7042 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_OTHER
7043 : SelectPatternNaNBehaviour::RETURNS_NAN;
7044}
7045
7046bool CombinerHelper::matchFPSelectToMinMax(Register Dst, Register Cond,
7047 Register TrueVal, Register FalseVal,
7048 BuildFnTy &MatchInfo) const {
7049 // Match: select (fcmp cond x, y) x, y
7050 // select (fcmp cond x, y) y, x
7051 // And turn it into fminnum/fmaxnum or fmin/fmax based off of the condition.
7052 LLT DstTy = MRI.getType(Reg: Dst);
7053 // Bail out early on pointers, since we'll never want to fold to a min/max.
7054 if (DstTy.isPointer())
7055 return false;
7056 // Match a floating point compare with a less-than/greater-than predicate.
7057 // TODO: Allow multiple users of the compare if they are all selects.
7058 CmpInst::Predicate Pred;
7059 Register CmpLHS, CmpRHS;
7060 if (!mi_match(
7061 R: Cond, MRI,
7062 P: m_OneUse(SP: m_GFCmp(P: m_Pred(P&: Pred), L: m_Reg(R&: CmpLHS), R: m_Reg(R&: CmpRHS)))) ||
7063 CmpInst::isEquality(pred: Pred))
7064 return false;
7065 SelectPatternNaNBehaviour ResWithKnownNaNInfo =
7066 computeRetValAgainstNaN(LHS: CmpLHS, RHS: CmpRHS, IsOrderedComparison: CmpInst::isOrdered(predicate: Pred));
7067 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::NOT_APPLICABLE)
7068 return false;
7069 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
7070 std::swap(a&: CmpLHS, b&: CmpRHS);
7071 Pred = CmpInst::getSwappedPredicate(pred: Pred);
7072 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_NAN)
7073 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_OTHER;
7074 else if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_OTHER)
7075 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_NAN;
7076 }
7077 if (TrueVal != CmpLHS || FalseVal != CmpRHS)
7078 return false;
7079 // Decide what type of max/min this should be based off of the predicate.
7080 unsigned Opc = getFPMinMaxOpcForSelect(Pred, DstTy, VsNaNRetVal: ResWithKnownNaNInfo);
7081 if (!Opc || !isLegal(Query: {Opc, {DstTy}}))
7082 return false;
7083 // Comparisons between signed zero and zero may have different results...
7084 // unless we have fmaximum/fminimum. In that case, we know -0 < 0.
7085 if (Opc != TargetOpcode::G_FMAXIMUM && Opc != TargetOpcode::G_FMINIMUM) {
7086 // We don't know if a comparison between two 0s will give us a consistent
7087 // result. Be conservative and only proceed if at least one side is
7088 // non-zero.
7089 auto KnownNonZeroSide = getFConstantVRegValWithLookThrough(VReg: CmpLHS, MRI);
7090 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero()) {
7091 KnownNonZeroSide = getFConstantVRegValWithLookThrough(VReg: CmpRHS, MRI);
7092 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero())
7093 return false;
7094 }
7095 }
7096 MatchInfo = [=](MachineIRBuilder &B) {
7097 B.buildInstr(Opc, DstOps: {Dst}, SrcOps: {CmpLHS, CmpRHS});
7098 };
7099 return true;
7100}
7101
7102bool CombinerHelper::matchSimplifySelectToMinMax(MachineInstr &MI,
7103 BuildFnTy &MatchInfo) const {
7104 // TODO: Handle integer cases.
7105 assert(MI.getOpcode() == TargetOpcode::G_SELECT);
7106 // Condition may be fed by a truncated compare.
7107 Register Cond = MI.getOperand(i: 1).getReg();
7108 Register MaybeTrunc;
7109 if (mi_match(R: Cond, MRI, P: m_OneUse(SP: m_GTrunc(Src: m_Reg(R&: MaybeTrunc)))))
7110 Cond = MaybeTrunc;
7111 Register Dst = MI.getOperand(i: 0).getReg();
7112 Register TrueVal = MI.getOperand(i: 2).getReg();
7113 Register FalseVal = MI.getOperand(i: 3).getReg();
7114 return matchFPSelectToMinMax(Dst, Cond, TrueVal, FalseVal, MatchInfo);
7115}
7116
7117bool CombinerHelper::matchRedundantBinOpInEquality(MachineInstr &MI,
7118 BuildFnTy &MatchInfo) const {
7119 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
7120 // (X + Y) == X --> Y == 0
7121 // (X + Y) != X --> Y != 0
7122 // (X - Y) == X --> Y == 0
7123 // (X - Y) != X --> Y != 0
7124 // (X ^ Y) == X --> Y == 0
7125 // (X ^ Y) != X --> Y != 0
7126 Register Dst = MI.getOperand(i: 0).getReg();
7127 CmpInst::Predicate Pred;
7128 Register X, Y, OpLHS, OpRHS;
7129 bool MatchedSub = mi_match(
7130 R: Dst, MRI,
7131 P: m_c_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: X), R: m_GSub(L: m_Reg(R&: OpLHS), R: m_Reg(R&: Y))));
7132 if (MatchedSub && X != OpLHS)
7133 return false;
7134 if (!MatchedSub) {
7135 if (!mi_match(R: Dst, MRI,
7136 P: m_c_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: X),
7137 R: m_any_of(preds: m_GAdd(L: m_Reg(R&: OpLHS), R: m_Reg(R&: OpRHS)),
7138 preds: m_GXor(L: m_Reg(R&: OpLHS), R: m_Reg(R&: OpRHS))))))
7139 return false;
7140 Y = X == OpLHS ? OpRHS : X == OpRHS ? OpLHS : Register();
7141 }
7142 MatchInfo = [=](MachineIRBuilder &B) {
7143 auto Zero = B.buildConstant(Res: MRI.getType(Reg: Y), Val: 0);
7144 B.buildICmp(Pred, Res: Dst, Op0: Y, Op1: Zero);
7145 };
7146 return CmpInst::isEquality(pred: Pred) && Y.isValid();
7147}
7148
7149/// Return the minimum useless shift amount that results in complete loss of the
7150/// source value. Return std::nullopt when it cannot determine a value.
7151static std::optional<unsigned>
7152getMinUselessShift(KnownBits ValueKB, unsigned Opcode,
7153 std::optional<int64_t> &Result) {
7154 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR ||
7155 Opcode == TargetOpcode::G_ASHR) &&
7156 "Expect G_SHL, G_LSHR or G_ASHR.");
7157 auto SignificantBits = 0;
7158 switch (Opcode) {
7159 case TargetOpcode::G_SHL:
7160 SignificantBits = ValueKB.countMinTrailingZeros();
7161 Result = 0;
7162 break;
7163 case TargetOpcode::G_LSHR:
7164 Result = 0;
7165 SignificantBits = ValueKB.countMinLeadingZeros();
7166 break;
7167 case TargetOpcode::G_ASHR:
7168 if (ValueKB.isNonNegative()) {
7169 SignificantBits = ValueKB.countMinLeadingZeros();
7170 Result = 0;
7171 } else if (ValueKB.isNegative()) {
7172 SignificantBits = ValueKB.countMinLeadingOnes();
7173 Result = -1;
7174 } else {
7175 // Cannot determine shift result.
7176 Result = std::nullopt;
7177 }
7178 break;
7179 default:
7180 break;
7181 }
7182 return ValueKB.getBitWidth() - SignificantBits;
7183}
7184
7185bool CombinerHelper::matchShiftsTooBig(MachineInstr &MI,
7186 BuildFnTy &MatchInfo) const {
7187 Register Dst = MI.getOperand(i: 0).getReg();
7188 Register ShiftVal = MI.getOperand(i: 1).getReg();
7189 Register ShiftReg = MI.getOperand(i: 2).getReg();
7190 LLT ResTy = MRI.getType(Reg: Dst);
7191 std::optional<int64_t> Result;
7192 auto IsShiftTooBig = [&](const Constant *C) {
7193 auto *CI = dyn_cast<ConstantInt>(Val: C);
7194 if (!CI)
7195 return false;
7196 if (CI->uge(Num: ResTy.getScalarSizeInBits())) {
7197 Result = std::nullopt;
7198 return true;
7199 }
7200 auto OptMaxUsefulShift =
7201 getMinUselessShift(ValueKB: VT->getKnownBits(R: ShiftVal), Opcode: MI.getOpcode(), Result);
7202 return OptMaxUsefulShift && CI->uge(Num: *OptMaxUsefulShift);
7203 };
7204 if (!matchUnaryPredicate(MRI, Reg: ShiftReg, Match: IsShiftTooBig))
7205 return false;
7206
7207 if (!Result) {
7208 MatchInfo = [Dst](MachineIRBuilder &B) { B.buildUndef(Res: Dst); };
7209 return true;
7210 }
7211
7212 int64_t Val = *Result;
7213 MatchInfo = [Dst, Val](MachineIRBuilder &B) { B.buildConstant(Res: Dst, Val); };
7214 return true;
7215}
7216
7217bool CombinerHelper::matchCommuteConstantToRHS(MachineInstr &MI) const {
7218 unsigned LHSOpndIdx = 1;
7219 unsigned RHSOpndIdx = 2;
7220 switch (MI.getOpcode()) {
7221 case TargetOpcode::G_UADDO:
7222 case TargetOpcode::G_SADDO:
7223 case TargetOpcode::G_UMULO:
7224 case TargetOpcode::G_SMULO:
7225 LHSOpndIdx = 2;
7226 RHSOpndIdx = 3;
7227 break;
7228 default:
7229 break;
7230 }
7231 Register LHS = MI.getOperand(i: LHSOpndIdx).getReg();
7232 Register RHS = MI.getOperand(i: RHSOpndIdx).getReg();
7233 MachineInstr *LHSDef, *RHSDef;
7234 if (!mi_match(R: LHS, MRI, P: m_MInstr(MI&: LHSDef)) ||
7235 !mi_match(R: RHS, MRI, P: m_MInstr(MI&: RHSDef)))
7236 return false;
7237
7238 if (!isConstantOrConstantVector(MI: *LHSDef, MRI, /*AllowFP=*/false,
7239 /*AllowOpaqueConstants=*/false)) {
7240 // Skip commuting if LHS is not a constant or a vector of constants. But,
7241 // LHS may be a G_CONSTANT_FOLD_BARRIER. If so we commute as long as we
7242 // don't already have a constant on the RHS.
7243 if (LHSDef->getOpcode() != TargetOpcode::G_CONSTANT_FOLD_BARRIER)
7244 return false;
7245 }
7246 // Commute as long as RHS is not a constant, a vector of constants or
7247 // G_CONSTANT_FOLD_BARRIER.
7248 return RHSDef->getOpcode() != TargetOpcode::G_CONSTANT_FOLD_BARRIER &&
7249 !isConstantOrConstantVector(MI: *RHSDef, MRI, /*AllowFP=*/false,
7250 /*AllowOpaqueConstants=*/false);
7251}
7252
7253bool CombinerHelper::matchCommuteFPConstantToRHS(MachineInstr &MI) const {
7254 Register LHS = MI.getOperand(i: 1).getReg();
7255 Register RHS = MI.getOperand(i: 2).getReg();
7256 std::optional<FPValueAndVReg> ValAndVReg;
7257 if (!mi_match(R: LHS, MRI, P: m_GFCstOrSplat(FPValReg&: ValAndVReg)))
7258 return false;
7259 return !mi_match(R: RHS, MRI, P: m_GFCstOrSplat(FPValReg&: ValAndVReg));
7260}
7261
7262bool CombinerHelper::isOneOrOneSplat(Register Src, bool AllowUndefs) const {
7263 LLT SrcTy = MRI.getType(Reg: Src);
7264 if (SrcTy.isFixedVector())
7265 return isConstantSplatVector(Src, SplatValue: 1, AllowUndefs);
7266 if (SrcTy.isScalar()) {
7267 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Reg: Src, MRI) != nullptr)
7268 return true;
7269 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7270 return IConstant && IConstant->Value == 1;
7271 }
7272 return false; // scalable vector
7273}
7274
7275bool CombinerHelper::isZeroOrZeroSplat(Register Src, bool AllowUndefs) const {
7276 LLT SrcTy = MRI.getType(Reg: Src);
7277 if (SrcTy.isFixedVector())
7278 return isConstantSplatVector(Src, SplatValue: 0, AllowUndefs);
7279 if (SrcTy.isScalar()) {
7280 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Reg: Src, MRI) != nullptr)
7281 return true;
7282 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7283 return IConstant && IConstant->Value == 0;
7284 }
7285 return false; // scalable vector
7286}
7287
7288// Ignores COPYs during conformance checks.
7289// FIXME scalable vectors.
7290bool CombinerHelper::isConstantSplatVector(Register Src, int64_t SplatValue,
7291 bool AllowUndefs) const {
7292 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
7293 if (!BuildVector)
7294 return false;
7295 unsigned NumSources = BuildVector->getNumSources();
7296
7297 for (unsigned I = 0; I < NumSources; ++I) {
7298 GImplicitDef *ImplicitDef =
7299 getOpcodeDef<GImplicitDef>(Reg: BuildVector->getSourceReg(I), MRI);
7300 if (ImplicitDef && AllowUndefs)
7301 continue;
7302 if (ImplicitDef && !AllowUndefs)
7303 return false;
7304 std::optional<ValueAndVReg> IConstant =
7305 getIConstantVRegValWithLookThrough(VReg: BuildVector->getSourceReg(I), MRI);
7306 if (IConstant && IConstant->Value == SplatValue)
7307 continue;
7308 return false;
7309 }
7310 return true;
7311}
7312
7313// Ignores COPYs during lookups.
7314// FIXME scalable vectors
7315std::optional<APInt>
7316CombinerHelper::getConstantOrConstantSplatVector(Register Src) const {
7317 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7318 if (IConstant)
7319 return IConstant->Value;
7320
7321 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
7322 if (!BuildVector)
7323 return std::nullopt;
7324 unsigned NumSources = BuildVector->getNumSources();
7325
7326 std::optional<APInt> Value = std::nullopt;
7327 for (unsigned I = 0; I < NumSources; ++I) {
7328 std::optional<ValueAndVReg> IConstant =
7329 getIConstantVRegValWithLookThrough(VReg: BuildVector->getSourceReg(I), MRI);
7330 if (!IConstant)
7331 return std::nullopt;
7332 if (!Value)
7333 Value = IConstant->Value;
7334 else if (*Value != IConstant->Value)
7335 return std::nullopt;
7336 }
7337 return Value;
7338}
7339
7340// FIXME G_SPLAT_VECTOR
7341bool CombinerHelper::isConstantOrConstantVectorI(Register Src) const {
7342 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7343 if (IConstant)
7344 return true;
7345
7346 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
7347 if (!BuildVector)
7348 return false;
7349
7350 unsigned NumSources = BuildVector->getNumSources();
7351 for (unsigned I = 0; I < NumSources; ++I) {
7352 std::optional<ValueAndVReg> IConstant =
7353 getIConstantVRegValWithLookThrough(VReg: BuildVector->getSourceReg(I), MRI);
7354 if (!IConstant)
7355 return false;
7356 }
7357 return true;
7358}
7359
7360// TODO: use knownbits to determine zeros
7361bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select,
7362 BuildFnTy &MatchInfo) const {
7363 uint32_t Flags = Select->getFlags();
7364 Register Dest = Select->getReg(Idx: 0);
7365 Register Cond = Select->getCondReg();
7366 Register True = Select->getTrueReg();
7367 Register False = Select->getFalseReg();
7368 LLT CondTy = MRI.getType(Reg: Select->getCondReg());
7369 LLT TrueTy = MRI.getType(Reg: Select->getTrueReg());
7370
7371 // We only do this combine for scalar boolean conditions.
7372 if (CondTy != LLT::scalar(SizeInBits: 1))
7373 return false;
7374
7375 if (TrueTy.isPointer())
7376 return false;
7377
7378 // Both are scalars.
7379 std::optional<ValueAndVReg> TrueOpt =
7380 getIConstantVRegValWithLookThrough(VReg: True, MRI);
7381 std::optional<ValueAndVReg> FalseOpt =
7382 getIConstantVRegValWithLookThrough(VReg: False, MRI);
7383
7384 if (!TrueOpt || !FalseOpt)
7385 return false;
7386
7387 APInt TrueValue = TrueOpt->Value;
7388 APInt FalseValue = FalseOpt->Value;
7389
7390 // select Cond, 1, 0 --> zext (Cond)
7391 if (TrueValue.isOne() && FalseValue.isZero()) {
7392 MatchInfo = [=](MachineIRBuilder &B) {
7393 B.setInstrAndDebugLoc(*Select);
7394 B.buildZExtOrTrunc(Res: Dest, Op: Cond);
7395 };
7396 return true;
7397 }
7398
7399 // select Cond, -1, 0 --> sext (Cond)
7400 if (TrueValue.isAllOnes() && FalseValue.isZero()) {
7401 MatchInfo = [=](MachineIRBuilder &B) {
7402 B.setInstrAndDebugLoc(*Select);
7403 B.buildSExtOrTrunc(Res: Dest, Op: Cond);
7404 };
7405 return true;
7406 }
7407
7408 // select Cond, 0, 1 --> zext (!Cond)
7409 if (TrueValue.isZero() && FalseValue.isOne()) {
7410 MatchInfo = [=](MachineIRBuilder &B) {
7411 B.setInstrAndDebugLoc(*Select);
7412 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7413 B.buildNot(Dst: Inner, Src0: Cond);
7414 B.buildZExtOrTrunc(Res: Dest, Op: Inner);
7415 };
7416 return true;
7417 }
7418
7419 // select Cond, 0, -1 --> sext (!Cond)
7420 if (TrueValue.isZero() && FalseValue.isAllOnes()) {
7421 MatchInfo = [=](MachineIRBuilder &B) {
7422 B.setInstrAndDebugLoc(*Select);
7423 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7424 B.buildNot(Dst: Inner, Src0: Cond);
7425 B.buildSExtOrTrunc(Res: Dest, Op: Inner);
7426 };
7427 return true;
7428 }
7429
7430 // select Cond, C1, C1-1 --> add (zext Cond), C1-1
7431 if (TrueValue - 1 == FalseValue) {
7432 MatchInfo = [=](MachineIRBuilder &B) {
7433 B.setInstrAndDebugLoc(*Select);
7434 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7435 B.buildZExtOrTrunc(Res: Inner, Op: Cond);
7436 B.buildAdd(Dst: Dest, Src0: Inner, Src1: False);
7437 };
7438 return true;
7439 }
7440
7441 // select Cond, C1, C1+1 --> add (sext Cond), C1+1
7442 if (TrueValue + 1 == FalseValue) {
7443 MatchInfo = [=](MachineIRBuilder &B) {
7444 B.setInstrAndDebugLoc(*Select);
7445 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7446 B.buildSExtOrTrunc(Res: Inner, Op: Cond);
7447 B.buildAdd(Dst: Dest, Src0: Inner, Src1: False);
7448 };
7449 return true;
7450 }
7451
7452 // select Cond, Pow2, 0 --> (zext Cond) << log2(Pow2)
7453 if (TrueValue.isPowerOf2() && FalseValue.isZero()) {
7454 MatchInfo = [=](MachineIRBuilder &B) {
7455 B.setInstrAndDebugLoc(*Select);
7456 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7457 B.buildZExtOrTrunc(Res: Inner, Op: Cond);
7458 // The shift amount must be scalar.
7459 LLT ShiftTy = TrueTy.isVector() ? TrueTy.getElementType() : TrueTy;
7460 auto ShAmtC = B.buildConstant(Res: ShiftTy, Val: TrueValue.exactLogBase2());
7461 B.buildShl(Dst: Dest, Src0: Inner, Src1: ShAmtC, Flags);
7462 };
7463 return true;
7464 }
7465
7466 // select Cond, 0, Pow2 --> (zext (!Cond)) << log2(Pow2)
7467 if (FalseValue.isPowerOf2() && TrueValue.isZero()) {
7468 MatchInfo = [=](MachineIRBuilder &B) {
7469 B.setInstrAndDebugLoc(*Select);
7470 Register Not = MRI.createGenericVirtualRegister(Ty: CondTy);
7471 B.buildNot(Dst: Not, Src0: Cond);
7472 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7473 B.buildZExtOrTrunc(Res: Inner, Op: Not);
7474 // The shift amount must be scalar.
7475 LLT ShiftTy = TrueTy.isVector() ? TrueTy.getElementType() : TrueTy;
7476 auto ShAmtC = B.buildConstant(Res: ShiftTy, Val: FalseValue.exactLogBase2());
7477 B.buildShl(Dst: Dest, Src0: Inner, Src1: ShAmtC, Flags);
7478 };
7479 return true;
7480 }
7481
7482 // select Cond, -1, C --> or (sext Cond), C
7483 if (TrueValue.isAllOnes()) {
7484 MatchInfo = [=](MachineIRBuilder &B) {
7485 B.setInstrAndDebugLoc(*Select);
7486 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7487 B.buildSExtOrTrunc(Res: Inner, Op: Cond);
7488 B.buildOr(Dst: Dest, Src0: Inner, Src1: False, Flags);
7489 };
7490 return true;
7491 }
7492
7493 // select Cond, C, -1 --> or (sext (not Cond)), C
7494 if (FalseValue.isAllOnes()) {
7495 MatchInfo = [=](MachineIRBuilder &B) {
7496 B.setInstrAndDebugLoc(*Select);
7497 Register Not = MRI.createGenericVirtualRegister(Ty: CondTy);
7498 B.buildNot(Dst: Not, Src0: Cond);
7499 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7500 B.buildSExtOrTrunc(Res: Inner, Op: Not);
7501 B.buildOr(Dst: Dest, Src0: Inner, Src1: True, Flags);
7502 };
7503 return true;
7504 }
7505
7506 return false;
7507}
7508
7509// TODO: use knownbits to determine zeros
7510bool CombinerHelper::tryFoldBoolSelectToLogic(GSelect *Select,
7511 BuildFnTy &MatchInfo) const {
7512 uint32_t Flags = Select->getFlags();
7513 Register DstReg = Select->getReg(Idx: 0);
7514 Register Cond = Select->getCondReg();
7515 Register True = Select->getTrueReg();
7516 Register False = Select->getFalseReg();
7517 LLT CondTy = MRI.getType(Reg: Select->getCondReg());
7518 LLT TrueTy = MRI.getType(Reg: Select->getTrueReg());
7519
7520 // Boolean or fixed vector of booleans.
7521 if (CondTy.isScalableVector() ||
7522 (CondTy.isFixedVector() &&
7523 CondTy.getElementType().getScalarSizeInBits() != 1) ||
7524 CondTy.getScalarSizeInBits() != 1)
7525 return false;
7526
7527 if (CondTy != TrueTy)
7528 return false;
7529
7530 // select Cond, Cond, F --> or Cond, F
7531 // select Cond, 1, F --> or Cond, F
7532 if ((Cond == True) || isOneOrOneSplat(Src: True, /* AllowUndefs */ true)) {
7533 MatchInfo = [=](MachineIRBuilder &B) {
7534 B.setInstrAndDebugLoc(*Select);
7535 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7536 B.buildZExtOrTrunc(Res: Ext, Op: Cond);
7537 auto FreezeFalse = B.buildFreeze(Dst: TrueTy, Src: False);
7538 B.buildOr(Dst: DstReg, Src0: Ext, Src1: FreezeFalse, Flags);
7539 };
7540 return true;
7541 }
7542
7543 // select Cond, T, Cond --> and Cond, T
7544 // select Cond, T, 0 --> and Cond, T
7545 if ((Cond == False) || isZeroOrZeroSplat(Src: False, /* AllowUndefs */ true)) {
7546 MatchInfo = [=](MachineIRBuilder &B) {
7547 B.setInstrAndDebugLoc(*Select);
7548 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7549 B.buildZExtOrTrunc(Res: Ext, Op: Cond);
7550 auto FreezeTrue = B.buildFreeze(Dst: TrueTy, Src: True);
7551 B.buildAnd(Dst: DstReg, Src0: Ext, Src1: FreezeTrue);
7552 };
7553 return true;
7554 }
7555
7556 // select Cond, T, 1 --> or (not Cond), T
7557 if (isOneOrOneSplat(Src: False, /* AllowUndefs */ true)) {
7558 MatchInfo = [=](MachineIRBuilder &B) {
7559 B.setInstrAndDebugLoc(*Select);
7560 // First the not.
7561 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7562 B.buildNot(Dst: Inner, Src0: Cond);
7563 // Then an ext to match the destination register.
7564 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7565 B.buildZExtOrTrunc(Res: Ext, Op: Inner);
7566 auto FreezeTrue = B.buildFreeze(Dst: TrueTy, Src: True);
7567 B.buildOr(Dst: DstReg, Src0: Ext, Src1: FreezeTrue, Flags);
7568 };
7569 return true;
7570 }
7571
7572 // select Cond, 0, F --> and (not Cond), F
7573 if (isZeroOrZeroSplat(Src: True, /* AllowUndefs */ true)) {
7574 MatchInfo = [=](MachineIRBuilder &B) {
7575 B.setInstrAndDebugLoc(*Select);
7576 // First the not.
7577 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7578 B.buildNot(Dst: Inner, Src0: Cond);
7579 // Then an ext to match the destination register.
7580 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7581 B.buildZExtOrTrunc(Res: Ext, Op: Inner);
7582 auto FreezeFalse = B.buildFreeze(Dst: TrueTy, Src: False);
7583 B.buildAnd(Dst: DstReg, Src0: Ext, Src1: FreezeFalse);
7584 };
7585 return true;
7586 }
7587
7588 return false;
7589}
7590
7591bool CombinerHelper::matchSelectIMinMax(const MachineOperand &MO,
7592 BuildFnTy &MatchInfo) const {
7593 Register DstReg = MO.getReg();
7594 Register CondReg, True, False;
7595 if (!mi_match(R: DstReg, MRI,
7596 P: m_GISelect(Src0: m_Reg(R&: CondReg), Src1: m_Reg(R&: True), Src2: m_Reg(R&: False))))
7597 return false;
7598
7599 CmpInst::Predicate Pred;
7600 Register CmpLHS, CmpRHS;
7601 if (!mi_match(R: CondReg, MRI,
7602 P: m_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: CmpLHS), R: m_Reg(R&: CmpRHS))))
7603 return false;
7604
7605 LLT DstTy = MRI.getType(Reg: DstReg);
7606 if (DstTy.isPointerOrPointerVector())
7607 return false;
7608
7609 // We want to fold the icmp and replace the select.
7610 if (!MRI.hasOneNonDBGUse(RegNo: CondReg))
7611 return false;
7612
7613 // We need a larger or smaller predicate for
7614 // canonicalization.
7615 if (CmpInst::isEquality(pred: Pred))
7616 return false;
7617
7618 // We can swap CmpLHS and CmpRHS for higher hitrate.
7619 if (True == CmpRHS && False == CmpLHS) {
7620 std::swap(a&: CmpLHS, b&: CmpRHS);
7621 Pred = CmpInst::getSwappedPredicate(pred: Pred);
7622 }
7623
7624 // (icmp X, Y) ? X : Y -> integer minmax.
7625 // see matchSelectPattern in ValueTracking.
7626 // Legality between G_SELECT and integer minmax can differ.
7627 if (True != CmpLHS || False != CmpRHS)
7628 return false;
7629
7630 switch (Pred) {
7631 case ICmpInst::ICMP_UGT:
7632 case ICmpInst::ICMP_UGE: {
7633 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_UMAX, DstTy}))
7634 return false;
7635 MatchInfo = [=](MachineIRBuilder &B) { B.buildUMax(Dst: DstReg, Src0: True, Src1: False); };
7636 return true;
7637 }
7638 case ICmpInst::ICMP_SGT:
7639 case ICmpInst::ICMP_SGE: {
7640 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SMAX, DstTy}))
7641 return false;
7642 MatchInfo = [=](MachineIRBuilder &B) { B.buildSMax(Dst: DstReg, Src0: True, Src1: False); };
7643 return true;
7644 }
7645 case ICmpInst::ICMP_ULT:
7646 case ICmpInst::ICMP_ULE: {
7647 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_UMIN, DstTy}))
7648 return false;
7649 MatchInfo = [=](MachineIRBuilder &B) { B.buildUMin(Dst: DstReg, Src0: True, Src1: False); };
7650 return true;
7651 }
7652 case ICmpInst::ICMP_SLT:
7653 case ICmpInst::ICMP_SLE: {
7654 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SMIN, DstTy}))
7655 return false;
7656 MatchInfo = [=](MachineIRBuilder &B) { B.buildSMin(Dst: DstReg, Src0: True, Src1: False); };
7657 return true;
7658 }
7659 default:
7660 return false;
7661 }
7662}
7663
7664// (neg (min/max x, (neg x))) --> (max/min x, (neg x))
7665bool CombinerHelper::matchSimplifyNegMinMax(MachineInstr &MI,
7666 BuildFnTy &MatchInfo) const {
7667 assert(MI.getOpcode() == TargetOpcode::G_SUB);
7668 Register DestReg = MI.getOperand(i: 0).getReg();
7669 LLT DestTy = MRI.getType(Reg: DestReg);
7670
7671 Register X;
7672 Register Sub0;
7673 auto NegPattern = m_all_of(preds: m_Neg(Src: m_DeferredReg(R&: X)), preds: m_Reg(R&: Sub0));
7674 if (mi_match(R: DestReg, MRI,
7675 P: m_Neg(Src: m_OneUse(SP: m_any_of(preds: m_GSMin(L: m_Reg(R&: X), R: NegPattern),
7676 preds: m_GSMax(L: m_Reg(R&: X), R: NegPattern),
7677 preds: m_GUMin(L: m_Reg(R&: X), R: NegPattern),
7678 preds: m_GUMax(L: m_Reg(R&: X), R: NegPattern)))))) {
7679 MachineInstr *MinMaxMI;
7680 if (!mi_match(R: MI.getOperand(i: 2).getReg(), MRI, P: m_MInstr(MI&: MinMaxMI)))
7681 return false;
7682 unsigned NewOpc = getInverseGMinMaxOpcode(MinMaxOpc: MinMaxMI->getOpcode());
7683 if (isLegal(Query: {NewOpc, {DestTy}})) {
7684 MatchInfo = [=](MachineIRBuilder &B) {
7685 B.buildInstr(Opc: NewOpc, DstOps: {DestReg}, SrcOps: {X, Sub0});
7686 };
7687 return true;
7688 }
7689 }
7690
7691 return false;
7692}
7693
7694bool CombinerHelper::matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo) const {
7695 GSelect *Select = cast<GSelect>(Val: &MI);
7696
7697 if (tryFoldSelectOfConstants(Select, MatchInfo))
7698 return true;
7699
7700 if (tryFoldBoolSelectToLogic(Select, MatchInfo))
7701 return true;
7702
7703 return false;
7704}
7705
7706/// Fold (icmp Pred1 V1, C1) && (icmp Pred2 V2, C2)
7707/// or (icmp Pred1 V1, C1) || (icmp Pred2 V2, C2)
7708/// into a single comparison using range-based reasoning.
7709/// see InstCombinerImpl::foldAndOrOfICmpsUsingRanges.
7710bool CombinerHelper::tryFoldAndOrOrICmpsUsingRanges(
7711 GLogicalBinOp *Logic, BuildFnTy &MatchInfo) const {
7712 assert(Logic->getOpcode() != TargetOpcode::G_XOR && "unexpected xor");
7713 bool IsAnd = Logic->getOpcode() == TargetOpcode::G_AND;
7714 Register DstReg = Logic->getReg(Idx: 0);
7715 Register LHS = Logic->getLHSReg();
7716 Register RHS = Logic->getRHSReg();
7717 unsigned Flags = Logic->getFlags();
7718
7719 // We need an G_ICMP on the LHS register.
7720 GICmp *Cmp1 = getOpcodeDef<GICmp>(Reg: LHS, MRI);
7721 if (!Cmp1)
7722 return false;
7723
7724 // We need an G_ICMP on the RHS register.
7725 GICmp *Cmp2 = getOpcodeDef<GICmp>(Reg: RHS, MRI);
7726 if (!Cmp2)
7727 return false;
7728
7729 // We want to fold the icmps.
7730 if (!MRI.hasOneNonDBGUse(RegNo: Cmp1->getReg(Idx: 0)) ||
7731 !MRI.hasOneNonDBGUse(RegNo: Cmp2->getReg(Idx: 0)))
7732 return false;
7733
7734 APInt C1;
7735 APInt C2;
7736 std::optional<ValueAndVReg> MaybeC1 =
7737 getIConstantVRegValWithLookThrough(VReg: Cmp1->getRHSReg(), MRI);
7738 if (!MaybeC1)
7739 return false;
7740 C1 = MaybeC1->Value;
7741
7742 std::optional<ValueAndVReg> MaybeC2 =
7743 getIConstantVRegValWithLookThrough(VReg: Cmp2->getRHSReg(), MRI);
7744 if (!MaybeC2)
7745 return false;
7746 C2 = MaybeC2->Value;
7747
7748 Register R1 = Cmp1->getLHSReg();
7749 Register R2 = Cmp2->getLHSReg();
7750 CmpInst::Predicate Pred1 = Cmp1->getCond();
7751 CmpInst::Predicate Pred2 = Cmp2->getCond();
7752 LLT CmpTy = MRI.getType(Reg: Cmp1->getReg(Idx: 0));
7753 LLT CmpOperandTy = MRI.getType(Reg: R1);
7754
7755 if (CmpOperandTy.isPointer())
7756 return false;
7757
7758 // We build ands, adds, and constants of type CmpOperandTy.
7759 // They must be legal to build.
7760 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_AND, CmpOperandTy}) ||
7761 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ADD, CmpOperandTy}) ||
7762 !isConstantLegalOrBeforeLegalizer(Ty: CmpOperandTy))
7763 return false;
7764
7765 // Look through add of a constant offset on R1, R2, or both operands. This
7766 // allows us to interpret the R + C' < C'' range idiom into a proper range.
7767 std::optional<APInt> Offset1;
7768 std::optional<APInt> Offset2;
7769 if (R1 != R2) {
7770 if (GAdd *Add = getOpcodeDef<GAdd>(Reg: R1, MRI)) {
7771 std::optional<ValueAndVReg> MaybeOffset1 =
7772 getIConstantVRegValWithLookThrough(VReg: Add->getRHSReg(), MRI);
7773 if (MaybeOffset1) {
7774 R1 = Add->getLHSReg();
7775 Offset1 = MaybeOffset1->Value;
7776 }
7777 }
7778 if (GAdd *Add = getOpcodeDef<GAdd>(Reg: R2, MRI)) {
7779 std::optional<ValueAndVReg> MaybeOffset2 =
7780 getIConstantVRegValWithLookThrough(VReg: Add->getRHSReg(), MRI);
7781 if (MaybeOffset2) {
7782 R2 = Add->getLHSReg();
7783 Offset2 = MaybeOffset2->Value;
7784 }
7785 }
7786 }
7787
7788 if (R1 != R2)
7789 return false;
7790
7791 // We calculate the icmp ranges including maybe offsets.
7792 ConstantRange CR1 = ConstantRange::makeExactICmpRegion(
7793 Pred: IsAnd ? ICmpInst::getInversePredicate(pred: Pred1) : Pred1, Other: C1);
7794 if (Offset1)
7795 CR1 = CR1.subtract(CI: *Offset1);
7796
7797 ConstantRange CR2 = ConstantRange::makeExactICmpRegion(
7798 Pred: IsAnd ? ICmpInst::getInversePredicate(pred: Pred2) : Pred2, Other: C2);
7799 if (Offset2)
7800 CR2 = CR2.subtract(CI: *Offset2);
7801
7802 bool CreateMask = false;
7803 APInt LowerDiff;
7804 std::optional<ConstantRange> CR = CR1.exactUnionWith(CR: CR2);
7805 if (!CR) {
7806 // We need non-wrapping ranges.
7807 if (CR1.isWrappedSet() || CR2.isWrappedSet())
7808 return false;
7809
7810 // Check whether we have equal-size ranges that only differ by one bit.
7811 // In that case we can apply a mask to map one range onto the other.
7812 LowerDiff = CR1.getLower() ^ CR2.getLower();
7813 APInt UpperDiff = (CR1.getUpper() - 1) ^ (CR2.getUpper() - 1);
7814 APInt CR1Size = CR1.getUpper() - CR1.getLower();
7815 if (!LowerDiff.isPowerOf2() || LowerDiff != UpperDiff ||
7816 CR1Size != CR2.getUpper() - CR2.getLower())
7817 return false;
7818
7819 CR = CR1.getLower().ult(RHS: CR2.getLower()) ? CR1 : CR2;
7820 CreateMask = true;
7821 }
7822
7823 if (IsAnd)
7824 CR = CR->inverse();
7825
7826 CmpInst::Predicate NewPred;
7827 APInt NewC, Offset;
7828 CR->getEquivalentICmp(Pred&: NewPred, RHS&: NewC, Offset);
7829
7830 // We take the result type of one of the original icmps, CmpTy, for
7831 // the to be build icmp. The operand type, CmpOperandTy, is used for
7832 // the other instructions and constants to be build. The types of
7833 // the parameters and output are the same for add and and. CmpTy
7834 // and the type of DstReg might differ. That is why we zext or trunc
7835 // the icmp into the destination register.
7836
7837 MatchInfo = [=](MachineIRBuilder &B) {
7838 if (CreateMask && Offset != 0) {
7839 auto TildeLowerDiff = B.buildConstant(Res: CmpOperandTy, Val: ~LowerDiff);
7840 auto And = B.buildAnd(Dst: CmpOperandTy, Src0: R1, Src1: TildeLowerDiff); // the mask.
7841 auto OffsetC = B.buildConstant(Res: CmpOperandTy, Val: Offset);
7842 auto Add = B.buildAdd(Dst: CmpOperandTy, Src0: And, Src1: OffsetC, Flags);
7843 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
7844 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: Add, Op1: NewCon);
7845 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
7846 } else if (CreateMask && Offset == 0) {
7847 auto TildeLowerDiff = B.buildConstant(Res: CmpOperandTy, Val: ~LowerDiff);
7848 auto And = B.buildAnd(Dst: CmpOperandTy, Src0: R1, Src1: TildeLowerDiff); // the mask.
7849 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
7850 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: And, Op1: NewCon);
7851 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
7852 } else if (!CreateMask && Offset != 0) {
7853 auto OffsetC = B.buildConstant(Res: CmpOperandTy, Val: Offset);
7854 auto Add = B.buildAdd(Dst: CmpOperandTy, Src0: R1, Src1: OffsetC, Flags);
7855 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
7856 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: Add, Op1: NewCon);
7857 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
7858 } else if (!CreateMask && Offset == 0) {
7859 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
7860 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: R1, Op1: NewCon);
7861 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
7862 } else {
7863 llvm_unreachable("unexpected configuration of CreateMask and Offset");
7864 }
7865 };
7866 return true;
7867}
7868
7869bool CombinerHelper::tryFoldLogicOfFCmps(GLogicalBinOp *Logic,
7870 BuildFnTy &MatchInfo) const {
7871 assert(Logic->getOpcode() != TargetOpcode::G_XOR && "unexpecte xor");
7872 Register DestReg = Logic->getReg(Idx: 0);
7873 Register LHS = Logic->getLHSReg();
7874 Register RHS = Logic->getRHSReg();
7875 bool IsAnd = Logic->getOpcode() == TargetOpcode::G_AND;
7876
7877 // We need a compare on the LHS register.
7878 GFCmp *Cmp1 = getOpcodeDef<GFCmp>(Reg: LHS, MRI);
7879 if (!Cmp1)
7880 return false;
7881
7882 // We need a compare on the RHS register.
7883 GFCmp *Cmp2 = getOpcodeDef<GFCmp>(Reg: RHS, MRI);
7884 if (!Cmp2)
7885 return false;
7886
7887 LLT CmpTy = MRI.getType(Reg: Cmp1->getReg(Idx: 0));
7888 LLT CmpOperandTy = MRI.getType(Reg: Cmp1->getLHSReg());
7889
7890 // We build one fcmp, want to fold the fcmps, replace the logic op,
7891 // and the fcmps must have the same shape.
7892 if (!isLegalOrBeforeLegalizer(
7893 Query: {TargetOpcode::G_FCMP, {CmpTy, CmpOperandTy}}) ||
7894 !MRI.hasOneNonDBGUse(RegNo: Logic->getReg(Idx: 0)) ||
7895 !MRI.hasOneNonDBGUse(RegNo: Cmp1->getReg(Idx: 0)) ||
7896 !MRI.hasOneNonDBGUse(RegNo: Cmp2->getReg(Idx: 0)) ||
7897 MRI.getType(Reg: Cmp1->getLHSReg()) != MRI.getType(Reg: Cmp2->getLHSReg()))
7898 return false;
7899
7900 CmpInst::Predicate PredL = Cmp1->getCond();
7901 CmpInst::Predicate PredR = Cmp2->getCond();
7902 Register LHS0 = Cmp1->getLHSReg();
7903 Register LHS1 = Cmp1->getRHSReg();
7904 Register RHS0 = Cmp2->getLHSReg();
7905 Register RHS1 = Cmp2->getRHSReg();
7906
7907 if (LHS0 == RHS1 && LHS1 == RHS0) {
7908 // Swap RHS operands to match LHS.
7909 PredR = CmpInst::getSwappedPredicate(pred: PredR);
7910 std::swap(a&: RHS0, b&: RHS1);
7911 }
7912
7913 if (LHS0 == RHS0 && LHS1 == RHS1) {
7914 // We determine the new predicate.
7915 unsigned CmpCodeL = getFCmpCode(CC: PredL);
7916 unsigned CmpCodeR = getFCmpCode(CC: PredR);
7917 unsigned NewPred = IsAnd ? CmpCodeL & CmpCodeR : CmpCodeL | CmpCodeR;
7918 unsigned Flags = Cmp1->getFlags() | Cmp2->getFlags();
7919 MatchInfo = [=](MachineIRBuilder &B) {
7920 // The fcmp predicates fill the lower part of the enum.
7921 FCmpInst::Predicate Pred = static_cast<FCmpInst::Predicate>(NewPred);
7922 if (Pred == FCmpInst::FCMP_FALSE &&
7923 isConstantLegalOrBeforeLegalizer(Ty: CmpTy)) {
7924 auto False = B.buildConstant(Res: CmpTy, Val: 0);
7925 B.buildZExtOrTrunc(Res: DestReg, Op: False);
7926 } else if (Pred == FCmpInst::FCMP_TRUE &&
7927 isConstantLegalOrBeforeLegalizer(Ty: CmpTy)) {
7928 auto True =
7929 B.buildConstant(Res: CmpTy, Val: getICmpTrueVal(TLI: getTargetLowering(),
7930 IsVector: CmpTy.isVector() /*isVector*/,
7931 IsFP: true /*isFP*/));
7932 B.buildZExtOrTrunc(Res: DestReg, Op: True);
7933 } else { // We take the predicate without predicate optimizations.
7934 auto Cmp = B.buildFCmp(Pred, Res: CmpTy, Op0: LHS0, Op1: LHS1, Flags);
7935 B.buildZExtOrTrunc(Res: DestReg, Op: Cmp);
7936 }
7937 };
7938 return true;
7939 }
7940
7941 return false;
7942}
7943
7944bool CombinerHelper::matchAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const {
7945 GAnd *And = cast<GAnd>(Val: &MI);
7946
7947 if (tryFoldAndOrOrICmpsUsingRanges(Logic: And, MatchInfo))
7948 return true;
7949
7950 if (tryFoldLogicOfFCmps(Logic: And, MatchInfo))
7951 return true;
7952
7953 return false;
7954}
7955
7956bool CombinerHelper::matchOr(MachineInstr &MI, BuildFnTy &MatchInfo) const {
7957 GOr *Or = cast<GOr>(Val: &MI);
7958
7959 if (tryFoldAndOrOrICmpsUsingRanges(Logic: Or, MatchInfo))
7960 return true;
7961
7962 if (tryFoldLogicOfFCmps(Logic: Or, MatchInfo))
7963 return true;
7964
7965 return false;
7966}
7967
7968bool CombinerHelper::matchAddOverflow(MachineInstr &MI,
7969 BuildFnTy &MatchInfo) const {
7970 GAddCarryOut *Add = cast<GAddCarryOut>(Val: &MI);
7971
7972 // Addo has no flags
7973 Register Dst = Add->getReg(Idx: 0);
7974 Register Carry = Add->getReg(Idx: 1);
7975 Register LHS = Add->getLHSReg();
7976 Register RHS = Add->getRHSReg();
7977 bool IsSigned = Add->isSigned();
7978 LLT DstTy = MRI.getType(Reg: Dst);
7979 LLT CarryTy = MRI.getType(Reg: Carry);
7980
7981 // Canonicalize constant to RHS.
7982 if (isConstantOrConstantVectorI(Src: LHS) && !isConstantOrConstantVectorI(Src: RHS)) {
7983 if (IsSigned) {
7984 MatchInfo = [=](MachineIRBuilder &B) {
7985 B.buildSAddo(Res: Dst, CarryOut: Carry, Op0: RHS, Op1: LHS);
7986 };
7987 return true;
7988 }
7989 // !IsSigned
7990 MatchInfo = [=](MachineIRBuilder &B) {
7991 B.buildUAddo(Res: Dst, CarryOut: Carry, Op0: RHS, Op1: LHS);
7992 };
7993 return true;
7994 }
7995
7996 std::optional<APInt> MaybeLHS = getConstantOrConstantSplatVector(Src: LHS);
7997 std::optional<APInt> MaybeRHS = getConstantOrConstantSplatVector(Src: RHS);
7998
7999 // Fold addo(c1, c2) -> c3, carry.
8000 if (MaybeLHS && MaybeRHS && isConstantLegalOrBeforeLegalizer(Ty: DstTy) &&
8001 isConstantLegalOrBeforeLegalizer(Ty: CarryTy)) {
8002 bool Overflow;
8003 APInt Result = IsSigned ? MaybeLHS->sadd_ov(RHS: *MaybeRHS, Overflow)
8004 : MaybeLHS->uadd_ov(RHS: *MaybeRHS, Overflow);
8005 MatchInfo = [=](MachineIRBuilder &B) {
8006 B.buildConstant(Res: Dst, Val: Result);
8007 B.buildConstant(Res: Carry, Val: Overflow);
8008 };
8009 return true;
8010 }
8011
8012 // Fold (addo x, 0) -> x, no carry
8013 if (MaybeRHS && *MaybeRHS == 0 && isConstantLegalOrBeforeLegalizer(Ty: CarryTy)) {
8014 MatchInfo = [=](MachineIRBuilder &B) {
8015 B.buildCopy(Res: Dst, Op: LHS);
8016 B.buildConstant(Res: Carry, Val: 0);
8017 };
8018 return true;
8019 }
8020
8021 // Given 2 constant operands whose sum does not overflow:
8022 // uaddo (X +nuw C0), C1 -> uaddo X, C0 + C1
8023 // saddo (X +nsw C0), C1 -> saddo X, C0 + C1
8024 GAdd *AddLHS = getOpcodeDef<GAdd>(Reg: LHS, MRI);
8025 if (MaybeRHS && AddLHS && MRI.hasOneNonDBGUse(RegNo: Add->getReg(Idx: 0)) &&
8026 ((IsSigned && AddLHS->getFlag(Flag: MachineInstr::MIFlag::NoSWrap)) ||
8027 (!IsSigned && AddLHS->getFlag(Flag: MachineInstr::MIFlag::NoUWrap)))) {
8028 std::optional<APInt> MaybeAddRHS =
8029 getConstantOrConstantSplatVector(Src: AddLHS->getRHSReg());
8030 if (MaybeAddRHS) {
8031 bool Overflow;
8032 APInt NewC = IsSigned ? MaybeAddRHS->sadd_ov(RHS: *MaybeRHS, Overflow)
8033 : MaybeAddRHS->uadd_ov(RHS: *MaybeRHS, Overflow);
8034 if (!Overflow && isConstantLegalOrBeforeLegalizer(Ty: DstTy)) {
8035 if (IsSigned) {
8036 MatchInfo = [=](MachineIRBuilder &B) {
8037 auto ConstRHS = B.buildConstant(Res: DstTy, Val: NewC);
8038 B.buildSAddo(Res: Dst, CarryOut: Carry, Op0: AddLHS->getLHSReg(), Op1: ConstRHS);
8039 };
8040 return true;
8041 }
8042 // !IsSigned
8043 MatchInfo = [=](MachineIRBuilder &B) {
8044 auto ConstRHS = B.buildConstant(Res: DstTy, Val: NewC);
8045 B.buildUAddo(Res: Dst, CarryOut: Carry, Op0: AddLHS->getLHSReg(), Op1: ConstRHS);
8046 };
8047 return true;
8048 }
8049 }
8050 };
8051
8052 // We try to combine addo to non-overflowing add.
8053 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ADD, {DstTy}}) ||
8054 !isConstantLegalOrBeforeLegalizer(Ty: CarryTy))
8055 return false;
8056
8057 // We try to combine uaddo to non-overflowing add.
8058 if (!IsSigned) {
8059 ConstantRange CRLHS =
8060 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: LHS), /*IsSigned=*/false);
8061 ConstantRange CRRHS =
8062 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: RHS), /*IsSigned=*/false);
8063
8064 switch (CRLHS.unsignedAddMayOverflow(Other: CRRHS)) {
8065 case ConstantRange::OverflowResult::MayOverflow:
8066 return false;
8067 case ConstantRange::OverflowResult::NeverOverflows: {
8068 MatchInfo = [=](MachineIRBuilder &B) {
8069 B.buildAdd(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoUWrap);
8070 B.buildConstant(Res: Carry, Val: 0);
8071 };
8072 return true;
8073 }
8074 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
8075 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
8076 MatchInfo = [=](MachineIRBuilder &B) {
8077 B.buildAdd(Dst, Src0: LHS, Src1: RHS);
8078 B.buildConstant(Res: Carry, Val: 1);
8079 };
8080 return true;
8081 }
8082 }
8083 return false;
8084 }
8085
8086 // We try to combine saddo to non-overflowing add.
8087
8088 // If LHS and RHS each have at least two sign bits, then there is no signed
8089 // overflow.
8090 if (VT->computeNumSignBits(R: RHS) > 1 && VT->computeNumSignBits(R: LHS) > 1) {
8091 MatchInfo = [=](MachineIRBuilder &B) {
8092 B.buildAdd(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoSWrap);
8093 B.buildConstant(Res: Carry, Val: 0);
8094 };
8095 return true;
8096 }
8097
8098 ConstantRange CRLHS =
8099 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: LHS), /*IsSigned=*/true);
8100 ConstantRange CRRHS =
8101 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: RHS), /*IsSigned=*/true);
8102
8103 switch (CRLHS.signedAddMayOverflow(Other: CRRHS)) {
8104 case ConstantRange::OverflowResult::MayOverflow:
8105 return false;
8106 case ConstantRange::OverflowResult::NeverOverflows: {
8107 MatchInfo = [=](MachineIRBuilder &B) {
8108 B.buildAdd(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoSWrap);
8109 B.buildConstant(Res: Carry, Val: 0);
8110 };
8111 return true;
8112 }
8113 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
8114 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
8115 MatchInfo = [=](MachineIRBuilder &B) {
8116 B.buildAdd(Dst, Src0: LHS, Src1: RHS);
8117 B.buildConstant(Res: Carry, Val: 1);
8118 };
8119 return true;
8120 }
8121 }
8122
8123 return false;
8124}
8125
8126void CombinerHelper::applyBuildFnMO(const MachineOperand &MO,
8127 BuildFnTy &MatchInfo) const {
8128 MachineInstr *Root = getDefIgnoringCopies(Reg: MO.getReg(), MRI);
8129 MatchInfo(Builder);
8130 Root->eraseFromParent();
8131}
8132
8133bool CombinerHelper::matchFPowIExpansion(MachineInstr &MI,
8134 int64_t Exponent) const {
8135 bool OptForSize = MI.getMF()->getFunction().hasOptSize();
8136 return getTargetLowering().isBeneficialToExpandPowI(Exponent, OptForSize);
8137}
8138
8139void CombinerHelper::applyExpandFPowI(MachineInstr &MI,
8140 int64_t Exponent) const {
8141 auto [Dst, Base] = MI.getFirst2Regs();
8142 LLT Ty = MRI.getType(Reg: Dst);
8143 int64_t ExpVal = Exponent;
8144
8145 if (ExpVal == 0) {
8146 Builder.buildFConstant(Res: Dst, Val: 1.0);
8147 MI.removeFromParent();
8148 return;
8149 }
8150
8151 if (ExpVal < 0)
8152 ExpVal = -ExpVal;
8153
8154 // We use the simple binary decomposition method from SelectionDAG ExpandPowI
8155 // to generate the multiply sequence. There are more optimal ways to do this
8156 // (for example, powi(x,15) generates one more multiply than it should), but
8157 // this has the benefit of being both really simple and much better than a
8158 // libcall.
8159 std::optional<SrcOp> Res;
8160 SrcOp CurSquare = Base;
8161 while (ExpVal > 0) {
8162 if (ExpVal & 1) {
8163 if (!Res)
8164 Res = CurSquare;
8165 else
8166 Res = Builder.buildFMul(Dst: Ty, Src0: *Res, Src1: CurSquare);
8167 }
8168
8169 CurSquare = Builder.buildFMul(Dst: Ty, Src0: CurSquare, Src1: CurSquare);
8170 ExpVal >>= 1;
8171 }
8172
8173 // If the original exponent was negative, invert the result, producing
8174 // 1/(x*x*x).
8175 if (Exponent < 0)
8176 Res = Builder.buildFDiv(Dst: Ty, Src0: Builder.buildFConstant(Res: Ty, Val: 1.0), Src1: *Res,
8177 Flags: MI.getFlags());
8178
8179 Builder.buildCopy(Res: Dst, Op: *Res);
8180 MI.eraseFromParent();
8181}
8182
8183bool CombinerHelper::matchFoldAPlusC1MinusC2(const MachineInstr &MI,
8184 BuildFnTy &MatchInfo) const {
8185 // fold (A+C1)-C2 -> A+(C1-C2)
8186 const GSub *Sub = cast<GSub>(Val: &MI);
8187 Register A, C1Reg;
8188 if (!mi_match(R: Sub->getLHSReg(), MRI, P: m_GAdd(L: m_Reg(R&: A), R: m_Reg(R&: C1Reg))))
8189 return false;
8190
8191 if (!MRI.hasOneNonDBGUse(RegNo: Sub->getLHSReg()))
8192 return false;
8193
8194 APInt C2 = getIConstantFromReg(VReg: Sub->getRHSReg(), MRI);
8195 APInt C1 = getIConstantFromReg(VReg: C1Reg, MRI);
8196
8197 Register Dst = Sub->getReg(Idx: 0);
8198 LLT DstTy = MRI.getType(Reg: Dst);
8199
8200 MatchInfo = [=](MachineIRBuilder &B) {
8201 auto Const = B.buildConstant(Res: DstTy, Val: C1 - C2);
8202 B.buildAdd(Dst, Src0: A, Src1: Const);
8203 };
8204
8205 return true;
8206}
8207
8208bool CombinerHelper::matchFoldC2MinusAPlusC1(const MachineInstr &MI,
8209 BuildFnTy &MatchInfo) const {
8210 // fold C2-(A+C1) -> (C2-C1)-A
8211 const GSub *Sub = cast<GSub>(Val: &MI);
8212 Register A, C1Reg;
8213 if (!mi_match(R: Sub->getRHSReg(), MRI, P: m_GAdd(L: m_Reg(R&: A), R: m_Reg(R&: C1Reg))))
8214 return false;
8215
8216 if (!MRI.hasOneNonDBGUse(RegNo: Sub->getRHSReg()))
8217 return false;
8218
8219 APInt C2 = getIConstantFromReg(VReg: Sub->getLHSReg(), MRI);
8220 APInt C1 = getIConstantFromReg(VReg: C1Reg, MRI);
8221
8222 Register Dst = Sub->getReg(Idx: 0);
8223 LLT DstTy = MRI.getType(Reg: Dst);
8224
8225 MatchInfo = [=](MachineIRBuilder &B) {
8226 auto Const = B.buildConstant(Res: DstTy, Val: C2 - C1);
8227 B.buildSub(Dst, Src0: Const, Src1: A);
8228 };
8229
8230 return true;
8231}
8232
8233bool CombinerHelper::matchFoldAMinusC1MinusC2(const MachineInstr &MI,
8234 BuildFnTy &MatchInfo) const {
8235 // fold (A-C1)-C2 -> A-(C1+C2)
8236 const GSub *Sub1 = cast<GSub>(Val: &MI);
8237 Register A, C1Reg;
8238 if (!mi_match(R: Sub1->getLHSReg(), MRI, P: m_GSub(L: m_Reg(R&: A), R: m_Reg(R&: C1Reg))))
8239 return false;
8240
8241 if (!MRI.hasOneNonDBGUse(RegNo: Sub1->getLHSReg()))
8242 return false;
8243
8244 APInt C2 = getIConstantFromReg(VReg: Sub1->getRHSReg(), MRI);
8245 APInt C1 = getIConstantFromReg(VReg: C1Reg, MRI);
8246
8247 Register Dst = Sub1->getReg(Idx: 0);
8248 LLT DstTy = MRI.getType(Reg: Dst);
8249
8250 MatchInfo = [=](MachineIRBuilder &B) {
8251 auto Const = B.buildConstant(Res: DstTy, Val: C1 + C2);
8252 B.buildSub(Dst, Src0: A, Src1: Const);
8253 };
8254
8255 return true;
8256}
8257
8258bool CombinerHelper::matchFoldC1Minus2MinusC2(const MachineInstr &MI,
8259 BuildFnTy &MatchInfo) const {
8260 // fold (C1-A)-C2 -> (C1-C2)-A
8261 const GSub *Sub1 = cast<GSub>(Val: &MI);
8262 Register C1Reg, A;
8263 if (!mi_match(R: Sub1->getLHSReg(), MRI, P: m_GSub(L: m_Reg(R&: C1Reg), R: m_Reg(R&: A))))
8264 return false;
8265
8266 if (!MRI.hasOneNonDBGUse(RegNo: Sub1->getLHSReg()))
8267 return false;
8268
8269 APInt C2 = getIConstantFromReg(VReg: Sub1->getRHSReg(), MRI);
8270 APInt C1 = getIConstantFromReg(VReg: C1Reg, MRI);
8271
8272 Register Dst = Sub1->getReg(Idx: 0);
8273 LLT DstTy = MRI.getType(Reg: Dst);
8274
8275 MatchInfo = [=](MachineIRBuilder &B) {
8276 auto Const = B.buildConstant(Res: DstTy, Val: C1 - C2);
8277 B.buildSub(Dst, Src0: Const, Src1: A);
8278 };
8279
8280 return true;
8281}
8282
8283bool CombinerHelper::matchFoldAMinusC1PlusC2(const MachineInstr &MI,
8284 BuildFnTy &MatchInfo) const {
8285 // fold ((A-C1)+C2) -> (A+(C2-C1))
8286 const GAdd *Add = cast<GAdd>(Val: &MI);
8287 Register A, C1Reg;
8288 if (!mi_match(R: Add->getLHSReg(), MRI, P: m_GSub(L: m_Reg(R&: A), R: m_Reg(R&: C1Reg))))
8289 return false;
8290
8291 if (!MRI.hasOneNonDBGUse(RegNo: Add->getLHSReg()))
8292 return false;
8293
8294 APInt C2 = getIConstantFromReg(VReg: Add->getRHSReg(), MRI);
8295 APInt C1 = getIConstantFromReg(VReg: C1Reg, MRI);
8296
8297 Register Dst = Add->getReg(Idx: 0);
8298 LLT DstTy = MRI.getType(Reg: Dst);
8299
8300 MatchInfo = [=](MachineIRBuilder &B) {
8301 auto Const = B.buildConstant(Res: DstTy, Val: C2 - C1);
8302 B.buildAdd(Dst, Src0: A, Src1: Const);
8303 };
8304
8305 return true;
8306}
8307
8308bool CombinerHelper::matchUnmergeValuesAnyExtBuildVector(
8309 const MachineInstr &MI, BuildFnTy &MatchInfo) const {
8310 const GUnmerge *Unmerge = cast<GUnmerge>(Val: &MI);
8311
8312 if (!MRI.hasOneNonDBGUse(RegNo: Unmerge->getSourceReg()))
8313 return false;
8314
8315 LLT DstTy = MRI.getType(Reg: Unmerge->getReg(Idx: 0));
8316
8317 // $bv:_(<8 x s8>) = G_BUILD_VECTOR ....
8318 // $any:_(<8 x s16>) = G_ANYEXT $bv
8319 // $uv:_(<4 x s16>), $uv1:_(<4 x s16>) = G_UNMERGE_VALUES $any
8320 //
8321 // ->
8322 //
8323 // $any:_(s16) = G_ANYEXT $bv[0]
8324 // $any1:_(s16) = G_ANYEXT $bv[1]
8325 // $any2:_(s16) = G_ANYEXT $bv[2]
8326 // $any3:_(s16) = G_ANYEXT $bv[3]
8327 // $any4:_(s16) = G_ANYEXT $bv[4]
8328 // $any5:_(s16) = G_ANYEXT $bv[5]
8329 // $any6:_(s16) = G_ANYEXT $bv[6]
8330 // $any7:_(s16) = G_ANYEXT $bv[7]
8331 // $uv:_(<4 x s16>) = G_BUILD_VECTOR $any, $any1, $any2, $any3
8332 // $uv1:_(<4 x s16>) = G_BUILD_VECTOR $any4, $any5, $any6, $any7
8333
8334 // We want to unmerge into vectors.
8335 if (!DstTy.isFixedVector())
8336 return false;
8337
8338 Register AnySrcReg;
8339 if (!mi_match(R: Unmerge->getSourceReg(), MRI, P: m_GAnyExt(Src: m_Reg(R&: AnySrcReg))))
8340 return false;
8341
8342 GBuildVector *BV;
8343 if (mi_match(R: AnySrcReg, MRI, P: m_GBuildVector(Inst&: BV))) {
8344 // G_UNMERGE_VALUES G_ANYEXT G_BUILD_VECTOR
8345
8346 if (!MRI.hasOneNonDBGUse(RegNo: BV->getReg(Idx: 0)))
8347 return false;
8348
8349 // FIXME: check element types?
8350 if (BV->getNumSources() % Unmerge->getNumDefs() != 0)
8351 return false;
8352
8353 LLT BigBvTy = MRI.getType(Reg: BV->getReg(Idx: 0));
8354 LLT SmallBvTy = DstTy;
8355 LLT SmallBvElemenTy = SmallBvTy.getElementType();
8356
8357 if (!isLegalOrBeforeLegalizer(
8358 Query: {TargetOpcode::G_BUILD_VECTOR, {SmallBvTy, SmallBvElemenTy}}))
8359 return false;
8360
8361 // We check the legality of scalar anyext.
8362 if (!isLegalOrBeforeLegalizer(
8363 Query: {TargetOpcode::G_ANYEXT,
8364 {SmallBvElemenTy, BigBvTy.getElementType()}}))
8365 return false;
8366
8367 MatchInfo = [=](MachineIRBuilder &B) {
8368 // Build into each G_UNMERGE_VALUES def
8369 // a small build vector with anyext from the source build vector.
8370 for (unsigned I = 0; I < Unmerge->getNumDefs(); ++I) {
8371 SmallVector<Register> Ops;
8372 for (unsigned J = 0; J < SmallBvTy.getNumElements(); ++J) {
8373 Register SourceArray =
8374 BV->getSourceReg(I: I * SmallBvTy.getNumElements() + J);
8375 auto AnyExt = B.buildAnyExt(Res: SmallBvElemenTy, Op: SourceArray);
8376 Ops.push_back(Elt: AnyExt.getReg(Idx: 0));
8377 }
8378 B.buildBuildVector(Res: Unmerge->getOperand(i: I).getReg(), Ops);
8379 };
8380 };
8381 return true;
8382 };
8383
8384 return false;
8385}
8386
8387bool CombinerHelper::matchShuffleUndefRHS(MachineInstr &MI,
8388 BuildFnTy &MatchInfo) const {
8389
8390 bool Changed = false;
8391 auto &Shuffle = cast<GShuffleVector>(Val&: MI);
8392 ArrayRef<int> OrigMask = Shuffle.getMask();
8393 SmallVector<int, 16> NewMask;
8394 const LLT SrcTy = MRI.getType(Reg: Shuffle.getSrc1Reg());
8395 const unsigned NumSrcElems = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
8396 const unsigned NumDstElts = OrigMask.size();
8397 for (unsigned i = 0; i != NumDstElts; ++i) {
8398 int Idx = OrigMask[i];
8399 if (Idx >= (int)NumSrcElems) {
8400 Idx = -1;
8401 Changed = true;
8402 }
8403 NewMask.push_back(Elt: Idx);
8404 }
8405
8406 if (!Changed)
8407 return false;
8408
8409 MatchInfo = [&, NewMask = std::move(NewMask)](MachineIRBuilder &B) {
8410 B.buildShuffleVector(Res: MI.getOperand(i: 0), Src1: MI.getOperand(i: 1), Src2: MI.getOperand(i: 2),
8411 Mask: std::move(NewMask));
8412 };
8413
8414 return true;
8415}
8416
8417static void commuteMask(MutableArrayRef<int> Mask, const unsigned NumElems) {
8418 const unsigned MaskSize = Mask.size();
8419 for (unsigned I = 0; I < MaskSize; ++I) {
8420 int Idx = Mask[I];
8421 if (Idx < 0)
8422 continue;
8423
8424 if (Idx < (int)NumElems)
8425 Mask[I] = Idx + NumElems;
8426 else
8427 Mask[I] = Idx - NumElems;
8428 }
8429}
8430
8431bool CombinerHelper::isZeroOrOneVector(Register Reg) const {
8432 if (!MRI.getType(Reg).isFixedVector())
8433 return false;
8434 return matchUnaryPredicate(MRI, Reg, Match: [](const Constant *C) {
8435 auto *CI = dyn_cast_or_null<ConstantInt>(Val: C);
8436 return CI && (CI->isZero() || CI->isOne());
8437 });
8438}
8439
8440bool CombinerHelper::matchShuffleDisjointMask(MachineInstr &MI,
8441 BuildFnTy &MatchInfo) const {
8442
8443 auto &Shuffle = cast<GShuffleVector>(Val&: MI);
8444 // If any of the two inputs is already undef, don't check the mask again to
8445 // prevent infinite loop
8446 if (getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: Shuffle.getSrc1Reg(), MRI))
8447 return false;
8448
8449 if (getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: Shuffle.getSrc2Reg(), MRI))
8450 return false;
8451
8452 const LLT DstTy = MRI.getType(Reg: Shuffle.getReg(Idx: 0));
8453 const LLT Src1Ty = MRI.getType(Reg: Shuffle.getSrc1Reg());
8454 if (!isLegalOrBeforeLegalizer(
8455 Query: {TargetOpcode::G_SHUFFLE_VECTOR, {DstTy, Src1Ty}}))
8456 return false;
8457
8458 ArrayRef<int> Mask = Shuffle.getMask();
8459 const unsigned NumSrcElems = Src1Ty.getNumElements();
8460
8461 bool TouchesSrc1 = false;
8462 bool TouchesSrc2 = false;
8463 const unsigned NumElems = Mask.size();
8464 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
8465 if (Mask[Idx] < 0)
8466 continue;
8467
8468 if (Mask[Idx] < (int)NumSrcElems)
8469 TouchesSrc1 = true;
8470 else
8471 TouchesSrc2 = true;
8472 }
8473
8474 if (TouchesSrc1 == TouchesSrc2)
8475 return false;
8476
8477 Register NewSrc1 = Shuffle.getSrc1Reg();
8478 SmallVector<int, 16> NewMask(Mask);
8479 if (TouchesSrc2) {
8480 NewSrc1 = Shuffle.getSrc2Reg();
8481 commuteMask(Mask: NewMask, NumElems: NumSrcElems);
8482 }
8483
8484 MatchInfo = [=, &Shuffle](MachineIRBuilder &B) {
8485 auto Undef = B.buildUndef(Res: Src1Ty);
8486 B.buildShuffleVector(Res: Shuffle.getReg(Idx: 0), Src1: NewSrc1, Src2: Undef, Mask: NewMask);
8487 };
8488
8489 return true;
8490}
8491
8492bool CombinerHelper::matchSuboCarryOut(const MachineInstr &MI,
8493 BuildFnTy &MatchInfo) const {
8494 const GSubCarryOut *Subo = cast<GSubCarryOut>(Val: &MI);
8495
8496 Register Dst = Subo->getReg(Idx: 0);
8497 Register LHS = Subo->getLHSReg();
8498 Register RHS = Subo->getRHSReg();
8499 Register Carry = Subo->getCarryOutReg();
8500 LLT DstTy = MRI.getType(Reg: Dst);
8501 LLT CarryTy = MRI.getType(Reg: Carry);
8502
8503 // Check legality before known bits.
8504 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SUB, {DstTy}}) ||
8505 !isConstantLegalOrBeforeLegalizer(Ty: CarryTy))
8506 return false;
8507
8508 ConstantRange KBLHS =
8509 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: LHS),
8510 /* IsSigned=*/Subo->isSigned());
8511 ConstantRange KBRHS =
8512 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: RHS),
8513 /* IsSigned=*/Subo->isSigned());
8514
8515 if (Subo->isSigned()) {
8516 // G_SSUBO
8517 switch (KBLHS.signedSubMayOverflow(Other: KBRHS)) {
8518 case ConstantRange::OverflowResult::MayOverflow:
8519 return false;
8520 case ConstantRange::OverflowResult::NeverOverflows: {
8521 MatchInfo = [=](MachineIRBuilder &B) {
8522 B.buildSub(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoSWrap);
8523 B.buildConstant(Res: Carry, Val: 0);
8524 };
8525 return true;
8526 }
8527 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
8528 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
8529 MatchInfo = [=](MachineIRBuilder &B) {
8530 B.buildSub(Dst, Src0: LHS, Src1: RHS);
8531 B.buildConstant(Res: Carry, Val: getICmpTrueVal(TLI: getTargetLowering(),
8532 /*isVector=*/IsVector: CarryTy.isVector(),
8533 /*isFP=*/IsFP: false));
8534 };
8535 return true;
8536 }
8537 }
8538 return false;
8539 }
8540
8541 // G_USUBO
8542 switch (KBLHS.unsignedSubMayOverflow(Other: KBRHS)) {
8543 case ConstantRange::OverflowResult::MayOverflow:
8544 return false;
8545 case ConstantRange::OverflowResult::NeverOverflows: {
8546 MatchInfo = [=](MachineIRBuilder &B) {
8547 B.buildSub(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoUWrap);
8548 B.buildConstant(Res: Carry, Val: 0);
8549 };
8550 return true;
8551 }
8552 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
8553 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
8554 MatchInfo = [=](MachineIRBuilder &B) {
8555 B.buildSub(Dst, Src0: LHS, Src1: RHS);
8556 B.buildConstant(Res: Carry, Val: getICmpTrueVal(TLI: getTargetLowering(),
8557 /*isVector=*/IsVector: CarryTy.isVector(),
8558 /*isFP=*/IsFP: false));
8559 };
8560 return true;
8561 }
8562 }
8563
8564 return false;
8565}
8566
8567// Fold (ctlz (xor x, (sra x, bitwidth-1))) -> (add (ctls x), 1).
8568// Fold (ctlz (or (shl (xor x, (sra x, bitwidth-1)), 1), 1) -> (ctls x)
8569bool CombinerHelper::matchCtls(MachineInstr &CtlzMI,
8570 BuildFnTy &MatchInfo) const {
8571 assert((CtlzMI.getOpcode() == TargetOpcode::G_CTLZ ||
8572 CtlzMI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_POISON) &&
8573 "Expected G_CTLZ variant");
8574
8575 const Register Dst = CtlzMI.getOperand(i: 0).getReg();
8576 Register Src = CtlzMI.getOperand(i: 1).getReg();
8577
8578 LLT Ty = MRI.getType(Reg: Dst);
8579 LLT SrcTy = MRI.getType(Reg: Src);
8580
8581 if (!(Ty.isValid() && Ty.isScalar()))
8582 return false;
8583
8584 if (!LI)
8585 return false;
8586
8587 SmallVector<LLT, 2> QueryTypes = {Ty, SrcTy};
8588 LegalityQuery Query(TargetOpcode::G_CTLS, QueryTypes);
8589
8590 switch (LI->getAction(Query).Action) {
8591 default:
8592 return false;
8593 case LegalizeActions::Legal:
8594 case LegalizeActions::Custom:
8595 case LegalizeActions::WidenScalar:
8596 break;
8597 }
8598
8599 // Src = or(shl(V, 1), 1) -> Src=V; NeedAdd = False
8600 Register V;
8601 bool NeedAdd = true;
8602 if (mi_match(R: Src, MRI,
8603 P: m_OneUse(SP: m_GOr(L: m_OneUse(SP: m_GShl(L: m_Reg(R&: V), R: m_SpecificICst(RequestedValue: 1))),
8604 R: m_SpecificICst(RequestedValue: 1))))) {
8605 NeedAdd = false;
8606 Src = V;
8607 }
8608
8609 unsigned BitWidth = Ty.getScalarSizeInBits();
8610
8611 Register X;
8612 if (!mi_match(R: Src, MRI,
8613 P: m_OneUse(SP: m_GXor(L: m_Reg(R&: X), R: m_OneUse(SP: m_GAShr(
8614 L: m_DeferredReg(R&: X),
8615 R: m_SpecificICst(RequestedValue: BitWidth - 1)))))))
8616 return false;
8617
8618 MatchInfo = [=](MachineIRBuilder &B) {
8619 if (!NeedAdd) {
8620 B.buildCTLS(Dst, Src0: X);
8621 return;
8622 }
8623
8624 auto Ctls = B.buildCTLS(Dst: Ty, Src0: X);
8625 auto One = B.buildConstant(Res: Ty, Val: 1);
8626
8627 B.buildAdd(Dst, Src0: Ctls, Src1: One);
8628 };
8629
8630 return true;
8631}
8632
8633bool CombinerHelper::isConstantNaN(Register Reg) const {
8634 const ConstantFP *C = getConstantFPVRegVal(VReg: Reg, MRI);
8635 return C && C->getValueAPF().isNaN();
8636}
8637
8638bool CombinerHelper::matchCountZeroToZeroPoison(MachineInstr &MI, unsigned Opc,
8639 unsigned ZeroPoisonOpc) const {
8640 const bool IsCtlzPair = Opc == TargetOpcode::G_CTLZ &&
8641 ZeroPoisonOpc == TargetOpcode::G_CTLZ_ZERO_POISON;
8642 const bool IsCttzPair = Opc == TargetOpcode::G_CTTZ &&
8643 ZeroPoisonOpc == TargetOpcode::G_CTTZ_ZERO_POISON;
8644 if (MI.getOpcode() != Opc || (!IsCtlzPair && !IsCttzPair))
8645 return false;
8646 if (!VT)
8647 return false;
8648
8649 Register Src = MI.getOperand(i: 1).getReg();
8650 if (!VT->isKnownNeverZero(R: Src))
8651 return false;
8652
8653 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
8654 LLT SrcTy = MRI.getType(Reg: Src);
8655 return isLegalOrBeforeLegalizer(Query: {ZeroPoisonOpc, {DstTy, SrcTy}});
8656}
8657
8658static APInt getDemandedBitsForLogicalOp(unsigned Opcode, const APInt &Demanded,
8659 const KnownBits &OtherKnown) {
8660 assert((Opcode == TargetOpcode::G_AND || Opcode == TargetOpcode::G_OR) &&
8661 "Expected G_AND or G_OR");
8662 return Demanded &
8663 ~(Opcode == TargetOpcode::G_AND ? OtherKnown.Zero : OtherKnown.One);
8664}
8665
8666static std::optional<unsigned>
8667getValidConstShiftAmt(const std::optional<APInt> &Amt, unsigned BW) {
8668 if (!Amt || Amt->uge(RHS: BW))
8669 return std::nullopt;
8670 return Amt->getZExtValue();
8671}
8672
8673static APInt getDemandedSrcBitsForShiftConst(unsigned Opcode,
8674 const APInt &DemandedBits,
8675 unsigned ShAmt) {
8676 assert(ShAmt < DemandedBits.getBitWidth() &&
8677 "shift amount must be less than the bit width");
8678 switch (Opcode) {
8679 case TargetOpcode::G_SHL:
8680 return DemandedBits.lshr(shiftAmt: ShAmt);
8681 case TargetOpcode::G_LSHR:
8682 return DemandedBits.shl(shiftAmt: ShAmt);
8683 case TargetOpcode::G_ASHR: {
8684 APInt Src = DemandedBits.shl(shiftAmt: ShAmt);
8685 if (DemandedBits.countLeadingZeros() < ShAmt)
8686 Src.setSignBit();
8687 return Src;
8688 }
8689 default:
8690 llvm_unreachable("not a shift opcode");
8691 }
8692}
8693
8694Register CombinerHelper::simplifyMultipleUseDemandedBits(
8695 Register R, const APInt &DemandedBits, unsigned Depth) const {
8696 assert(R.isVirtual() && "Expected a virtual register");
8697 assert(!DemandedBits.isZero() && "Zero demand is handled by the caller");
8698 if (Depth >= MaxAnalysisRecursionDepth)
8699 return Register();
8700
8701 [[maybe_unused]] LLT Ty = MRI.getType(Reg: R);
8702 assert(Ty.isValid() && "Expected a typed register");
8703 assert(DemandedBits.getBitWidth() == Ty.getScalarSizeInBits() &&
8704 "DemandedBits width must match the register scalar type");
8705
8706 MachineInstr *DefMI = MRI.getVRegDef(Reg: R);
8707 assert(DefMI && "Expected a definition in generic SSA MIR");
8708 if (!isa<GAnd>(Val: DefMI) && !isa<GOr>(Val: DefMI))
8709 return Register();
8710
8711 unsigned Opcode = DefMI->getOpcode();
8712 auto *Logic = cast<GLogicalBinOp>(Val: DefMI);
8713 Register LHS = Logic->getLHSReg();
8714 Register RHS = Logic->getRHSReg();
8715 assert(MRI.getType(LHS) == Ty && MRI.getType(RHS) == Ty &&
8716 "Logical operands must match the result type");
8717
8718 auto LookThrough = [&](Register X, Register CReg) -> Register {
8719 std::optional<APInt> C = getConstantOrConstantSplatVector(Src: CReg);
8720 if (!C || C->getBitWidth() != DemandedBits.getBitWidth())
8721 return Register();
8722 bool BypassToX = Opcode == TargetOpcode::G_AND
8723 ? DemandedBits.isSubsetOf(RHS: *C)
8724 : DemandedBits.isSubsetOf(RHS: ~*C);
8725 if (BypassToX) {
8726 if (Register Deeper =
8727 simplifyMultipleUseDemandedBits(R: X, DemandedBits, Depth: Depth + 1))
8728 return Deeper;
8729 return X;
8730 }
8731 bool BypassToC = Opcode == TargetOpcode::G_AND
8732 ? DemandedBits.isSubsetOf(RHS: ~*C)
8733 : DemandedBits.isSubsetOf(RHS: *C);
8734 if (BypassToC)
8735 return CReg;
8736 return Register();
8737 };
8738
8739 if (Register Repl = LookThrough(LHS, RHS))
8740 return Repl;
8741 return LookThrough(RHS, LHS);
8742}
8743
8744bool CombinerHelper::simplifyDemandedBitsImpl(MachineInstr &MI, unsigned OpNo,
8745 const APInt &DemandedBits,
8746 KnownBits &Known, unsigned Depth,
8747 bool DoRewrite) const {
8748 Known = KnownBits(DemandedBits.getBitWidth());
8749 assert(MI.getOperand(OpNo).isReg() && !MI.getOperand(OpNo).isDef() &&
8750 "Expected a register use");
8751 Register OpReg = MI.getOperand(i: OpNo).getReg();
8752 assert(OpReg.isVirtual() && "Expected a virtual register");
8753 LLT OpTy = MRI.getType(Reg: OpReg);
8754 assert(OpTy.isValid() && "Expected a typed register");
8755 assert(DemandedBits.getBitWidth() == OpTy.getScalarSizeInBits() &&
8756 "DemandedBits width must match the operand scalar type");
8757
8758 MachineInstr *DefMI = MRI.getVRegDef(Reg: OpReg);
8759 assert(DefMI && "Expected a definition in generic SSA MIR");
8760 // Undemanded bits can still affect poison-generating flags in the user.
8761 auto DropPoisonFlags = [&]() {
8762 if (!DoRewrite ||
8763 !(MI.getFlags() & MachineInstr::getPoisonGeneratingFlags()))
8764 return;
8765 Observer.changingInstr(MI);
8766 MI.clearFlags(flags: MachineInstr::getPoisonGeneratingFlags());
8767 Observer.changedInstr(MI);
8768 };
8769 // Debug users can observe bits that are not demanded by other users.
8770 auto InvalidateDebugUses = [&](const APInt &Demanded) {
8771 if (DoRewrite && !Demanded.isAllOnes())
8772 MRI.markUsesInDebugValueAsUndef(Reg: OpReg);
8773 };
8774 auto Rewrite = [&](Register Repl) {
8775 if (!DoRewrite || Repl == OpReg)
8776 return Repl != OpReg;
8777 DropPoisonFlags();
8778 if (DefMI->getNumExplicitDefs() == 1 && MRI.hasOneNonDBGUse(RegNo: OpReg)) {
8779 InvalidateDebugUses(DemandedBits);
8780 replaceRegWith(MRI, FromReg: OpReg, ToReg: Repl);
8781 eraseInst(MI&: *DefMI);
8782 } else {
8783 replaceRegOpWith(MRI, FromRegOp&: MI.getOperand(i: OpNo), ToReg: Repl);
8784 }
8785 return true;
8786 };
8787
8788 if (DemandedBits.isZero()) {
8789 if (DefMI->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
8790 return false;
8791 if (!DoRewrite)
8792 return true;
8793 MachineBasicBlock &SaveMBB = Builder.getMBB();
8794 MachineBasicBlock::iterator SavePt = Builder.getInsertPt();
8795 DebugLoc SaveDL = Builder.getDL();
8796 Builder.setInstrAndDebugLoc(MI);
8797 Register Undef = Builder.buildUndef(Res: OpTy).getReg(Idx: 0);
8798 Builder.setInsertPt(MBB&: SaveMBB, II: SavePt);
8799 Builder.setDebugLoc(SaveDL);
8800 // The producer may have side effects; replace only this use.
8801 DropPoisonFlags();
8802 replaceRegOpWith(MRI, FromRegOp&: MI.getOperand(i: OpNo), ToReg: Undef);
8803 return true;
8804 }
8805
8806 if (!VT)
8807 return false;
8808
8809 unsigned BW = DemandedBits.getBitWidth();
8810 auto GiveUp = [&]() {
8811 APInt DemandedElts = OpTy.isFixedVector()
8812 ? APInt::getAllOnes(numBits: OpTy.getNumElements())
8813 : APInt(1, 1);
8814 Known = VT->getKnownBits(R: OpReg, DemandedElts, Depth);
8815 return false;
8816 };
8817
8818 if (Depth >= MaxAnalysisRecursionDepth || DefMI->getNumExplicitDefs() != 1)
8819 return GiveUp();
8820
8821 // Shared defs require full demand unless this use can be rerouted to an
8822 // existing value that agrees on its demanded bits.
8823 APInt Demanded = DemandedBits;
8824 if (!MRI.hasOneNonDBGUse(RegNo: OpReg) && !Demanded.isAllOnes()) {
8825 if (Register Repl =
8826 simplifyMultipleUseDemandedBits(R: OpReg, DemandedBits: Demanded, Depth)) {
8827 Known = VT->getKnownBits(R: Repl);
8828 if (Rewrite(Repl))
8829 return true;
8830 }
8831 Demanded = APInt::getAllOnes(numBits: BW);
8832 }
8833
8834 unsigned Opcode = DefMI->getOpcode();
8835 switch (Opcode) {
8836 case TargetOpcode::G_AND:
8837 case TargetOpcode::G_OR: {
8838 auto *Logic = cast<GLogicalBinOp>(Val: DefMI);
8839 Register LHS = Logic->getLHSReg();
8840 Register RHS = Logic->getRHSReg();
8841 assert(MRI.getType(LHS) == OpTy && MRI.getType(RHS) == OpTy &&
8842 "Logical operands must match the result type");
8843 auto SimplifyWithConst = [&](Register X,
8844 Register CReg) -> std::optional<Register> {
8845 std::optional<APInt> C = getConstantOrConstantSplatVector(Src: CReg);
8846 if (!C)
8847 return std::nullopt;
8848 assert(C->getBitWidth() == BW && "Constant width must match its type");
8849 // Check redundancy before rewriting X can change its known bits.
8850 if (Opcode == TargetOpcode::G_AND) {
8851 if (Demanded.isSubsetOf(RHS: *C | VT->getKnownBits(R: X).Zero))
8852 return X;
8853 if (Demanded.isSubsetOf(RHS: ~*C))
8854 return CReg;
8855 return std::nullopt;
8856 }
8857 if (Demanded.isSubsetOf(RHS: ~*C | VT->getKnownBits(R: X).One))
8858 return X;
8859 if (Demanded.isSubsetOf(RHS: *C))
8860 return CReg;
8861 return std::nullopt;
8862 };
8863
8864 // Compute known bits before a rewrite can erase the def.
8865 if (std::optional<Register> Repl = SimplifyWithConst(LHS, RHS)) {
8866 Known = VT->getKnownBits(R: *Repl);
8867 if (Rewrite(*Repl))
8868 return true;
8869 }
8870 if (std::optional<Register> Repl = SimplifyWithConst(RHS, LHS)) {
8871 Known = VT->getKnownBits(R: *Repl);
8872 if (Rewrite(*Repl))
8873 return true;
8874 }
8875
8876 KnownBits RHSKnown(BW);
8877 bool Changed = simplifyDemandedBitsImpl(MI&: *DefMI, /*OpNo=*/2, DemandedBits: Demanded,
8878 Known&: RHSKnown, Depth: Depth + 1, DoRewrite);
8879 APInt LHSDemand = getDemandedBitsForLogicalOp(Opcode, Demanded, OtherKnown: RHSKnown);
8880 KnownBits LHSKnown(BW);
8881 Changed |= simplifyDemandedBitsImpl(MI&: *DefMI, /*OpNo=*/1, DemandedBits: LHSDemand, Known&: LHSKnown,
8882 Depth: Depth + 1, DoRewrite);
8883 if (!Changed && MRI.hasOneNonDBGUse(RegNo: RHS) &&
8884 !getConstantOrConstantSplatVector(Src: RHS)) {
8885 // Reduced-demand LHS facts cannot justify reducing the RHS demand.
8886 APInt RHSDemand =
8887 LHSDemand == Demanded
8888 ? getDemandedBitsForLogicalOp(Opcode, Demanded, OtherKnown: LHSKnown)
8889 : getDemandedBitsForLogicalOp(
8890 Opcode, Demanded,
8891 OtherKnown: VT->getKnownBits(
8892 R: LHS,
8893 DemandedElts: OpTy.isFixedVector()
8894 ? APInt::getAllOnes(numBits: OpTy.getNumElements())
8895 : APInt(1, 1),
8896 Depth: Depth + 1));
8897 if (RHSDemand != Demanded) {
8898 // Keep full-demand RHS facts if the reduced-demand retry fails.
8899 KnownBits RetryKnown(BW);
8900 Changed = simplifyDemandedBitsImpl(MI&: *DefMI, /*OpNo=*/2, DemandedBits: RHSDemand,
8901 Known&: RetryKnown, Depth: Depth + 1, DoRewrite);
8902 }
8903 }
8904 if (Changed) {
8905 DropPoisonFlags();
8906 InvalidateDebugUses(Demanded);
8907 }
8908 Known = Opcode == TargetOpcode::G_AND ? LHSKnown & RHSKnown
8909 : LHSKnown | RHSKnown;
8910 return Changed;
8911 }
8912 case TargetOpcode::G_SHL:
8913 case TargetOpcode::G_LSHR:
8914 case TargetOpcode::G_ASHR: {
8915 std::optional<unsigned> ShAmt = getValidConstShiftAmt(
8916 Amt: getConstantOrConstantSplatVector(Src: DefMI->getOperand(i: 2).getReg()), BW);
8917 if (!ShAmt)
8918 return GiveUp();
8919
8920 APInt SrcDemand = getDemandedSrcBitsForShiftConst(Opcode, DemandedBits: Demanded, ShAmt: *ShAmt);
8921 KnownBits SrcKnown(BW);
8922 bool Changed = simplifyDemandedBitsImpl(MI&: *DefMI, /*OpNo=*/1, DemandedBits: SrcDemand,
8923 Known&: SrcKnown, Depth: Depth + 1, DoRewrite);
8924 if (Changed) {
8925 DropPoisonFlags();
8926 InvalidateDebugUses(Demanded);
8927 }
8928 KnownBits AmtKnown = KnownBits::makeConstant(C: APInt(BW, *ShAmt));
8929 switch (Opcode) {
8930 case TargetOpcode::G_SHL:
8931 Known = KnownBits::shl(LHS: SrcKnown, RHS: AmtKnown);
8932 break;
8933 case TargetOpcode::G_LSHR:
8934 Known = KnownBits::lshr(LHS: SrcKnown, RHS: AmtKnown);
8935 break;
8936 case TargetOpcode::G_ASHR: {
8937 // ASHR and LSHR agree outside sign fill or for nonnegative inputs.
8938 LLT AmtTy = MRI.getType(Reg: DefMI->getOperand(i: 2).getReg());
8939 bool LShrLegal = isPreLegalize() || !LI ||
8940 isLegal(Query: {TargetOpcode::G_LSHR, {OpTy, AmtTy}});
8941 if ((Demanded.countLeadingZeros() >= *ShAmt ||
8942 SrcKnown.isNonNegative()) &&
8943 LShrLegal) {
8944 if (DoRewrite) {
8945 MachineBasicBlock &SaveMBB = Builder.getMBB();
8946 MachineBasicBlock::iterator SavePt = Builder.getInsertPt();
8947 DebugLoc SaveDL = Builder.getDL();
8948 Builder.setInstrAndDebugLoc(*DefMI);
8949 auto Lshr = Builder.buildLShr(Dst: OpTy, Src0: DefMI->getOperand(i: 1).getReg(),
8950 Src1: DefMI->getOperand(i: 2).getReg());
8951 Builder.setInsertPt(MBB&: SaveMBB, II: SavePt);
8952 Builder.setDebugLoc(SaveDL);
8953 Rewrite(Lshr.getReg(Idx: 0));
8954 Known = KnownBits::lshr(LHS: SrcKnown, RHS: AmtKnown);
8955 }
8956 return true;
8957 }
8958 Known = KnownBits::ashr(LHS: SrcKnown, RHS: AmtKnown);
8959 break;
8960 }
8961 }
8962 return Changed;
8963 }
8964 default:
8965 return GiveUp();
8966 }
8967}
8968
8969bool CombinerHelper::simplifyDemandedBits(MachineInstr &MI, unsigned OpNo,
8970 const APInt &DemandedBits,
8971 KnownBits &Known,
8972 unsigned Depth) const {
8973 return simplifyDemandedBitsImpl(MI, OpNo, DemandedBits, Known, Depth,
8974 /*DoRewrite=*/true);
8975}
8976
8977bool CombinerHelper::matchSimplifyDemandedBits(MachineInstr &MI,
8978 BuildFnTy &MatchInfo) const {
8979 Register Dst = MI.getOperand(i: 0).getReg();
8980 LLT Ty = MRI.getType(Reg: Dst);
8981 assert(Ty.isValid() && "Expected a typed generic result");
8982 if (!VT || Ty.isVector())
8983 return false;
8984
8985 APInt RootDemand = APInt::getAllOnes(numBits: Ty.getScalarSizeInBits());
8986 auto Probe = [&](unsigned OpNo, const APInt &OpDemand, KnownBits &Known) {
8987 if (!simplifyDemandedBitsImpl(MI, OpNo, DemandedBits: OpDemand, Known, /*Depth=*/0,
8988 /*DoRewrite=*/false))
8989 return false;
8990
8991 MatchInfo = [this, &MI, OpNo, OpDemand](MachineIRBuilder &) {
8992 KnownBits K(OpDemand.getBitWidth());
8993 simplifyDemandedBits(MI, OpNo, DemandedBits: OpDemand, Known&: K);
8994 };
8995 return true;
8996 };
8997
8998 unsigned Opcode = MI.getOpcode();
8999 switch (Opcode) {
9000 case TargetOpcode::G_SHL:
9001 case TargetOpcode::G_LSHR:
9002 case TargetOpcode::G_ASHR: {
9003 std::optional<unsigned> ShAmt = getValidConstShiftAmt(
9004 Amt: getConstantOrConstantSplatVector(Src: MI.getOperand(i: 2).getReg()),
9005 BW: RootDemand.getBitWidth());
9006 if (!ShAmt)
9007 return false;
9008 APInt SrcDemand =
9009 getDemandedSrcBitsForShiftConst(Opcode, DemandedBits: RootDemand, ShAmt: *ShAmt);
9010 KnownBits SrcKnown(RootDemand.getBitWidth());
9011 return Probe(/*OpNo=*/1, SrcDemand, SrcKnown);
9012 }
9013 default:
9014 return false;
9015 case TargetOpcode::G_AND:
9016 case TargetOpcode::G_OR: {
9017 // Let redundant_and/redundant_or fold the root before operand rewrites can
9018 // discard the known bits that make it redundant.
9019 KnownBits L = VT->getKnownBits(R: MI.getOperand(i: 1).getReg());
9020 KnownBits R = VT->getKnownBits(R: MI.getOperand(i: 2).getReg());
9021 bool RootRedundant = Opcode == TargetOpcode::G_AND
9022 ? RootDemand.isSubsetOf(RHS: L.Zero | R.One) ||
9023 RootDemand.isSubsetOf(RHS: R.Zero | L.One)
9024 : RootDemand.isSubsetOf(RHS: L.One | R.Zero) ||
9025 RootDemand.isSubsetOf(RHS: R.One | L.Zero);
9026 if (RootRedundant)
9027 return false;
9028
9029 KnownBits RHSKnown(RootDemand.getBitWidth());
9030 if (Probe(/*OpNo=*/2, RootDemand, RHSKnown))
9031 return true;
9032
9033 APInt LHSDemand = getDemandedBitsForLogicalOp(Opcode, Demanded: RootDemand, OtherKnown: RHSKnown);
9034
9035 KnownBits LHSKnown(RootDemand.getBitWidth());
9036 if (Probe(/*OpNo=*/1, LHSDemand, LHSKnown))
9037 return true;
9038
9039 Register RHS = MI.getOperand(i: 2).getReg();
9040 if (!MRI.hasOneNonDBGUse(RegNo: RHS) || getConstantOrConstantSplatVector(Src: RHS))
9041 return false;
9042
9043 APInt RHSDemand =
9044 LHSDemand == RootDemand
9045 ? getDemandedBitsForLogicalOp(Opcode, Demanded: RootDemand, OtherKnown: LHSKnown)
9046 : getDemandedBitsForLogicalOp(
9047 Opcode, Demanded: RootDemand,
9048 OtherKnown: VT->getKnownBits(R: MI.getOperand(i: 1).getReg()));
9049 KnownBits RetryKnown(RootDemand.getBitWidth());
9050 return RHSDemand != RootDemand && Probe(/*OpNo=*/2, RHSDemand, RetryKnown);
9051 }
9052 }
9053}
9054