1//===-- lib/CodeGen/GlobalISel/GICombinerHelper.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
9#include "llvm/ADT/APFloat.h"
10#include "llvm/ADT/STLExtras.h"
11#include "llvm/ADT/SetVector.h"
12#include "llvm/ADT/SmallBitVector.h"
13#include "llvm/Analysis/CmpInstAnalysis.h"
14#include "llvm/Analysis/ValueTracking.h"
15#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
16#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
17#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
18#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
19#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
20#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
21#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22#include "llvm/CodeGen/GlobalISel/Utils.h"
23#include "llvm/CodeGen/LowLevelTypeUtils.h"
24#include "llvm/CodeGen/MachineBasicBlock.h"
25#include "llvm/CodeGen/MachineDominators.h"
26#include "llvm/CodeGen/MachineInstr.h"
27#include "llvm/CodeGen/MachineMemOperand.h"
28#include "llvm/CodeGen/MachineRegisterInfo.h"
29#include "llvm/CodeGen/Register.h"
30#include "llvm/CodeGen/RegisterBankInfo.h"
31#include "llvm/CodeGen/TargetInstrInfo.h"
32#include "llvm/CodeGen/TargetLowering.h"
33#include "llvm/CodeGen/TargetOpcodes.h"
34#include "llvm/IR/ConstantRange.h"
35#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/InstrTypes.h"
37#include "llvm/Support/Casting.h"
38#include "llvm/Support/DivisionByConstantInfo.h"
39#include "llvm/Support/ErrorHandling.h"
40#include "llvm/Support/KnownBits.h"
41#include "llvm/Support/MathExtras.h"
42#include "llvm/Target/TargetMachine.h"
43#include <cmath>
44#include <optional>
45#include <tuple>
46
47#define DEBUG_TYPE "gi-combiner"
48
49using namespace llvm;
50using namespace MIPatternMatch;
51
52// Option to allow testing of the combiner while no targets know about indexed
53// addressing.
54static cl::opt<bool>
55 ForceLegalIndexing("force-legal-indexing", cl::Hidden, cl::init(Val: false),
56 cl::desc("Force all indexed operations to be "
57 "legal for the GlobalISel combiner"));
58
59CombinerHelper::CombinerHelper(GISelChangeObserver &Observer,
60 MachineIRBuilder &B, bool IsPreLegalize,
61 GISelValueTracking *VT,
62 MachineDominatorTree *MDT,
63 const LegalizerInfo *LI)
64 : Builder(B), MRI(Builder.getMF().getRegInfo()), Observer(Observer), VT(VT),
65 MDT(MDT), IsPreLegalize(IsPreLegalize), LI(LI),
66 TII(Builder.getMF().getSubtarget().getInstrInfo()),
67 RBI(Builder.getMF().getSubtarget().getRegBankInfo()),
68 TRI(Builder.getMF().getSubtarget().getRegisterInfo()) {
69 (void)this->VT;
70}
71
72const TargetLowering &CombinerHelper::getTargetLowering() const {
73 return *Builder.getMF().getSubtarget().getTargetLowering();
74}
75
76const MachineFunction &CombinerHelper::getMachineFunction() const {
77 return Builder.getMF();
78}
79
80LLVMContext &CombinerHelper::getContext() const { return Builder.getContext(); }
81
82/// \returns The little endian in-memory byte position of byte \p I in a
83/// \p ByteWidth bytes wide type.
84///
85/// E.g. Given a 4-byte type x, x[0] -> byte 0
86static unsigned littleEndianByteAt(const unsigned ByteWidth, const unsigned I) {
87 assert(I < ByteWidth && "I must be in [0, ByteWidth)");
88 return I;
89}
90
91/// Determines the LogBase2 value for a non-null input value using the
92/// transform: LogBase2(V) = (EltBits - 1) - ctlz(V).
93static Register buildLogBase2(Register V, MachineIRBuilder &MIB) {
94 auto &MRI = *MIB.getMRI();
95 LLT Ty = MRI.getType(Reg: V);
96 auto Ctlz = MIB.buildCTLZ(Dst: Ty, Src0: V);
97 auto Base = MIB.buildConstant(Res: Ty, Val: Ty.getScalarSizeInBits() - 1);
98 return MIB.buildSub(Dst: Ty, Src0: Base, Src1: Ctlz).getReg(Idx: 0);
99}
100
101/// \returns The big endian in-memory byte position of byte \p I in a
102/// \p ByteWidth bytes wide type.
103///
104/// E.g. Given a 4-byte type x, x[0] -> byte 3
105static unsigned bigEndianByteAt(const unsigned ByteWidth, const unsigned I) {
106 assert(I < ByteWidth && "I must be in [0, ByteWidth)");
107 return ByteWidth - I - 1;
108}
109
110/// Given a map from byte offsets in memory to indices in a load/store,
111/// determine if that map corresponds to a little or big endian byte pattern.
112///
113/// \param MemOffset2Idx maps memory offsets to address offsets.
114/// \param LowestIdx is the lowest index in \p MemOffset2Idx.
115///
116/// \returns true if the map corresponds to a big endian byte pattern, false if
117/// it corresponds to a little endian byte pattern, and std::nullopt otherwise.
118///
119/// E.g. given a 32-bit type x, and x[AddrOffset], the in-memory byte patterns
120/// are as follows:
121///
122/// AddrOffset Little endian Big endian
123/// 0 0 3
124/// 1 1 2
125/// 2 2 1
126/// 3 3 0
127static std::optional<bool>
128isBigEndian(const SmallDenseMap<int64_t, int64_t, 8> &MemOffset2Idx,
129 int64_t LowestIdx) {
130 // Need at least two byte positions to decide on endianness.
131 unsigned Width = MemOffset2Idx.size();
132 if (Width < 2)
133 return std::nullopt;
134 bool BigEndian = true, LittleEndian = true;
135 for (unsigned MemOffset = 0; MemOffset < Width; ++ MemOffset) {
136 auto MemOffsetAndIdx = MemOffset2Idx.find(Val: MemOffset);
137 if (MemOffsetAndIdx == MemOffset2Idx.end())
138 return std::nullopt;
139 const int64_t Idx = MemOffsetAndIdx->second - LowestIdx;
140 assert(Idx >= 0 && "Expected non-negative byte offset?");
141 LittleEndian &= Idx == littleEndianByteAt(ByteWidth: Width, I: MemOffset);
142 BigEndian &= Idx == bigEndianByteAt(ByteWidth: Width, I: MemOffset);
143 if (!BigEndian && !LittleEndian)
144 return std::nullopt;
145 }
146
147 assert((BigEndian != LittleEndian) &&
148 "Pattern cannot be both big and little endian!");
149 return BigEndian;
150}
151
152bool CombinerHelper::isPreLegalize() const { return IsPreLegalize; }
153
154bool CombinerHelper::isLegal(const LegalityQuery &Query) const {
155 assert(LI && "Must have LegalizerInfo to query isLegal!");
156 return LI->getAction(Query).Action == LegalizeActions::Legal;
157}
158
159bool CombinerHelper::isLegalOrBeforeLegalizer(
160 const LegalityQuery &Query) const {
161 return isPreLegalize() || isLegal(Query);
162}
163
164bool CombinerHelper::isLegalOrHasWidenScalar(const LegalityQuery &Query) const {
165 return isLegal(Query) ||
166 LI->getAction(Query).Action == LegalizeActions::WidenScalar;
167}
168
169bool CombinerHelper::isLegalOrHasFewerElements(
170 const LegalityQuery &Query) const {
171 LegalizeAction Action = LI->getAction(Query).Action;
172 return Action == LegalizeActions::Legal ||
173 Action == LegalizeActions::FewerElements;
174}
175
176bool CombinerHelper::isConstantLegalOrBeforeLegalizer(const LLT Ty) const {
177 if (!Ty.isVector())
178 return isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_CONSTANT, {Ty}});
179 // Vector constants are represented as a G_BUILD_VECTOR of scalar G_CONSTANTs.
180 if (isPreLegalize())
181 return true;
182 LLT EltTy = Ty.getElementType();
183 return isLegal(Query: {TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}}) &&
184 isLegal(Query: {TargetOpcode::G_CONSTANT, {EltTy}});
185}
186
187void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg,
188 Register ToReg) const {
189 Observer.changingAllUsesOfReg(MRI, Reg: FromReg);
190
191 if (MRI.constrainRegAttrs(Reg: ToReg, ConstrainingReg: FromReg))
192 MRI.replaceRegWith(FromReg, ToReg);
193 else
194 Builder.buildCopy(Res: FromReg, Op: ToReg);
195
196 Observer.finishedChangingAllUsesOfReg();
197}
198
199void CombinerHelper::replaceRegOpWith(MachineRegisterInfo &MRI,
200 MachineOperand &FromRegOp,
201 Register ToReg) const {
202 assert(FromRegOp.getParent() && "Expected an operand in an MI");
203 Observer.changingInstr(MI&: *FromRegOp.getParent());
204
205 FromRegOp.setReg(ToReg);
206
207 Observer.changedInstr(MI&: *FromRegOp.getParent());
208}
209
210void CombinerHelper::replaceOpcodeWith(MachineInstr &FromMI,
211 unsigned ToOpcode) const {
212 Observer.changingInstr(MI&: FromMI);
213
214 FromMI.setDesc(Builder.getTII().get(Opcode: ToOpcode));
215
216 Observer.changedInstr(MI&: FromMI);
217}
218
219const RegisterBank *CombinerHelper::getRegBank(Register Reg) const {
220 return RBI->getRegBank(Reg, MRI, TRI: *TRI);
221}
222
223void CombinerHelper::setRegBank(Register Reg,
224 const RegisterBank *RegBank) const {
225 if (RegBank)
226 MRI.setRegBank(Reg, RegBank: *RegBank);
227}
228
229bool CombinerHelper::matchCombineCopy(MachineInstr &MI) const {
230 if (MI.getOpcode() != TargetOpcode::COPY)
231 return false;
232 Register DstReg = MI.getOperand(i: 0).getReg();
233 Register SrcReg = MI.getOperand(i: 1).getReg();
234 return canReplaceReg(DstReg, SrcReg, MRI);
235}
236void CombinerHelper::applyCombineCopy(MachineInstr &MI) const {
237 Register DstReg = MI.getOperand(i: 0).getReg();
238 Register SrcReg = MI.getOperand(i: 1).getReg();
239 replaceRegWith(MRI, FromReg: DstReg, ToReg: SrcReg);
240 MI.eraseFromParent();
241}
242
243bool CombinerHelper::matchFreezeOfSingleMaybePoisonOperand(
244 MachineInstr &MI, BuildFnTy &MatchInfo) const {
245 assert(MI.getOpcode() == TargetOpcode::G_FREEZE && "Invalid instruction");
246
247 // Ported from InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating.
248 Register DstOp = MI.getOperand(i: 0).getReg();
249 Register OrigOp = MI.getOperand(i: 1).getReg();
250
251 if (!MRI.hasOneNonDBGUse(RegNo: OrigOp))
252 return false;
253
254 MachineInstr *OrigDef;
255 if (!mi_match(R: OrigOp, MRI, P: m_MInstr(MI&: OrigDef)))
256 return false;
257 // Even if only a single operand of the PHI is not guaranteed non-poison,
258 // moving freeze() backwards across a PHI can cause optimization issues for
259 // other users of that operand.
260 //
261 // Moving freeze() from one of the output registers of a G_UNMERGE_VALUES to
262 // the source register is unprofitable because it makes the freeze() more
263 // strict than is necessary (it would affect the whole register instead of
264 // just the subreg being frozen).
265 if (OrigDef->isPHI() || isa<GUnmerge>(Val: OrigDef))
266 return false;
267
268 if (canCreateUndefOrPoison(Reg: OrigOp, MRI,
269 /*ConsiderFlagsAndMetadata=*/false))
270 return false;
271
272 std::optional<MachineOperand> MaybePoisonOperand;
273 for (MachineOperand &Operand : OrigDef->uses()) {
274 if (!Operand.isReg())
275 return false;
276
277 if (isGuaranteedNotToBeUndefOrPoison(Reg: Operand.getReg(), MRI))
278 continue;
279
280 if (!MaybePoisonOperand)
281 MaybePoisonOperand = Operand;
282 else {
283 // We have more than one maybe-poison operand. Moving the freeze is
284 // unsafe.
285 return false;
286 }
287 }
288
289 // Eliminate freeze if all operands are guaranteed non-poison.
290 if (!MaybePoisonOperand) {
291 MatchInfo = [=](MachineIRBuilder &B) {
292 Observer.changingInstr(MI&: *OrigDef);
293 cast<GenericMachineInstr>(Val: OrigDef)->dropPoisonGeneratingFlags();
294 Observer.changedInstr(MI&: *OrigDef);
295 B.buildCopy(Res: DstOp, Op: OrigOp);
296 };
297 return true;
298 }
299
300 Register MaybePoisonOperandReg = MaybePoisonOperand->getReg();
301 LLT MaybePoisonOperandRegTy = MRI.getType(Reg: MaybePoisonOperandReg);
302
303 if (!isLegalOrBeforeLegalizer(
304 Query: {TargetOpcode::G_FREEZE, {MaybePoisonOperandRegTy}}))
305 return false;
306
307 MatchInfo = [=](MachineIRBuilder &B) mutable {
308 Observer.changingInstr(MI&: *OrigDef);
309 cast<GenericMachineInstr>(Val: OrigDef)->dropPoisonGeneratingFlags();
310 Observer.changedInstr(MI&: *OrigDef);
311 B.setInsertPt(MBB&: *OrigDef->getParent(), II: OrigDef->getIterator());
312 auto Freeze = B.buildFreeze(Dst: MaybePoisonOperandRegTy, Src: MaybePoisonOperandReg);
313 replaceRegOpWith(
314 MRI, FromRegOp&: *OrigDef->findRegisterUseOperand(Reg: MaybePoisonOperandReg, TRI),
315 ToReg: Freeze.getReg(Idx: 0));
316 replaceRegWith(MRI, FromReg: DstOp, ToReg: OrigOp);
317 };
318 return true;
319}
320
321bool CombinerHelper::matchCombineConcatVectors(
322 MachineInstr &MI, SmallVector<Register> &Ops) const {
323 assert(MI.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
324 "Invalid instruction");
325 bool IsUndef = true;
326 MachineInstr *Undef = nullptr;
327
328 // Walk over all the operands of concat vectors and check if they are
329 // build_vector themselves or undef.
330 // Then collect their operands in Ops.
331 for (const MachineOperand &MO : MI.uses()) {
332 Register Reg = MO.getReg();
333 MachineInstr *Def;
334 if (!mi_match(R: Reg, MRI, P: m_MInstr(MI&: Def)))
335 return false;
336 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
337 return false;
338 switch (Def->getOpcode()) {
339 case TargetOpcode::G_BUILD_VECTOR:
340 IsUndef = false;
341 // Remember the operands of the build_vector to fold
342 // them into the yet-to-build flattened concat vectors.
343 for (const MachineOperand &BuildVecMO : Def->uses())
344 Ops.push_back(Elt: BuildVecMO.getReg());
345 break;
346 case TargetOpcode::G_IMPLICIT_DEF: {
347 LLT OpType = MRI.getType(Reg);
348 // Keep one undef value for all the undef operands.
349 if (!Undef) {
350 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
351 Undef = Builder.buildUndef(Res: OpType.getScalarType());
352 }
353 assert(MRI.getType(Undef->getOperand(0).getReg()) ==
354 OpType.getScalarType() &&
355 "All undefs should have the same type");
356 // Break the undef vector in as many scalar elements as needed
357 // for the flattening.
358 for (unsigned EltIdx = 0, EltEnd = OpType.getNumElements();
359 EltIdx != EltEnd; ++EltIdx)
360 Ops.push_back(Elt: Undef->getOperand(i: 0).getReg());
361 break;
362 }
363 default:
364 return false;
365 }
366 }
367
368 // Check if the combine is illegal
369 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
370 if (!isLegalOrBeforeLegalizer(
371 Query: {TargetOpcode::G_BUILD_VECTOR, {DstTy, MRI.getType(Reg: Ops[0])}})) {
372 return false;
373 }
374
375 if (IsUndef)
376 Ops.clear();
377
378 return true;
379}
380void CombinerHelper::applyCombineConcatVectors(
381 MachineInstr &MI, SmallVector<Register> &Ops) const {
382 // We determined that the concat_vectors can be flatten.
383 // Generate the flattened build_vector.
384 Register DstReg = MI.getOperand(i: 0).getReg();
385 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
386 Register NewDstReg = MRI.cloneVirtualRegister(VReg: DstReg);
387
388 // Note: IsUndef is sort of redundant. We could have determine it by
389 // checking that at all Ops are undef. Alternatively, we could have
390 // generate a build_vector of undefs and rely on another combine to
391 // clean that up. For now, given we already gather this information
392 // in matchCombineConcatVectors, just save compile time and issue the
393 // right thing.
394 if (Ops.empty())
395 Builder.buildUndef(Res: NewDstReg);
396 else
397 Builder.buildBuildVector(Res: NewDstReg, Ops);
398 replaceRegWith(MRI, FromReg: DstReg, ToReg: NewDstReg);
399 MI.eraseFromParent();
400}
401
402bool CombinerHelper::matchCombineBuildVectorOfBitcast(
403 MachineInstr &MI, SmallVector<Register> &Ops) const {
404 auto &BV = cast<GBuildVector>(Val&: MI);
405
406 // Look at the first operand for a unmerge(bitcast) from a scalar type.
407 GUnmerge *Unmerge = getOpcodeDef<GUnmerge>(Reg: BV.getSourceReg(I: 0), MRI);
408 if (!Unmerge || Unmerge->getReg(Idx: 0) != BV.getSourceReg(I: 0))
409 return false;
410 Register BCSrc;
411 if (!mi_match(R: Unmerge->getSourceReg(), MRI, P: m_GBitcast(Src: m_Reg(R&: BCSrc))))
412 return false;
413 LLT InputTy = MRI.getType(Reg: BCSrc);
414 unsigned Factor = Unmerge->getNumDefs();
415 if (!InputTy.isScalar() || BV.getNumSources() % Factor != 0)
416 return false;
417
418 // Check if the build_vector is legal
419 LLT BVDstTy = LLT::fixed_vector(NumElements: BV.getNumSources() / Factor, ScalarTy: InputTy);
420 if (!isLegal(Query: {TargetOpcode::G_BUILD_VECTOR, {BVDstTy, InputTy}}))
421 return false;
422
423 // Check all other operands are bitcasts or undef.
424 for (unsigned Idx = 0; Idx < BV.getNumSources(); Idx += Factor) {
425 GUnmerge *Unmerge = getOpcodeDef<GUnmerge>(Reg: BV.getSourceReg(I: Idx), MRI);
426 if (!all_of(Range: iota_range<unsigned>(0, Factor, false), P: [&](unsigned J) {
427 if (mi_match(R: BV.getSourceReg(I: Idx + J), MRI, P: m_GImplicitDef()))
428 return true;
429 return Unmerge && BV.getSourceReg(I: Idx + J) == Unmerge->getReg(Idx: J);
430 }))
431 return false;
432 if (!Unmerge)
433 Ops.push_back(Elt: 0);
434 else {
435 Register BCSrc;
436 if (!mi_match(
437 R: Unmerge->getSourceReg(), MRI,
438 P: m_GBitcast(Src: m_all_of(preds: m_Reg(R&: BCSrc), preds: m_SpecificType(Ty: InputTy)))))
439 return false;
440 Ops.push_back(Elt: BCSrc);
441 }
442 }
443
444 return true;
445}
446
447void CombinerHelper::applyCombineBuildVectorOfBitcast(
448 MachineInstr &MI, SmallVector<Register> &Ops) const {
449 LLT SrcTy = MRI.getType(Reg: Ops[0]);
450 // Build undef if any operations require it.
451 Register Undef = 0;
452 for (Register &Op : Ops) {
453 if (!Op) {
454 if (!Undef)
455 Undef = Builder.buildUndef(Res: SrcTy).getReg(Idx: 0);
456 Op = Undef;
457 }
458 }
459
460 LLT BVDstTy = LLT::fixed_vector(NumElements: Ops.size(), ScalarTy: SrcTy);
461 auto BV = Builder.buildBuildVector(Res: BVDstTy, Ops);
462 Builder.buildBitcast(Dst: MI.getOperand(i: 0).getReg(), Src: BV);
463 MI.eraseFromParent();
464}
465
466void CombinerHelper::applyCombineShuffleToBuildVector(MachineInstr &MI) const {
467 auto &Shuffle = cast<GShuffleVector>(Val&: MI);
468
469 Register SrcVec1 = Shuffle.getSrc1Reg();
470 Register SrcVec2 = Shuffle.getSrc2Reg();
471 LLT EltTy = MRI.getType(Reg: SrcVec1).getElementType();
472 int Width = MRI.getType(Reg: SrcVec1).getNumElements();
473
474 auto Unmerge1 = Builder.buildUnmerge(Res: EltTy, Op: SrcVec1);
475 auto Unmerge2 = Builder.buildUnmerge(Res: EltTy, Op: SrcVec2);
476
477 SmallVector<Register> Extracts;
478 // Select only applicable elements from unmerged values.
479 for (int Val : Shuffle.getMask()) {
480 if (Val == -1)
481 Extracts.push_back(Elt: Builder.buildUndef(Res: EltTy).getReg(Idx: 0));
482 else if (Val < Width)
483 Extracts.push_back(Elt: Unmerge1.getReg(Idx: Val));
484 else
485 Extracts.push_back(Elt: Unmerge2.getReg(Idx: Val - Width));
486 }
487 assert(Extracts.size() > 0 && "Expected at least one element in the shuffle");
488 if (Extracts.size() == 1)
489 Builder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: Extracts[0]);
490 else
491 Builder.buildBuildVector(Res: MI.getOperand(i: 0).getReg(), Ops: Extracts);
492 MI.eraseFromParent();
493}
494
495bool CombinerHelper::matchCombineShuffleConcat(
496 MachineInstr &MI, SmallVector<Register> &Ops) const {
497 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
498 GConcatVectors *ConcatMI1, *ConcatMI2;
499 if (!mi_match(R: MI.getOperand(i: 1).getReg(), MRI, P: m_GConcatVectors(Inst&: ConcatMI1)) ||
500 !mi_match(R: MI.getOperand(i: 2).getReg(), MRI, P: m_GConcatVectors(Inst&: ConcatMI2)))
501 return false;
502
503 // Check that the sources of the Concat instructions have the same type
504 if (MRI.getType(Reg: ConcatMI1->getSourceReg(I: 0)) !=
505 MRI.getType(Reg: ConcatMI2->getSourceReg(I: 0)))
506 return false;
507
508 LLT ConcatSrcTy = MRI.getType(Reg: ConcatMI1->getReg(Idx: 1));
509 LLT ShuffleSrcTy1 = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
510 unsigned ConcatSrcNumElt = ConcatSrcTy.getNumElements();
511 for (unsigned i = 0; i < Mask.size(); i += ConcatSrcNumElt) {
512 // Check if the index takes a whole source register from G_CONCAT_VECTORS
513 // Assumes that all Sources of G_CONCAT_VECTORS are the same type
514 if (Mask[i] == -1) {
515 for (unsigned j = 1; j < ConcatSrcNumElt; j++) {
516 if (i + j >= Mask.size())
517 return false;
518 if (Mask[i + j] != -1)
519 return false;
520 }
521 if (!isLegalOrBeforeLegalizer(
522 Query: {TargetOpcode::G_IMPLICIT_DEF, {ConcatSrcTy}}))
523 return false;
524 Ops.push_back(Elt: 0);
525 } else if (Mask[i] % ConcatSrcNumElt == 0) {
526 for (unsigned j = 1; j < ConcatSrcNumElt; j++) {
527 if (i + j >= Mask.size())
528 return false;
529 if (Mask[i + j] != Mask[i] + static_cast<int>(j))
530 return false;
531 }
532 // Retrieve the source register from its respective G_CONCAT_VECTORS
533 // instruction
534 if (Mask[i] < ShuffleSrcTy1.getNumElements()) {
535 Ops.push_back(Elt: ConcatMI1->getSourceReg(I: Mask[i] / ConcatSrcNumElt));
536 } else {
537 Ops.push_back(Elt: ConcatMI2->getSourceReg(I: Mask[i] / ConcatSrcNumElt -
538 ConcatMI1->getNumSources()));
539 }
540 } else {
541 return false;
542 }
543 }
544
545 if (!isLegalOrBeforeLegalizer(
546 Query: {TargetOpcode::G_CONCAT_VECTORS,
547 {MRI.getType(Reg: MI.getOperand(i: 0).getReg()), ConcatSrcTy}}))
548 return false;
549
550 return !Ops.empty();
551}
552
553void CombinerHelper::applyCombineShuffleConcat(
554 MachineInstr &MI, SmallVector<Register> &Ops) const {
555 LLT SrcTy;
556 for (Register &Reg : Ops) {
557 if (Reg != 0)
558 SrcTy = MRI.getType(Reg);
559 }
560 assert(SrcTy.isValid() && "Unexpected full undef vector in concat combine");
561
562 Register UndefReg = 0;
563
564 for (Register &Reg : Ops) {
565 if (Reg == 0) {
566 if (UndefReg == 0)
567 UndefReg = Builder.buildUndef(Res: SrcTy).getReg(Idx: 0);
568 Reg = UndefReg;
569 }
570 }
571
572 if (Ops.size() > 1)
573 Builder.buildConcatVectors(Res: MI.getOperand(i: 0).getReg(), Ops);
574 else
575 Builder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: Ops[0]);
576 MI.eraseFromParent();
577}
578
579bool CombinerHelper::matchCombineShuffleVector(
580 MachineInstr &MI, SmallVectorImpl<Register> &Ops) const {
581 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
582 "Invalid instruction kind");
583 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
584 Register Src1 = MI.getOperand(i: 1).getReg();
585 LLT SrcType = MRI.getType(Reg: Src1);
586
587 unsigned DstNumElts = DstType.getNumElements();
588 unsigned SrcNumElts = SrcType.getNumElements();
589
590 // If the resulting vector is smaller than the size of the source
591 // vectors being concatenated, we won't be able to replace the
592 // shuffle vector into a concat_vectors.
593 //
594 // Note: We may still be able to produce a concat_vectors fed by
595 // extract_vector_elt and so on. It is less clear that would
596 // be better though, so don't bother for now.
597 //
598 // If the destination is a scalar, the size of the sources doesn't
599 // matter. we will lower the shuffle to a plain copy. This will
600 // work only if the source and destination have the same size. But
601 // that's covered by the next condition.
602 //
603 // TODO: If the size between the source and destination don't match
604 // we could still emit an extract vector element in that case.
605 if (DstNumElts < 2 * SrcNumElts)
606 return false;
607
608 // Check that the shuffle mask can be broken evenly between the
609 // different sources.
610 if (DstNumElts % SrcNumElts != 0)
611 return false;
612
613 // Mask length is a multiple of the source vector length.
614 // Check if the shuffle is some kind of concatenation of the input
615 // vectors.
616 unsigned NumConcat = DstNumElts / SrcNumElts;
617 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
618 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
619 for (unsigned i = 0; i != DstNumElts; ++i) {
620 int Idx = Mask[i];
621 // Undef value.
622 if (Idx < 0)
623 continue;
624 // Ensure the indices in each SrcType sized piece are sequential and that
625 // the same source is used for the whole piece.
626 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
627 (ConcatSrcs[i / SrcNumElts] >= 0 &&
628 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts)))
629 return false;
630 // Remember which source this index came from.
631 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
632 }
633
634 // The shuffle is concatenating multiple vectors together.
635 // Collect the different operands for that.
636 Register UndefReg;
637 Register Src2 = MI.getOperand(i: 2).getReg();
638 for (auto Src : ConcatSrcs) {
639 if (Src < 0) {
640 if (!UndefReg) {
641 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
642 UndefReg = Builder.buildUndef(Res: SrcType).getReg(Idx: 0);
643 }
644 Ops.push_back(Elt: UndefReg);
645 } else if (Src == 0)
646 Ops.push_back(Elt: Src1);
647 else
648 Ops.push_back(Elt: Src2);
649 }
650 return true;
651}
652
653void CombinerHelper::applyCombineShuffleVector(MachineInstr &MI,
654 ArrayRef<Register> Ops) const {
655 Register DstReg = MI.getOperand(i: 0).getReg();
656 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
657 Register NewDstReg = MRI.cloneVirtualRegister(VReg: DstReg);
658
659 if (Ops.size() == 1)
660 Builder.buildCopy(Res: NewDstReg, Op: Ops[0]);
661 else
662 Builder.buildMergeLikeInstr(Res: NewDstReg, Ops);
663
664 replaceRegWith(MRI, FromReg: DstReg, ToReg: NewDstReg);
665 MI.eraseFromParent();
666}
667
668namespace {
669
670/// Select a preference between two uses. CurrentUse is the current preference
671/// while *ForCandidate is attributes of the candidate under consideration.
672PreferredTuple ChoosePreferredUse(MachineInstr &LoadMI,
673 PreferredTuple &CurrentUse,
674 const LLT TyForCandidate,
675 unsigned OpcodeForCandidate,
676 MachineInstr *MIForCandidate) {
677 if (!CurrentUse.Ty.isValid()) {
678 if (CurrentUse.ExtendOpcode == OpcodeForCandidate ||
679 CurrentUse.ExtendOpcode == TargetOpcode::G_ANYEXT)
680 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
681 return CurrentUse;
682 }
683
684 // We permit the extend to hoist through basic blocks but this is only
685 // sensible if the target has extending loads. If you end up lowering back
686 // into a load and extend during the legalizer then the end result is
687 // hoisting the extend up to the load.
688
689 // Prefer defined extensions to undefined extensions as these are more
690 // likely to reduce the number of instructions.
691 if (OpcodeForCandidate == TargetOpcode::G_ANYEXT &&
692 CurrentUse.ExtendOpcode != TargetOpcode::G_ANYEXT)
693 return CurrentUse;
694 else if (CurrentUse.ExtendOpcode == TargetOpcode::G_ANYEXT &&
695 OpcodeForCandidate != TargetOpcode::G_ANYEXT)
696 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
697
698 // Prefer sign extensions to zero extensions as sign-extensions tend to be
699 // more expensive. Don't do this if the load is already a zero-extend load
700 // though, otherwise we'll rewrite a zero-extend load into a sign-extend
701 // later.
702 if (!isa<GZExtLoad>(Val: LoadMI) && CurrentUse.Ty == TyForCandidate) {
703 if (CurrentUse.ExtendOpcode == TargetOpcode::G_SEXT &&
704 OpcodeForCandidate == TargetOpcode::G_ZEXT)
705 return CurrentUse;
706 else if (CurrentUse.ExtendOpcode == TargetOpcode::G_ZEXT &&
707 OpcodeForCandidate == TargetOpcode::G_SEXT)
708 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
709 }
710
711 // This is potentially target specific. We've chosen the largest type
712 // because G_TRUNC is usually free. One potential catch with this is that
713 // some targets have a reduced number of larger registers than smaller
714 // registers and this choice potentially increases the live-range for the
715 // larger value.
716 if (TyForCandidate.getSizeInBits() > CurrentUse.Ty.getSizeInBits()) {
717 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
718 }
719 return CurrentUse;
720}
721
722/// Find a suitable place to insert some instructions and insert them. This
723/// function accounts for special cases like inserting before a PHI node.
724/// The current strategy for inserting before PHI's is to duplicate the
725/// instructions for each predecessor. However, while that's ok for G_TRUNC
726/// on most targets since it generally requires no code, other targets/cases may
727/// want to try harder to find a dominating block.
728static void InsertInsnsWithoutSideEffectsBeforeUse(
729 MachineIRBuilder &Builder, MachineInstr &DefMI, MachineOperand &UseMO,
730 std::function<void(MachineBasicBlock *, MachineBasicBlock::iterator,
731 MachineOperand &UseMO)>
732 Inserter) {
733 MachineInstr &UseMI = *UseMO.getParent();
734
735 MachineBasicBlock *InsertBB = UseMI.getParent();
736
737 // If the use is a PHI then we want the predecessor block instead.
738 if (UseMI.isPHI()) {
739 MachineOperand *PredBB = std::next(x: &UseMO);
740 InsertBB = PredBB->getMBB();
741 }
742
743 // If the block is the same block as the def then we want to insert just after
744 // the def instead of at the start of the block.
745 if (InsertBB == DefMI.getParent()) {
746 MachineBasicBlock::iterator InsertPt = &DefMI;
747 Inserter(InsertBB, std::next(x: InsertPt), UseMO);
748 return;
749 }
750
751 // Otherwise we want the start of the BB
752 Inserter(InsertBB, InsertBB->getFirstNonPHI(), UseMO);
753}
754} // end anonymous namespace
755
756bool CombinerHelper::tryCombineExtendingLoads(MachineInstr &MI) const {
757 PreferredTuple Preferred;
758 if (matchCombineExtendingLoads(MI, MatchInfo&: Preferred)) {
759 applyCombineExtendingLoads(MI, MatchInfo&: Preferred);
760 return true;
761 }
762 return false;
763}
764
765static unsigned getExtLoadOpcForExtend(unsigned ExtOpc) {
766 unsigned CandidateLoadOpc;
767 switch (ExtOpc) {
768 case TargetOpcode::G_ANYEXT:
769 CandidateLoadOpc = TargetOpcode::G_LOAD;
770 break;
771 case TargetOpcode::G_SEXT:
772 CandidateLoadOpc = TargetOpcode::G_SEXTLOAD;
773 break;
774 case TargetOpcode::G_ZEXT:
775 CandidateLoadOpc = TargetOpcode::G_ZEXTLOAD;
776 break;
777 default:
778 llvm_unreachable("Unexpected extend opc");
779 }
780 return CandidateLoadOpc;
781}
782
783bool CombinerHelper::matchCombineExtendingLoads(
784 MachineInstr &MI, PreferredTuple &Preferred) const {
785 // We match the loads and follow the uses to the extend instead of matching
786 // the extends and following the def to the load. This is because the load
787 // must remain in the same position for correctness (unless we also add code
788 // to find a safe place to sink it) whereas the extend is freely movable.
789 // It also prevents us from duplicating the load for the volatile case or just
790 // for performance.
791 GAnyLoad *LoadMI = dyn_cast<GAnyLoad>(Val: &MI);
792 if (!LoadMI)
793 return false;
794
795 Register LoadReg = LoadMI->getDstReg();
796
797 LLT LoadValueTy = MRI.getType(Reg: LoadReg);
798 if (!LoadValueTy.isScalar())
799 return false;
800
801 // Most architectures are going to legalize <s8 loads into at least a 1 byte
802 // load, and the MMOs can only describe memory accesses in multiples of bytes.
803 // If we try to perform extload combining on those, we can end up with
804 // %a(s8) = extload %ptr (load 1 byte from %ptr)
805 // ... which is an illegal extload instruction.
806 if (LoadValueTy.getSizeInBits() < 8)
807 return false;
808
809 // For non power-of-2 types, they will very likely be legalized into multiple
810 // loads. Don't bother trying to match them into extending loads.
811 if (!llvm::has_single_bit<uint32_t>(Value: LoadValueTy.getSizeInBits()))
812 return false;
813
814 // Find the preferred type aside from the any-extends (unless it's the only
815 // one) and non-extending ops. We'll emit an extending load to that type and
816 // and emit a variant of (extend (trunc X)) for the others according to the
817 // relative type sizes. At the same time, pick an extend to use based on the
818 // extend involved in the chosen type.
819 unsigned PreferredOpcode =
820 isa<GLoad>(Val: &MI)
821 ? TargetOpcode::G_ANYEXT
822 : isa<GSExtLoad>(Val: &MI) ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
823 Preferred = {.Ty: LLT(), .ExtendOpcode: PreferredOpcode, .MI: nullptr};
824 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: LoadReg)) {
825 if (UseMI.getOpcode() == TargetOpcode::G_SEXT ||
826 UseMI.getOpcode() == TargetOpcode::G_ZEXT ||
827 (UseMI.getOpcode() == TargetOpcode::G_ANYEXT)) {
828 const auto &MMO = LoadMI->getMMO();
829 // Don't do anything for atomics.
830 if (MMO.isAtomic())
831 continue;
832 // Check for legality.
833 if (!isPreLegalize()) {
834 LegalityQuery::MemDesc MMDesc(MMO);
835 unsigned CandidateLoadOpc = getExtLoadOpcForExtend(ExtOpc: UseMI.getOpcode());
836 LLT UseTy = MRI.getType(Reg: UseMI.getOperand(i: 0).getReg());
837 LLT SrcTy = MRI.getType(Reg: LoadMI->getPointerReg());
838 if (LI->getAction(Query: {CandidateLoadOpc, {UseTy, SrcTy}, {MMDesc}})
839 .Action != LegalizeActions::Legal)
840 continue;
841 }
842 Preferred = ChoosePreferredUse(LoadMI&: MI, CurrentUse&: Preferred,
843 TyForCandidate: MRI.getType(Reg: UseMI.getOperand(i: 0).getReg()),
844 OpcodeForCandidate: UseMI.getOpcode(), MIForCandidate: &UseMI);
845 }
846 }
847
848 // There were no extends
849 if (!Preferred.MI)
850 return false;
851 // It should be impossible to chose an extend without selecting a different
852 // type since by definition the result of an extend is larger.
853 assert(Preferred.Ty != LoadValueTy && "Extending to same type?");
854
855 LLVM_DEBUG(dbgs() << "Preferred use is: " << *Preferred.MI);
856 return true;
857}
858
859void CombinerHelper::applyCombineExtendingLoads(
860 MachineInstr &MI, PreferredTuple &Preferred) const {
861 // Rewrite the load to the chosen extending load.
862 Register ChosenDstReg = Preferred.MI->getOperand(i: 0).getReg();
863
864 // Inserter to insert a truncate back to the original type at a given point
865 // with some basic CSE to limit truncate duplication to one per BB.
866 DenseMap<MachineBasicBlock *, MachineInstr *> EmittedInsns;
867 auto InsertTruncAt = [&](MachineBasicBlock *InsertIntoBB,
868 MachineBasicBlock::iterator InsertBefore,
869 MachineOperand &UseMO) {
870 MachineInstr *PreviouslyEmitted = EmittedInsns.lookup(Val: InsertIntoBB);
871 if (PreviouslyEmitted) {
872 Observer.changingInstr(MI&: *UseMO.getParent());
873 UseMO.setReg(PreviouslyEmitted->getOperand(i: 0).getReg());
874 Observer.changedInstr(MI&: *UseMO.getParent());
875 return;
876 }
877
878 Builder.setInsertPt(MBB&: *InsertIntoBB, II: InsertBefore);
879 Register NewDstReg = MRI.cloneVirtualRegister(VReg: MI.getOperand(i: 0).getReg());
880 MachineInstr *NewMI = Builder.buildTrunc(Res: NewDstReg, Op: ChosenDstReg);
881 EmittedInsns[InsertIntoBB] = NewMI;
882 replaceRegOpWith(MRI, FromRegOp&: UseMO, ToReg: NewDstReg);
883 };
884
885 Observer.changingInstr(MI);
886 unsigned LoadOpc = getExtLoadOpcForExtend(ExtOpc: Preferred.ExtendOpcode);
887 MI.setDesc(Builder.getTII().get(Opcode: LoadOpc));
888
889 // Rewrite all the uses to fix up the types.
890 auto &LoadValue = MI.getOperand(i: 0);
891 SmallVector<MachineOperand *, 4> Uses(
892 llvm::make_pointer_range(Range: MRI.use_operands(Reg: LoadValue.getReg())));
893
894 for (auto *UseMO : Uses) {
895 MachineInstr *UseMI = UseMO->getParent();
896
897 // If the extend is compatible with the preferred extend then we should fix
898 // up the type and extend so that it uses the preferred use.
899 if (UseMI->getOpcode() == Preferred.ExtendOpcode ||
900 UseMI->getOpcode() == TargetOpcode::G_ANYEXT) {
901 Register UseDstReg = UseMI->getOperand(i: 0).getReg();
902 MachineOperand &UseSrcMO = UseMI->getOperand(i: 1);
903 const LLT UseDstTy = MRI.getType(Reg: UseDstReg);
904 if (UseDstReg != ChosenDstReg) {
905 if (Preferred.Ty == UseDstTy) {
906 // If the use has the same type as the preferred use, then merge
907 // the vregs and erase the extend. For example:
908 // %1:_(s8) = G_LOAD ...
909 // %2:_(s32) = G_SEXT %1(s8)
910 // %3:_(s32) = G_ANYEXT %1(s8)
911 // ... = ... %3(s32)
912 // rewrites to:
913 // %2:_(s32) = G_SEXTLOAD ...
914 // ... = ... %2(s32)
915 replaceRegWith(MRI, FromReg: UseDstReg, ToReg: ChosenDstReg);
916 Observer.erasingInstr(MI&: *UseMO->getParent());
917 UseMO->getParent()->eraseFromParent();
918 } else if (Preferred.Ty.getSizeInBits() < UseDstTy.getSizeInBits()) {
919 // If the preferred size is smaller, then keep the extend but extend
920 // from the result of the extending load. For example:
921 // %1:_(s8) = G_LOAD ...
922 // %2:_(s32) = G_SEXT %1(s8)
923 // %3:_(s64) = G_ANYEXT %1(s8)
924 // ... = ... %3(s64)
925 /// rewrites to:
926 // %2:_(s32) = G_SEXTLOAD ...
927 // %3:_(s64) = G_ANYEXT %2:_(s32)
928 // ... = ... %3(s64)
929 replaceRegOpWith(MRI, FromRegOp&: UseSrcMO, ToReg: ChosenDstReg);
930 } else {
931 // If the preferred size is large, then insert a truncate. For
932 // example:
933 // %1:_(s8) = G_LOAD ...
934 // %2:_(s64) = G_SEXT %1(s8)
935 // %3:_(s32) = G_ZEXT %1(s8)
936 // ... = ... %3(s32)
937 /// rewrites to:
938 // %2:_(s64) = G_SEXTLOAD ...
939 // %4:_(s8) = G_TRUNC %2:_(s32)
940 // %3:_(s64) = G_ZEXT %2:_(s8)
941 // ... = ... %3(s64)
942 InsertInsnsWithoutSideEffectsBeforeUse(Builder, DefMI&: MI, UseMO&: *UseMO,
943 Inserter: InsertTruncAt);
944 }
945 continue;
946 }
947 // The use is (one of) the uses of the preferred use we chose earlier.
948 // We're going to update the load to def this value later so just erase
949 // the old extend.
950 Observer.erasingInstr(MI&: *UseMO->getParent());
951 UseMO->getParent()->eraseFromParent();
952 continue;
953 }
954
955 // The use isn't an extend. Truncate back to the type we originally loaded.
956 // This is free on many targets.
957 InsertInsnsWithoutSideEffectsBeforeUse(Builder, DefMI&: MI, UseMO&: *UseMO, Inserter: InsertTruncAt);
958 }
959
960 MI.getOperand(i: 0).setReg(ChosenDstReg);
961 Observer.changedInstr(MI);
962}
963
964bool CombinerHelper::matchCombineLoadWithAndMask(MachineInstr &MI,
965 BuildFnTy &MatchInfo) const {
966 assert(MI.getOpcode() == TargetOpcode::G_AND);
967
968 // If we have the following code:
969 // %mask = G_CONSTANT 255
970 // %ld = G_LOAD %ptr, (load s16)
971 // %and = G_AND %ld, %mask
972 //
973 // Try to fold it into
974 // %ld = G_ZEXTLOAD %ptr, (load s8)
975
976 Register Dst = MI.getOperand(i: 0).getReg();
977 if (MRI.getType(Reg: Dst).isVector())
978 return false;
979
980 auto MaybeMask =
981 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
982 if (!MaybeMask)
983 return false;
984
985 APInt MaskVal = MaybeMask->Value;
986
987 if (!MaskVal.isMask())
988 return false;
989
990 Register SrcReg = MI.getOperand(i: 1).getReg();
991 // Don't use getOpcodeDef() here since intermediate instructions may have
992 // multiple users.
993 GAnyLoad *LoadMI;
994 Register PtrReg;
995 const MachineMemOperand *MMO;
996 if (!mi_match(R: SrcReg, MRI, P: m_GAnyLoad(Inst&: LoadMI, Ptr: m_Reg(R&: PtrReg), MMO: m_MMO(MMO))))
997 return false;
998
999 Register LoadReg = LoadMI->getDstReg();
1000 LLT RegTy = MRI.getType(Reg: LoadReg);
1001 unsigned RegSize = RegTy.getSizeInBits();
1002 unsigned LoadSizeBits = MMO->getSizeInBits().getValue();
1003 unsigned MaskSizeBits = MaskVal.countr_one();
1004
1005 if ((isa<GSExtLoad>(Val: LoadMI) || MaskSizeBits < LoadSizeBits) &&
1006 !MRI.hasOneNonDBGUse(RegNo: LoadReg))
1007 return false;
1008
1009 // The mask may not be larger than the in-memory type, as it might cover sign
1010 // extended bits
1011 if (MaskSizeBits > LoadSizeBits)
1012 return false;
1013
1014 // If the mask covers the whole destination register, there's nothing to
1015 // extend
1016 if (MaskSizeBits >= RegSize)
1017 return false;
1018
1019 // Most targets cannot deal with loads of size < 8 and need to re-legalize to
1020 // at least byte loads. Avoid creating such loads here
1021 if (MaskSizeBits < 8 || !isPowerOf2_32(Value: MaskSizeBits))
1022 return false;
1023
1024 LegalityQuery::MemDesc MemDesc(*MMO);
1025
1026 // Don't modify the memory access size if this is atomic/volatile, but we can
1027 // still adjust the opcode to indicate the high bit behavior.
1028 if (!MMO->isAtomic() && !MMO->isVolatile())
1029 MemDesc.MemoryTy = LLT::scalar(SizeInBits: MaskSizeBits);
1030 else if (LoadSizeBits > MaskSizeBits || LoadSizeBits == RegSize)
1031 return false;
1032
1033 // TODO: Could check if it's legal with the reduced or original memory size.
1034 if (!isLegalOrBeforeLegalizer(
1035 Query: {TargetOpcode::G_ZEXTLOAD, {RegTy, MRI.getType(Reg: PtrReg)}, {MemDesc}}))
1036 return false;
1037
1038 MatchInfo = [=](MachineIRBuilder &B) {
1039 B.setInstrAndDebugLoc(*LoadMI);
1040 auto &MF = B.getMF();
1041 auto PtrInfo = MMO->getPointerInfo();
1042 auto *NewMMO = MF.getMachineMemOperand(MMO, PtrInfo, Ty: MemDesc.MemoryTy);
1043 B.buildLoadInstr(Opcode: TargetOpcode::G_ZEXTLOAD, Res: Dst, Addr: PtrReg, MMO&: *NewMMO);
1044 replaceRegWith(MRI, FromReg: LoadReg, ToReg: Dst);
1045 LoadMI->eraseFromParent();
1046 };
1047 return true;
1048}
1049
1050bool CombinerHelper::isPredecessor(const MachineInstr &DefMI,
1051 const MachineInstr &UseMI) const {
1052 assert(!DefMI.isDebugInstr() && !UseMI.isDebugInstr() &&
1053 "shouldn't consider debug uses");
1054 assert(DefMI.getParent() == UseMI.getParent());
1055 if (&DefMI == &UseMI)
1056 return true;
1057 const MachineBasicBlock &MBB = *DefMI.getParent();
1058 auto DefOrUse = find_if(Range: MBB, P: [&DefMI, &UseMI](const MachineInstr &MI) {
1059 return &MI == &DefMI || &MI == &UseMI;
1060 });
1061 if (DefOrUse == MBB.end())
1062 llvm_unreachable("Block must contain both DefMI and UseMI!");
1063 return &*DefOrUse == &DefMI;
1064}
1065
1066bool CombinerHelper::dominates(const MachineInstr &DefMI,
1067 const MachineInstr &UseMI) const {
1068 assert(!DefMI.isDebugInstr() && !UseMI.isDebugInstr() &&
1069 "shouldn't consider debug uses");
1070 if (MDT)
1071 return MDT->dominates(A: &DefMI, B: &UseMI);
1072 else if (DefMI.getParent() != UseMI.getParent())
1073 return false;
1074
1075 return isPredecessor(DefMI, UseMI);
1076}
1077
1078bool CombinerHelper::matchSextTruncSextLoad(MachineInstr &MI) const {
1079 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1080 Register SrcReg = MI.getOperand(i: 1).getReg();
1081 Register LoadUser = SrcReg;
1082
1083 if (MRI.getType(Reg: SrcReg).isVector())
1084 return false;
1085
1086 Register TruncSrc;
1087 if (mi_match(R: SrcReg, MRI, P: m_GTrunc(Src: m_Reg(R&: TruncSrc))))
1088 LoadUser = TruncSrc;
1089
1090 uint64_t SizeInBits = MI.getOperand(i: 2).getImm();
1091 // If the source is a G_SEXTLOAD from the same bit width, then we don't
1092 // need any extend at all, just a truncate.
1093 if (auto *LoadMI = getOpcodeDef<GSExtLoad>(Reg: LoadUser, MRI)) {
1094 // If truncating more than the original extended value, abort.
1095 auto LoadSizeBits = LoadMI->getMemSizeInBits();
1096 if (TruncSrc &&
1097 MRI.getType(Reg: TruncSrc).getSizeInBits() < LoadSizeBits.getValue())
1098 return false;
1099 if (LoadSizeBits == SizeInBits)
1100 return true;
1101 }
1102 return false;
1103}
1104
1105bool CombinerHelper::matchSextInRegOfLoad(
1106 MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) const {
1107 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1108
1109 Register DstReg = MI.getOperand(i: 0).getReg();
1110 LLT RegTy = MRI.getType(Reg: DstReg);
1111
1112 // Only supports scalars for now.
1113 if (RegTy.isVector())
1114 return false;
1115
1116 Register SrcReg = MI.getOperand(i: 1).getReg();
1117 Register PtrReg;
1118 const MachineMemOperand *MMO;
1119 if (!mi_match(R: SrcReg, MRI, P: m_GLoad(Ptr: m_Reg(R&: PtrReg), MMO: m_MMO(MMO))))
1120 return false;
1121
1122 uint64_t MemBits = MMO->getSizeInBits().getValue();
1123 uint64_t ExtFrom = MI.getOperand(i: 2).getImm();
1124
1125 if (MemBits > ExtFrom && !MRI.hasOneNonDBGUse(RegNo: SrcReg))
1126 return false;
1127
1128 // If the sign extend extends from a narrower width than the load's width,
1129 // then we can narrow the load width when we combine to a G_SEXTLOAD.
1130 // Avoid widening the load at all.
1131 unsigned NewSizeBits = std::min(a: ExtFrom, b: MemBits);
1132
1133 // Don't generate G_SEXTLOADs with a < 1 byte width.
1134 if (NewSizeBits < 8)
1135 return false;
1136 // Don't bother creating a non-power-2 sextload, it will likely be broken up
1137 // anyway for most targets.
1138 if (!isPowerOf2_32(Value: NewSizeBits))
1139 return false;
1140
1141 LegalityQuery::MemDesc MMDesc(*MMO);
1142
1143 // Don't modify the memory access size if this is atomic/volatile, but we can
1144 // still adjust the opcode to indicate the high bit behavior.
1145 if (!MMO->isAtomic() && !MMO->isVolatile())
1146 MMDesc.MemoryTy = LLT::scalar(SizeInBits: NewSizeBits);
1147 else if (MemBits > NewSizeBits || MemBits == RegTy.getSizeInBits())
1148 return false;
1149
1150 // TODO: Could check if it's legal with the reduced or original memory size.
1151 if (!isLegalOrBeforeLegalizer(
1152 Query: {TargetOpcode::G_SEXTLOAD, {RegTy, MRI.getType(Reg: PtrReg)}, {MMDesc}}))
1153 return false;
1154
1155 MatchInfo = std::make_tuple(args&: SrcReg, args&: NewSizeBits);
1156 return true;
1157}
1158
1159void CombinerHelper::applySextInRegOfLoad(
1160 MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) const {
1161 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1162 Register LoadReg;
1163 unsigned ScalarSizeBits;
1164 std::tie(args&: LoadReg, args&: ScalarSizeBits) = MatchInfo;
1165 GLoad *LoadDef = cast<GLoad>(Val: MRI.getVRegDef(Reg: LoadReg));
1166
1167 // If we have the following:
1168 // %ld = G_LOAD %ptr, (load 2)
1169 // %ext = G_SEXT_INREG %ld, 8
1170 // ==>
1171 // %ld = G_SEXTLOAD %ptr (load 1)
1172
1173 auto &MMO = LoadDef->getMMO();
1174 Builder.setInstrAndDebugLoc(*LoadDef);
1175 auto &MF = Builder.getMF();
1176 auto PtrInfo = MMO.getPointerInfo();
1177 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Size: ScalarSizeBits / 8);
1178 Builder.buildLoadInstr(Opcode: TargetOpcode::G_SEXTLOAD, Res: MI.getOperand(i: 0).getReg(),
1179 Addr: LoadDef->getPointerReg(), MMO&: *NewMMO);
1180 replaceRegWith(MRI, FromReg: LoadReg, ToReg: MI.getOperand(i: 0).getReg());
1181 MI.eraseFromParent();
1182
1183 // Not all loads can be deleted, so make sure the old one is removed.
1184 LoadDef->eraseFromParent();
1185}
1186
1187/// Return true if 'MI' is a load or a store that may be fold it's address
1188/// operand into the load / store addressing mode.
1189static bool canFoldInAddressingMode(GLoadStore *MI, const TargetLowering &TLI,
1190 MachineRegisterInfo &MRI) {
1191 TargetLowering::AddrMode AM;
1192 auto *MF = MI->getMF();
1193 auto *Addr = getOpcodeDef<GPtrAdd>(Reg: MI->getPointerReg(), MRI);
1194 if (!Addr)
1195 return false;
1196
1197 AM.HasBaseReg = true;
1198 if (auto CstOff = getIConstantVRegVal(VReg: Addr->getOffsetReg(), MRI))
1199 AM.BaseOffs = CstOff->getSExtValue(); // [reg +/- imm]
1200 else
1201 AM.Scale = 1; // [reg +/- reg]
1202
1203 return TLI.isLegalAddressingMode(
1204 DL: MF->getDataLayout(), AM,
1205 Ty: getTypeForLLT(Ty: MI->getMMO().getMemoryType(),
1206 C&: MF->getFunction().getContext()),
1207 AddrSpace: MI->getMMO().getAddrSpace());
1208}
1209
1210static unsigned getIndexedOpc(unsigned LdStOpc) {
1211 switch (LdStOpc) {
1212 case TargetOpcode::G_LOAD:
1213 return TargetOpcode::G_INDEXED_LOAD;
1214 case TargetOpcode::G_STORE:
1215 return TargetOpcode::G_INDEXED_STORE;
1216 case TargetOpcode::G_ZEXTLOAD:
1217 return TargetOpcode::G_INDEXED_ZEXTLOAD;
1218 case TargetOpcode::G_SEXTLOAD:
1219 return TargetOpcode::G_INDEXED_SEXTLOAD;
1220 default:
1221 llvm_unreachable("Unexpected opcode");
1222 }
1223}
1224
1225bool CombinerHelper::isIndexedLoadStoreLegal(GLoadStore &LdSt) const {
1226 // Check for legality.
1227 LLT PtrTy = MRI.getType(Reg: LdSt.getPointerReg());
1228 LLT Ty = MRI.getType(Reg: LdSt.getReg(Idx: 0));
1229 LLT MemTy = LdSt.getMMO().getMemoryType();
1230 SmallVector<LegalityQuery::MemDesc, 2> MemDescrs(
1231 {{MemTy, MemTy.getSizeInBits().getKnownMinValue(),
1232 AtomicOrdering::NotAtomic, AtomicOrdering::NotAtomic}});
1233 unsigned IndexedOpc = getIndexedOpc(LdStOpc: LdSt.getOpcode());
1234 SmallVector<LLT> OpTys;
1235 if (IndexedOpc == TargetOpcode::G_INDEXED_STORE)
1236 OpTys = {PtrTy, Ty, Ty};
1237 else
1238 OpTys = {Ty, PtrTy}; // For G_INDEXED_LOAD, G_INDEXED_[SZ]EXTLOAD
1239
1240 LegalityQuery Q(IndexedOpc, OpTys, MemDescrs);
1241 return isLegal(Query: Q);
1242}
1243
1244static cl::opt<unsigned> PostIndexUseThreshold(
1245 "post-index-use-threshold", cl::Hidden, cl::init(Val: 32),
1246 cl::desc("Number of uses of a base pointer to check before it is no longer "
1247 "considered for post-indexing."));
1248
1249bool CombinerHelper::findPostIndexCandidate(GLoadStore &LdSt, Register &Addr,
1250 Register &Base, Register &Offset,
1251 bool &RematOffset) const {
1252 // We're looking for the following pattern, for either load or store:
1253 // %baseptr:_(p0) = ...
1254 // G_STORE %val(s64), %baseptr(p0)
1255 // %offset:_(s64) = G_CONSTANT i64 -256
1256 // %new_addr:_(p0) = G_PTR_ADD %baseptr, %offset(s64)
1257 const auto &TLI = getTargetLowering();
1258
1259 Register Ptr = LdSt.getPointerReg();
1260 // If the store is the only use, don't bother.
1261 if (MRI.hasOneNonDBGUse(RegNo: Ptr))
1262 return false;
1263
1264 if (!isIndexedLoadStoreLegal(LdSt))
1265 return false;
1266
1267 if (getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Ptr, MRI))
1268 return false;
1269
1270 MachineInstr *StoredValDef = getDefIgnoringCopies(Reg: LdSt.getReg(Idx: 0), MRI);
1271 MachineInstr *PtrDef;
1272 if (!mi_match(R: Ptr, MRI, P: m_MInstr(MI&: PtrDef)))
1273 return false;
1274
1275 unsigned NumUsesChecked = 0;
1276 for (auto &Use : MRI.use_nodbg_instructions(Reg: Ptr)) {
1277 if (++NumUsesChecked > PostIndexUseThreshold)
1278 return false; // Try to avoid exploding compile time.
1279
1280 auto *PtrAdd = dyn_cast<GPtrAdd>(Val: &Use);
1281 // The use itself might be dead. This can happen during combines if DCE
1282 // hasn't had a chance to run yet. Don't allow it to form an indexed op.
1283 if (!PtrAdd || MRI.use_nodbg_empty(RegNo: PtrAdd->getReg(Idx: 0)))
1284 continue;
1285
1286 // Check the user of this isn't the store, otherwise we'd be generate a
1287 // indexed store defining its own use.
1288 if (StoredValDef == &Use)
1289 continue;
1290
1291 Offset = PtrAdd->getOffsetReg();
1292 if (!ForceLegalIndexing &&
1293 !TLI.isIndexingLegal(MI&: LdSt, Base: PtrAdd->getBaseReg(), Offset,
1294 /*IsPre*/ false, MRI))
1295 continue;
1296
1297 // Make sure the offset calculation is before the potentially indexed op.
1298 MachineInstr *OffsetDef;
1299 if (!mi_match(R: Offset, MRI, P: m_MInstr(MI&: OffsetDef)))
1300 continue;
1301 RematOffset = false;
1302 if (!dominates(DefMI: *OffsetDef, UseMI: LdSt)) {
1303 // If the offset however is just a G_CONSTANT, we can always just
1304 // rematerialize it where we need it.
1305 if (OffsetDef->getOpcode() != TargetOpcode::G_CONSTANT)
1306 continue;
1307 RematOffset = true;
1308 }
1309
1310 for (auto &BasePtrUse : MRI.use_nodbg_instructions(Reg: PtrAdd->getBaseReg())) {
1311 if (&BasePtrUse == PtrDef)
1312 continue;
1313
1314 // If the user is a later load/store that can be post-indexed, then don't
1315 // combine this one.
1316 auto *BasePtrLdSt = dyn_cast<GLoadStore>(Val: &BasePtrUse);
1317 if (BasePtrLdSt && BasePtrLdSt != &LdSt &&
1318 dominates(DefMI: LdSt, UseMI: *BasePtrLdSt) &&
1319 isIndexedLoadStoreLegal(LdSt&: *BasePtrLdSt))
1320 return false;
1321
1322 // Now we're looking for the key G_PTR_ADD instruction, which contains
1323 // the offset add that we want to fold.
1324 if (auto *BasePtrUseDef = dyn_cast<GPtrAdd>(Val: &BasePtrUse)) {
1325 Register PtrAddDefReg = BasePtrUseDef->getReg(Idx: 0);
1326 for (auto &BaseUseUse : MRI.use_nodbg_instructions(Reg: PtrAddDefReg)) {
1327 // If the use is in a different block, then we may produce worse code
1328 // due to the extra register pressure.
1329 if (BaseUseUse.getParent() != LdSt.getParent())
1330 return false;
1331
1332 if (auto *UseUseLdSt = dyn_cast<GLoadStore>(Val: &BaseUseUse))
1333 if (canFoldInAddressingMode(MI: UseUseLdSt, TLI, MRI))
1334 return false;
1335 }
1336 if (!dominates(DefMI: LdSt, UseMI: BasePtrUse))
1337 return false; // All use must be dominated by the load/store.
1338 }
1339 }
1340
1341 Addr = PtrAdd->getReg(Idx: 0);
1342 Base = PtrAdd->getBaseReg();
1343 return true;
1344 }
1345
1346 return false;
1347}
1348
1349bool CombinerHelper::findPreIndexCandidate(GLoadStore &LdSt, Register &Addr,
1350 Register &Base,
1351 Register &Offset) const {
1352 auto &MF = *LdSt.getParent()->getParent();
1353 const auto &TLI = *MF.getSubtarget().getTargetLowering();
1354
1355 Addr = LdSt.getPointerReg();
1356 if (!mi_match(R: Addr, MRI, P: m_GPtrAdd(L: m_Reg(R&: Base), R: m_Reg(R&: Offset))) ||
1357 MRI.hasOneNonDBGUse(RegNo: Addr))
1358 return false;
1359
1360 if (!ForceLegalIndexing &&
1361 !TLI.isIndexingLegal(MI&: LdSt, Base, Offset, /*IsPre*/ true, MRI))
1362 return false;
1363
1364 if (!isIndexedLoadStoreLegal(LdSt))
1365 return false;
1366
1367 MachineInstr *BaseDef = getDefIgnoringCopies(Reg: Base, MRI);
1368 if (BaseDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
1369 return false;
1370
1371 if (auto *St = dyn_cast<GStore>(Val: &LdSt)) {
1372 // Would require a copy.
1373 if (Base == St->getValueReg())
1374 return false;
1375
1376 // We're expecting one use of Addr in MI, but it could also be the
1377 // value stored, which isn't actually dominated by the instruction.
1378 if (St->getValueReg() == Addr)
1379 return false;
1380 }
1381
1382 // Avoid increasing cross-block register pressure.
1383 for (auto &AddrUse : MRI.use_nodbg_instructions(Reg: Addr))
1384 if (AddrUse.getParent() != LdSt.getParent())
1385 return false;
1386
1387 // FIXME: check whether all uses of the base pointer are constant PtrAdds.
1388 // That might allow us to end base's liveness here by adjusting the constant.
1389 bool RealUse = false;
1390 for (auto &AddrUse : MRI.use_nodbg_instructions(Reg: Addr)) {
1391 if (!dominates(DefMI: LdSt, UseMI: AddrUse))
1392 return false; // All use must be dominated by the load/store.
1393
1394 // If Ptr may be folded in addressing mode of other use, then it's
1395 // not profitable to do this transformation.
1396 if (auto *UseLdSt = dyn_cast<GLoadStore>(Val: &AddrUse)) {
1397 if (!canFoldInAddressingMode(MI: UseLdSt, TLI, MRI))
1398 RealUse = true;
1399 } else {
1400 RealUse = true;
1401 }
1402 }
1403 return RealUse;
1404}
1405
1406bool CombinerHelper::matchCombineExtractedVectorLoad(
1407 MachineInstr &MI, BuildFnTy &MatchInfo) const {
1408 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
1409
1410 // Check if there is a load that defines the vector being extracted from.
1411 auto *LoadMI = getOpcodeDef<GLoad>(Reg: MI.getOperand(i: 1).getReg(), MRI);
1412 if (!LoadMI)
1413 return false;
1414
1415 Register Vector = MI.getOperand(i: 1).getReg();
1416 LLT VecEltTy = MRI.getType(Reg: Vector).getElementType();
1417
1418 assert(MRI.getType(MI.getOperand(0).getReg()) == VecEltTy);
1419
1420 // Checking whether we should reduce the load width.
1421 if (!MRI.hasOneNonDBGUse(RegNo: Vector))
1422 return false;
1423
1424 // Check if the defining load is simple.
1425 if (!LoadMI->isSimple())
1426 return false;
1427
1428 // If the vector element type is not a multiple of a byte then we are unable
1429 // to correctly compute an address to load only the extracted element as a
1430 // scalar.
1431 if (!VecEltTy.isByteSized())
1432 return false;
1433
1434 // Check for load fold barriers between the extraction and the load.
1435 if (MI.getParent() != LoadMI->getParent())
1436 return false;
1437 const unsigned MaxIter = 20;
1438 unsigned Iter = 0;
1439 for (const auto &Instr :
1440 instructionsWithoutDebug(It: LoadMI->getIterator(), End: MI.getIterator())) {
1441 if (Instr.isLoadFoldBarrier())
1442 return false;
1443 if (Iter++ == MaxIter)
1444 return false;
1445 }
1446
1447 // Check if the new load that we are going to create is legal
1448 // if we are in the post-legalization phase.
1449 MachineMemOperand MMO = LoadMI->getMMO();
1450 Align Alignment = MMO.getAlign();
1451 MachinePointerInfo PtrInfo;
1452 uint64_t Offset;
1453
1454 // Finding the appropriate PtrInfo if offset is a known constant.
1455 // This is required to create the memory operand for the narrowed load.
1456 // This machine memory operand object helps us infer about legality
1457 // before we proceed to combine the instruction.
1458 if (auto CVal = getIConstantVRegVal(VReg: Vector, MRI)) {
1459 int Elt = CVal->getZExtValue();
1460 // FIXME: should be (ABI size)*Elt.
1461 Offset = VecEltTy.getSizeInBits() * Elt / 8;
1462 PtrInfo = MMO.getPointerInfo().getWithOffset(O: Offset);
1463 } else {
1464 // Discard the pointer info except the address space because the memory
1465 // operand can't represent this new access since the offset is variable.
1466 Offset = VecEltTy.getSizeInBits() / 8;
1467 PtrInfo = MachinePointerInfo(MMO.getPointerInfo().getAddrSpace());
1468 }
1469
1470 Alignment = commonAlignment(A: Alignment, Offset);
1471
1472 Register VecPtr = LoadMI->getPointerReg();
1473 LLT PtrTy = MRI.getType(Reg: VecPtr);
1474
1475 MachineFunction &MF = *MI.getMF();
1476 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Ty: VecEltTy);
1477
1478 LegalityQuery::MemDesc MMDesc(*NewMMO);
1479
1480 if (!isLegalOrBeforeLegalizer(
1481 Query: {TargetOpcode::G_LOAD, {VecEltTy, PtrTy}, {MMDesc}}))
1482 return false;
1483
1484 // Load must be allowed and fast on the target.
1485 LLVMContext &C = MF.getFunction().getContext();
1486 auto &DL = MF.getDataLayout();
1487 unsigned Fast = 0;
1488 if (!getTargetLowering().allowsMemoryAccess(Context&: C, DL, Ty: VecEltTy, MMO: *NewMMO,
1489 Fast: &Fast) ||
1490 !Fast)
1491 return false;
1492
1493 Register Result = MI.getOperand(i: 0).getReg();
1494 Register Index = MI.getOperand(i: 2).getReg();
1495
1496 MatchInfo = [=](MachineIRBuilder &B) {
1497 GISelObserverWrapper DummyObserver;
1498 LegalizerHelper Helper(B.getMF(), DummyObserver, B);
1499 //// Get pointer to the vector element.
1500 Register finalPtr = Helper.getVectorElementPointer(
1501 VecPtr: LoadMI->getPointerReg(), VecTy: MRI.getType(Reg: LoadMI->getOperand(i: 0).getReg()),
1502 Index);
1503 // New G_LOAD instruction.
1504 B.buildLoad(Res: Result, Addr: finalPtr, PtrInfo, Alignment);
1505 // Remove original GLOAD instruction.
1506 LoadMI->eraseFromParent();
1507 };
1508
1509 return true;
1510}
1511
1512bool CombinerHelper::matchCombineIndexedLoadStore(
1513 MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) const {
1514 auto &LdSt = cast<GLoadStore>(Val&: MI);
1515
1516 if (LdSt.isAtomic())
1517 return false;
1518
1519 MatchInfo.IsPre = findPreIndexCandidate(LdSt, Addr&: MatchInfo.Addr, Base&: MatchInfo.Base,
1520 Offset&: MatchInfo.Offset);
1521 if (!MatchInfo.IsPre &&
1522 !findPostIndexCandidate(LdSt, Addr&: MatchInfo.Addr, Base&: MatchInfo.Base,
1523 Offset&: MatchInfo.Offset, RematOffset&: MatchInfo.RematOffset))
1524 return false;
1525
1526 return true;
1527}
1528
1529void CombinerHelper::applyCombineIndexedLoadStore(
1530 MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) const {
1531 MachineInstr &AddrDef = *MRI.getVRegDef(Reg: MatchInfo.Addr);
1532 unsigned Opcode = MI.getOpcode();
1533 bool IsStore = Opcode == TargetOpcode::G_STORE;
1534 unsigned NewOpcode = getIndexedOpc(LdStOpc: Opcode);
1535
1536 // If the offset constant didn't happen to dominate the load/store, we can
1537 // just clone it as needed.
1538 if (MatchInfo.RematOffset) {
1539 auto *OldCst = MRI.getVRegDef(Reg: MatchInfo.Offset);
1540 auto NewCst = Builder.buildConstant(Res: MRI.getType(Reg: MatchInfo.Offset),
1541 Val: *OldCst->getOperand(i: 1).getCImm());
1542 MatchInfo.Offset = NewCst.getReg(Idx: 0);
1543 }
1544
1545 auto MIB = Builder.buildInstr(Opcode: NewOpcode);
1546 if (IsStore) {
1547 MIB.addDef(RegNo: MatchInfo.Addr);
1548 MIB.addUse(RegNo: MI.getOperand(i: 0).getReg());
1549 } else {
1550 MIB.addDef(RegNo: MI.getOperand(i: 0).getReg());
1551 MIB.addDef(RegNo: MatchInfo.Addr);
1552 }
1553
1554 MIB.addUse(RegNo: MatchInfo.Base);
1555 MIB.addUse(RegNo: MatchInfo.Offset);
1556 MIB.addImm(Val: MatchInfo.IsPre);
1557 MIB->cloneMemRefs(MF&: *MI.getMF(), MI);
1558 MI.eraseFromParent();
1559 AddrDef.eraseFromParent();
1560
1561 LLVM_DEBUG(dbgs() << " Combinined to indexed operation");
1562}
1563
1564bool CombinerHelper::matchCombineDivRem(MachineInstr &MI,
1565 MachineInstr *&OtherMI) const {
1566 unsigned Opcode = MI.getOpcode();
1567 bool IsDiv, IsSigned;
1568
1569 switch (Opcode) {
1570 default:
1571 llvm_unreachable("Unexpected opcode!");
1572 case TargetOpcode::G_SDIV:
1573 case TargetOpcode::G_UDIV: {
1574 IsDiv = true;
1575 IsSigned = Opcode == TargetOpcode::G_SDIV;
1576 break;
1577 }
1578 case TargetOpcode::G_SREM:
1579 case TargetOpcode::G_UREM: {
1580 IsDiv = false;
1581 IsSigned = Opcode == TargetOpcode::G_SREM;
1582 break;
1583 }
1584 }
1585
1586 Register Src1 = MI.getOperand(i: 1).getReg();
1587 unsigned DivOpcode, RemOpcode, DivremOpcode;
1588 if (IsSigned) {
1589 DivOpcode = TargetOpcode::G_SDIV;
1590 RemOpcode = TargetOpcode::G_SREM;
1591 DivremOpcode = TargetOpcode::G_SDIVREM;
1592 } else {
1593 DivOpcode = TargetOpcode::G_UDIV;
1594 RemOpcode = TargetOpcode::G_UREM;
1595 DivremOpcode = TargetOpcode::G_UDIVREM;
1596 }
1597
1598 if (!isLegalOrBeforeLegalizer(Query: {DivremOpcode, {MRI.getType(Reg: Src1)}}))
1599 return false;
1600
1601 // Combine:
1602 // %div:_ = G_[SU]DIV %src1:_, %src2:_
1603 // %rem:_ = G_[SU]REM %src1:_, %src2:_
1604 // into:
1605 // %div:_, %rem:_ = G_[SU]DIVREM %src1:_, %src2:_
1606
1607 // Combine:
1608 // %rem:_ = G_[SU]REM %src1:_, %src2:_
1609 // %div:_ = G_[SU]DIV %src1:_, %src2:_
1610 // into:
1611 // %div:_, %rem:_ = G_[SU]DIVREM %src1:_, %src2:_
1612
1613 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: Src1)) {
1614 if (MI.getParent() == UseMI.getParent() &&
1615 ((IsDiv && UseMI.getOpcode() == RemOpcode) ||
1616 (!IsDiv && UseMI.getOpcode() == DivOpcode)) &&
1617 matchEqualDefs(MOP1: MI.getOperand(i: 2), MOP2: UseMI.getOperand(i: 2)) &&
1618 matchEqualDefs(MOP1: MI.getOperand(i: 1), MOP2: UseMI.getOperand(i: 1))) {
1619 OtherMI = &UseMI;
1620 return true;
1621 }
1622 }
1623
1624 return false;
1625}
1626
1627void CombinerHelper::applyCombineDivRem(MachineInstr &MI,
1628 MachineInstr *&OtherMI) const {
1629 unsigned Opcode = MI.getOpcode();
1630 assert(OtherMI && "OtherMI shouldn't be empty.");
1631
1632 Register DestDivReg, DestRemReg;
1633 if (Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_UDIV) {
1634 DestDivReg = MI.getOperand(i: 0).getReg();
1635 DestRemReg = OtherMI->getOperand(i: 0).getReg();
1636 } else {
1637 DestDivReg = OtherMI->getOperand(i: 0).getReg();
1638 DestRemReg = MI.getOperand(i: 0).getReg();
1639 }
1640
1641 bool IsSigned =
1642 Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM;
1643
1644 // Check which instruction is first in the block so we don't break def-use
1645 // deps by "moving" the instruction incorrectly. Also keep track of which
1646 // instruction is first so we pick it's operands, avoiding use-before-def
1647 // bugs.
1648 MachineInstr *FirstInst = dominates(DefMI: MI, UseMI: *OtherMI) ? &MI : OtherMI;
1649 Builder.setInstrAndDebugLoc(*FirstInst);
1650
1651 Builder.buildInstr(Opc: IsSigned ? TargetOpcode::G_SDIVREM
1652 : TargetOpcode::G_UDIVREM,
1653 DstOps: {DestDivReg, DestRemReg},
1654 SrcOps: { FirstInst->getOperand(i: 1), FirstInst->getOperand(i: 2) });
1655 MI.eraseFromParent();
1656 OtherMI->eraseFromParent();
1657}
1658
1659bool CombinerHelper::matchOptBrCondByInvertingCond(
1660 MachineInstr &MI, MachineInstr *&BrCond) const {
1661 assert(MI.getOpcode() == TargetOpcode::G_BR);
1662
1663 // Try to match the following:
1664 // bb1:
1665 // G_BRCOND %c1, %bb2
1666 // G_BR %bb3
1667 // bb2:
1668 // ...
1669 // bb3:
1670
1671 // The above pattern does not have a fall through to the successor bb2, always
1672 // resulting in a branch no matter which path is taken. Here we try to find
1673 // and replace that pattern with conditional branch to bb3 and otherwise
1674 // fallthrough to bb2. This is generally better for branch predictors.
1675
1676 MachineBasicBlock *MBB = MI.getParent();
1677 MachineBasicBlock::iterator BrIt(MI);
1678 if (BrIt == MBB->begin())
1679 return false;
1680 assert(std::next(BrIt) == MBB->end() && "expected G_BR to be a terminator");
1681
1682 BrCond = &*std::prev(x: BrIt);
1683 if (BrCond->getOpcode() != TargetOpcode::G_BRCOND)
1684 return false;
1685
1686 // Check that the next block is the conditional branch target. Also make sure
1687 // that it isn't the same as the G_BR's target (otherwise, this will loop.)
1688 MachineBasicBlock *BrCondTarget = BrCond->getOperand(i: 1).getMBB();
1689 return BrCondTarget != MI.getOperand(i: 0).getMBB() &&
1690 MBB->isLayoutSuccessor(MBB: BrCondTarget);
1691}
1692
1693void CombinerHelper::applyOptBrCondByInvertingCond(
1694 MachineInstr &MI, MachineInstr *&BrCond) const {
1695 MachineBasicBlock *BrTarget = MI.getOperand(i: 0).getMBB();
1696 Builder.setInstrAndDebugLoc(*BrCond);
1697 LLT Ty = MRI.getType(Reg: BrCond->getOperand(i: 0).getReg());
1698 // FIXME: Does int/fp matter for this? If so, we might need to restrict
1699 // this to i1 only since we might not know for sure what kind of
1700 // compare generated the condition value.
1701 auto True = Builder.buildConstant(
1702 Res: Ty, Val: getICmpTrueVal(TLI: getTargetLowering(), IsVector: false, IsFP: false));
1703 auto Xor = Builder.buildXor(Dst: Ty, Src0: BrCond->getOperand(i: 0), Src1: True);
1704
1705 auto *FallthroughBB = BrCond->getOperand(i: 1).getMBB();
1706 Observer.changingInstr(MI);
1707 MI.getOperand(i: 0).setMBB(FallthroughBB);
1708 Observer.changedInstr(MI);
1709
1710 // Change the conditional branch to use the inverted condition and
1711 // new target block.
1712 Observer.changingInstr(MI&: *BrCond);
1713 BrCond->getOperand(i: 0).setReg(Xor.getReg(Idx: 0));
1714 BrCond->getOperand(i: 1).setMBB(BrTarget);
1715 Observer.changedInstr(MI&: *BrCond);
1716}
1717
1718bool CombinerHelper::matchCombineMemCpyFamily(
1719 MachineInstr &MI, MemCpyFamilyLoweringInfo &MatchInfo,
1720 unsigned MaxLen) const {
1721 auto &[Dst, Src, KnownLen, Alignment, DstAlignCanChange, MemOps] = MatchInfo;
1722 return canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, Alignment,
1723 DstAlignCanChange, MemOps);
1724}
1725
1726void CombinerHelper::applyCombineMemCpyFamily(
1727 MachineInstr &MI, MemCpyFamilyLoweringInfo &MatchInfo) const {
1728 auto &[Dst, Src, KnownLen, Alignment, DstAlignCanChange, MemOps] = MatchInfo;
1729 MachineIRBuilder HelperBuilder(MI);
1730 GISelObserverWrapper DummyObserver;
1731 LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);
1732 bool Changed = Helper.lowerMemCpyFamily(MI, Dst, Src, KnownLen, Alignment,
1733 DstAlignCanChange, MemOps) ==
1734 LegalizerHelper::LegalizeResult::Legalized;
1735 assert(Changed && "expected memcpy-family instruction to lower");
1736 (void)Changed;
1737}
1738
1739bool CombinerHelper::tryCombineMemCpyFamily(MachineInstr &MI,
1740 unsigned MaxLen) const {
1741 MachineIRBuilder HelperBuilder(MI);
1742 GISelObserverWrapper DummyObserver;
1743 LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);
1744 return Helper.lowerMemCpyFamily(MI, MaxLen) ==
1745 LegalizerHelper::LegalizeResult::Legalized;
1746}
1747
1748static APFloat constantFoldFpUnary(const MachineInstr &MI,
1749 const MachineRegisterInfo &MRI,
1750 const APFloat &Val) {
1751 APFloat Result(Val);
1752 switch (MI.getOpcode()) {
1753 default:
1754 llvm_unreachable("Unexpected opcode!");
1755 case TargetOpcode::G_FNEG: {
1756 Result.changeSign();
1757 return Result;
1758 }
1759 case TargetOpcode::G_FABS: {
1760 Result.clearSign();
1761 return Result;
1762 }
1763 case TargetOpcode::G_FCEIL:
1764 Result.roundToIntegral(RM: APFloat::rmTowardPositive);
1765 return Result;
1766 case TargetOpcode::G_FFLOOR:
1767 Result.roundToIntegral(RM: APFloat::rmTowardNegative);
1768 return Result;
1769 case TargetOpcode::G_INTRINSIC_TRUNC:
1770 Result.roundToIntegral(RM: APFloat::rmTowardZero);
1771 return Result;
1772 case TargetOpcode::G_INTRINSIC_ROUND:
1773 Result.roundToIntegral(RM: APFloat::rmNearestTiesToAway);
1774 return Result;
1775 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
1776 Result.roundToIntegral(RM: APFloat::rmNearestTiesToEven);
1777 return Result;
1778 case TargetOpcode::G_FRINT:
1779 case TargetOpcode::G_FNEARBYINT:
1780 // Use default rounding mode (round to nearest, ties to even)
1781 Result.roundToIntegral(RM: APFloat::rmNearestTiesToEven);
1782 return Result;
1783 case TargetOpcode::G_FPEXT:
1784 case TargetOpcode::G_FPTRUNC: {
1785 bool Unused;
1786 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1787 Result.convert(ToSemantics: getFltSemanticForLLT(Ty: DstTy), RM: APFloat::rmNearestTiesToEven,
1788 losesInfo: &Unused);
1789 return Result;
1790 }
1791 case TargetOpcode::G_FSQRT: {
1792 bool Unused;
1793 Result.convert(ToSemantics: APFloat::IEEEdouble(), RM: APFloat::rmNearestTiesToEven,
1794 losesInfo: &Unused);
1795 Result = APFloat(sqrt(x: Result.convertToDouble()));
1796 break;
1797 }
1798 case TargetOpcode::G_FLOG2: {
1799 bool Unused;
1800 Result.convert(ToSemantics: APFloat::IEEEdouble(), RM: APFloat::rmNearestTiesToEven,
1801 losesInfo: &Unused);
1802 Result = APFloat(log2(x: Result.convertToDouble()));
1803 break;
1804 }
1805 }
1806 // Convert `APFloat` to appropriate IEEE type depending on `DstTy`. Otherwise,
1807 // `buildFConstant` will assert on size mismatch. Only `G_FSQRT`, and
1808 // `G_FLOG2` reach here.
1809 bool Unused;
1810 Result.convert(ToSemantics: Val.getSemantics(), RM: APFloat::rmNearestTiesToEven, losesInfo: &Unused);
1811 return Result;
1812}
1813
1814void CombinerHelper::applyCombineConstantFoldFpUnary(
1815 MachineInstr &MI, const ConstantFP *Cst) const {
1816 APFloat Folded = constantFoldFpUnary(MI, MRI, Val: Cst->getValue());
1817 const ConstantFP *NewCst = ConstantFP::get(Context&: Builder.getContext(), V: Folded);
1818 Builder.buildFConstant(Res: MI.getOperand(i: 0), Val: *NewCst);
1819 MI.eraseFromParent();
1820}
1821
1822bool CombinerHelper::matchPtrAddImmedChain(MachineInstr &MI,
1823 PtrAddChain &MatchInfo) const {
1824 // We're trying to match the following pattern:
1825 // %t1 = G_PTR_ADD %base, G_CONSTANT imm1
1826 // %root = G_PTR_ADD %t1, G_CONSTANT imm2
1827 // -->
1828 // %root = G_PTR_ADD %base, G_CONSTANT (imm1 + imm2)
1829
1830 if (MI.getOpcode() != TargetOpcode::G_PTR_ADD)
1831 return false;
1832
1833 Register Add2 = MI.getOperand(i: 1).getReg();
1834 Register Imm1 = MI.getOperand(i: 2).getReg();
1835 auto MaybeImmVal = getIConstantVRegValWithLookThrough(VReg: Imm1, MRI);
1836 if (!MaybeImmVal)
1837 return false;
1838
1839 Register Base, Imm2;
1840 uint32_t LHSPtrAddFlags;
1841 if (!mi_match(R: Add2, MRI,
1842 P: m_GPtrAdd(L: m_Reg(R&: Base), R: m_Reg(R&: Imm2), Flags: m_MIFlags(Flags&: LHSPtrAddFlags))))
1843 return false;
1844
1845 auto MaybeImm2Val = getIConstantVRegValWithLookThrough(VReg: Imm2, MRI);
1846 if (!MaybeImm2Val)
1847 return false;
1848
1849 // Check if the new combined immediate forms an illegal addressing mode.
1850 // Do not combine if it was legal before but would get illegal.
1851 // To do so, we need to find a load/store user of the pointer to get
1852 // the access type.
1853 Type *AccessTy = nullptr;
1854 auto &MF = *MI.getMF();
1855 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg())) {
1856 if (auto *LdSt = dyn_cast<GLoadStore>(Val: &UseMI)) {
1857 AccessTy = getTypeForLLT(Ty: MRI.getType(Reg: LdSt->getReg(Idx: 0)),
1858 C&: MF.getFunction().getContext());
1859 break;
1860 }
1861 }
1862 TargetLoweringBase::AddrMode AMNew;
1863 APInt CombinedImm = MaybeImmVal->Value + MaybeImm2Val->Value;
1864 AMNew.BaseOffs = CombinedImm.getSExtValue();
1865 if (AccessTy) {
1866 AMNew.HasBaseReg = true;
1867 TargetLoweringBase::AddrMode AMOld;
1868 AMOld.BaseOffs = MaybeImmVal->Value.getSExtValue();
1869 AMOld.HasBaseReg = true;
1870 unsigned AS = MRI.getType(Reg: Add2).getAddressSpace();
1871 const auto &TLI = *MF.getSubtarget().getTargetLowering();
1872 if (TLI.isLegalAddressingMode(DL: MF.getDataLayout(), AM: AMOld, Ty: AccessTy, AddrSpace: AS) &&
1873 !TLI.isLegalAddressingMode(DL: MF.getDataLayout(), AM: AMNew, Ty: AccessTy, AddrSpace: AS))
1874 return false;
1875 }
1876
1877 // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
1878 // inbounds, reaching the same result in one G_PTR_ADD is also inbounds.
1879 // The nusw constraints are satisfied because imm1+imm2 cannot exceed the
1880 // largest signed integer that fits into the index type, which is the maximum
1881 // size of allocated objects according to the IR Language Reference.
1882 unsigned PtrAddFlags = MI.getFlags();
1883 bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
1884 bool IsInBounds =
1885 PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::InBounds;
1886 unsigned Flags = 0;
1887 if (IsNoUWrap)
1888 Flags |= MachineInstr::MIFlag::NoUWrap;
1889 if (IsInBounds) {
1890 Flags |= MachineInstr::MIFlag::InBounds;
1891 Flags |= MachineInstr::MIFlag::NoUSWrap;
1892 }
1893
1894 // Pass the combined immediate to the apply function.
1895 MatchInfo.Imm = AMNew.BaseOffs;
1896 MatchInfo.Base = Base;
1897 MatchInfo.Bank = getRegBank(Reg: Imm2);
1898 MatchInfo.Flags = Flags;
1899 return true;
1900}
1901
1902void CombinerHelper::applyPtrAddImmedChain(MachineInstr &MI,
1903 PtrAddChain &MatchInfo) const {
1904 assert(MI.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD");
1905 MachineIRBuilder MIB(MI);
1906 LLT OffsetTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
1907 auto NewOffset = MIB.buildConstant(Res: OffsetTy, Val: MatchInfo.Imm);
1908 setRegBank(Reg: NewOffset.getReg(Idx: 0), RegBank: MatchInfo.Bank);
1909 Observer.changingInstr(MI);
1910 MI.getOperand(i: 1).setReg(MatchInfo.Base);
1911 MI.getOperand(i: 2).setReg(NewOffset.getReg(Idx: 0));
1912 MI.setFlags(MatchInfo.Flags);
1913 Observer.changedInstr(MI);
1914}
1915
1916bool CombinerHelper::matchShiftImmedChain(MachineInstr &MI,
1917 RegisterImmPair &MatchInfo) const {
1918 // We're trying to match the following pattern with any of
1919 // G_SHL/G_ASHR/G_LSHR/G_SSHLSAT/G_USHLSAT shift instructions:
1920 // %t1 = SHIFT %base, G_CONSTANT imm1
1921 // %root = SHIFT %t1, G_CONSTANT imm2
1922 // -->
1923 // %root = SHIFT %base, G_CONSTANT (imm1 + imm2)
1924
1925 unsigned Opcode = MI.getOpcode();
1926 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1927 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1928 Opcode == TargetOpcode::G_USHLSAT) &&
1929 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1930
1931 Register Shl2 = MI.getOperand(i: 1).getReg();
1932 Register Imm1 = MI.getOperand(i: 2).getReg();
1933 auto MaybeImmVal = getIConstantVRegValWithLookThrough(VReg: Imm1, MRI);
1934 if (!MaybeImmVal)
1935 return false;
1936
1937 MachineInstr *Shl2Def;
1938 if (!mi_match(R: Shl2, MRI, P: m_MInstr(MI&: Shl2Def)) || Shl2Def->getOpcode() != Opcode)
1939 return false;
1940
1941 Register Base = Shl2Def->getOperand(i: 1).getReg();
1942 Register Imm2 = Shl2Def->getOperand(i: 2).getReg();
1943 auto MaybeImm2Val = getIConstantVRegValWithLookThrough(VReg: Imm2, MRI);
1944 if (!MaybeImm2Val)
1945 return false;
1946
1947 // Pass the combined immediate to the apply function.
1948 MatchInfo.Imm =
1949 (MaybeImmVal->Value.getZExtValue() + MaybeImm2Val->Value).getZExtValue();
1950 MatchInfo.Reg = Base;
1951
1952 // There is no simple replacement for a saturating unsigned left shift that
1953 // exceeds the scalar size.
1954 if (Opcode == TargetOpcode::G_USHLSAT &&
1955 MatchInfo.Imm >= MRI.getType(Reg: Shl2).getScalarSizeInBits())
1956 return false;
1957
1958 return true;
1959}
1960
1961void CombinerHelper::applyShiftImmedChain(MachineInstr &MI,
1962 RegisterImmPair &MatchInfo) const {
1963 unsigned Opcode = MI.getOpcode();
1964 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1965 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1966 Opcode == TargetOpcode::G_USHLSAT) &&
1967 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1968
1969 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1970 unsigned const ScalarSizeInBits = Ty.getScalarSizeInBits();
1971 auto Imm = MatchInfo.Imm;
1972
1973 if (Imm >= ScalarSizeInBits) {
1974 // Any logical shift that exceeds scalar size will produce zero.
1975 if (Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR) {
1976 Builder.buildConstant(Res: MI.getOperand(i: 0), Val: 0);
1977 MI.eraseFromParent();
1978 return;
1979 }
1980 // Arithmetic shift and saturating signed left shift have no effect beyond
1981 // scalar size.
1982 Imm = ScalarSizeInBits - 1;
1983 }
1984
1985 LLT ImmTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
1986 Register NewImm = Builder.buildConstant(Res: ImmTy, Val: Imm).getReg(Idx: 0);
1987 Observer.changingInstr(MI);
1988 MI.getOperand(i: 1).setReg(MatchInfo.Reg);
1989 MI.getOperand(i: 2).setReg(NewImm);
1990 Observer.changedInstr(MI);
1991}
1992
1993bool CombinerHelper::matchShiftOfShiftedLogic(
1994 MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo) const {
1995 // We're trying to match the following pattern with any of
1996 // G_SHL/G_ASHR/G_LSHR/G_USHLSAT/G_SSHLSAT shift instructions in combination
1997 // with any of G_AND/G_OR/G_XOR logic instructions.
1998 // %t1 = SHIFT %X, G_CONSTANT C0
1999 // %t2 = LOGIC %t1, %Y
2000 // %root = SHIFT %t2, G_CONSTANT C1
2001 // -->
2002 // %t3 = SHIFT %X, G_CONSTANT (C0+C1)
2003 // %t4 = SHIFT %Y, G_CONSTANT C1
2004 // %root = LOGIC %t3, %t4
2005 unsigned ShiftOpcode = MI.getOpcode();
2006 assert((ShiftOpcode == TargetOpcode::G_SHL ||
2007 ShiftOpcode == TargetOpcode::G_ASHR ||
2008 ShiftOpcode == TargetOpcode::G_LSHR ||
2009 ShiftOpcode == TargetOpcode::G_USHLSAT ||
2010 ShiftOpcode == TargetOpcode::G_SSHLSAT) &&
2011 "Expected G_SHL, G_ASHR, G_LSHR, G_USHLSAT and G_SSHLSAT");
2012
2013 // Match a one-use bitwise logic op.
2014 Register LogicDest = MI.getOperand(i: 1).getReg();
2015 if (!MRI.hasOneNonDBGUse(RegNo: LogicDest))
2016 return false;
2017
2018 MachineInstr *LogicMI;
2019 if (!mi_match(R: LogicDest, MRI, P: m_MInstr(MI&: LogicMI)))
2020 return false;
2021 unsigned LogicOpcode = LogicMI->getOpcode();
2022 if (LogicOpcode != TargetOpcode::G_AND && LogicOpcode != TargetOpcode::G_OR &&
2023 LogicOpcode != TargetOpcode::G_XOR)
2024 return false;
2025
2026 // Find a matching one-use shift by constant.
2027 const Register C1 = MI.getOperand(i: 2).getReg();
2028 auto MaybeImmVal = getIConstantVRegValWithLookThrough(VReg: C1, MRI);
2029 if (!MaybeImmVal || MaybeImmVal->Value == 0)
2030 return false;
2031
2032 const uint64_t C1Val = MaybeImmVal->Value.getZExtValue();
2033
2034 auto matchFirstShift = [&](const MachineInstr *MI, uint64_t &ShiftVal) {
2035 // Shift should match previous one and should be a one-use.
2036 if (MI->getOpcode() != ShiftOpcode ||
2037 !MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
2038 return false;
2039
2040 // Must be a constant.
2041 auto MaybeImmVal =
2042 getIConstantVRegValWithLookThrough(VReg: MI->getOperand(i: 2).getReg(), MRI);
2043 if (!MaybeImmVal)
2044 return false;
2045
2046 ShiftVal = MaybeImmVal->Value.getSExtValue();
2047 return true;
2048 };
2049
2050 // Logic ops are commutative, so check each operand for a match.
2051 Register LogicMIReg1 = LogicMI->getOperand(i: 1).getReg();
2052 MachineInstr *LogicMIOp1;
2053 Register LogicMIReg2 = LogicMI->getOperand(i: 2).getReg();
2054 MachineInstr *LogicMIOp2;
2055 if (!mi_match(R: LogicMIReg1, MRI, P: m_MInstr(MI&: LogicMIOp1)) ||
2056 !mi_match(R: LogicMIReg2, MRI, P: m_MInstr(MI&: LogicMIOp2)))
2057 return false;
2058 uint64_t C0Val;
2059
2060 if (matchFirstShift(LogicMIOp1, C0Val)) {
2061 MatchInfo.LogicNonShiftReg = LogicMIReg2;
2062 MatchInfo.Shift2 = LogicMIOp1;
2063 } else if (matchFirstShift(LogicMIOp2, C0Val)) {
2064 MatchInfo.LogicNonShiftReg = LogicMIReg1;
2065 MatchInfo.Shift2 = LogicMIOp2;
2066 } else
2067 return false;
2068
2069 MatchInfo.ValSum = C0Val + C1Val;
2070
2071 // The fold is not valid if the sum of the shift values exceeds bitwidth.
2072 if (MatchInfo.ValSum >= MRI.getType(Reg: LogicDest).getScalarSizeInBits())
2073 return false;
2074
2075 MatchInfo.Logic = LogicMI;
2076 return true;
2077}
2078
2079void CombinerHelper::applyShiftOfShiftedLogic(
2080 MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo) const {
2081 unsigned Opcode = MI.getOpcode();
2082 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
2083 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_USHLSAT ||
2084 Opcode == TargetOpcode::G_SSHLSAT) &&
2085 "Expected G_SHL, G_ASHR, G_LSHR, G_USHLSAT and G_SSHLSAT");
2086
2087 LLT ShlType = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
2088 LLT DestType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2089
2090 Register Const = Builder.buildConstant(Res: ShlType, Val: MatchInfo.ValSum).getReg(Idx: 0);
2091
2092 Register Shift1Base = MatchInfo.Shift2->getOperand(i: 1).getReg();
2093 Register Shift1 =
2094 Builder.buildInstr(Opc: Opcode, DstOps: {DestType}, SrcOps: {Shift1Base, Const}).getReg(Idx: 0);
2095
2096 // If LogicNonShiftReg is the same to Shift1Base, and shift1 const is the same
2097 // to MatchInfo.Shift2 const, CSEMIRBuilder will reuse the old shift1 when
2098 // build shift2. So, if we erase MatchInfo.Shift2 at the end, actually we
2099 // remove old shift1. And it will cause crash later. So erase it earlier to
2100 // avoid the crash.
2101 MatchInfo.Shift2->eraseFromParent();
2102
2103 Register Shift2Const = MI.getOperand(i: 2).getReg();
2104 Register Shift2 = Builder
2105 .buildInstr(Opc: Opcode, DstOps: {DestType},
2106 SrcOps: {MatchInfo.LogicNonShiftReg, Shift2Const})
2107 .getReg(Idx: 0);
2108
2109 Register Dest = MI.getOperand(i: 0).getReg();
2110 Builder.buildInstr(Opc: MatchInfo.Logic->getOpcode(), DstOps: {Dest}, SrcOps: {Shift1, Shift2});
2111
2112 // This was one use so it's safe to remove it.
2113 MatchInfo.Logic->eraseFromParent();
2114
2115 MI.eraseFromParent();
2116}
2117
2118bool CombinerHelper::isDesirableToCommuteWithShift(
2119 const MachineInstr &MI) const {
2120 return getTargetLowering().isDesirableToCommuteWithShift(MI,
2121 IsAfterLegal: !isPreLegalize());
2122}
2123
2124bool CombinerHelper::matchLshrOfTruncOfLshr(MachineInstr &MI,
2125 LshrOfTruncOfLshr &MatchInfo,
2126 MachineInstr &ShiftMI) const {
2127 assert(MI.getOpcode() == TargetOpcode::G_LSHR && "Expected a G_LSHR");
2128
2129 Register N0 = MI.getOperand(i: 1).getReg();
2130 Register N1 = MI.getOperand(i: 2).getReg();
2131 unsigned OpSizeInBits = MRI.getType(Reg: N0).getScalarSizeInBits();
2132
2133 APInt N1C, N001C;
2134 if (!mi_match(R: N1, MRI, P: m_ICstOrSplat(Cst&: N1C)))
2135 return false;
2136 auto N001 = ShiftMI.getOperand(i: 2).getReg();
2137 if (!mi_match(R: N001, MRI, P: m_ICstOrSplat(Cst&: N001C)))
2138 return false;
2139
2140 if (N001C.getBitWidth() > N1C.getBitWidth())
2141 N1C = N1C.zext(width: N001C.getBitWidth());
2142 else
2143 N001C = N001C.zext(width: N1C.getBitWidth());
2144
2145 Register InnerShift = ShiftMI.getOperand(i: 0).getReg();
2146 LLT InnerShiftTy = MRI.getType(Reg: InnerShift);
2147 uint64_t InnerShiftSize = InnerShiftTy.getScalarSizeInBits();
2148 if ((N1C + N001C).ult(RHS: InnerShiftSize)) {
2149 MatchInfo.Src = ShiftMI.getOperand(i: 1).getReg();
2150 MatchInfo.ShiftAmt = N1C + N001C;
2151 MatchInfo.ShiftAmtTy = MRI.getType(Reg: N001);
2152 MatchInfo.InnerShiftTy = InnerShiftTy;
2153
2154 if ((N001C + OpSizeInBits) == InnerShiftSize)
2155 return true;
2156 if (MRI.hasOneUse(RegNo: N0) && MRI.hasOneUse(RegNo: InnerShift)) {
2157 MatchInfo.Mask = true;
2158 MatchInfo.MaskVal = APInt(N1C.getBitWidth(), OpSizeInBits) - N1C;
2159 return true;
2160 }
2161 }
2162 return false;
2163}
2164
2165void CombinerHelper::applyLshrOfTruncOfLshr(
2166 MachineInstr &MI, LshrOfTruncOfLshr &MatchInfo) const {
2167 assert(MI.getOpcode() == TargetOpcode::G_LSHR && "Expected a G_LSHR");
2168
2169 Register Dst = MI.getOperand(i: 0).getReg();
2170 auto ShiftAmt =
2171 Builder.buildConstant(Res: MatchInfo.ShiftAmtTy, Val: MatchInfo.ShiftAmt);
2172 auto Shift =
2173 Builder.buildLShr(Dst: MatchInfo.InnerShiftTy, Src0: MatchInfo.Src, Src1: ShiftAmt);
2174 if (MatchInfo.Mask == true) {
2175 APInt MaskVal =
2176 APInt::getLowBitsSet(numBits: MatchInfo.InnerShiftTy.getScalarSizeInBits(),
2177 loBitsSet: MatchInfo.MaskVal.getZExtValue());
2178 auto Mask = Builder.buildConstant(Res: MatchInfo.InnerShiftTy, Val: MaskVal);
2179 auto And = Builder.buildAnd(Dst: MatchInfo.InnerShiftTy, Src0: Shift, Src1: Mask);
2180 Builder.buildTrunc(Res: Dst, Op: And);
2181 } else
2182 Builder.buildTrunc(Res: Dst, Op: Shift);
2183 MI.eraseFromParent();
2184}
2185
2186bool CombinerHelper::matchCombineMulToShl(MachineInstr &MI,
2187 unsigned &ShiftVal) const {
2188 assert(MI.getOpcode() == TargetOpcode::G_MUL && "Expected a G_MUL");
2189 auto MaybeImmVal =
2190 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
2191 if (!MaybeImmVal)
2192 return false;
2193
2194 ShiftVal = MaybeImmVal->Value.exactLogBase2();
2195 return (static_cast<int32_t>(ShiftVal) != -1);
2196}
2197
2198void CombinerHelper::applyCombineMulToShl(MachineInstr &MI,
2199 unsigned &ShiftVal) const {
2200 assert(MI.getOpcode() == TargetOpcode::G_MUL && "Expected a G_MUL");
2201 MachineIRBuilder MIB(MI);
2202 LLT ShiftTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2203 auto ShiftCst = MIB.buildConstant(Res: ShiftTy, Val: ShiftVal);
2204 Observer.changingInstr(MI);
2205 MI.setDesc(MIB.getTII().get(Opcode: TargetOpcode::G_SHL));
2206 MI.getOperand(i: 2).setReg(ShiftCst.getReg(Idx: 0));
2207 if (ShiftVal == ShiftTy.getScalarSizeInBits() - 1)
2208 MI.clearFlag(Flag: MachineInstr::MIFlag::NoSWrap);
2209 Observer.changedInstr(MI);
2210}
2211
2212bool CombinerHelper::matchCombineSubToAdd(MachineInstr &MI,
2213 BuildFnTy &MatchInfo) const {
2214 GSub &Sub = cast<GSub>(Val&: MI);
2215
2216 LLT Ty = MRI.getType(Reg: Sub.getReg(Idx: 0));
2217
2218 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ADD, {Ty}}))
2219 return false;
2220
2221 if (!isConstantLegalOrBeforeLegalizer(Ty))
2222 return false;
2223
2224 APInt Imm = getIConstantFromReg(VReg: Sub.getRHSReg(), MRI);
2225
2226 MatchInfo = [=, &MI](MachineIRBuilder &B) {
2227 auto NegCst = B.buildConstant(Res: Ty, Val: -Imm);
2228 Observer.changingInstr(MI);
2229 MI.setDesc(B.getTII().get(Opcode: TargetOpcode::G_ADD));
2230 MI.getOperand(i: 2).setReg(NegCst.getReg(Idx: 0));
2231 MI.clearFlag(Flag: MachineInstr::MIFlag::NoUWrap);
2232 if (Imm.isMinSignedValue())
2233 MI.clearFlags(flags: MachineInstr::MIFlag::NoSWrap);
2234 Observer.changedInstr(MI);
2235 };
2236 return true;
2237}
2238
2239// shl ([sza]ext x), y => zext (shl x, y), if shift does not overflow source
2240bool CombinerHelper::matchCombineShlOfExtend(MachineInstr &MI,
2241 RegisterImmPair &MatchData) const {
2242 assert(MI.getOpcode() == TargetOpcode::G_SHL && VT);
2243 if (!getTargetLowering().isDesirableToPullExtFromShl(MI))
2244 return false;
2245
2246 Register LHS = MI.getOperand(i: 1).getReg();
2247
2248 Register ExtSrc;
2249 if (!mi_match(R: LHS, MRI, P: m_GAnyExt(Src: m_Reg(R&: ExtSrc))) &&
2250 !mi_match(R: LHS, MRI, P: m_GZExt(Src: m_Reg(R&: ExtSrc))) &&
2251 !mi_match(R: LHS, MRI, P: m_GSExt(Src: m_Reg(R&: ExtSrc))))
2252 return false;
2253
2254 Register RHS = MI.getOperand(i: 2).getReg();
2255 auto MaybeShiftAmtVal = isConstantOrConstantSplatVector(Def: RHS, MRI);
2256 if (!MaybeShiftAmtVal)
2257 return false;
2258
2259 if (LI) {
2260 LLT SrcTy = MRI.getType(Reg: ExtSrc);
2261
2262 // We only really care about the legality with the shifted value. We can
2263 // pick any type the constant shift amount, so ask the target what to
2264 // use. Otherwise we would have to guess and hope it is reported as legal.
2265 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: SrcTy);
2266 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SHL, {SrcTy, ShiftAmtTy}}))
2267 return false;
2268 }
2269
2270 int64_t ShiftAmt = MaybeShiftAmtVal->getSExtValue();
2271 MatchData.Reg = ExtSrc;
2272 MatchData.Imm = ShiftAmt;
2273
2274 unsigned MinLeadingZeros = VT->getKnownZeroes(R: ExtSrc).countl_one();
2275 unsigned SrcTySize = MRI.getType(Reg: ExtSrc).getScalarSizeInBits();
2276 return MinLeadingZeros >= ShiftAmt && ShiftAmt < SrcTySize;
2277}
2278
2279void CombinerHelper::applyCombineShlOfExtend(
2280 MachineInstr &MI, const RegisterImmPair &MatchData) const {
2281 Register ExtSrcReg = MatchData.Reg;
2282 int64_t ShiftAmtVal = MatchData.Imm;
2283
2284 LLT ExtSrcTy = MRI.getType(Reg: ExtSrcReg);
2285 auto ShiftAmt = Builder.buildConstant(Res: ExtSrcTy, Val: ShiftAmtVal);
2286 auto NarrowShift =
2287 Builder.buildShl(Dst: ExtSrcTy, Src0: ExtSrcReg, Src1: ShiftAmt, Flags: MI.getFlags());
2288 Builder.buildZExt(Res: MI.getOperand(i: 0), Op: NarrowShift);
2289 MI.eraseFromParent();
2290}
2291
2292static Register peekThroughBitcast(Register Reg,
2293 const MachineRegisterInfo &MRI) {
2294 while (mi_match(R: Reg, MRI, P: m_GBitcast(Src: m_Reg(R&: Reg))))
2295 ;
2296
2297 return Reg;
2298}
2299
2300bool CombinerHelper::matchCombineUnmergeMergeToPlainValues(
2301 MachineInstr &MI, SmallVectorImpl<Register> &Operands) const {
2302 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2303 "Expected an unmerge");
2304 auto &Unmerge = cast<GUnmerge>(Val&: MI);
2305 Register SrcReg = peekThroughBitcast(Reg: Unmerge.getSourceReg(), MRI);
2306
2307 auto *SrcInstr = getOpcodeDef<GMergeLikeInstr>(Reg: SrcReg, MRI);
2308 if (!SrcInstr)
2309 return false;
2310
2311 // Check the source type of the merge.
2312 LLT SrcMergeTy = MRI.getType(Reg: SrcInstr->getSourceReg(I: 0));
2313 LLT Dst0Ty = MRI.getType(Reg: Unmerge.getReg(Idx: 0));
2314 bool SameSize = Dst0Ty.getSizeInBits() == SrcMergeTy.getSizeInBits();
2315 if (SrcMergeTy != Dst0Ty && !SameSize)
2316 return false;
2317 // They are the same now (modulo a bitcast).
2318 // We can collect all the src registers.
2319 for (unsigned Idx = 0; Idx < SrcInstr->getNumSources(); ++Idx)
2320 Operands.push_back(Elt: SrcInstr->getSourceReg(I: Idx));
2321 return true;
2322}
2323
2324void CombinerHelper::applyCombineUnmergeMergeToPlainValues(
2325 MachineInstr &MI, SmallVectorImpl<Register> &Operands) const {
2326 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2327 "Expected an unmerge");
2328 assert((MI.getNumOperands() - 1 == Operands.size()) &&
2329 "Not enough operands to replace all defs");
2330 unsigned NumElems = MI.getNumOperands() - 1;
2331
2332 LLT SrcTy = MRI.getType(Reg: Operands[0]);
2333 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2334 bool CanReuseInputDirectly = DstTy == SrcTy;
2335 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2336 Register DstReg = MI.getOperand(i: Idx).getReg();
2337 Register SrcReg = Operands[Idx];
2338
2339 // This combine may run after RegBankSelect, so we need to be aware of
2340 // register banks.
2341 const auto &DstCB = MRI.getRegClassOrRegBank(Reg: DstReg);
2342 if (!DstCB.isNull() && DstCB != MRI.getRegClassOrRegBank(Reg: SrcReg)) {
2343 SrcReg = Builder.buildCopy(Res: MRI.getType(Reg: SrcReg), Op: SrcReg).getReg(Idx: 0);
2344 MRI.setRegClassOrRegBank(Reg: SrcReg, RCOrRB: DstCB);
2345 }
2346
2347 if (CanReuseInputDirectly)
2348 replaceRegWith(MRI, FromReg: DstReg, ToReg: SrcReg);
2349 else
2350 Builder.buildCast(Dst: DstReg, Src: SrcReg);
2351 }
2352 MI.eraseFromParent();
2353}
2354
2355bool CombinerHelper::matchCombineUnmergeConstant(
2356 MachineInstr &MI, SmallVectorImpl<APInt> &Csts) const {
2357 unsigned SrcIdx = MI.getNumOperands() - 1;
2358 Register SrcReg = MI.getOperand(i: SrcIdx).getReg();
2359 // Break down the big constant in smaller ones.
2360 APInt Val;
2361 if (!mi_match(R: SrcReg, MRI, P: m_GConstantOrFConstantBits(Bits&: Val)))
2362 return false;
2363
2364 LLT Dst0Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2365 unsigned ShiftAmt = Dst0Ty.getSizeInBits();
2366 // Unmerge a constant.
2367 for (unsigned Idx = 0; Idx != SrcIdx; ++Idx) {
2368 Csts.emplace_back(Args: Val.trunc(width: ShiftAmt));
2369 Val = Val.lshr(shiftAmt: ShiftAmt);
2370 }
2371
2372 return true;
2373}
2374
2375void CombinerHelper::applyCombineUnmergeConstant(
2376 MachineInstr &MI, SmallVectorImpl<APInt> &Csts) const {
2377 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2378 "Expected an unmerge");
2379 assert((MI.getNumOperands() - 1 == Csts.size()) &&
2380 "Not enough operands to replace all defs");
2381 unsigned NumElems = MI.getNumOperands() - 1;
2382 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2383 Register DstReg = MI.getOperand(i: Idx).getReg();
2384 Builder.buildConstant(Res: DstReg, Val: Csts[Idx]);
2385 }
2386
2387 MI.eraseFromParent();
2388}
2389
2390bool CombinerHelper::matchCombineUnmergeUndef(
2391 MachineInstr &MI,
2392 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
2393 unsigned SrcIdx = MI.getNumOperands() - 1;
2394 Register SrcReg = MI.getOperand(i: SrcIdx).getReg();
2395 MatchInfo = [&MI](MachineIRBuilder &B) {
2396 unsigned NumElems = MI.getNumOperands() - 1;
2397 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2398 Register DstReg = MI.getOperand(i: Idx).getReg();
2399 B.buildUndef(Res: DstReg);
2400 }
2401 };
2402 return mi_match(R: SrcReg, MRI, P: m_GImplicitDef());
2403}
2404
2405bool CombinerHelper::matchCombineUnmergeWithDeadLanesToTrunc(
2406 MachineInstr &MI) const {
2407 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2408 "Expected an unmerge");
2409 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isScalar() ||
2410 !MRI.getType(Reg: MI.getOperand(i: MI.getNumDefs()).getReg()).isScalar())
2411 return false;
2412 // Check that all the lanes are dead except the first one.
2413 for (unsigned Idx = 1, EndIdx = MI.getNumDefs(); Idx != EndIdx; ++Idx) {
2414 if (!MRI.use_nodbg_empty(RegNo: MI.getOperand(i: Idx).getReg()))
2415 return false;
2416 }
2417 return true;
2418}
2419
2420void CombinerHelper::applyCombineUnmergeWithDeadLanesToTrunc(
2421 MachineInstr &MI) const {
2422 Register SrcReg = MI.getOperand(i: MI.getNumDefs()).getReg();
2423 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2424 Builder.buildTrunc(Res: Dst0Reg, Op: SrcReg);
2425 MI.eraseFromParent();
2426}
2427
2428bool CombinerHelper::matchCombineUnmergeZExtToZExt(MachineInstr &MI) const {
2429 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2430 "Expected an unmerge");
2431 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2432 LLT Dst0Ty = MRI.getType(Reg: Dst0Reg);
2433 // G_ZEXT on vector applies to each lane, so it will
2434 // affect all destinations. Therefore we won't be able
2435 // to simplify the unmerge to just the first definition.
2436 if (Dst0Ty.isVector())
2437 return false;
2438 Register SrcReg = MI.getOperand(i: MI.getNumDefs()).getReg();
2439 LLT SrcTy = MRI.getType(Reg: SrcReg);
2440 if (SrcTy.isVector())
2441 return false;
2442
2443 Register ZExtSrcReg;
2444 if (!mi_match(R: SrcReg, MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg))))
2445 return false;
2446
2447 // Finally we can replace the first definition with
2448 // a zext of the source if the definition is big enough to hold
2449 // all of ZExtSrc bits.
2450 LLT ZExtSrcTy = MRI.getType(Reg: ZExtSrcReg);
2451 return ZExtSrcTy.getSizeInBits() <= Dst0Ty.getSizeInBits();
2452}
2453
2454void CombinerHelper::applyCombineUnmergeZExtToZExt(MachineInstr &MI) const {
2455 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2456 "Expected an unmerge");
2457
2458 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2459
2460 GZext *ZExtInstr =
2461 cast<GZext>(Val: MRI.getVRegDef(Reg: MI.getOperand(i: MI.getNumDefs()).getReg()));
2462 Register ZExtSrcReg = ZExtInstr->getSrcReg();
2463 LLT Dst0Ty = MRI.getType(Reg: Dst0Reg);
2464 LLT ZExtSrcTy = MRI.getType(Reg: ZExtSrcReg);
2465
2466 if (Dst0Ty.getSizeInBits() > ZExtSrcTy.getSizeInBits()) {
2467 Builder.buildZExt(Res: Dst0Reg, Op: ZExtSrcReg);
2468 } else {
2469 assert(Dst0Ty.getSizeInBits() == ZExtSrcTy.getSizeInBits() &&
2470 "ZExt src doesn't fit in destination");
2471 replaceRegWith(MRI, FromReg: Dst0Reg, ToReg: ZExtSrcReg);
2472 }
2473
2474 Register ZeroReg;
2475 for (unsigned Idx = 1, EndIdx = MI.getNumDefs(); Idx != EndIdx; ++Idx) {
2476 if (!ZeroReg)
2477 ZeroReg = Builder.buildConstant(Res: Dst0Ty, Val: 0).getReg(Idx: 0);
2478 replaceRegWith(MRI, FromReg: MI.getOperand(i: Idx).getReg(), ToReg: ZeroReg);
2479 }
2480 MI.eraseFromParent();
2481}
2482
2483bool CombinerHelper::matchCombineShiftToUnmerge(MachineInstr &MI,
2484 unsigned TargetShiftSize,
2485 unsigned &ShiftVal) const {
2486 assert((MI.getOpcode() == TargetOpcode::G_SHL ||
2487 MI.getOpcode() == TargetOpcode::G_LSHR ||
2488 MI.getOpcode() == TargetOpcode::G_ASHR) && "Expected a shift");
2489
2490 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2491 if (Ty.isVector()) // TODO:
2492 return false;
2493
2494 // Don't narrow further than the requested size.
2495 unsigned Size = Ty.getSizeInBits();
2496 if (Size <= TargetShiftSize)
2497 return false;
2498
2499 auto MaybeImmVal =
2500 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
2501 if (!MaybeImmVal)
2502 return false;
2503
2504 ShiftVal = MaybeImmVal->Value.getSExtValue();
2505 return ShiftVal >= Size / 2 && ShiftVal < Size;
2506}
2507
2508void CombinerHelper::applyCombineShiftToUnmerge(
2509 MachineInstr &MI, const unsigned &ShiftVal) const {
2510 Register DstReg = MI.getOperand(i: 0).getReg();
2511 Register SrcReg = MI.getOperand(i: 1).getReg();
2512 LLT Ty = MRI.getType(Reg: SrcReg);
2513 unsigned Size = Ty.getSizeInBits();
2514 unsigned HalfSize = Size / 2;
2515 assert(ShiftVal >= HalfSize);
2516
2517 LLT HalfTy = Ty.changeElementSize(NewEltSize: HalfSize);
2518
2519 auto Unmerge = Builder.buildUnmerge(Res: HalfTy, Op: SrcReg);
2520 unsigned NarrowShiftAmt = ShiftVal - HalfSize;
2521
2522 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
2523 Register Narrowed = Unmerge.getReg(Idx: 1);
2524
2525 // dst = G_LSHR s64:x, C for C >= 32
2526 // =>
2527 // lo, hi = G_UNMERGE_VALUES x
2528 // dst = G_MERGE_VALUES (G_LSHR hi, C - 32), 0
2529
2530 if (NarrowShiftAmt != 0) {
2531 Narrowed = Builder.buildLShr(Dst: HalfTy, Src0: Narrowed,
2532 Src1: Builder.buildConstant(Res: HalfTy, Val: NarrowShiftAmt)).getReg(Idx: 0);
2533 }
2534
2535 auto Zero = Builder.buildConstant(Res: HalfTy, Val: 0);
2536 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Narrowed, Zero});
2537 } else if (MI.getOpcode() == TargetOpcode::G_SHL) {
2538 Register Narrowed = Unmerge.getReg(Idx: 0);
2539 // dst = G_SHL s64:x, C for C >= 32
2540 // =>
2541 // lo, hi = G_UNMERGE_VALUES x
2542 // dst = G_MERGE_VALUES 0, (G_SHL hi, C - 32)
2543 if (NarrowShiftAmt != 0) {
2544 Narrowed = Builder.buildShl(Dst: HalfTy, Src0: Narrowed,
2545 Src1: Builder.buildConstant(Res: HalfTy, Val: NarrowShiftAmt)).getReg(Idx: 0);
2546 }
2547
2548 auto Zero = Builder.buildConstant(Res: HalfTy, Val: 0);
2549 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Zero, Narrowed});
2550 } else {
2551 assert(MI.getOpcode() == TargetOpcode::G_ASHR);
2552 auto Hi = Builder.buildAShr(
2553 Dst: HalfTy, Src0: Unmerge.getReg(Idx: 1),
2554 Src1: Builder.buildConstant(Res: HalfTy, Val: HalfSize - 1));
2555
2556 if (ShiftVal == HalfSize) {
2557 // (G_ASHR i64:x, 32) ->
2558 // G_MERGE_VALUES hi_32(x), (G_ASHR hi_32(x), 31)
2559 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Unmerge.getReg(Idx: 1), Hi});
2560 } else if (ShiftVal == Size - 1) {
2561 // Don't need a second shift.
2562 // (G_ASHR i64:x, 63) ->
2563 // %narrowed = (G_ASHR hi_32(x), 31)
2564 // G_MERGE_VALUES %narrowed, %narrowed
2565 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Hi, Hi});
2566 } else {
2567 auto Lo = Builder.buildAShr(
2568 Dst: HalfTy, Src0: Unmerge.getReg(Idx: 1),
2569 Src1: Builder.buildConstant(Res: HalfTy, Val: ShiftVal - HalfSize));
2570
2571 // (G_ASHR i64:x, C) ->, for C >= 32
2572 // G_MERGE_VALUES (G_ASHR hi_32(x), C - 32), (G_ASHR hi_32(x), 31)
2573 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Lo, Hi});
2574 }
2575 }
2576
2577 MI.eraseFromParent();
2578}
2579
2580bool CombinerHelper::tryCombineShiftToUnmerge(
2581 MachineInstr &MI, unsigned TargetShiftAmount) const {
2582 unsigned ShiftAmt;
2583 if (matchCombineShiftToUnmerge(MI, TargetShiftSize: TargetShiftAmount, ShiftVal&: ShiftAmt)) {
2584 applyCombineShiftToUnmerge(MI, ShiftVal: ShiftAmt);
2585 return true;
2586 }
2587
2588 return false;
2589}
2590
2591bool CombinerHelper::matchCombineAnyExtTrunc(MachineInstr &MI,
2592 Register &Reg) const {
2593 assert(MI.getOpcode() == TargetOpcode::G_ANYEXT && "Expected a G_ANYEXT");
2594 Register DstReg = MI.getOperand(i: 0).getReg();
2595 Register SrcReg = MI.getOperand(i: 1).getReg();
2596 Register OriginalSrcReg = getSrcRegIgnoringCopies(Reg: SrcReg, MRI);
2597 if (OriginalSrcReg.isValid())
2598 SrcReg = OriginalSrcReg;
2599 LLT DstTy = MRI.getType(Reg: DstReg);
2600 return mi_match(R: SrcReg, MRI,
2601 P: m_GTrunc(Src: m_all_of(preds: m_Reg(R&: Reg), preds: m_SpecificType(Ty: DstTy)))) &&
2602 canReplaceReg(DstReg, SrcReg: Reg, MRI);
2603}
2604
2605bool CombinerHelper::matchCombineZextTrunc(MachineInstr &MI,
2606 Register &Reg) const {
2607 assert(MI.getOpcode() == TargetOpcode::G_ZEXT && "Expected a G_ZEXT");
2608 Register DstReg = MI.getOperand(i: 0).getReg();
2609 Register SrcReg = MI.getOperand(i: 1).getReg();
2610 LLT DstTy = MRI.getType(Reg: DstReg);
2611 if (mi_match(R: SrcReg, MRI,
2612 P: m_GTrunc(Src: m_all_of(preds: m_Reg(R&: Reg), preds: m_SpecificType(Ty: DstTy)))) &&
2613 canReplaceReg(DstReg, SrcReg: Reg, MRI)) {
2614 unsigned DstSize = DstTy.getScalarSizeInBits();
2615 unsigned SrcSize = MRI.getType(Reg: SrcReg).getScalarSizeInBits();
2616 return VT->getKnownBits(R: Reg).countMinLeadingZeros() >= DstSize - SrcSize;
2617 }
2618 return false;
2619}
2620
2621static LLT getMidVTForTruncRightShiftCombine(LLT ShiftTy, LLT TruncTy) {
2622 const unsigned ShiftSize = ShiftTy.getScalarSizeInBits();
2623 const unsigned TruncSize = TruncTy.getScalarSizeInBits();
2624
2625 // ShiftTy > 32 > TruncTy -> 32
2626 if (ShiftSize > 32 && TruncSize < 32)
2627 return ShiftTy.changeElementSize(NewEltSize: 32);
2628
2629 // TODO: We could also reduce to 16 bits, but that's more target-dependent.
2630 // Some targets like it, some don't, some only like it under certain
2631 // conditions/processor versions, etc.
2632 // A TL hook might be needed for this.
2633
2634 // Don't combine
2635 return ShiftTy;
2636}
2637
2638bool CombinerHelper::matchCombineTruncOfShift(
2639 MachineInstr &MI, std::pair<MachineInstr *, LLT> &MatchInfo) const {
2640 assert(MI.getOpcode() == TargetOpcode::G_TRUNC && "Expected a G_TRUNC");
2641 Register DstReg = MI.getOperand(i: 0).getReg();
2642 Register SrcReg = MI.getOperand(i: 1).getReg();
2643
2644 if (!MRI.hasOneNonDBGUse(RegNo: SrcReg))
2645 return false;
2646
2647 LLT SrcTy = MRI.getType(Reg: SrcReg);
2648 LLT DstTy = MRI.getType(Reg: DstReg);
2649
2650 MachineInstr *SrcMI = getDefIgnoringCopies(Reg: SrcReg, MRI);
2651 const auto &TL = getTargetLowering();
2652
2653 LLT NewShiftTy;
2654 switch (SrcMI->getOpcode()) {
2655 default:
2656 return false;
2657 case TargetOpcode::G_SHL: {
2658 NewShiftTy = DstTy;
2659
2660 // Make sure new shift amount is legal.
2661 KnownBits Known = VT->getKnownBits(R: SrcMI->getOperand(i: 2).getReg());
2662 if (Known.getMaxValue().uge(RHS: NewShiftTy.getScalarSizeInBits()))
2663 return false;
2664 break;
2665 }
2666 case TargetOpcode::G_LSHR:
2667 case TargetOpcode::G_ASHR: {
2668 // For right shifts, we conservatively do not do the transform if the TRUNC
2669 // has any STORE users. The reason is that if we change the type of the
2670 // shift, we may break the truncstore combine.
2671 //
2672 // TODO: Fix truncstore combine to handle (trunc(lshr (trunc x), k)).
2673 for (auto &User : MRI.use_instructions(Reg: DstReg))
2674 if (User.getOpcode() == TargetOpcode::G_STORE)
2675 return false;
2676
2677 NewShiftTy = getMidVTForTruncRightShiftCombine(ShiftTy: SrcTy, TruncTy: DstTy);
2678 if (NewShiftTy == SrcTy)
2679 return false;
2680
2681 // Make sure we won't lose information by truncating the high bits.
2682 KnownBits Known = VT->getKnownBits(R: SrcMI->getOperand(i: 2).getReg());
2683 if (Known.getMaxValue().ugt(RHS: NewShiftTy.getScalarSizeInBits() -
2684 DstTy.getScalarSizeInBits()))
2685 return false;
2686 break;
2687 }
2688 }
2689
2690 if (!isLegalOrBeforeLegalizer(
2691 Query: {SrcMI->getOpcode(),
2692 {NewShiftTy, TL.getPreferredShiftAmountTy(ShiftValueTy: NewShiftTy)}}))
2693 return false;
2694
2695 MatchInfo = std::make_pair(x&: SrcMI, y&: NewShiftTy);
2696 return true;
2697}
2698
2699void CombinerHelper::applyCombineTruncOfShift(
2700 MachineInstr &MI, std::pair<MachineInstr *, LLT> &MatchInfo) const {
2701 MachineInstr *ShiftMI = MatchInfo.first;
2702 LLT NewShiftTy = MatchInfo.second;
2703
2704 Register Dst = MI.getOperand(i: 0).getReg();
2705 LLT DstTy = MRI.getType(Reg: Dst);
2706
2707 Register ShiftAmt = ShiftMI->getOperand(i: 2).getReg();
2708 Register ShiftSrc = ShiftMI->getOperand(i: 1).getReg();
2709 ShiftSrc = Builder.buildTrunc(Res: NewShiftTy, Op: ShiftSrc).getReg(Idx: 0);
2710
2711 const auto &TL = getTargetLowering();
2712 LLT PrefShiftTy = TL.getPreferredShiftAmountTy(ShiftValueTy: NewShiftTy);
2713 if (MRI.getType(Reg: ShiftAmt) != PrefShiftTy)
2714 ShiftAmt = Builder.buildZExtOrTrunc(Res: PrefShiftTy, Op: ShiftAmt).getReg(Idx: 0);
2715
2716 Register NewShift =
2717 Builder
2718 .buildInstr(Opc: ShiftMI->getOpcode(), DstOps: {NewShiftTy}, SrcOps: {ShiftSrc, ShiftAmt})
2719 .getReg(Idx: 0);
2720
2721 if (NewShiftTy == DstTy)
2722 replaceRegWith(MRI, FromReg: Dst, ToReg: NewShift);
2723 else
2724 Builder.buildTrunc(Res: Dst, Op: NewShift);
2725
2726 eraseInst(MI);
2727}
2728
2729bool CombinerHelper::matchAllExplicitUsesAreUndef(MachineInstr &MI) const {
2730 return all_of(Range: MI.explicit_uses(), P: [this](const MachineOperand &MO) {
2731 return !MO.isReg() ||
2732 getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: MO.getReg(), MRI);
2733 });
2734}
2735
2736bool CombinerHelper::matchUndefShuffleVectorMask(MachineInstr &MI) const {
2737 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
2738 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
2739 return all_of(Range&: Mask, P: [](int Elt) { return Elt < 0; });
2740}
2741
2742bool CombinerHelper::matchInsertExtractVecEltOutOfBounds(
2743 MachineInstr &MI) const {
2744 assert((MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT ||
2745 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT) &&
2746 "Expected an insert/extract element op");
2747 LLT VecTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
2748 if (VecTy.isScalableVector())
2749 return false;
2750
2751 unsigned IdxIdx =
2752 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
2753 auto Idx = getIConstantVRegVal(VReg: MI.getOperand(i: IdxIdx).getReg(), MRI);
2754 if (!Idx)
2755 return false;
2756 return Idx->getZExtValue() >= VecTy.getNumElements();
2757}
2758
2759void CombinerHelper::eraseInst(MachineInstr &MI) const { MI.eraseFromParent(); }
2760
2761bool CombinerHelper::matchEqualDefs(const MachineOperand &MOP1,
2762 const MachineOperand &MOP2) const {
2763 if (!MOP1.isReg() || !MOP2.isReg())
2764 return false;
2765 auto InstAndDef1 = getDefSrcRegIgnoringCopies(Reg: MOP1.getReg(), MRI);
2766 if (!InstAndDef1)
2767 return false;
2768 auto InstAndDef2 = getDefSrcRegIgnoringCopies(Reg: MOP2.getReg(), MRI);
2769 if (!InstAndDef2)
2770 return false;
2771 MachineInstr *I1 = InstAndDef1->MI;
2772 MachineInstr *I2 = InstAndDef2->MI;
2773
2774 // Handle a case like this:
2775 //
2776 // %0:_(s64), %1:_(s64) = G_UNMERGE_VALUES %2:_(<2 x s64>)
2777 //
2778 // Even though %0 and %1 are produced by the same instruction they are not
2779 // the same values.
2780 if (I1 == I2)
2781 return MOP1.getReg() == MOP2.getReg();
2782
2783 // If we have an instruction which loads or stores, we can't guarantee that
2784 // it is identical.
2785 //
2786 // For example, we may have
2787 //
2788 // %x1 = G_LOAD %addr (load N from @somewhere)
2789 // ...
2790 // call @foo
2791 // ...
2792 // %x2 = G_LOAD %addr (load N from @somewhere)
2793 // ...
2794 // %or = G_OR %x1, %x2
2795 //
2796 // It's possible that @foo will modify whatever lives at the address we're
2797 // loading from. To be safe, let's just assume that all loads and stores
2798 // are different (unless we have something which is guaranteed to not
2799 // change.)
2800 if (I1->mayLoadOrStore() && !I1->isDereferenceableInvariantLoad())
2801 return false;
2802
2803 // If both instructions are loads or stores, they are equal only if both
2804 // are dereferenceable invariant loads with the same number of bits.
2805 if (I1->mayLoadOrStore() && I2->mayLoadOrStore()) {
2806 GLoadStore *LS1 = dyn_cast<GLoadStore>(Val: I1);
2807 GLoadStore *LS2 = dyn_cast<GLoadStore>(Val: I2);
2808 if (!LS1 || !LS2)
2809 return false;
2810
2811 if (!I2->isDereferenceableInvariantLoad() ||
2812 (LS1->getMemSizeInBits() != LS2->getMemSizeInBits()))
2813 return false;
2814 }
2815
2816 // Check for physical registers on the instructions first to avoid cases
2817 // like this:
2818 //
2819 // %a = COPY $physreg
2820 // ...
2821 // SOMETHING implicit-def $physreg
2822 // ...
2823 // %b = COPY $physreg
2824 //
2825 // These copies are not equivalent.
2826 if (any_of(Range: I1->uses(), P: [](const MachineOperand &MO) {
2827 return MO.isReg() && MO.getReg().isPhysical();
2828 })) {
2829 // Check if we have a case like this:
2830 //
2831 // %a = COPY $physreg
2832 // %b = COPY %a
2833 //
2834 // In this case, I1 and I2 will both be equal to %a = COPY $physreg.
2835 // From that, we know that they must have the same value, since they must
2836 // have come from the same COPY.
2837 return I1->isIdenticalTo(Other: *I2);
2838 }
2839
2840 // We don't have any physical registers, so we don't necessarily need the
2841 // same vreg defs.
2842 //
2843 // On the off-chance that there's some target instruction feeding into the
2844 // instruction, let's use produceSameValue instead of isIdenticalTo.
2845 if (Builder.getTII().produceSameValue(MI0: *I1, MI1: *I2, MRI: &MRI)) {
2846 // Handle instructions with multiple defs that produce same values. Values
2847 // are same for operands with same index.
2848 // %0:_(s8), %1:_(s8), %2:_(s8), %3:_(s8) = G_UNMERGE_VALUES %4:_(<4 x s8>)
2849 // %5:_(s8), %6:_(s8), %7:_(s8), %8:_(s8) = G_UNMERGE_VALUES %4:_(<4 x s8>)
2850 // I1 and I2 are different instructions but produce same values,
2851 // %1 and %6 are same, %1 and %7 are not the same value.
2852 return I1->findRegisterDefOperandIdx(Reg: InstAndDef1->Reg, /*TRI=*/nullptr) ==
2853 I2->findRegisterDefOperandIdx(Reg: InstAndDef2->Reg, /*TRI=*/nullptr);
2854 }
2855 return false;
2856}
2857
2858bool CombinerHelper::matchConstantOp(const MachineOperand &MOP,
2859 int64_t C) const {
2860 if (!MOP.isReg())
2861 return false;
2862 auto MaybeCst = isConstantOrConstantSplatVector(Def: MOP.getReg(), MRI);
2863 // Compare as APInt so constants wider than 64 bits (e.g. i128 2) still match.
2864 return MaybeCst &&
2865 APInt::isSameValue(I1: *MaybeCst, I2: APInt(64, C, /*isSigned=*/true),
2866 /*SignedCompare=*/true);
2867}
2868
2869bool CombinerHelper::matchConstantFPOp(const MachineOperand &MOP,
2870 double C) const {
2871 if (!MOP.isReg())
2872 return false;
2873 std::optional<FPValueAndVReg> MaybeCst;
2874 if (!mi_match(R: MOP.getReg(), MRI, P: m_GFCstOrSplat(FPValReg&: MaybeCst)))
2875 return false;
2876
2877 return MaybeCst->Value.isExactlyValue(V: C);
2878}
2879
2880void CombinerHelper::replaceSingleDefInstWithOperand(MachineInstr &MI,
2881 unsigned OpIdx) const {
2882 assert(MI.getNumExplicitDefs() == 1 && "Expected one explicit def?");
2883 Register OldReg = MI.getOperand(i: 0).getReg();
2884 Register Replacement = MI.getOperand(i: OpIdx).getReg();
2885 assert(canReplaceReg(OldReg, Replacement, MRI) && "Cannot replace register?");
2886 replaceRegWith(MRI, FromReg: OldReg, ToReg: Replacement);
2887 MI.eraseFromParent();
2888}
2889
2890void CombinerHelper::replaceSingleDefInstWithReg(MachineInstr &MI,
2891 Register Replacement) const {
2892 assert(MI.getNumExplicitDefs() == 1 && "Expected one explicit def?");
2893 Register OldReg = MI.getOperand(i: 0).getReg();
2894 assert(canReplaceReg(OldReg, Replacement, MRI) && "Cannot replace register?");
2895 replaceRegWith(MRI, FromReg: OldReg, ToReg: Replacement);
2896 MI.eraseFromParent();
2897}
2898
2899bool CombinerHelper::matchConstantLargerBitWidth(MachineInstr &MI,
2900 unsigned ConstIdx) const {
2901 Register ConstReg = MI.getOperand(i: ConstIdx).getReg();
2902 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2903
2904 // Get the shift amount
2905 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstReg, MRI);
2906 if (!VRegAndVal)
2907 return false;
2908
2909 // Return true of shift amount >= Bitwidth
2910 return (VRegAndVal->Value.uge(RHS: DstTy.getSizeInBits()));
2911}
2912
2913void CombinerHelper::applyFunnelShiftConstantModulo(MachineInstr &MI) const {
2914 assert((MI.getOpcode() == TargetOpcode::G_FSHL ||
2915 MI.getOpcode() == TargetOpcode::G_FSHR) &&
2916 "This is not a funnel shift operation");
2917
2918 Register ConstReg = MI.getOperand(i: 3).getReg();
2919 LLT ConstTy = MRI.getType(Reg: ConstReg);
2920 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2921
2922 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstReg, MRI);
2923 assert((VRegAndVal) && "Value is not a constant");
2924
2925 // Calculate the new Shift Amount = Old Shift Amount % BitWidth
2926 APInt NewConst = VRegAndVal->Value.urem(
2927 RHS: APInt(ConstTy.getSizeInBits(), DstTy.getScalarSizeInBits()));
2928
2929 auto NewConstInstr = Builder.buildConstant(Res: ConstTy, Val: NewConst.getZExtValue());
2930 Builder.buildInstr(
2931 Opc: MI.getOpcode(), DstOps: {MI.getOperand(i: 0)},
2932 SrcOps: {MI.getOperand(i: 1), MI.getOperand(i: 2), NewConstInstr.getReg(Idx: 0)});
2933
2934 MI.eraseFromParent();
2935}
2936
2937bool CombinerHelper::matchSelectSameVal(MachineInstr &MI) const {
2938 assert(MI.getOpcode() == TargetOpcode::G_SELECT);
2939 // Match (cond ? x : x)
2940 return matchEqualDefs(MOP1: MI.getOperand(i: 2), MOP2: MI.getOperand(i: 3)) &&
2941 canReplaceReg(DstReg: MI.getOperand(i: 0).getReg(), SrcReg: MI.getOperand(i: 2).getReg(),
2942 MRI);
2943}
2944
2945bool CombinerHelper::matchOperandIsKnownToBeAPowerOfTwo(
2946 const MachineOperand &MO, bool OrNegative) const {
2947 return isKnownToBeAPowerOfTwo(Val: MO.getReg(), MRI, ValueTracking: VT, OrNegative);
2948}
2949
2950void CombinerHelper::replaceInstWithFConstant(MachineInstr &MI,
2951 double C) const {
2952 assert(MI.getNumDefs() == 1 && "Expected only one def?");
2953 Builder.buildFConstant(Res: MI.getOperand(i: 0), Val: C);
2954 MI.eraseFromParent();
2955}
2956
2957void CombinerHelper::replaceInstWithConstant(MachineInstr &MI,
2958 int64_t C) const {
2959 assert(MI.getNumDefs() == 1 && "Expected only one def?");
2960 Builder.buildConstant(Res: MI.getOperand(i: 0), Val: C);
2961 MI.eraseFromParent();
2962}
2963
2964void CombinerHelper::replaceInstWithConstant(MachineInstr &MI, APInt C) const {
2965 assert(MI.getNumDefs() == 1 && "Expected only one def?");
2966 Builder.buildConstant(Res: MI.getOperand(i: 0), Val: C);
2967 MI.eraseFromParent();
2968}
2969
2970void CombinerHelper::replaceInstWithFConstant(MachineInstr &MI,
2971 ConstantFP *CFP) const {
2972 assert(MI.getNumDefs() == 1 && "Expected only one def?");
2973 Builder.buildFConstant(Res: MI.getOperand(i: 0), Val: CFP->getValueAPF());
2974 MI.eraseFromParent();
2975}
2976
2977void CombinerHelper::replaceInstWithUndef(MachineInstr &MI) const {
2978 assert(MI.getNumDefs() == 1 && "Expected only one def?");
2979 Builder.buildUndef(Res: MI.getOperand(i: 0));
2980 MI.eraseFromParent();
2981}
2982
2983bool CombinerHelper::matchCombineInsertVecElts(
2984 MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo) const {
2985 assert(MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT &&
2986 "Invalid opcode");
2987 Register DstReg = MI.getOperand(i: 0).getReg();
2988 LLT DstTy = MRI.getType(Reg: DstReg);
2989 assert(DstTy.isVector() && "Invalid G_INSERT_VECTOR_ELT?");
2990
2991 if (DstTy.isScalableVector())
2992 return false;
2993
2994 unsigned NumElts = DstTy.getNumElements();
2995 // If this MI is part of a sequence of insert_vec_elts, then
2996 // don't do the combine in the middle of the sequence.
2997 if (MRI.hasOneUse(RegNo: DstReg) && MRI.use_instr_begin(RegNo: DstReg)->getOpcode() ==
2998 TargetOpcode::G_INSERT_VECTOR_ELT)
2999 return false;
3000 MachineInstr *CurrInst = &MI;
3001 MachineInstr *TmpInst;
3002 int64_t IntImm;
3003 Register TmpReg;
3004 MatchInfo.resize(N: NumElts);
3005 while (mi_match(
3006 MI&: *CurrInst, MRI,
3007 P: m_GInsertVecElt(Src0: m_MInstr(MI&: TmpInst), Src1: m_Reg(R&: TmpReg), Src2: m_ICst(Cst&: IntImm)))) {
3008 if (IntImm >= NumElts || IntImm < 0)
3009 return false;
3010 if (!MatchInfo[IntImm])
3011 MatchInfo[IntImm] = TmpReg;
3012 CurrInst = TmpInst;
3013 }
3014 // Variable index.
3015 if (CurrInst->getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
3016 return false;
3017 if (TmpInst->getOpcode() == TargetOpcode::G_BUILD_VECTOR) {
3018 for (unsigned I = 1; I < TmpInst->getNumOperands(); ++I) {
3019 if (!MatchInfo[I - 1].isValid())
3020 MatchInfo[I - 1] = TmpInst->getOperand(i: I).getReg();
3021 }
3022 return true;
3023 }
3024 // If we didn't end in a G_IMPLICIT_DEF and the source is not fully
3025 // overwritten, bail out.
3026 return TmpInst->getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
3027 all_of(Range&: MatchInfo, P: [](Register Reg) { return !!Reg; });
3028}
3029
3030void CombinerHelper::applyCombineInsertVecElts(
3031 MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo) const {
3032 Register UndefReg;
3033 auto GetUndef = [&]() {
3034 if (UndefReg)
3035 return UndefReg;
3036 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3037 UndefReg = Builder.buildUndef(Res: DstTy.getScalarType()).getReg(Idx: 0);
3038 return UndefReg;
3039 };
3040 for (Register &Reg : MatchInfo) {
3041 if (!Reg)
3042 Reg = GetUndef();
3043 }
3044 Builder.buildBuildVector(Res: MI.getOperand(i: 0).getReg(), Ops: MatchInfo);
3045 MI.eraseFromParent();
3046}
3047
3048bool CombinerHelper::matchBinopWithNegInner(Register MInner, Register Other,
3049 unsigned RootOpc, Register Dst,
3050 LLT Ty,
3051 BuildFnTy &MatchInfo) const {
3052 /// Helper function for matchBinopWithNeg: tries to match one commuted form
3053 /// of `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`.
3054 MachineInstr *InnerDef;
3055 if (!mi_match(R: MInner, MRI, P: m_MInstr(MI&: InnerDef)))
3056 return false;
3057
3058 unsigned InnerOpc = InnerDef->getOpcode();
3059 if (InnerOpc != TargetOpcode::G_ADD && InnerOpc != TargetOpcode::G_SUB)
3060 return false;
3061
3062 if (!MRI.hasOneNonDBGUse(RegNo: MInner))
3063 return false;
3064
3065 Register InnerLHS = InnerDef->getOperand(i: 1).getReg();
3066 Register InnerRHS = InnerDef->getOperand(i: 2).getReg();
3067 Register NotSrc;
3068 Register B, C;
3069
3070 // Check if either operand is ~b
3071 auto TryMatch = [&](Register MaybeNot, Register Other) {
3072 if (mi_match(R: MaybeNot, MRI, P: m_Not(Src: m_Reg(R&: NotSrc)))) {
3073 if (!MRI.hasOneNonDBGUse(RegNo: MaybeNot))
3074 return false;
3075 B = NotSrc;
3076 C = Other;
3077 return true;
3078 }
3079 return false;
3080 };
3081
3082 // For SUB, the not must be the LHS. For ADD, it can be either operand.
3083 if (!TryMatch(InnerLHS, InnerRHS) &&
3084 !(InnerOpc == TargetOpcode::G_ADD && TryMatch(InnerRHS, InnerLHS)))
3085 return false;
3086
3087 // Flip add/sub
3088 unsigned FlippedOpc = (InnerOpc == TargetOpcode::G_ADD) ? TargetOpcode::G_SUB
3089 : TargetOpcode::G_ADD;
3090
3091 Register A = Other;
3092 MatchInfo = [=](MachineIRBuilder &Builder) {
3093 auto NewInner = Builder.buildInstr(Opc: FlippedOpc, DstOps: {Ty}, SrcOps: {B, C});
3094 auto NewNot = Builder.buildNot(Dst: Ty, Src0: NewInner);
3095 Builder.buildInstr(Opc: RootOpc, DstOps: {Dst}, SrcOps: {A, NewNot});
3096 };
3097 return true;
3098}
3099
3100bool CombinerHelper::matchBinopWithNeg(MachineInstr &MI,
3101 BuildFnTy &MatchInfo) const {
3102 // Fold `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`
3103 // Root MI is one of G_AND, G_OR, G_XOR.
3104 // We also look for commuted forms of operations. Pattern shouldn't apply
3105 // if there are multiple reasons of inner operations.
3106
3107 unsigned RootOpc = MI.getOpcode();
3108 Register Dst = MI.getOperand(i: 0).getReg();
3109 LLT Ty = MRI.getType(Reg: Dst);
3110
3111 Register LHS = MI.getOperand(i: 1).getReg();
3112 Register RHS = MI.getOperand(i: 2).getReg();
3113 // Check the commuted and uncommuted forms of the operation.
3114 return matchBinopWithNegInner(MInner: LHS, Other: RHS, RootOpc, Dst, Ty, MatchInfo) ||
3115 matchBinopWithNegInner(MInner: RHS, Other: LHS, RootOpc, Dst, Ty, MatchInfo);
3116}
3117
3118bool CombinerHelper::matchHoistLogicOpWithSameOpcodeHands(
3119 MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) const {
3120 // Matches: logic (hand x, ...), (hand y, ...) -> hand (logic x, y), ...
3121 //
3122 // Creates the new hand + logic instruction (but does not insert them.)
3123 //
3124 // On success, MatchInfo is populated with the new instructions. These are
3125 // inserted in applyHoistLogicOpWithSameOpcodeHands.
3126 unsigned LogicOpcode = MI.getOpcode();
3127 assert(LogicOpcode == TargetOpcode::G_AND ||
3128 LogicOpcode == TargetOpcode::G_OR ||
3129 LogicOpcode == TargetOpcode::G_XOR);
3130 MachineIRBuilder MIB(MI);
3131 Register Dst = MI.getOperand(i: 0).getReg();
3132 Register LHSReg = MI.getOperand(i: 1).getReg();
3133 Register RHSReg = MI.getOperand(i: 2).getReg();
3134
3135 // Don't recompute anything.
3136 if (!MRI.hasOneNonDBGUse(RegNo: LHSReg) || !MRI.hasOneNonDBGUse(RegNo: RHSReg))
3137 return false;
3138
3139 // Make sure we have (hand x, ...), (hand y, ...)
3140 MachineInstr *LeftHandInst = getDefIgnoringCopies(Reg: LHSReg, MRI);
3141 MachineInstr *RightHandInst = getDefIgnoringCopies(Reg: RHSReg, MRI);
3142 if (!LeftHandInst || !RightHandInst)
3143 return false;
3144 unsigned HandOpcode = LeftHandInst->getOpcode();
3145 if (HandOpcode != RightHandInst->getOpcode())
3146 return false;
3147 if (LeftHandInst->getNumOperands() < 2 ||
3148 !LeftHandInst->getOperand(i: 1).isReg() ||
3149 RightHandInst->getNumOperands() < 2 ||
3150 !RightHandInst->getOperand(i: 1).isReg())
3151 return false;
3152
3153 // Make sure the types match up, and if we're doing this post-legalization,
3154 // we end up with legal types.
3155 Register X = LeftHandInst->getOperand(i: 1).getReg();
3156 Register Y = RightHandInst->getOperand(i: 1).getReg();
3157 LLT XTy = MRI.getType(Reg: X);
3158 LLT YTy = MRI.getType(Reg: Y);
3159 if (!XTy.isValid() || XTy != YTy)
3160 return false;
3161
3162 // Optional extra source register.
3163 Register ExtraHandOpSrcReg;
3164 switch (HandOpcode) {
3165 default:
3166 return false;
3167 case TargetOpcode::G_ANYEXT:
3168 case TargetOpcode::G_SEXT:
3169 case TargetOpcode::G_ZEXT: {
3170 // Match: logic (ext X), (ext Y) --> ext (logic X, Y)
3171 break;
3172 }
3173 case TargetOpcode::G_TRUNC: {
3174 // Match: logic (trunc X), (trunc Y) -> trunc (logic X, Y)
3175 const MachineFunction *MF = MI.getMF();
3176 LLVMContext &Ctx = MF->getFunction().getContext();
3177
3178 LLT DstTy = MRI.getType(Reg: Dst);
3179 const TargetLowering &TLI = getTargetLowering();
3180
3181 // Be extra careful sinking truncate. If it's free, there's no benefit in
3182 // widening a binop.
3183 if (TLI.isZExtFree(FromTy: DstTy, ToTy: XTy, Ctx) && TLI.isTruncateFree(FromTy: XTy, ToTy: DstTy, Ctx))
3184 return false;
3185 break;
3186 }
3187 case TargetOpcode::G_AND:
3188 case TargetOpcode::G_ASHR:
3189 case TargetOpcode::G_LSHR:
3190 case TargetOpcode::G_SHL: {
3191 // Match: logic (binop x, z), (binop y, z) -> binop (logic x, y), z
3192 MachineOperand &ZOp = LeftHandInst->getOperand(i: 2);
3193 if (!matchEqualDefs(MOP1: ZOp, MOP2: RightHandInst->getOperand(i: 2)))
3194 return false;
3195 ExtraHandOpSrcReg = ZOp.getReg();
3196 break;
3197 }
3198 }
3199
3200 if (!isLegalOrBeforeLegalizer(Query: {LogicOpcode, {XTy, YTy}}))
3201 return false;
3202
3203 // Record the steps to build the new instructions.
3204 //
3205 // Steps to build (logic x, y)
3206 auto NewLogicDst = MRI.createGenericVirtualRegister(Ty: XTy);
3207 OperandBuildSteps LogicBuildSteps = {
3208 [=](MachineInstrBuilder &MIB) { MIB.addDef(RegNo: NewLogicDst); },
3209 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: X); },
3210 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Y); }};
3211 InstructionBuildSteps LogicSteps(LogicOpcode, LogicBuildSteps);
3212
3213 // Steps to build hand (logic x, y), ...z
3214 OperandBuildSteps HandBuildSteps = {
3215 [=](MachineInstrBuilder &MIB) { MIB.addDef(RegNo: Dst); },
3216 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: NewLogicDst); }};
3217 if (ExtraHandOpSrcReg.isValid())
3218 HandBuildSteps.push_back(
3219 Elt: [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ExtraHandOpSrcReg); });
3220 InstructionBuildSteps HandSteps(HandOpcode, HandBuildSteps);
3221
3222 MatchInfo = InstructionStepsMatchInfo({LogicSteps, HandSteps});
3223 return true;
3224}
3225
3226void CombinerHelper::applyBuildInstructionSteps(
3227 MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) const {
3228 assert(MatchInfo.InstrsToBuild.size() &&
3229 "Expected at least one instr to build?");
3230 for (auto &InstrToBuild : MatchInfo.InstrsToBuild) {
3231 assert(InstrToBuild.Opcode && "Expected a valid opcode?");
3232 assert(InstrToBuild.OperandFns.size() && "Expected at least one operand?");
3233 MachineInstrBuilder Instr = Builder.buildInstr(Opcode: InstrToBuild.Opcode);
3234 for (auto &OperandFn : InstrToBuild.OperandFns)
3235 OperandFn(Instr);
3236 }
3237 MI.eraseFromParent();
3238}
3239
3240bool CombinerHelper::matchAshrShlToSextInreg(
3241 MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {
3242 assert(MI.getOpcode() == TargetOpcode::G_ASHR);
3243 int64_t ShlCst, AshrCst;
3244 Register Src;
3245 if (!mi_match(R: MI.getOperand(i: 0).getReg(), MRI,
3246 P: m_GAShr(L: m_GShl(L: m_Reg(R&: Src), R: m_ICstOrSplat(Cst&: ShlCst)),
3247 R: m_ICstOrSplat(Cst&: AshrCst))))
3248 return false;
3249 if (ShlCst != AshrCst)
3250 return false;
3251 if (!isLegalOrBeforeLegalizer(
3252 Query: {TargetOpcode::G_SEXT_INREG,
3253 {MRI.getType(Reg: Src)},
3254 {},
3255 {MRI.getType(Reg: Src).getScalarSizeInBits() - ShlCst}}))
3256 return false;
3257 MatchInfo = std::make_tuple(args&: Src, args&: ShlCst);
3258 return true;
3259}
3260
3261void CombinerHelper::applyAshShlToSextInreg(
3262 MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {
3263 assert(MI.getOpcode() == TargetOpcode::G_ASHR);
3264 Register Src;
3265 int64_t ShiftAmt;
3266 std::tie(args&: Src, args&: ShiftAmt) = MatchInfo;
3267 unsigned Size = MRI.getType(Reg: Src).getScalarSizeInBits();
3268 Builder.buildSExtInReg(Res: MI.getOperand(i: 0).getReg(), Op: Src, ImmOp: Size - ShiftAmt);
3269 MI.eraseFromParent();
3270}
3271
3272/// and(and(x, C1), C2) -> C1&C2 ? and(x, C1&C2) : 0
3273bool CombinerHelper::matchOverlappingAnd(
3274 MachineInstr &MI,
3275 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
3276 assert(MI.getOpcode() == TargetOpcode::G_AND);
3277
3278 Register Dst = MI.getOperand(i: 0).getReg();
3279 LLT Ty = MRI.getType(Reg: Dst);
3280
3281 Register R;
3282 int64_t C1;
3283 int64_t C2;
3284 if (!mi_match(
3285 R: Dst, MRI,
3286 P: m_GAnd(L: m_GAnd(L: m_Reg(R), R: m_ICst(Cst&: C1)), R: m_ICst(Cst&: C2))))
3287 return false;
3288
3289 MatchInfo = [=](MachineIRBuilder &B) {
3290 if (C1 & C2) {
3291 B.buildAnd(Dst, Src0: R, Src1: B.buildConstant(Res: Ty, Val: C1 & C2));
3292 return;
3293 }
3294 auto Zero = B.buildConstant(Res: Ty, Val: 0);
3295 replaceRegWith(MRI, FromReg: Dst, ToReg: Zero->getOperand(i: 0).getReg());
3296 };
3297 return true;
3298}
3299
3300bool CombinerHelper::matchRedundantAnd(MachineInstr &MI,
3301 Register &Replacement) const {
3302 // Given
3303 //
3304 // %y:_(sN) = G_SOMETHING
3305 // %x:_(sN) = G_SOMETHING
3306 // %res:_(sN) = G_AND %x, %y
3307 //
3308 // Eliminate the G_AND when it is known that x & y == x or x & y == y.
3309 //
3310 // Patterns like this can appear as a result of legalization. E.g.
3311 //
3312 // %cmp:_(s32) = G_ICMP intpred(pred), %x(s32), %y
3313 // %one:_(s32) = G_CONSTANT i32 1
3314 // %and:_(s32) = G_AND %cmp, %one
3315 //
3316 // In this case, G_ICMP only produces a single bit, so x & 1 == x.
3317 assert(MI.getOpcode() == TargetOpcode::G_AND);
3318 if (!VT)
3319 return false;
3320
3321 Register AndDst = MI.getOperand(i: 0).getReg();
3322 Register LHS = MI.getOperand(i: 1).getReg();
3323 Register RHS = MI.getOperand(i: 2).getReg();
3324
3325 // Check the RHS (maybe a constant) first, and if we have no KnownBits there,
3326 // we can't do anything. If we do, then it depends on whether we have
3327 // KnownBits on the LHS.
3328 KnownBits RHSBits = VT->getKnownBits(R: RHS);
3329 if (RHSBits.isUnknown())
3330 return false;
3331
3332 KnownBits LHSBits = VT->getKnownBits(R: LHS);
3333
3334 // Check that x & Mask == x.
3335 // x & 1 == x, always
3336 // x & 0 == x, only if x is also 0
3337 // Meaning Mask has no effect if every bit is either one in Mask or zero in x.
3338 //
3339 // Check if we can replace AndDst with the LHS of the G_AND
3340 if (canReplaceReg(DstReg: AndDst, SrcReg: LHS, MRI) &&
3341 (LHSBits.Zero | RHSBits.One).isAllOnes()) {
3342 Replacement = LHS;
3343 return true;
3344 }
3345
3346 // Check if we can replace AndDst with the RHS of the G_AND
3347 if (canReplaceReg(DstReg: AndDst, SrcReg: RHS, MRI) &&
3348 (LHSBits.One | RHSBits.Zero).isAllOnes()) {
3349 Replacement = RHS;
3350 return true;
3351 }
3352
3353 return false;
3354}
3355
3356bool CombinerHelper::matchRedundantOr(MachineInstr &MI,
3357 Register &Replacement) const {
3358 // Given
3359 //
3360 // %y:_(sN) = G_SOMETHING
3361 // %x:_(sN) = G_SOMETHING
3362 // %res:_(sN) = G_OR %x, %y
3363 //
3364 // Eliminate the G_OR when it is known that x | y == x or x | y == y.
3365 assert(MI.getOpcode() == TargetOpcode::G_OR);
3366 if (!VT)
3367 return false;
3368
3369 Register OrDst = MI.getOperand(i: 0).getReg();
3370 Register LHS = MI.getOperand(i: 1).getReg();
3371 Register RHS = MI.getOperand(i: 2).getReg();
3372
3373 KnownBits LHSBits = VT->getKnownBits(R: LHS);
3374 KnownBits RHSBits = VT->getKnownBits(R: RHS);
3375
3376 // Check that x | Mask == x.
3377 // x | 0 == x, always
3378 // x | 1 == x, only if x is also 1
3379 // Meaning Mask has no effect if every bit is either zero in Mask or one in x.
3380 //
3381 // Check if we can replace OrDst with the LHS of the G_OR
3382 if (canReplaceReg(DstReg: OrDst, SrcReg: LHS, MRI) &&
3383 (LHSBits.One | RHSBits.Zero).isAllOnes()) {
3384 Replacement = LHS;
3385 return true;
3386 }
3387
3388 // Check if we can replace OrDst with the RHS of the G_OR
3389 if (canReplaceReg(DstReg: OrDst, SrcReg: RHS, MRI) &&
3390 (LHSBits.Zero | RHSBits.One).isAllOnes()) {
3391 Replacement = RHS;
3392 return true;
3393 }
3394
3395 return false;
3396}
3397
3398bool CombinerHelper::matchRedundantSExtInReg(MachineInstr &MI) const {
3399 // If the input is already sign extended, just drop the extension.
3400 Register Src = MI.getOperand(i: 1).getReg();
3401 unsigned ExtBits = MI.getOperand(i: 2).getImm();
3402 unsigned TypeSize = MRI.getType(Reg: Src).getScalarSizeInBits();
3403 return VT->computeNumSignBits(R: Src) >= (TypeSize - ExtBits + 1);
3404}
3405
3406static bool isConstValidTrue(const TargetLowering &TLI, unsigned ScalarSizeBits,
3407 int64_t Cst, bool IsVector, bool IsFP) {
3408 // For i1, Cst will always be -1 regardless of boolean contents.
3409 return (ScalarSizeBits == 1 && Cst == -1) ||
3410 isConstTrueVal(TLI, Val: Cst, IsVector, IsFP);
3411}
3412
3413// This pattern aims to match the following shape to avoid extra mov
3414// instructions
3415// G_BUILD_VECTOR(
3416// G_UNMERGE_VALUES(src, 0)
3417// G_UNMERGE_VALUES(src, 1)
3418// G_IMPLICIT_DEF
3419// G_IMPLICIT_DEF
3420// )
3421// ->
3422// G_CONCAT_VECTORS(
3423// src,
3424// undef
3425// )
3426bool CombinerHelper::matchCombineBuildUnmerge(MachineInstr &MI,
3427 MachineRegisterInfo &MRI,
3428 Register &UnmergeSrc) const {
3429 auto &BV = cast<GBuildVector>(Val&: MI);
3430
3431 unsigned BuildUseCount = BV.getNumSources();
3432 if (BuildUseCount % 2 != 0)
3433 return false;
3434
3435 unsigned NumUnmerge = BuildUseCount / 2;
3436
3437 auto *Unmerge = getOpcodeDef<GUnmerge>(Reg: BV.getSourceReg(I: 0), MRI);
3438
3439 // Check the first operand is an unmerge and has the correct number of
3440 // operands
3441 if (!Unmerge || Unmerge->getNumDefs() != NumUnmerge)
3442 return false;
3443
3444 UnmergeSrc = Unmerge->getSourceReg();
3445
3446 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3447 LLT UnmergeSrcTy = MRI.getType(Reg: UnmergeSrc);
3448
3449 if (!UnmergeSrcTy.isVector())
3450 return false;
3451
3452 // Ensure we only generate legal instructions post-legalizer
3453 if (!IsPreLegalize &&
3454 !isLegal(Query: {TargetOpcode::G_CONCAT_VECTORS, {DstTy, UnmergeSrcTy}}))
3455 return false;
3456
3457 // Check that all of the operands before the midpoint come from the same
3458 // unmerge and are in the same order as they are used in the build_vector
3459 for (unsigned I = 0; I < NumUnmerge; ++I) {
3460 auto MaybeUnmergeReg = BV.getSourceReg(I);
3461 auto *LoopUnmerge = getOpcodeDef<GUnmerge>(Reg: MaybeUnmergeReg, MRI);
3462
3463 if (!LoopUnmerge || LoopUnmerge != Unmerge)
3464 return false;
3465
3466 if (LoopUnmerge->getOperand(i: I).getReg() != MaybeUnmergeReg)
3467 return false;
3468 }
3469
3470 // Check that all of the unmerged values are used
3471 if (Unmerge->getNumDefs() != NumUnmerge)
3472 return false;
3473
3474 // Check that all of the operands after the mid point are undefs.
3475 for (unsigned I = NumUnmerge; I < BuildUseCount; ++I) {
3476 auto *Undef = getDefIgnoringCopies(Reg: BV.getSourceReg(I), MRI);
3477
3478 if (Undef->getOpcode() != TargetOpcode::G_IMPLICIT_DEF)
3479 return false;
3480 }
3481
3482 return true;
3483}
3484
3485void CombinerHelper::applyCombineBuildUnmerge(MachineInstr &MI,
3486 MachineRegisterInfo &MRI,
3487 MachineIRBuilder &B,
3488 Register &UnmergeSrc) const {
3489 assert(UnmergeSrc && "Expected there to be one matching G_UNMERGE_VALUES");
3490 B.setInstrAndDebugLoc(MI);
3491
3492 Register UndefVec = B.buildUndef(Res: MRI.getType(Reg: UnmergeSrc)).getReg(Idx: 0);
3493 B.buildConcatVectors(Res: MI.getOperand(i: 0), Ops: {UnmergeSrc, UndefVec});
3494
3495 MI.eraseFromParent();
3496}
3497
3498// This combine tries to reduce the number of scalarised G_TRUNC instructions by
3499// using vector truncates instead
3500//
3501// EXAMPLE:
3502// %a(i32), %b(i32) = G_UNMERGE_VALUES %src(<2 x i32>)
3503// %T_a(i16) = G_TRUNC %a(i32)
3504// %T_b(i16) = G_TRUNC %b(i32)
3505// %Undef(i16) = G_IMPLICIT_DEF(i16)
3506// %dst(v4i16) = G_BUILD_VECTORS %T_a(i16), %T_b(i16), %Undef(i16), %Undef(i16)
3507//
3508// ===>
3509// %Undef(<2 x i32>) = G_IMPLICIT_DEF(<2 x i32>)
3510// %Mid(<4 x s32>) = G_CONCAT_VECTORS %src(<2 x i32>), %Undef(<2 x i32>)
3511// %dst(<4 x s16>) = G_TRUNC %Mid(<4 x s32>)
3512//
3513// Only matches sources made up of G_TRUNCs followed by G_IMPLICIT_DEFs
3514bool CombinerHelper::matchUseVectorTruncate(MachineInstr &MI,
3515 Register &MatchInfo) const {
3516 auto BuildMI = cast<GBuildVector>(Val: &MI);
3517 unsigned NumOperands = BuildMI->getNumSources();
3518 LLT DstTy = MRI.getType(Reg: BuildMI->getReg(Idx: 0));
3519
3520 // Check the G_BUILD_VECTOR sources
3521 unsigned I;
3522 GUnmerge *UnmergeMI = nullptr;
3523
3524 // Check all source TRUNCs come from the same UNMERGE instruction
3525 // and that the element order matches (BUILD_VECTOR position I
3526 // corresponds to UNMERGE result I)
3527 for (I = 0; I < NumOperands; ++I) {
3528 // Check if the G_TRUNC instructions all come from the same MI
3529 Register TruncSrcReg;
3530 if (!mi_match(R: BuildMI->getSourceReg(I), MRI, P: m_GTrunc(Src: m_Reg(R&: TruncSrcReg))))
3531 break;
3532
3533 if (!UnmergeMI) {
3534 if (!mi_match(R: TruncSrcReg, MRI, P: m_GUnmerge(Inst&: UnmergeMI)))
3535 return false;
3536 } else {
3537 MachineInstr *UnmergeSrcMI;
3538 if (!mi_match(R: TruncSrcReg, MRI, P: m_MInstr(MI&: UnmergeSrcMI)) ||
3539 UnmergeMI != UnmergeSrcMI)
3540 return false;
3541 }
3542 // Element order must match: position I must use UNMERGE result I.
3543 if (UnmergeMI->getOperand(i: I).getReg() != TruncSrcReg)
3544 return false;
3545 }
3546 if (I < 2)
3547 return false;
3548
3549 // Check the remaining source elements are only G_IMPLICIT_DEF
3550 for (; I < NumOperands; ++I) {
3551 if (!mi_match(R: BuildMI->getSourceReg(I), MRI, P: m_GImplicitDef()))
3552 return false;
3553 }
3554
3555 // Check the size of unmerge source
3556 MatchInfo = UnmergeMI->getSourceReg();
3557 LLT UnmergeSrcTy = MRI.getType(Reg: MatchInfo);
3558 if (!UnmergeSrcTy.isVector())
3559 return false;
3560
3561 if (!DstTy.getElementCount().isKnownMultipleOf(RHS: UnmergeSrcTy.getNumElements()))
3562 return false;
3563
3564 // Check the unmerge source and destination element types match
3565 LLT UnmergeSrcEltTy = UnmergeSrcTy.getElementType();
3566 Register UnmergeDstReg = UnmergeMI->getOperand(i: 0).getReg();
3567 LLT UnmergeDstEltTy = MRI.getType(Reg: UnmergeDstReg);
3568 if (UnmergeSrcEltTy != UnmergeDstEltTy)
3569 return false;
3570
3571 // Only generate legal instructions post-legalizer
3572 if (!IsPreLegalize) {
3573 LLT MidTy = DstTy.changeElementType(NewEltTy: UnmergeSrcTy.getScalarType());
3574
3575 if (DstTy.getElementCount() != UnmergeSrcTy.getElementCount() &&
3576 !isLegal(Query: {TargetOpcode::G_CONCAT_VECTORS, {MidTy, UnmergeSrcTy}}))
3577 return false;
3578
3579 if (!isLegal(Query: {TargetOpcode::G_TRUNC, {DstTy, MidTy}}))
3580 return false;
3581 }
3582
3583 return true;
3584}
3585
3586void CombinerHelper::applyUseVectorTruncate(MachineInstr &MI,
3587 Register &MatchInfo) const {
3588 Register MidReg;
3589 auto BuildMI = cast<GBuildVector>(Val: &MI);
3590 Register DstReg = BuildMI->getReg(Idx: 0);
3591 LLT DstTy = MRI.getType(Reg: DstReg);
3592 LLT UnmergeSrcTy = MRI.getType(Reg: MatchInfo);
3593 unsigned DstTyNumElt = DstTy.getNumElements();
3594 unsigned UnmergeSrcTyNumElt = UnmergeSrcTy.getNumElements();
3595
3596 // No need to pad vector if only G_TRUNC is needed
3597 if (DstTyNumElt / UnmergeSrcTyNumElt == 1) {
3598 MidReg = MatchInfo;
3599 } else {
3600 Register UndefReg = Builder.buildUndef(Res: UnmergeSrcTy).getReg(Idx: 0);
3601 SmallVector<Register> ConcatRegs = {MatchInfo};
3602 for (unsigned I = 1; I < DstTyNumElt / UnmergeSrcTyNumElt; ++I)
3603 ConcatRegs.push_back(Elt: UndefReg);
3604
3605 auto MidTy = DstTy.changeElementType(NewEltTy: UnmergeSrcTy.getScalarType());
3606 MidReg = Builder.buildConcatVectors(Res: MidTy, Ops: ConcatRegs).getReg(Idx: 0);
3607 }
3608
3609 Builder.buildTrunc(Res: DstReg, Op: MidReg);
3610 MI.eraseFromParent();
3611}
3612
3613bool CombinerHelper::matchNotCmp(
3614 MachineInstr &MI, SmallVectorImpl<Register> &RegsToNegate) const {
3615 assert(MI.getOpcode() == TargetOpcode::G_XOR);
3616 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3617 const auto &TLI = *Builder.getMF().getSubtarget().getTargetLowering();
3618 Register XorSrc;
3619 Register CstReg;
3620 // We match xor(src, true) here.
3621 if (!mi_match(R: MI.getOperand(i: 0).getReg(), MRI,
3622 P: m_GXor(L: m_Reg(R&: XorSrc), R: m_Reg(R&: CstReg))))
3623 return false;
3624
3625 if (!MRI.hasOneNonDBGUse(RegNo: XorSrc))
3626 return false;
3627
3628 // Check that XorSrc is the root of a tree of comparisons combined with ANDs
3629 // and ORs. The suffix of RegsToNegate starting from index I is used a work
3630 // list of tree nodes to visit.
3631 RegsToNegate.push_back(Elt: XorSrc);
3632 // Remember whether the comparisons are all integer or all floating point.
3633 bool IsInt = false;
3634 bool IsFP = false;
3635 for (unsigned I = 0; I < RegsToNegate.size(); ++I) {
3636 Register Reg = RegsToNegate[I];
3637 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
3638 return false;
3639 MachineInstr *Def;
3640 if (!mi_match(R: Reg, MRI, P: m_MInstr(MI&: Def)))
3641 return false;
3642 switch (Def->getOpcode()) {
3643 default:
3644 // Don't match if the tree contains anything other than ANDs, ORs and
3645 // comparisons.
3646 return false;
3647 case TargetOpcode::G_ICMP:
3648 if (IsFP)
3649 return false;
3650 IsInt = true;
3651 // When we apply the combine we will invert the predicate.
3652 break;
3653 case TargetOpcode::G_FCMP:
3654 if (IsInt)
3655 return false;
3656 IsFP = true;
3657 // When we apply the combine we will invert the predicate.
3658 break;
3659 case TargetOpcode::G_AND:
3660 case TargetOpcode::G_OR:
3661 // Implement De Morgan's laws:
3662 // ~(x & y) -> ~x | ~y
3663 // ~(x | y) -> ~x & ~y
3664 // When we apply the combine we will change the opcode and recursively
3665 // negate the operands.
3666 RegsToNegate.push_back(Elt: Def->getOperand(i: 1).getReg());
3667 RegsToNegate.push_back(Elt: Def->getOperand(i: 2).getReg());
3668 break;
3669 }
3670 }
3671
3672 // Now we know whether the comparisons are integer or floating point, check
3673 // the constant in the xor.
3674 int64_t Cst;
3675 if (Ty.isVector()) {
3676 int64_t SplatCst;
3677 if (!mi_match(R: CstReg, MRI, P: m_ICstOrSplat(Cst&: SplatCst)))
3678 return false;
3679 if (!isConstValidTrue(TLI, ScalarSizeBits: Ty.getScalarSizeInBits(), Cst: SplatCst, IsVector: true, IsFP))
3680 return false;
3681 } else {
3682 if (!mi_match(R: CstReg, MRI, P: m_ICst(Cst)))
3683 return false;
3684 if (!isConstValidTrue(TLI, ScalarSizeBits: Ty.getSizeInBits(), Cst, IsVector: false, IsFP))
3685 return false;
3686 }
3687
3688 return true;
3689}
3690
3691void CombinerHelper::applyNotCmp(
3692 MachineInstr &MI, SmallVectorImpl<Register> &RegsToNegate) const {
3693 for (Register Reg : RegsToNegate) {
3694 MachineInstr *Def = MRI.getVRegDef(Reg);
3695 Observer.changingInstr(MI&: *Def);
3696 // For each comparison, invert the opcode. For each AND and OR, change the
3697 // opcode.
3698 switch (Def->getOpcode()) {
3699 default:
3700 llvm_unreachable("Unexpected opcode");
3701 case TargetOpcode::G_ICMP:
3702 case TargetOpcode::G_FCMP: {
3703 MachineOperand &PredOp = Def->getOperand(i: 1);
3704 CmpInst::Predicate NewP = CmpInst::getInversePredicate(
3705 pred: (CmpInst::Predicate)PredOp.getPredicate());
3706 PredOp.setPredicate(NewP);
3707 break;
3708 }
3709 case TargetOpcode::G_AND:
3710 Def->setDesc(Builder.getTII().get(Opcode: TargetOpcode::G_OR));
3711 break;
3712 case TargetOpcode::G_OR:
3713 Def->setDesc(Builder.getTII().get(Opcode: TargetOpcode::G_AND));
3714 break;
3715 }
3716 Observer.changedInstr(MI&: *Def);
3717 }
3718
3719 replaceRegWith(MRI, FromReg: MI.getOperand(i: 0).getReg(), ToReg: MI.getOperand(i: 1).getReg());
3720 MI.eraseFromParent();
3721}
3722
3723bool CombinerHelper::matchXorOfAndWithSameReg(
3724 MachineInstr &MI, std::pair<Register, Register> &MatchInfo) const {
3725 // Match (xor (and x, y), y) (or any of its commuted cases)
3726 assert(MI.getOpcode() == TargetOpcode::G_XOR);
3727 Register &X = MatchInfo.first;
3728 Register &Y = MatchInfo.second;
3729 Register AndReg = MI.getOperand(i: 1).getReg();
3730 Register SharedReg = MI.getOperand(i: 2).getReg();
3731
3732 // Find a G_AND on either side of the G_XOR.
3733 // Look for one of
3734 //
3735 // (xor (and x, y), SharedReg)
3736 // (xor SharedReg, (and x, y))
3737 if (!mi_match(R: AndReg, MRI, P: m_GAnd(L: m_Reg(R&: X), R: m_Reg(R&: Y)))) {
3738 std::swap(a&: AndReg, b&: SharedReg);
3739 if (!mi_match(R: AndReg, MRI, P: m_GAnd(L: m_Reg(R&: X), R: m_Reg(R&: Y))))
3740 return false;
3741 }
3742
3743 // Only do this if we'll eliminate the G_AND.
3744 if (!MRI.hasOneNonDBGUse(RegNo: AndReg))
3745 return false;
3746
3747 // We can combine if SharedReg is the same as either the LHS or RHS of the
3748 // G_AND.
3749 if (Y != SharedReg)
3750 std::swap(a&: X, b&: Y);
3751 return Y == SharedReg;
3752}
3753
3754void CombinerHelper::applyXorOfAndWithSameReg(
3755 MachineInstr &MI, std::pair<Register, Register> &MatchInfo) const {
3756 // Fold (xor (and x, y), y) -> (and (not x), y)
3757 Register X, Y;
3758 std::tie(args&: X, args&: Y) = MatchInfo;
3759 auto Not = Builder.buildNot(Dst: MRI.getType(Reg: X), Src0: X);
3760 Observer.changingInstr(MI);
3761 MI.setDesc(Builder.getTII().get(Opcode: TargetOpcode::G_AND));
3762 MI.getOperand(i: 1).setReg(Not->getOperand(i: 0).getReg());
3763 MI.getOperand(i: 2).setReg(Y);
3764 Observer.changedInstr(MI);
3765}
3766
3767bool CombinerHelper::matchPtrAddZeroVector(MachineInstr &MI) const {
3768 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
3769 Register DstReg = PtrAdd.getReg(Idx: 0);
3770 LLT Ty = MRI.getType(Reg: DstReg);
3771 const DataLayout &DL = Builder.getMF().getDataLayout();
3772
3773 if (DL.isNonIntegralAddressSpace(AddrSpace: Ty.getScalarType().getAddressSpace()))
3774 return false;
3775
3776 if (!Ty.isVector())
3777 return false;
3778 const MachineInstr *VecMI;
3779 if (!mi_match(R: PtrAdd.getBaseReg(), MRI, P: m_MInstr(MI&: VecMI)))
3780 return false;
3781 return isBuildVectorAllZeros(MI: *VecMI, MRI);
3782}
3783
3784/// The second source operand is known to be a power of 2.
3785void CombinerHelper::applySimplifyURemByPow2(MachineInstr &MI) const {
3786 Register DstReg = MI.getOperand(i: 0).getReg();
3787 Register Src0 = MI.getOperand(i: 1).getReg();
3788 Register Pow2Src1 = MI.getOperand(i: 2).getReg();
3789 LLT Ty = MRI.getType(Reg: DstReg);
3790
3791 // Fold (urem x, pow2) -> (and x, pow2-1)
3792 auto NegOne = Builder.buildConstant(Res: Ty, Val: -1);
3793 auto Add = Builder.buildAdd(Dst: Ty, Src0: Pow2Src1, Src1: NegOne);
3794 Builder.buildAnd(Dst: DstReg, Src0, Src1: Add);
3795 MI.eraseFromParent();
3796}
3797
3798bool CombinerHelper::matchFoldBinOpIntoSelect(MachineInstr &MI,
3799 unsigned &SelectOpNo) const {
3800 Register LHS = MI.getOperand(i: 1).getReg();
3801 Register RHS = MI.getOperand(i: 2).getReg();
3802
3803 Register OtherOperandReg = RHS;
3804 SelectOpNo = 1;
3805 Register SelectTrue, SelectFalse;
3806
3807 // Don't do this unless the old select is going away. We want to eliminate the
3808 // binary operator, not replace a binop with a select.
3809 if (!mi_match(R: LHS, MRI,
3810 P: m_GISelect(Src0: m_Reg(), Src1: m_Reg(R&: SelectTrue), Src2: m_Reg(R&: SelectFalse))) ||
3811 !MRI.hasOneNonDBGUse(RegNo: LHS)) {
3812 OtherOperandReg = LHS;
3813 SelectOpNo = 2;
3814 if (!mi_match(R: RHS, MRI,
3815 P: m_GISelect(Src0: m_Reg(), Src1: m_Reg(R&: SelectTrue), Src2: m_Reg(R&: SelectFalse))) ||
3816 !MRI.hasOneNonDBGUse(RegNo: RHS))
3817 return false;
3818 }
3819
3820 MachineInstr *SelectLHS, *SelectRHS;
3821 if (!mi_match(R: SelectTrue, MRI, P: m_MInstr(MI&: SelectLHS)) ||
3822 !mi_match(R: SelectFalse, MRI, P: m_MInstr(MI&: SelectRHS)))
3823 return false;
3824
3825 if (!isConstantOrConstantVector(MI: *SelectLHS, MRI,
3826 /*AllowFP*/ true,
3827 /*AllowOpaqueConstants*/ false))
3828 return false;
3829 if (!isConstantOrConstantVector(MI: *SelectRHS, MRI,
3830 /*AllowFP*/ true,
3831 /*AllowOpaqueConstants*/ false))
3832 return false;
3833
3834 unsigned BinOpcode = MI.getOpcode();
3835
3836 // We know that one of the operands is a select of constants. Now verify that
3837 // the other binary operator operand is either a constant, or we can handle a
3838 // variable.
3839 bool CanFoldNonConst =
3840 (BinOpcode == TargetOpcode::G_AND || BinOpcode == TargetOpcode::G_OR) &&
3841 (isNullOrNullSplat(MI: *SelectLHS, MRI) ||
3842 isAllOnesOrAllOnesSplat(MI: *SelectLHS, MRI)) &&
3843 (isNullOrNullSplat(MI: *SelectRHS, MRI) ||
3844 isAllOnesOrAllOnesSplat(MI: *SelectRHS, MRI));
3845 if (CanFoldNonConst)
3846 return true;
3847
3848 MachineInstr *OtherOperandDef;
3849 if (!mi_match(R: OtherOperandReg, MRI, P: m_MInstr(MI&: OtherOperandDef)))
3850 return false;
3851 return isConstantOrConstantVector(MI: *OtherOperandDef, MRI,
3852 /*AllowFP*/ true,
3853 /*AllowOpaqueConstants*/ false);
3854}
3855
3856/// \p SelectOperand is the operand in binary operator \p MI that is the select
3857/// to fold.
3858void CombinerHelper::applyFoldBinOpIntoSelect(
3859 MachineInstr &MI, const unsigned &SelectOperand) const {
3860 Register Dst = MI.getOperand(i: 0).getReg();
3861 Register LHS = MI.getOperand(i: 1).getReg();
3862 Register RHS = MI.getOperand(i: 2).getReg();
3863 GSelect *Select =
3864 cast<GSelect>(Val: MRI.getVRegDef(Reg: MI.getOperand(i: SelectOperand).getReg()));
3865
3866 Register SelectCond = Select->getCondReg();
3867 Register SelectTrue = Select->getTrueReg();
3868 Register SelectFalse = Select->getFalseReg();
3869
3870 LLT Ty = MRI.getType(Reg: Dst);
3871 unsigned BinOpcode = MI.getOpcode();
3872
3873 Register FoldTrue, FoldFalse;
3874
3875 // We have a select-of-constants followed by a binary operator with a
3876 // constant. Eliminate the binop by pulling the constant math into the select.
3877 // Example: add (select Cond, CT, CF), CBO --> select Cond, CT + CBO, CF + CBO
3878 if (SelectOperand == 1) {
3879 // TODO: SelectionDAG verifies this actually constant folds before
3880 // committing to the combine.
3881
3882 FoldTrue = Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {SelectTrue, RHS}).getReg(Idx: 0);
3883 FoldFalse =
3884 Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {SelectFalse, RHS}).getReg(Idx: 0);
3885 } else {
3886 FoldTrue = Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {LHS, SelectTrue}).getReg(Idx: 0);
3887 FoldFalse =
3888 Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {LHS, SelectFalse}).getReg(Idx: 0);
3889 }
3890
3891 Builder.buildSelect(Res: Dst, Tst: SelectCond, Op0: FoldTrue, Op1: FoldFalse, Flags: MI.getFlags());
3892 MI.eraseFromParent();
3893}
3894
3895std::optional<SmallVector<Register, 8>>
3896CombinerHelper::findCandidatesForLoadOrCombine(const MachineInstr *Root) const {
3897 assert(Root->getOpcode() == TargetOpcode::G_OR && "Expected G_OR only!");
3898 // We want to detect if Root is part of a tree which represents a bunch
3899 // of loads being merged into a larger load. We'll try to recognize patterns
3900 // like, for example:
3901 //
3902 // Reg Reg
3903 // \ /
3904 // OR_1 Reg
3905 // \ /
3906 // OR_2
3907 // \ Reg
3908 // .. /
3909 // Root
3910 //
3911 // Reg Reg Reg Reg
3912 // \ / \ /
3913 // OR_1 OR_2
3914 // \ /
3915 // \ /
3916 // ...
3917 // Root
3918 //
3919 // Each "Reg" may have been produced by a load + some arithmetic. This
3920 // function will save each of them.
3921 SmallVector<Register, 8> RegsToVisit;
3922 SmallVector<const MachineInstr *, 7> Ors = {Root};
3923
3924 // In the "worst" case, we're dealing with a load for each byte. So, there
3925 // are at most #bytes - 1 ORs.
3926 const unsigned MaxIter =
3927 MRI.getType(Reg: Root->getOperand(i: 0).getReg()).getSizeInBytes() - 1;
3928 for (unsigned Iter = 0; Iter < MaxIter; ++Iter) {
3929 if (Ors.empty())
3930 break;
3931 const MachineInstr *Curr = Ors.pop_back_val();
3932 Register OrLHS = Curr->getOperand(i: 1).getReg();
3933 Register OrRHS = Curr->getOperand(i: 2).getReg();
3934
3935 // In the combine, we want to elimate the entire tree.
3936 if (!MRI.hasOneNonDBGUse(RegNo: OrLHS) || !MRI.hasOneNonDBGUse(RegNo: OrRHS))
3937 return std::nullopt;
3938
3939 // If it's a G_OR, save it and continue to walk. If it's not, then it's
3940 // something that may be a load + arithmetic.
3941 if (const MachineInstr *Or = getOpcodeDef(Opcode: TargetOpcode::G_OR, Reg: OrLHS, MRI))
3942 Ors.push_back(Elt: Or);
3943 else
3944 RegsToVisit.push_back(Elt: OrLHS);
3945 if (const MachineInstr *Or = getOpcodeDef(Opcode: TargetOpcode::G_OR, Reg: OrRHS, MRI))
3946 Ors.push_back(Elt: Or);
3947 else
3948 RegsToVisit.push_back(Elt: OrRHS);
3949 }
3950
3951 // We're going to try and merge each register into a wider power-of-2 type,
3952 // so we ought to have an even number of registers.
3953 if (RegsToVisit.empty() || RegsToVisit.size() % 2 != 0)
3954 return std::nullopt;
3955 return RegsToVisit;
3956}
3957
3958/// Helper function for findLoadOffsetsForLoadOrCombine.
3959///
3960/// Check if \p Reg is the result of loading a \p MemSizeInBits wide value,
3961/// and then moving that value into a specific byte offset.
3962///
3963/// e.g. x[i] << 24
3964///
3965/// \returns The load instruction and the byte offset it is moved into.
3966static std::optional<std::pair<GZExtLoad *, int64_t>>
3967matchLoadAndBytePosition(Register Reg, unsigned MemSizeInBits,
3968 const MachineRegisterInfo &MRI) {
3969 assert(MRI.hasOneNonDBGUse(Reg) &&
3970 "Expected Reg to only have one non-debug use?");
3971 Register MaybeLoad;
3972 int64_t Shift;
3973 if (!mi_match(R: Reg, MRI,
3974 P: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: MaybeLoad), R: m_ICst(Cst&: Shift))))) {
3975 Shift = 0;
3976 MaybeLoad = Reg;
3977 }
3978
3979 if (Shift % MemSizeInBits != 0)
3980 return std::nullopt;
3981
3982 // TODO: Handle other types of loads.
3983 auto *Load = getOpcodeDef<GZExtLoad>(Reg: MaybeLoad, MRI);
3984 if (!Load)
3985 return std::nullopt;
3986
3987 if (!Load->isUnordered() || Load->getMemSizeInBits() != MemSizeInBits)
3988 return std::nullopt;
3989
3990 return std::make_pair(x&: Load, y: Shift / MemSizeInBits);
3991}
3992
3993std::optional<std::tuple<GZExtLoad *, int64_t, GZExtLoad *>>
3994CombinerHelper::findLoadOffsetsForLoadOrCombine(
3995 SmallDenseMap<int64_t, int64_t, 8> &MemOffset2Idx,
3996 const SmallVector<Register, 8> &RegsToVisit,
3997 const unsigned MemSizeInBits) const {
3998
3999 // Each load found for the pattern. There should be one for each RegsToVisit.
4000 SmallSetVector<const MachineInstr *, 8> Loads;
4001
4002 // The lowest index used in any load. (The lowest "i" for each x[i].)
4003 int64_t LowestIdx = INT64_MAX;
4004
4005 // The load which uses the lowest index.
4006 GZExtLoad *LowestIdxLoad = nullptr;
4007
4008 // Keeps track of the load indices we see. We shouldn't see any indices twice.
4009 SmallSet<int64_t, 8> SeenIdx;
4010
4011 // Ensure each load is in the same MBB.
4012 // TODO: Support multiple MachineBasicBlocks.
4013 MachineBasicBlock *MBB = nullptr;
4014 const MachineMemOperand *MMO = nullptr;
4015
4016 // Earliest instruction-order load in the pattern.
4017 GZExtLoad *EarliestLoad = nullptr;
4018
4019 // Latest instruction-order load in the pattern.
4020 GZExtLoad *LatestLoad = nullptr;
4021
4022 // Base pointer which every load should share.
4023 Register BasePtr;
4024
4025 // We want to find a load for each register. Each load should have some
4026 // appropriate bit twiddling arithmetic. During this loop, we will also keep
4027 // track of the load which uses the lowest index. Later, we will check if we
4028 // can use its pointer in the final, combined load.
4029 for (auto Reg : RegsToVisit) {
4030 // Find the load, and find the position that it will end up in (e.g. a
4031 // shifted) value.
4032 auto LoadAndPos = matchLoadAndBytePosition(Reg, MemSizeInBits, MRI);
4033 if (!LoadAndPos)
4034 return std::nullopt;
4035 GZExtLoad *Load;
4036 int64_t DstPos;
4037 std::tie(args&: Load, args&: DstPos) = *LoadAndPos;
4038
4039 // TODO: Handle multiple MachineBasicBlocks. Currently not handled because
4040 // it is difficult to check for stores/calls/etc between loads.
4041 MachineBasicBlock *LoadMBB = Load->getParent();
4042 if (!MBB)
4043 MBB = LoadMBB;
4044 if (LoadMBB != MBB)
4045 return std::nullopt;
4046
4047 // Make sure that the MachineMemOperands of every seen load are compatible.
4048 auto &LoadMMO = Load->getMMO();
4049 if (!MMO)
4050 MMO = &LoadMMO;
4051 if (MMO->getAddrSpace() != LoadMMO.getAddrSpace())
4052 return std::nullopt;
4053
4054 // Find out what the base pointer and index for the load is.
4055 Register LoadPtr;
4056 int64_t Idx;
4057 if (!mi_match(R: Load->getOperand(i: 1).getReg(), MRI,
4058 P: m_GPtrAdd(L: m_Reg(R&: LoadPtr), R: m_ICst(Cst&: Idx)))) {
4059 LoadPtr = Load->getOperand(i: 1).getReg();
4060 Idx = 0;
4061 }
4062
4063 // Don't combine things like a[i], a[i] -> a bigger load.
4064 if (!SeenIdx.insert(V: Idx).second)
4065 return std::nullopt;
4066
4067 // Every load must share the same base pointer; don't combine things like:
4068 //
4069 // a[i], b[i + 1] -> a bigger load.
4070 if (!BasePtr.isValid())
4071 BasePtr = LoadPtr;
4072 if (BasePtr != LoadPtr)
4073 return std::nullopt;
4074
4075 if (Idx < LowestIdx) {
4076 LowestIdx = Idx;
4077 LowestIdxLoad = Load;
4078 }
4079
4080 // Keep track of the byte offset that this load ends up at. If we have seen
4081 // the byte offset, then stop here. We do not want to combine:
4082 //
4083 // a[i] << 16, a[i + k] << 16 -> a bigger load.
4084 if (!MemOffset2Idx.try_emplace(Key: DstPos, Args&: Idx).second)
4085 return std::nullopt;
4086 Loads.insert(X: Load);
4087
4088 // Keep track of the position of the earliest/latest loads in the pattern.
4089 // We will check that there are no load fold barriers between them later
4090 // on.
4091 //
4092 // FIXME: Is there a better way to check for load fold barriers?
4093 if (!EarliestLoad || dominates(DefMI: *Load, UseMI: *EarliestLoad))
4094 EarliestLoad = Load;
4095 if (!LatestLoad || dominates(DefMI: *LatestLoad, UseMI: *Load))
4096 LatestLoad = Load;
4097 }
4098
4099 // We found a load for each register. Let's check if each load satisfies the
4100 // pattern.
4101 assert(Loads.size() == RegsToVisit.size() &&
4102 "Expected to find a load for each register?");
4103 assert(EarliestLoad != LatestLoad && EarliestLoad &&
4104 LatestLoad && "Expected at least two loads?");
4105
4106 // Check if there are any stores, calls, etc. between any of the loads. If
4107 // there are, then we can't safely perform the combine.
4108 //
4109 // MaxIter is chosen based off the (worst case) number of iterations it
4110 // typically takes to succeed in the LLVM test suite plus some padding.
4111 //
4112 // FIXME: Is there a better way to check for load fold barriers?
4113 const unsigned MaxIter = 20;
4114 unsigned Iter = 0;
4115 for (const auto &MI : instructionsWithoutDebug(It: EarliestLoad->getIterator(),
4116 End: LatestLoad->getIterator())) {
4117 if (Loads.count(key: &MI))
4118 continue;
4119 if (MI.isLoadFoldBarrier())
4120 return std::nullopt;
4121 if (Iter++ == MaxIter)
4122 return std::nullopt;
4123 }
4124
4125 return std::make_tuple(args&: LowestIdxLoad, args&: LowestIdx, args&: LatestLoad);
4126}
4127
4128bool CombinerHelper::matchLoadOrCombine(
4129 MachineInstr &MI,
4130 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4131 assert(MI.getOpcode() == TargetOpcode::G_OR);
4132 MachineFunction &MF = *MI.getMF();
4133 // Assuming a little-endian target, transform:
4134 // s8 *a = ...
4135 // s32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
4136 // =>
4137 // s32 val = *((i32)a)
4138 //
4139 // s8 *a = ...
4140 // s32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
4141 // =>
4142 // s32 val = BSWAP(*((s32)a))
4143 Register Dst = MI.getOperand(i: 0).getReg();
4144 LLT Ty = MRI.getType(Reg: Dst);
4145 if (Ty.isVector())
4146 return false;
4147
4148 // We need to combine at least two loads into this type. Since the smallest
4149 // possible load is into a byte, we need at least a 16-bit wide type.
4150 const unsigned WideMemSizeInBits = Ty.getSizeInBits();
4151 if (WideMemSizeInBits < 16 || WideMemSizeInBits % 8 != 0)
4152 return false;
4153
4154 // Match a collection of non-OR instructions in the pattern.
4155 auto RegsToVisit = findCandidatesForLoadOrCombine(Root: &MI);
4156 if (!RegsToVisit)
4157 return false;
4158
4159 // We have a collection of non-OR instructions. Figure out how wide each of
4160 // the small loads should be based off of the number of potential loads we
4161 // found.
4162 const unsigned NarrowMemSizeInBits = WideMemSizeInBits / RegsToVisit->size();
4163 if (NarrowMemSizeInBits % 8 != 0)
4164 return false;
4165
4166 // Check if each register feeding into each OR is a load from the same
4167 // base pointer + some arithmetic.
4168 //
4169 // e.g. a[0], a[1] << 8, a[2] << 16, etc.
4170 //
4171 // Also verify that each of these ends up putting a[i] into the same memory
4172 // offset as a load into a wide type would.
4173 SmallDenseMap<int64_t, int64_t, 8> MemOffset2Idx;
4174 GZExtLoad *LowestIdxLoad, *LatestLoad;
4175 int64_t LowestIdx;
4176 auto MaybeLoadInfo = findLoadOffsetsForLoadOrCombine(
4177 MemOffset2Idx, RegsToVisit: *RegsToVisit, MemSizeInBits: NarrowMemSizeInBits);
4178 if (!MaybeLoadInfo)
4179 return false;
4180 std::tie(args&: LowestIdxLoad, args&: LowestIdx, args&: LatestLoad) = *MaybeLoadInfo;
4181
4182 // We have a bunch of loads being OR'd together. Using the addresses + offsets
4183 // we found before, check if this corresponds to a big or little endian byte
4184 // pattern. If it does, then we can represent it using a load + possibly a
4185 // BSWAP.
4186 bool IsBigEndianTarget = MF.getDataLayout().isBigEndian();
4187 std::optional<bool> IsBigEndian = isBigEndian(MemOffset2Idx, LowestIdx);
4188 if (!IsBigEndian)
4189 return false;
4190 bool NeedsBSwap = IsBigEndianTarget != *IsBigEndian;
4191 if (NeedsBSwap && !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_BSWAP, {Ty}}))
4192 return false;
4193
4194 // Make sure that the load from the lowest index produces offset 0 in the
4195 // final value.
4196 //
4197 // This ensures that we won't combine something like this:
4198 //
4199 // load x[i] -> byte 2
4200 // load x[i+1] -> byte 0 ---> wide_load x[i]
4201 // load x[i+2] -> byte 1
4202 const unsigned NumLoadsInTy = WideMemSizeInBits / NarrowMemSizeInBits;
4203 const unsigned ZeroByteOffset =
4204 *IsBigEndian
4205 ? bigEndianByteAt(ByteWidth: NumLoadsInTy, I: 0)
4206 : littleEndianByteAt(ByteWidth: NumLoadsInTy, I: 0);
4207 auto ZeroOffsetIdx = MemOffset2Idx.find(Val: ZeroByteOffset);
4208 if (ZeroOffsetIdx == MemOffset2Idx.end() ||
4209 ZeroOffsetIdx->second != LowestIdx)
4210 return false;
4211
4212 // We wil reuse the pointer from the load which ends up at byte offset 0. It
4213 // may not use index 0.
4214 Register Ptr = LowestIdxLoad->getPointerReg();
4215 const MachineMemOperand &MMO = LowestIdxLoad->getMMO();
4216 LegalityQuery::MemDesc MMDesc(MMO);
4217 MMDesc.MemoryTy = Ty;
4218 if (!isLegalOrBeforeLegalizer(
4219 Query: {TargetOpcode::G_LOAD, {Ty, MRI.getType(Reg: Ptr)}, {MMDesc}}))
4220 return false;
4221 auto PtrInfo = MMO.getPointerInfo();
4222 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Size: WideMemSizeInBits / 8);
4223
4224 // Load must be allowed and fast on the target.
4225 LLVMContext &C = MF.getFunction().getContext();
4226 auto &DL = MF.getDataLayout();
4227 unsigned Fast = 0;
4228 if (!getTargetLowering().allowsMemoryAccess(Context&: C, DL, Ty, MMO: *NewMMO, Fast: &Fast) ||
4229 !Fast)
4230 return false;
4231
4232 MatchInfo = [=](MachineIRBuilder &MIB) {
4233 MIB.setInstrAndDebugLoc(*LatestLoad);
4234 Register LoadDst = NeedsBSwap ? MRI.cloneVirtualRegister(VReg: Dst) : Dst;
4235 MIB.buildLoad(Res: LoadDst, Addr: Ptr, MMO&: *NewMMO);
4236 if (NeedsBSwap)
4237 MIB.buildBSwap(Dst, Src0: LoadDst);
4238 };
4239 return true;
4240}
4241
4242bool CombinerHelper::matchExtendThroughPhis(MachineInstr &MI,
4243 MachineInstr *&ExtMI) const {
4244 auto &PHI = cast<GPhi>(Val&: MI);
4245 Register DstReg = PHI.getReg(Idx: 0);
4246
4247 // TODO: Extending a vector may be expensive, don't do this until heuristics
4248 // are better.
4249 if (MRI.getType(Reg: DstReg).isVector())
4250 return false;
4251
4252 // Try to match a phi, whose only use is an extend.
4253 if (!MRI.hasOneNonDBGUse(RegNo: DstReg))
4254 return false;
4255 ExtMI = &*MRI.use_instr_nodbg_begin(RegNo: DstReg);
4256 switch (ExtMI->getOpcode()) {
4257 case TargetOpcode::G_ANYEXT:
4258 return true; // G_ANYEXT is usually free.
4259 case TargetOpcode::G_ZEXT:
4260 case TargetOpcode::G_SEXT:
4261 break;
4262 default:
4263 return false;
4264 }
4265
4266 // If the target is likely to fold this extend away, don't propagate.
4267 if (Builder.getTII().isExtendLikelyToBeFolded(ExtMI&: *ExtMI, MRI))
4268 return false;
4269
4270 // We don't want to propagate the extends unless there's a good chance that
4271 // they'll be optimized in some way.
4272 // Collect the unique incoming values.
4273 SmallPtrSet<MachineInstr *, 4> InSrcs;
4274 for (unsigned I = 0; I < PHI.getNumIncomingValues(); ++I) {
4275 auto *DefMI = getDefIgnoringCopies(Reg: PHI.getIncomingValue(I), MRI);
4276 switch (DefMI->getOpcode()) {
4277 case TargetOpcode::G_LOAD:
4278 case TargetOpcode::G_TRUNC:
4279 case TargetOpcode::G_SEXT:
4280 case TargetOpcode::G_ZEXT:
4281 case TargetOpcode::G_ANYEXT:
4282 case TargetOpcode::G_CONSTANT:
4283 InSrcs.insert(Ptr: DefMI);
4284 // Don't try to propagate if there are too many places to create new
4285 // extends, chances are it'll increase code size.
4286 if (InSrcs.size() > 2)
4287 return false;
4288 break;
4289 default:
4290 return false;
4291 }
4292 }
4293 return true;
4294}
4295
4296void CombinerHelper::applyExtendThroughPhis(MachineInstr &MI,
4297 MachineInstr *&ExtMI) const {
4298 auto &PHI = cast<GPhi>(Val&: MI);
4299 Register DstReg = ExtMI->getOperand(i: 0).getReg();
4300 LLT ExtTy = MRI.getType(Reg: DstReg);
4301
4302 // Propagate the extension into the block of each incoming reg's block.
4303 // Use a SetVector here because PHIs can have duplicate edges, and we want
4304 // deterministic iteration order.
4305 SmallSetVector<MachineInstr *, 8> SrcMIs;
4306 SmallDenseMap<MachineInstr *, MachineInstr *, 8> OldToNewSrcMap;
4307 for (unsigned I = 0; I < PHI.getNumIncomingValues(); ++I) {
4308 auto SrcReg = PHI.getIncomingValue(I);
4309 MachineInstr *SrcMI;
4310 if (!mi_match(R: SrcReg, MRI, P: m_MInstr(MI&: SrcMI)))
4311 continue;
4312 if (!SrcMIs.insert(X: SrcMI))
4313 continue;
4314
4315 // Build an extend after each src inst.
4316 auto *MBB = SrcMI->getParent();
4317 MachineBasicBlock::iterator InsertPt = ++SrcMI->getIterator();
4318 if (InsertPt != MBB->end() && InsertPt->isPHI())
4319 InsertPt = MBB->getFirstNonPHI();
4320
4321 Builder.setInsertPt(MBB&: *SrcMI->getParent(), II: InsertPt);
4322 Builder.setDebugLoc(MI.getDebugLoc());
4323 auto NewExt = Builder.buildExtOrTrunc(ExtOpc: ExtMI->getOpcode(), Res: ExtTy, Op: SrcReg);
4324 OldToNewSrcMap[SrcMI] = NewExt;
4325 }
4326
4327 // Create a new phi with the extended inputs.
4328 Builder.setInstrAndDebugLoc(MI);
4329 auto NewPhi = Builder.buildInstrNoInsert(Opcode: TargetOpcode::G_PHI);
4330 NewPhi.addDef(RegNo: DstReg);
4331 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
4332 if (!MO.isReg()) {
4333 NewPhi.addMBB(MBB: MO.getMBB());
4334 continue;
4335 }
4336 auto *NewSrc = OldToNewSrcMap[MRI.getVRegDef(Reg: MO.getReg())];
4337 NewPhi.addUse(RegNo: NewSrc->getOperand(i: 0).getReg());
4338 }
4339 Builder.insertInstr(MIB: NewPhi);
4340 ExtMI->eraseFromParent();
4341}
4342
4343bool CombinerHelper::matchExtractVecEltBuildVec(MachineInstr &MI,
4344 Register &Reg) const {
4345 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
4346 // If we have a constant index, look for a G_BUILD_VECTOR source
4347 // and find the source register that the index maps to.
4348 Register SrcVec = MI.getOperand(i: 1).getReg();
4349 LLT SrcTy = MRI.getType(Reg: SrcVec);
4350 if (SrcTy.isScalableVector())
4351 return false;
4352
4353 auto Cst = getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
4354 if (!Cst || Cst->Value.getZExtValue() >= SrcTy.getNumElements())
4355 return false;
4356
4357 unsigned VecIdx = Cst->Value.getZExtValue();
4358
4359 // Check if we have a build_vector or build_vector_trunc with an optional
4360 // trunc in front.
4361 MachineInstr *SrcVecMI;
4362 Register TruncSrc;
4363 if (mi_match(R: SrcVec, MRI, P: m_GTrunc(Src: m_Reg(R&: TruncSrc)))) {
4364 if (!mi_match(R: TruncSrc, MRI, P: m_MInstr(MI&: SrcVecMI)))
4365 return false;
4366 } else if (!mi_match(R: SrcVec, MRI, P: m_MInstr(MI&: SrcVecMI)))
4367 return false;
4368
4369 if (SrcVecMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR &&
4370 SrcVecMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR_TRUNC)
4371 return false;
4372
4373 EVT Ty(getMVTForLLT(Ty: SrcTy));
4374 if (!MRI.hasOneNonDBGUse(RegNo: SrcVec) &&
4375 !getTargetLowering().aggressivelyPreferBuildVectorSources(VecVT: Ty))
4376 return false;
4377
4378 Reg = SrcVecMI->getOperand(i: VecIdx + 1).getReg();
4379 return true;
4380}
4381
4382void CombinerHelper::applyExtractVecEltBuildVec(MachineInstr &MI,
4383 Register &Reg) const {
4384 // Check the type of the register, since it may have come from a
4385 // G_BUILD_VECTOR_TRUNC.
4386 LLT ScalarTy = MRI.getType(Reg);
4387 Register DstReg = MI.getOperand(i: 0).getReg();
4388 LLT DstTy = MRI.getType(Reg: DstReg);
4389
4390 if (ScalarTy != DstTy) {
4391 assert(ScalarTy.getSizeInBits() > DstTy.getSizeInBits());
4392 Builder.buildTrunc(Res: DstReg, Op: Reg);
4393 MI.eraseFromParent();
4394 return;
4395 }
4396 replaceSingleDefInstWithReg(MI, Replacement: Reg);
4397}
4398
4399bool CombinerHelper::matchExtractAllEltsFromBuildVector(
4400 MachineInstr &MI,
4401 SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs) const {
4402 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4403 // This combine tries to find build_vector's which have every source element
4404 // extracted using G_EXTRACT_VECTOR_ELT. This can happen when transforms like
4405 // the masked load scalarization is run late in the pipeline. There's already
4406 // a combine for a similar pattern starting from the extract, but that
4407 // doesn't attempt to do it if there are multiple uses of the build_vector,
4408 // which in this case is true. Starting the combine from the build_vector
4409 // feels more natural than trying to find sibling nodes of extracts.
4410 // E.g.
4411 // %vec(<4 x s32>) = G_BUILD_VECTOR %s1(s32), %s2, %s3, %s4
4412 // %ext1 = G_EXTRACT_VECTOR_ELT %vec, 0
4413 // %ext2 = G_EXTRACT_VECTOR_ELT %vec, 1
4414 // %ext3 = G_EXTRACT_VECTOR_ELT %vec, 2
4415 // %ext4 = G_EXTRACT_VECTOR_ELT %vec, 3
4416 // ==>
4417 // replace ext{1,2,3,4} with %s{1,2,3,4}
4418
4419 Register DstReg = MI.getOperand(i: 0).getReg();
4420 LLT DstTy = MRI.getType(Reg: DstReg);
4421 unsigned NumElts = DstTy.getNumElements();
4422
4423 SmallBitVector ExtractedElts(NumElts);
4424 for (MachineInstr &II : MRI.use_nodbg_instructions(Reg: DstReg)) {
4425 if (II.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)
4426 return false;
4427 auto Cst = getIConstantVRegVal(VReg: II.getOperand(i: 2).getReg(), MRI);
4428 if (!Cst)
4429 return false;
4430 unsigned Idx = Cst->getZExtValue();
4431 if (Idx >= NumElts)
4432 return false; // Out of range.
4433 ExtractedElts.set(Idx);
4434 SrcDstPairs.emplace_back(
4435 Args: std::make_pair(x: MI.getOperand(i: Idx + 1).getReg(), y: &II));
4436 }
4437 // Match if every element was extracted.
4438 return ExtractedElts.all();
4439}
4440
4441void CombinerHelper::applyExtractAllEltsFromBuildVector(
4442 MachineInstr &MI,
4443 SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs) const {
4444 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4445 for (auto &Pair : SrcDstPairs) {
4446 auto *ExtMI = Pair.second;
4447 replaceRegWith(MRI, FromReg: ExtMI->getOperand(i: 0).getReg(), ToReg: Pair.first);
4448 ExtMI->eraseFromParent();
4449 }
4450 MI.eraseFromParent();
4451}
4452
4453void CombinerHelper::applyBuildFn(
4454 MachineInstr &MI,
4455 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4456 applyBuildFnNoErase(MI, MatchInfo);
4457 MI.eraseFromParent();
4458}
4459
4460void CombinerHelper::applyBuildFnNoErase(
4461 MachineInstr &MI,
4462 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4463 MatchInfo(Builder);
4464}
4465
4466bool CombinerHelper::matchOrShiftToFunnelShift(MachineInstr &MI,
4467 bool AllowScalarConstants,
4468 BuildFnTy &MatchInfo) const {
4469 assert(MI.getOpcode() == TargetOpcode::G_OR);
4470
4471 Register Dst = MI.getOperand(i: 0).getReg();
4472 LLT Ty = MRI.getType(Reg: Dst);
4473 unsigned BitWidth = Ty.getScalarSizeInBits();
4474
4475 Register ShlSrc, ShlAmt, LShrSrc, LShrAmt, Amt;
4476 unsigned FshOpc = 0;
4477
4478 // Match (or (shl ...), (lshr ...)).
4479 if (!mi_match(R: Dst, MRI,
4480 // m_GOr() handles the commuted version as well.
4481 P: m_GOr(L: m_GShl(L: m_Reg(R&: ShlSrc), R: m_Reg(R&: ShlAmt)),
4482 R: m_GLShr(L: m_Reg(R&: LShrSrc), R: m_Reg(R&: LShrAmt)))))
4483 return false;
4484
4485 // Given constants C0 and C1 such that C0 + C1 is bit-width:
4486 // (or (shl x, C0), (lshr y, C1)) -> (fshl x, y, C0) or (fshr x, y, C1)
4487 int64_t CstShlAmt = 0, CstLShrAmt;
4488 if (mi_match(R: ShlAmt, MRI, P: m_ICstOrSplat(Cst&: CstShlAmt)) &&
4489 mi_match(R: LShrAmt, MRI, P: m_ICstOrSplat(Cst&: CstLShrAmt)) &&
4490 CstShlAmt + CstLShrAmt == BitWidth) {
4491 FshOpc = TargetOpcode::G_FSHR;
4492 Amt = LShrAmt;
4493 } else if (mi_match(R: LShrAmt, MRI,
4494 P: m_GSub(L: m_SpecificICstOrSplat(RequestedValue: BitWidth), R: m_Reg(R&: Amt))) &&
4495 ShlAmt == Amt) {
4496 // (or (shl x, amt), (lshr y, (sub bw, amt))) -> (fshl x, y, amt)
4497 FshOpc = TargetOpcode::G_FSHL;
4498 } else if (mi_match(R: ShlAmt, MRI,
4499 P: m_GSub(L: m_SpecificICstOrSplat(RequestedValue: BitWidth), R: m_Reg(R&: Amt))) &&
4500 LShrAmt == Amt) {
4501 // (or (shl x, (sub bw, amt)), (lshr y, amt)) -> (fshr x, y, amt)
4502 FshOpc = TargetOpcode::G_FSHR;
4503 } else {
4504 return false;
4505 }
4506
4507 LLT AmtTy = MRI.getType(Reg: Amt);
4508 if (!isLegalOrBeforeLegalizer(Query: {FshOpc, {Ty, AmtTy}}) &&
4509 (!AllowScalarConstants || CstShlAmt == 0 || !Ty.isScalar()))
4510 return false;
4511
4512 MatchInfo = [=](MachineIRBuilder &B) {
4513 B.buildInstr(Opc: FshOpc, DstOps: {Dst}, SrcOps: {ShlSrc, LShrSrc, Amt});
4514 };
4515 return true;
4516}
4517
4518// Fold (rot x, c) -> (rot x, c % BitSize)
4519bool CombinerHelper::matchRotateOutOfRange(MachineInstr &MI) const {
4520 assert(MI.getOpcode() == TargetOpcode::G_ROTL ||
4521 MI.getOpcode() == TargetOpcode::G_ROTR);
4522 unsigned Bitsize =
4523 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getScalarSizeInBits();
4524 Register AmtReg = MI.getOperand(i: 2).getReg();
4525 bool OutOfRange = false;
4526 auto MatchOutOfRange = [Bitsize, &OutOfRange](const Constant *C) {
4527 if (auto *CI = dyn_cast<ConstantInt>(Val: C))
4528 OutOfRange |= CI->getValue().uge(RHS: Bitsize);
4529 return true;
4530 };
4531 return matchUnaryPredicate(MRI, Reg: AmtReg, Match: MatchOutOfRange) && OutOfRange;
4532}
4533
4534void CombinerHelper::applyRotateOutOfRange(MachineInstr &MI) const {
4535 assert(MI.getOpcode() == TargetOpcode::G_ROTL ||
4536 MI.getOpcode() == TargetOpcode::G_ROTR);
4537 unsigned Bitsize =
4538 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getScalarSizeInBits();
4539 Register Amt = MI.getOperand(i: 2).getReg();
4540 LLT AmtTy = MRI.getType(Reg: Amt);
4541 auto Bits = Builder.buildConstant(Res: AmtTy, Val: Bitsize);
4542 Amt = Builder.buildURem(Dst: AmtTy, Src0: MI.getOperand(i: 2).getReg(), Src1: Bits).getReg(Idx: 0);
4543 Observer.changingInstr(MI);
4544 MI.getOperand(i: 2).setReg(Amt);
4545 Observer.changedInstr(MI);
4546}
4547
4548bool CombinerHelper::matchICmpToTrueFalseKnownBits(MachineInstr &MI,
4549 int64_t &MatchInfo) const {
4550 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
4551 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
4552
4553 // We want to avoid calling KnownBits on the LHS if possible, as this combine
4554 // has no filter and runs on every G_ICMP instruction. We can avoid calling
4555 // KnownBits on the LHS in two cases:
4556 //
4557 // - The RHS is unknown: Constants are always on RHS. If the RHS is unknown
4558 // we cannot do any transforms so we can safely bail out early.
4559 // - The RHS is zero: we don't need to know the LHS to do unsigned <0 and
4560 // >=0.
4561 auto KnownRHS = VT->getKnownBits(R: MI.getOperand(i: 3).getReg());
4562 if (KnownRHS.isUnknown())
4563 return false;
4564
4565 std::optional<bool> KnownVal;
4566 if (KnownRHS.isZero()) {
4567 // ? uge 0 -> always true
4568 // ? ult 0 -> always false
4569 if (Pred == CmpInst::ICMP_UGE)
4570 KnownVal = true;
4571 else if (Pred == CmpInst::ICMP_ULT)
4572 KnownVal = false;
4573 }
4574
4575 if (!KnownVal) {
4576 auto KnownLHS = VT->getKnownBits(R: MI.getOperand(i: 2).getReg());
4577 KnownVal = ICmpInst::compare(LHS: KnownLHS, RHS: KnownRHS, Pred);
4578 }
4579
4580 if (!KnownVal)
4581 return false;
4582 MatchInfo =
4583 *KnownVal
4584 ? getICmpTrueVal(TLI: getTargetLowering(),
4585 /*IsVector = */
4586 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector(),
4587 /* IsFP = */ false)
4588 : 0;
4589 return true;
4590}
4591
4592bool CombinerHelper::matchICmpToLHSKnownBits(
4593 MachineInstr &MI,
4594 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4595 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
4596 // Given:
4597 //
4598 // %x = G_WHATEVER (... x is known to be 0 or 1 ...)
4599 // %cmp = G_ICMP ne %x, 0
4600 //
4601 // Or:
4602 //
4603 // %x = G_WHATEVER (... x is known to be 0 or 1 ...)
4604 // %cmp = G_ICMP eq %x, 1
4605 //
4606 // We can replace %cmp with %x assuming true is 1 on the target.
4607 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
4608 if (!CmpInst::isEquality(pred: Pred))
4609 return false;
4610 Register Dst = MI.getOperand(i: 0).getReg();
4611 LLT DstTy = MRI.getType(Reg: Dst);
4612 if (getICmpTrueVal(TLI: getTargetLowering(), IsVector: DstTy.isVector(),
4613 /* IsFP = */ false) != 1)
4614 return false;
4615 int64_t OneOrZero = Pred == CmpInst::ICMP_EQ;
4616 if (!mi_match(R: MI.getOperand(i: 3).getReg(), MRI, P: m_SpecificICst(RequestedValue: OneOrZero)))
4617 return false;
4618 Register LHS = MI.getOperand(i: 2).getReg();
4619 auto KnownLHS = VT->getKnownBits(R: LHS);
4620 if (KnownLHS.getMinValue() != 0 || KnownLHS.getMaxValue() != 1)
4621 return false;
4622 // Make sure replacing Dst with the LHS is a legal operation.
4623 LLT LHSTy = MRI.getType(Reg: LHS);
4624 unsigned LHSSize = LHSTy.getSizeInBits();
4625 unsigned DstSize = DstTy.getSizeInBits();
4626 unsigned Op = TargetOpcode::COPY;
4627 if (DstSize != LHSSize)
4628 Op = DstSize < LHSSize ? TargetOpcode::G_TRUNC : TargetOpcode::G_ZEXT;
4629 if (!isLegalOrBeforeLegalizer(Query: {Op, {DstTy, LHSTy}}))
4630 return false;
4631 MatchInfo = [=](MachineIRBuilder &B) { B.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {LHS}); };
4632 return true;
4633}
4634
4635// Replace (and (or x, c1), c2) with (and x, c2) iff c1 & c2 == 0
4636bool CombinerHelper::matchAndOrDisjointMask(
4637 MachineInstr &MI,
4638 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4639 assert(MI.getOpcode() == TargetOpcode::G_AND);
4640
4641 // Ignore vector types to simplify matching the two constants.
4642 // TODO: do this for vectors and scalars via a demanded bits analysis.
4643 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4644 if (Ty.isVector())
4645 return false;
4646
4647 Register Src;
4648 Register AndMaskReg;
4649 int64_t AndMaskBits;
4650 int64_t OrMaskBits;
4651 if (!mi_match(MI, MRI,
4652 P: m_GAnd(L: m_GOr(L: m_Reg(R&: Src), R: m_ICst(Cst&: OrMaskBits)),
4653 R: m_all_of(preds: m_ICst(Cst&: AndMaskBits), preds: m_Reg(R&: AndMaskReg)))))
4654 return false;
4655
4656 // Check if OrMask could turn on any bits in Src.
4657 if (AndMaskBits & OrMaskBits)
4658 return false;
4659
4660 MatchInfo = [=, &MI](MachineIRBuilder &B) {
4661 Observer.changingInstr(MI);
4662 // Canonicalize the result to have the constant on the RHS.
4663 if (MI.getOperand(i: 1).getReg() == AndMaskReg)
4664 MI.getOperand(i: 2).setReg(AndMaskReg);
4665 MI.getOperand(i: 1).setReg(Src);
4666 Observer.changedInstr(MI);
4667 };
4668 return true;
4669}
4670
4671/// Form a G_SBFX from a G_SEXT_INREG fed by a right shift.
4672bool CombinerHelper::matchBitfieldExtractFromSExtInReg(
4673 MachineInstr &MI,
4674 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4675 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
4676 Register Dst = MI.getOperand(i: 0).getReg();
4677 Register Src = MI.getOperand(i: 1).getReg();
4678 LLT Ty = MRI.getType(Reg: Src);
4679 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
4680 if (!LI || !LI->isLegalOrCustom(Query: {TargetOpcode::G_SBFX, {Ty, ExtractTy}}))
4681 return false;
4682 int64_t Width = MI.getOperand(i: 2).getImm();
4683 Register ShiftSrc;
4684 int64_t ShiftImm;
4685 if (!mi_match(
4686 R: Src, MRI,
4687 P: m_OneNonDBGUse(SP: m_any_of(preds: m_GAShr(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: ShiftImm)),
4688 preds: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: ShiftImm))))))
4689 return false;
4690 if (ShiftImm < 0 || ShiftImm + Width > Ty.getScalarSizeInBits())
4691 return false;
4692
4693 MatchInfo = [=](MachineIRBuilder &B) {
4694 auto Cst1 = B.buildConstant(Res: ExtractTy, Val: ShiftImm);
4695 auto Cst2 = B.buildConstant(Res: ExtractTy, Val: Width);
4696 B.buildSbfx(Dst, Src: ShiftSrc, LSB: Cst1, Width: Cst2);
4697 };
4698 return true;
4699}
4700
4701/// Form a G_UBFX from "(a srl b) & mask", where b and mask are constants.
4702bool CombinerHelper::matchBitfieldExtractFromAnd(MachineInstr &MI,
4703 BuildFnTy &MatchInfo) const {
4704 GAnd *And = cast<GAnd>(Val: &MI);
4705 Register Dst = And->getReg(Idx: 0);
4706 LLT Ty = MRI.getType(Reg: Dst);
4707 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
4708 // Note that isLegalOrBeforeLegalizer is stricter and does not take custom
4709 // into account.
4710 if (LI && !LI->isLegalOrCustom(Query: {TargetOpcode::G_UBFX, {Ty, ExtractTy}}))
4711 return false;
4712
4713 int64_t AndImm, LSBImm;
4714 Register ShiftSrc;
4715 const unsigned Size = Ty.getScalarSizeInBits();
4716 if (!mi_match(R: And->getReg(Idx: 0), MRI,
4717 P: m_GAnd(L: m_OneNonDBGUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: LSBImm))),
4718 R: m_ICst(Cst&: AndImm))))
4719 return false;
4720
4721 // AndImm is sign-extended to 64 bits by m_ICst; restrict it to the operand
4722 // width so an all-ones mask (a redundant AND) is not misread as a wider mask.
4723 uint64_t MaybeMask = static_cast<uint64_t>(AndImm);
4724 if (Size < 64)
4725 MaybeMask &= maskTrailingOnes<uint64_t>(N: Size);
4726
4727 // The mask is a mask of the low bits iff imm & (imm+1) == 0.
4728 if (MaybeMask & (MaybeMask + 1))
4729 return false;
4730
4731 // LSB must fit within the register.
4732 if (static_cast<uint64_t>(LSBImm) >= Size)
4733 return false;
4734
4735 uint64_t Width = APInt(Size, MaybeMask).countr_one();
4736 // The extracted field [LSB, LSB+Width) must fit within the register.
4737 // Otherwise this is a redundant AND (e.g. an all-ones mask combined with a
4738 // non-zero shift) that is better handled by other combines, and would form
4739 // an out-of-range bitfield extract.
4740 if (static_cast<uint64_t>(LSBImm) + Width > Size)
4741 return false;
4742
4743 MatchInfo = [=](MachineIRBuilder &B) {
4744 auto WidthCst = B.buildConstant(Res: ExtractTy, Val: Width);
4745 auto LSBCst = B.buildConstant(Res: ExtractTy, Val: LSBImm);
4746 B.buildInstr(Opc: TargetOpcode::G_UBFX, DstOps: {Dst}, SrcOps: {ShiftSrc, LSBCst, WidthCst});
4747 };
4748 return true;
4749}
4750
4751bool CombinerHelper::matchBitfieldExtractFromShr(
4752 MachineInstr &MI,
4753 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4754 const unsigned Opcode = MI.getOpcode();
4755 assert(Opcode == TargetOpcode::G_ASHR || Opcode == TargetOpcode::G_LSHR);
4756
4757 const Register Dst = MI.getOperand(i: 0).getReg();
4758
4759 const unsigned ExtrOpcode = Opcode == TargetOpcode::G_ASHR
4760 ? TargetOpcode::G_SBFX
4761 : TargetOpcode::G_UBFX;
4762
4763 // Check if the type we would use for the extract is legal
4764 LLT Ty = MRI.getType(Reg: Dst);
4765 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
4766 if (!LI || !LI->isLegalOrCustom(Query: {ExtrOpcode, {Ty, ExtractTy}}))
4767 return false;
4768
4769 Register ShlSrc;
4770 int64_t ShrAmt;
4771 int64_t ShlAmt;
4772 const unsigned Size = Ty.getScalarSizeInBits();
4773
4774 // Try to match shr (shl x, c1), c2
4775 if (!mi_match(R: Dst, MRI,
4776 P: m_BinOp(Opcode,
4777 L: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: ShlSrc), R: m_ICst(Cst&: ShlAmt))),
4778 R: m_ICst(Cst&: ShrAmt))))
4779 return false;
4780
4781 // Make sure that the shift sizes can fit a bitfield extract
4782 if (ShlAmt < 0 || ShlAmt > ShrAmt || ShrAmt >= Size)
4783 return false;
4784
4785 // Skip this combine if the G_SEXT_INREG combine could handle it
4786 if (Opcode == TargetOpcode::G_ASHR && ShlAmt == ShrAmt)
4787 return false;
4788
4789 // Calculate start position and width of the extract
4790 const int64_t Pos = ShrAmt - ShlAmt;
4791 const int64_t Width = Size - ShrAmt;
4792
4793 MatchInfo = [=](MachineIRBuilder &B) {
4794 auto WidthCst = B.buildConstant(Res: ExtractTy, Val: Width);
4795 auto PosCst = B.buildConstant(Res: ExtractTy, Val: Pos);
4796 B.buildInstr(Opc: ExtrOpcode, DstOps: {Dst}, SrcOps: {ShlSrc, PosCst, WidthCst});
4797 };
4798 return true;
4799}
4800
4801bool CombinerHelper::matchBitfieldExtractFromShrAnd(
4802 MachineInstr &MI,
4803 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4804 const unsigned Opcode = MI.getOpcode();
4805 assert(Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_ASHR);
4806
4807 const Register Dst = MI.getOperand(i: 0).getReg();
4808 LLT Ty = MRI.getType(Reg: Dst);
4809 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
4810 if (LI && !LI->isLegalOrCustom(Query: {TargetOpcode::G_UBFX, {Ty, ExtractTy}}))
4811 return false;
4812
4813 // Try to match shr (and x, c1), c2
4814 Register AndSrc;
4815 int64_t ShrAmt;
4816 int64_t SMask;
4817 if (!mi_match(R: Dst, MRI,
4818 P: m_BinOp(Opcode,
4819 L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: AndSrc), R: m_ICst(Cst&: SMask))),
4820 R: m_ICst(Cst&: ShrAmt))))
4821 return false;
4822
4823 const unsigned Size = Ty.getScalarSizeInBits();
4824 if (ShrAmt < 0 || ShrAmt >= Size)
4825 return false;
4826
4827 // If the shift subsumes the mask, emit the 0 directly.
4828 if (0 == (SMask >> ShrAmt)) {
4829 MatchInfo = [=](MachineIRBuilder &B) {
4830 B.buildConstant(Res: Dst, Val: 0);
4831 };
4832 return true;
4833 }
4834
4835 // Check that ubfx can do the extraction, with no holes in the mask.
4836 uint64_t UMask = SMask;
4837 UMask |= maskTrailingOnes<uint64_t>(N: ShrAmt);
4838 UMask &= maskTrailingOnes<uint64_t>(N: Size);
4839 if (!isMask_64(Value: UMask))
4840 return false;
4841
4842 // Calculate start position and width of the extract.
4843 const int64_t Pos = ShrAmt;
4844 const int64_t Width = llvm::countr_one(Value: UMask) - ShrAmt;
4845
4846 // It's preferable to keep the shift, rather than form G_SBFX.
4847 // TODO: remove the G_AND via demanded bits analysis.
4848 if (Opcode == TargetOpcode::G_ASHR && Width + ShrAmt == Size)
4849 return false;
4850
4851 MatchInfo = [=](MachineIRBuilder &B) {
4852 auto WidthCst = B.buildConstant(Res: ExtractTy, Val: Width);
4853 auto PosCst = B.buildConstant(Res: ExtractTy, Val: Pos);
4854 B.buildInstr(Opc: TargetOpcode::G_UBFX, DstOps: {Dst}, SrcOps: {AndSrc, PosCst, WidthCst});
4855 };
4856 return true;
4857}
4858
4859bool CombinerHelper::reassociationCanBreakAddressingModePattern(
4860 MachineInstr &MI) const {
4861 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
4862
4863 Register Src1Reg = PtrAdd.getBaseReg();
4864 auto *Src1Def = getOpcodeDef<GPtrAdd>(Reg: Src1Reg, MRI);
4865 if (!Src1Def)
4866 return false;
4867
4868 Register Src2Reg = PtrAdd.getOffsetReg();
4869
4870 if (MRI.hasOneNonDBGUse(RegNo: Src1Reg))
4871 return false;
4872
4873 auto C1 = getIConstantVRegVal(VReg: Src1Def->getOffsetReg(), MRI);
4874 if (!C1)
4875 return false;
4876 auto C2 = getIConstantVRegVal(VReg: Src2Reg, MRI);
4877 if (!C2)
4878 return false;
4879
4880 const APInt &C1APIntVal = *C1;
4881 const APInt &C2APIntVal = *C2;
4882 const int64_t CombinedValue = (C1APIntVal + C2APIntVal).getSExtValue();
4883
4884 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: PtrAdd.getReg(Idx: 0))) {
4885 // This combine may end up running before ptrtoint/inttoptr combines
4886 // manage to eliminate redundant conversions, so try to look through them.
4887 MachineInstr *ConvUseMI = &UseMI;
4888 unsigned ConvUseOpc = ConvUseMI->getOpcode();
4889 while (ConvUseOpc == TargetOpcode::G_INTTOPTR ||
4890 ConvUseOpc == TargetOpcode::G_PTRTOINT) {
4891 Register DefReg = ConvUseMI->getOperand(i: 0).getReg();
4892 if (!MRI.hasOneNonDBGUse(RegNo: DefReg))
4893 break;
4894 ConvUseMI = &*MRI.use_instr_nodbg_begin(RegNo: DefReg);
4895 ConvUseOpc = ConvUseMI->getOpcode();
4896 }
4897 auto *LdStMI = dyn_cast<GLoadStore>(Val: ConvUseMI);
4898 if (!LdStMI)
4899 continue;
4900 // Is x[offset2] already not a legal addressing mode? If so then
4901 // reassociating the constants breaks nothing (we test offset2 because
4902 // that's the one we hope to fold into the load or store).
4903 TargetLoweringBase::AddrMode AM;
4904 AM.HasBaseReg = true;
4905 AM.BaseOffs = C2APIntVal.getSExtValue();
4906 unsigned AS = MRI.getType(Reg: LdStMI->getPointerReg()).getAddressSpace();
4907 Type *AccessTy = getTypeForLLT(Ty: LdStMI->getMMO().getMemoryType(),
4908 C&: PtrAdd.getMF()->getFunction().getContext());
4909 const auto &TLI = *PtrAdd.getMF()->getSubtarget().getTargetLowering();
4910 if (!TLI.isLegalAddressingMode(DL: PtrAdd.getMF()->getDataLayout(), AM,
4911 Ty: AccessTy, AddrSpace: AS))
4912 continue;
4913
4914 // Would x[offset1+offset2] still be a legal addressing mode?
4915 AM.BaseOffs = CombinedValue;
4916 if (!TLI.isLegalAddressingMode(DL: PtrAdd.getMF()->getDataLayout(), AM,
4917 Ty: AccessTy, AddrSpace: AS))
4918 return true;
4919 }
4920
4921 return false;
4922}
4923
4924bool CombinerHelper::matchReassocConstantInnerRHS(GPtrAdd &MI,
4925 MachineInstr *RHS,
4926 BuildFnTy &MatchInfo) const {
4927 // G_PTR_ADD(BASE, G_ADD(X, C)) -> G_PTR_ADD(G_PTR_ADD(BASE, X), C)
4928 Register Src1Reg = MI.getOperand(i: 1).getReg();
4929 if (RHS->getOpcode() != TargetOpcode::G_ADD)
4930 return false;
4931 auto C2 = getIConstantVRegVal(VReg: RHS->getOperand(i: 2).getReg(), MRI);
4932 if (!C2)
4933 return false;
4934
4935 // If both additions are nuw, the reassociated additions are also nuw.
4936 // If the original G_PTR_ADD is additionally nusw, X and C are both not
4937 // negative, so BASE+X is between BASE and BASE+(X+C). The new G_PTR_ADDs are
4938 // therefore also nusw.
4939 // If the original G_PTR_ADD is additionally inbounds (which implies nusw),
4940 // the new G_PTR_ADDs are then also inbounds.
4941 unsigned PtrAddFlags = MI.getFlags();
4942 unsigned AddFlags = RHS->getFlags();
4943 bool IsNoUWrap = PtrAddFlags & AddFlags & MachineInstr::MIFlag::NoUWrap;
4944 bool IsNoUSWrap = IsNoUWrap && (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap);
4945 bool IsInBounds = IsNoUWrap && (PtrAddFlags & MachineInstr::MIFlag::InBounds);
4946 unsigned Flags = 0;
4947 if (IsNoUWrap)
4948 Flags |= MachineInstr::MIFlag::NoUWrap;
4949 if (IsNoUSWrap)
4950 Flags |= MachineInstr::MIFlag::NoUSWrap;
4951 if (IsInBounds)
4952 Flags |= MachineInstr::MIFlag::InBounds;
4953
4954 MatchInfo = [=, &MI](MachineIRBuilder &B) {
4955 LLT PtrTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4956
4957 auto NewBase =
4958 Builder.buildPtrAdd(Res: PtrTy, Op0: Src1Reg, Op1: RHS->getOperand(i: 1).getReg(), Flags);
4959 Observer.changingInstr(MI);
4960 MI.getOperand(i: 1).setReg(NewBase.getReg(Idx: 0));
4961 MI.getOperand(i: 2).setReg(RHS->getOperand(i: 2).getReg());
4962 MI.setFlags(Flags);
4963 Observer.changedInstr(MI);
4964 };
4965 return !reassociationCanBreakAddressingModePattern(MI);
4966}
4967
4968bool CombinerHelper::matchReassocConstantInnerLHS(GPtrAdd &MI,
4969 MachineInstr *LHS,
4970 MachineInstr *RHS,
4971 BuildFnTy &MatchInfo) const {
4972 // G_PTR_ADD (G_PTR_ADD X, C), Y) -> (G_PTR_ADD (G_PTR_ADD(X, Y), C)
4973 // if and only if (G_PTR_ADD X, C) has one use.
4974 Register LHSBase;
4975 std::optional<ValueAndVReg> LHSCstOff;
4976 if (!mi_match(R: MI.getBaseReg(), MRI,
4977 P: m_OneNonDBGUse(SP: m_GPtrAdd(L: m_Reg(R&: LHSBase), R: m_GCst(ValReg&: LHSCstOff)))))
4978 return false;
4979
4980 auto *LHSPtrAdd = cast<GPtrAdd>(Val: LHS);
4981
4982 // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
4983 // nuw and inbounds (which implies nusw), the offsets are both non-negative,
4984 // so the new G_PTR_ADDs are also inbounds.
4985 unsigned PtrAddFlags = MI.getFlags();
4986 unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();
4987 bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
4988 bool IsNoUSWrap = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &
4989 MachineInstr::MIFlag::NoUSWrap);
4990 bool IsInBounds = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &
4991 MachineInstr::MIFlag::InBounds);
4992 unsigned Flags = 0;
4993 if (IsNoUWrap)
4994 Flags |= MachineInstr::MIFlag::NoUWrap;
4995 if (IsNoUSWrap)
4996 Flags |= MachineInstr::MIFlag::NoUSWrap;
4997 if (IsInBounds)
4998 Flags |= MachineInstr::MIFlag::InBounds;
4999
5000 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5001 // When we change LHSPtrAdd's offset register we might cause it to use a reg
5002 // before its def. Sink the instruction so the outer PTR_ADD to ensure this
5003 // doesn't happen.
5004 LHSPtrAdd->moveBefore(MovePos: &MI);
5005 Register RHSReg = MI.getOffsetReg();
5006 // set VReg will cause type mismatch if it comes from extend/trunc
5007 auto NewCst = B.buildConstant(Res: MRI.getType(Reg: RHSReg), Val: LHSCstOff->Value);
5008 Observer.changingInstr(MI);
5009 MI.getOperand(i: 2).setReg(NewCst.getReg(Idx: 0));
5010 MI.setFlags(Flags);
5011 Observer.changedInstr(MI);
5012 Observer.changingInstr(MI&: *LHSPtrAdd);
5013 LHSPtrAdd->getOperand(i: 2).setReg(RHSReg);
5014 LHSPtrAdd->setFlags(Flags);
5015 Observer.changedInstr(MI&: *LHSPtrAdd);
5016 };
5017 return !reassociationCanBreakAddressingModePattern(MI);
5018}
5019
5020bool CombinerHelper::matchReassocFoldConstantsInSubTree(
5021 GPtrAdd &MI, MachineInstr *LHS, MachineInstr *RHS,
5022 BuildFnTy &MatchInfo) const {
5023 // G_PTR_ADD(G_PTR_ADD(BASE, C1), C2) -> G_PTR_ADD(BASE, C1+C2)
5024 auto *LHSPtrAdd = dyn_cast<GPtrAdd>(Val: LHS);
5025 if (!LHSPtrAdd)
5026 return false;
5027
5028 Register Src2Reg = MI.getOperand(i: 2).getReg();
5029 Register LHSSrc1 = LHSPtrAdd->getBaseReg();
5030 Register LHSSrc2 = LHSPtrAdd->getOffsetReg();
5031 auto C1 = getIConstantVRegVal(VReg: LHSSrc2, MRI);
5032 if (!C1)
5033 return false;
5034 auto C2 = getIConstantVRegVal(VReg: Src2Reg, MRI);
5035 if (!C2)
5036 return false;
5037
5038 // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
5039 // inbounds, reaching the same result in one G_PTR_ADD is also inbounds.
5040 // The nusw constraints are satisfied because imm1+imm2 cannot exceed the
5041 // largest signed integer that fits into the index type, which is the maximum
5042 // size of allocated objects according to the IR Language Reference.
5043 unsigned PtrAddFlags = MI.getFlags();
5044 unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();
5045 bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
5046 bool IsInBounds =
5047 PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::InBounds;
5048 unsigned Flags = 0;
5049 if (IsNoUWrap)
5050 Flags |= MachineInstr::MIFlag::NoUWrap;
5051 if (IsInBounds) {
5052 Flags |= MachineInstr::MIFlag::InBounds;
5053 Flags |= MachineInstr::MIFlag::NoUSWrap;
5054 }
5055
5056 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5057 auto NewCst = B.buildConstant(Res: MRI.getType(Reg: Src2Reg), Val: *C1 + *C2);
5058 Observer.changingInstr(MI);
5059 MI.getOperand(i: 1).setReg(LHSSrc1);
5060 MI.getOperand(i: 2).setReg(NewCst.getReg(Idx: 0));
5061 MI.setFlags(Flags);
5062 Observer.changedInstr(MI);
5063 };
5064 return !reassociationCanBreakAddressingModePattern(MI);
5065}
5066
5067bool CombinerHelper::matchReassocPtrAdd(MachineInstr &MI,
5068 BuildFnTy &MatchInfo) const {
5069 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
5070 // We're trying to match a few pointer computation patterns here for
5071 // re-association opportunities.
5072 // 1) Isolating a constant operand to be on the RHS, e.g.:
5073 // G_PTR_ADD(BASE, G_ADD(X, C)) -> G_PTR_ADD(G_PTR_ADD(BASE, X), C)
5074 //
5075 // 2) Folding two constants in each sub-tree as long as such folding
5076 // doesn't break a legal addressing mode.
5077 // G_PTR_ADD(G_PTR_ADD(BASE, C1), C2) -> G_PTR_ADD(BASE, C1+C2)
5078 //
5079 // 3) Move a constant from the LHS of an inner op to the RHS of the outer.
5080 // G_PTR_ADD (G_PTR_ADD X, C), Y) -> G_PTR_ADD (G_PTR_ADD(X, Y), C)
5081 // iif (G_PTR_ADD X, C) has one use.
5082 MachineInstr *LHS, *RHS;
5083 if (!mi_match(R: PtrAdd.getBaseReg(), MRI, P: m_MInstr(MI&: LHS)) ||
5084 !mi_match(R: PtrAdd.getOffsetReg(), MRI, P: m_MInstr(MI&: RHS)))
5085 return false;
5086
5087 // Try to match example 2.
5088 if (matchReassocFoldConstantsInSubTree(MI&: PtrAdd, LHS, RHS, MatchInfo))
5089 return true;
5090
5091 // Try to match example 3.
5092 if (matchReassocConstantInnerLHS(MI&: PtrAdd, LHS, RHS, MatchInfo))
5093 return true;
5094
5095 // Try to match example 1.
5096 if (matchReassocConstantInnerRHS(MI&: PtrAdd, RHS, MatchInfo))
5097 return true;
5098
5099 return false;
5100}
5101bool CombinerHelper::tryReassocBinOp(unsigned Opc, Register DstReg,
5102 Register OpLHS, Register OpRHS,
5103 BuildFnTy &MatchInfo) const {
5104 LLT OpRHSTy = MRI.getType(Reg: OpRHS);
5105 MachineInstr *OpLHSDef;
5106 if (!mi_match(R: OpLHS, MRI, P: m_MInstr(MI&: OpLHSDef)) || OpLHSDef->getOpcode() != Opc)
5107 return false;
5108
5109 Register OpLHSLHS = OpLHSDef->getOperand(i: 1).getReg();
5110 Register OpLHSRHS = OpLHSDef->getOperand(i: 2).getReg();
5111
5112 // If the inner op is (X op C), pull the constant out so it can be folded with
5113 // other constants in the expression tree. Folding is not guaranteed so we
5114 // might have (C1 op C2). In that case do not pull a constant out because it
5115 // won't help and can lead to infinite loops.
5116 if (isConstantOrConstantSplatVector(Def: OpLHSRHS, MRI) &&
5117 !isConstantOrConstantSplatVector(Def: OpLHSLHS, MRI)) {
5118 if (isConstantOrConstantSplatVector(Def: OpRHS, MRI)) {
5119 // (Opc (Opc X, C1), C2) -> (Opc X, (Opc C1, C2))
5120 MatchInfo = [=](MachineIRBuilder &B) {
5121 auto NewCst = B.buildInstr(Opc, DstOps: {OpRHSTy}, SrcOps: {OpLHSRHS, OpRHS});
5122 B.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {OpLHSLHS, NewCst});
5123 };
5124 return true;
5125 }
5126 if (getTargetLowering().isReassocProfitable(MRI, N0: OpLHS, N1: OpRHS)) {
5127 // Reassociate: (op (op x, c1), y) -> (op (op x, y), c1)
5128 // iff (op x, c1) has one use
5129 MatchInfo = [=](MachineIRBuilder &B) {
5130 auto NewLHSLHS = B.buildInstr(Opc, DstOps: {OpRHSTy}, SrcOps: {OpLHSLHS, OpRHS});
5131 B.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {NewLHSLHS, OpLHSRHS});
5132 };
5133 return true;
5134 }
5135 }
5136
5137 return false;
5138}
5139
5140bool CombinerHelper::matchReassocCommBinOp(MachineInstr &MI,
5141 BuildFnTy &MatchInfo) const {
5142 // We don't check if the reassociation will break a legal addressing mode
5143 // here since pointer arithmetic is handled by G_PTR_ADD.
5144 unsigned Opc = MI.getOpcode();
5145 Register DstReg = MI.getOperand(i: 0).getReg();
5146 Register LHSReg = MI.getOperand(i: 1).getReg();
5147 Register RHSReg = MI.getOperand(i: 2).getReg();
5148
5149 if (tryReassocBinOp(Opc, DstReg, OpLHS: LHSReg, OpRHS: RHSReg, MatchInfo))
5150 return true;
5151 if (tryReassocBinOp(Opc, DstReg, OpLHS: RHSReg, OpRHS: LHSReg, MatchInfo))
5152 return true;
5153 return false;
5154}
5155
5156bool CombinerHelper::matchConstantFoldCastOp(MachineInstr &MI,
5157 APInt &MatchInfo) const {
5158 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5159 Register SrcOp = MI.getOperand(i: 1).getReg();
5160
5161 if (auto MaybeCst = ConstantFoldCastOp(Opcode: MI.getOpcode(), DstTy, Op0: SrcOp, MRI)) {
5162 MatchInfo = *MaybeCst;
5163 return true;
5164 }
5165
5166 return false;
5167}
5168
5169bool CombinerHelper::matchConstantFoldUnaryIntOp(MachineInstr &MI,
5170 BuildFnTy &MatchInfo) const {
5171 Register Dst = MI.getOperand(i: 0).getReg();
5172 auto Csts = ConstantFoldUnaryIntOp(Opcode: MI.getOpcode(), DstTy: MRI.getType(Reg: Dst),
5173 Src: MI.getOperand(i: 1).getReg(), MRI);
5174 if (Csts.empty())
5175 return false;
5176
5177 MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
5178 if (Csts.size() == 1)
5179 B.buildConstant(Res: Dst, Val: Csts[0]);
5180 else
5181 B.buildBuildVectorConstant(Res: Dst, Ops: Csts);
5182 };
5183 return true;
5184}
5185
5186bool CombinerHelper::matchConstantFoldBinOp(MachineInstr &MI,
5187 APInt &MatchInfo) const {
5188 Register Op1 = MI.getOperand(i: 1).getReg();
5189 Register Op2 = MI.getOperand(i: 2).getReg();
5190 auto MaybeCst = ConstantFoldBinOp(Opcode: MI.getOpcode(), Op1, Op2, MRI);
5191 if (!MaybeCst)
5192 return false;
5193 MatchInfo = *MaybeCst;
5194 return true;
5195}
5196
5197bool CombinerHelper::matchConstantFoldFPBinOp(MachineInstr &MI,
5198 ConstantFP *&MatchInfo) const {
5199 Register Op1 = MI.getOperand(i: 1).getReg();
5200 Register Op2 = MI.getOperand(i: 2).getReg();
5201 auto MaybeCst = ConstantFoldFPBinOp(Opcode: MI.getOpcode(), Op1, Op2, MRI);
5202 if (!MaybeCst)
5203 return false;
5204 MatchInfo =
5205 ConstantFP::get(Context&: MI.getMF()->getFunction().getContext(), V: *MaybeCst);
5206 return true;
5207}
5208
5209bool CombinerHelper::matchConstantFoldFMA(MachineInstr &MI,
5210 ConstantFP *&MatchInfo) const {
5211 assert(MI.getOpcode() == TargetOpcode::G_FMA ||
5212 MI.getOpcode() == TargetOpcode::G_FMAD);
5213 auto [_, Op1, Op2, Op3] = MI.getFirst4Regs();
5214
5215 const ConstantFP *Op3Cst = getConstantFPVRegVal(VReg: Op3, MRI);
5216 if (!Op3Cst)
5217 return false;
5218
5219 const ConstantFP *Op2Cst = getConstantFPVRegVal(VReg: Op2, MRI);
5220 if (!Op2Cst)
5221 return false;
5222
5223 const ConstantFP *Op1Cst = getConstantFPVRegVal(VReg: Op1, MRI);
5224 if (!Op1Cst)
5225 return false;
5226
5227 APFloat Op1F = Op1Cst->getValueAPF();
5228 Op1F.fusedMultiplyAdd(Multiplicand: Op2Cst->getValueAPF(), Addend: Op3Cst->getValueAPF(),
5229 RM: APFloat::rmNearestTiesToEven);
5230 MatchInfo = ConstantFP::get(Context&: MI.getMF()->getFunction().getContext(), V: Op1F);
5231 return true;
5232}
5233
5234bool CombinerHelper::matchNarrowBinopFeedingAnd(
5235 MachineInstr &MI,
5236 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
5237 // Look for a binop feeding into an AND with a mask:
5238 //
5239 // %add = G_ADD %lhs, %rhs
5240 // %and = G_AND %add, 000...11111111
5241 //
5242 // Check if it's possible to perform the binop at a narrower width and zext
5243 // back to the original width like so:
5244 //
5245 // %narrow_lhs = G_TRUNC %lhs
5246 // %narrow_rhs = G_TRUNC %rhs
5247 // %narrow_add = G_ADD %narrow_lhs, %narrow_rhs
5248 // %new_add = G_ZEXT %narrow_add
5249 // %and = G_AND %new_add, 000...11111111
5250 //
5251 // This can allow later combines to eliminate the G_AND if it turns out
5252 // that the mask is irrelevant.
5253 assert(MI.getOpcode() == TargetOpcode::G_AND);
5254 Register Dst = MI.getOperand(i: 0).getReg();
5255 Register AndLHS = MI.getOperand(i: 1).getReg();
5256 Register AndRHS = MI.getOperand(i: 2).getReg();
5257 LLT WideTy = MRI.getType(Reg: Dst);
5258
5259 // If the potential binop has more than one use, then it's possible that one
5260 // of those uses will need its full width.
5261 if (!WideTy.isScalar() || !MRI.hasOneNonDBGUse(RegNo: AndLHS))
5262 return false;
5263
5264 // Check if the LHS feeding the AND is impacted by the high bits that we're
5265 // masking out.
5266 //
5267 // e.g. for 64-bit x, y:
5268 //
5269 // add_64(x, y) & 65535 == zext(add_16(trunc(x), trunc(y))) & 65535
5270 MachineInstr *LHSInst = getDefIgnoringCopies(Reg: AndLHS, MRI);
5271 if (!LHSInst)
5272 return false;
5273 unsigned LHSOpc = LHSInst->getOpcode();
5274 switch (LHSOpc) {
5275 default:
5276 return false;
5277 case TargetOpcode::G_ADD:
5278 case TargetOpcode::G_SUB:
5279 case TargetOpcode::G_MUL:
5280 case TargetOpcode::G_AND:
5281 case TargetOpcode::G_OR:
5282 case TargetOpcode::G_XOR:
5283 break;
5284 }
5285
5286 // Find the mask on the RHS.
5287 auto Cst = getIConstantVRegValWithLookThrough(VReg: AndRHS, MRI);
5288 if (!Cst)
5289 return false;
5290 auto Mask = Cst->Value;
5291 if (!Mask.isMask())
5292 return false;
5293
5294 // No point in combining if there's nothing to truncate.
5295 unsigned NarrowWidth = Mask.countr_one();
5296 if (NarrowWidth == WideTy.getSizeInBits())
5297 return false;
5298 LLT NarrowTy = LLT::integer(SizeInBits: NarrowWidth);
5299
5300 // Check if adding the zext + truncates could be harmful.
5301 auto &MF = *MI.getMF();
5302 const auto &TLI = getTargetLowering();
5303 LLVMContext &Ctx = MF.getFunction().getContext();
5304 if (!TLI.isTruncateFree(FromTy: WideTy, ToTy: NarrowTy, Ctx) ||
5305 !TLI.isZExtFree(FromTy: NarrowTy, ToTy: WideTy, Ctx))
5306 return false;
5307 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_TRUNC, {NarrowTy, WideTy}}) ||
5308 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ZEXT, {WideTy, NarrowTy}}))
5309 return false;
5310 Register BinOpLHS = LHSInst->getOperand(i: 1).getReg();
5311 Register BinOpRHS = LHSInst->getOperand(i: 2).getReg();
5312 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5313 auto NarrowLHS = Builder.buildTrunc(Res: NarrowTy, Op: BinOpLHS);
5314 auto NarrowRHS = Builder.buildTrunc(Res: NarrowTy, Op: BinOpRHS);
5315 auto NarrowBinOp =
5316 Builder.buildInstr(Opc: LHSOpc, DstOps: {NarrowTy}, SrcOps: {NarrowLHS, NarrowRHS});
5317 auto Ext = Builder.buildZExt(Res: WideTy, Op: NarrowBinOp);
5318 Observer.changingInstr(MI);
5319 MI.getOperand(i: 1).setReg(Ext.getReg(Idx: 0));
5320 Observer.changedInstr(MI);
5321 };
5322 return true;
5323}
5324
5325bool CombinerHelper::matchMulOBy0(MachineInstr &MI,
5326 BuildFnTy &MatchInfo) const {
5327 // (G_*MULO x, 0) -> 0 + no carry out
5328 assert(MI.getOpcode() == TargetOpcode::G_UMULO ||
5329 MI.getOpcode() == TargetOpcode::G_SMULO);
5330 if (!mi_match(R: MI.getOperand(i: 3).getReg(), MRI, P: m_SpecificICstOrSplat(RequestedValue: 0)))
5331 return false;
5332 Register Dst = MI.getOperand(i: 0).getReg();
5333 Register Carry = MI.getOperand(i: 1).getReg();
5334 if (!isConstantLegalOrBeforeLegalizer(Ty: MRI.getType(Reg: Dst)) ||
5335 !isConstantLegalOrBeforeLegalizer(Ty: MRI.getType(Reg: Carry)))
5336 return false;
5337 MatchInfo = [=](MachineIRBuilder &B) {
5338 B.buildConstant(Res: Dst, Val: 0);
5339 B.buildConstant(Res: Carry, Val: 0);
5340 };
5341 return true;
5342}
5343
5344bool CombinerHelper::matchSameRegOrICstOrSplat(Register A, Register B) const {
5345 Register SrcA = getSrcRegIgnoringCopies(Reg: A, MRI);
5346 Register SrcB = getSrcRegIgnoringCopies(Reg: B, MRI);
5347 if (SrcA.isValid() && SrcA == SrcB)
5348 return true;
5349 int64_t Cst;
5350 return mi_match(R: A, MRI, P: m_ICstOrSplat(Cst)) &&
5351 mi_match(R: B, MRI, P: m_SpecificICstOrSplat(RequestedValue: Cst));
5352}
5353
5354MachineInstr *CombinerHelper::buildUDivOrURemUsingMul(MachineInstr &MI) const {
5355 unsigned Opcode = MI.getOpcode();
5356 assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
5357 auto &UDivorRem = cast<GenericMachineInstr>(Val&: MI);
5358 Register Dst = UDivorRem.getReg(Idx: 0);
5359 Register LHS = UDivorRem.getReg(Idx: 1);
5360 Register RHS = UDivorRem.getReg(Idx: 2);
5361 LLT Ty = MRI.getType(Reg: Dst);
5362 LLT ScalarTy = Ty.getScalarType();
5363 const unsigned EltBits = ScalarTy.getScalarSizeInBits();
5364 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5365 LLT ScalarShiftAmtTy = ShiftAmtTy.getScalarType();
5366
5367 auto &MIB = Builder;
5368
5369 bool UseSRL = false;
5370 SmallVector<Register, 16> Shifts, Factors;
5371 auto *RHSDefInstr = cast<GenericMachineInstr>(Val: getDefIgnoringCopies(Reg: RHS, MRI));
5372 bool IsSplat = getIConstantSplatVal(MI: *RHSDefInstr, MRI).has_value();
5373
5374 auto BuildExactUDIVPattern = [&](const Constant *C) {
5375 // Don't recompute inverses for each splat element.
5376 if (IsSplat && !Factors.empty()) {
5377 Shifts.push_back(Elt: Shifts[0]);
5378 Factors.push_back(Elt: Factors[0]);
5379 return true;
5380 }
5381
5382 auto *CI = cast<ConstantInt>(Val: C);
5383 APInt Divisor = CI->getValue();
5384 unsigned Shift = Divisor.countr_zero();
5385 if (Shift) {
5386 Divisor.lshrInPlace(ShiftAmt: Shift);
5387 UseSRL = true;
5388 }
5389
5390 // Calculate the multiplicative inverse modulo BW.
5391 APInt Factor = Divisor.multiplicativeInverse();
5392 Shifts.push_back(Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: Shift).getReg(Idx: 0));
5393 Factors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Factor).getReg(Idx: 0));
5394 return true;
5395 };
5396
5397 if (MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
5398 // Collect all magic values from the build vector.
5399 if (!matchUnaryPredicate(MRI, Reg: RHS, Match: BuildExactUDIVPattern))
5400 llvm_unreachable("Expected unary predicate match to succeed");
5401
5402 Register Shift, Factor;
5403 if (Ty.isVector()) {
5404 Shift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: Shifts).getReg(Idx: 0);
5405 Factor = MIB.buildBuildVector(Res: Ty, Ops: Factors).getReg(Idx: 0);
5406 } else {
5407 Shift = Shifts[0];
5408 Factor = Factors[0];
5409 }
5410
5411 Register Res = LHS;
5412
5413 if (UseSRL)
5414 Res = MIB.buildLShr(Dst: Ty, Src0: Res, Src1: Shift, Flags: MachineInstr::IsExact).getReg(Idx: 0);
5415
5416 return MIB.buildMul(Dst: Ty, Src0: Res, Src1: Factor);
5417 }
5418
5419 unsigned KnownLeadingZeros =
5420 VT ? VT->getKnownBits(R: LHS).countMinLeadingZeros() : 0;
5421
5422 bool UseNPQ = false;
5423 SmallVector<Register, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
5424 auto BuildUDIVPattern = [&](const Constant *C) {
5425 auto *CI = cast<ConstantInt>(Val: C);
5426 const APInt &Divisor = CI->getValue();
5427
5428 bool SelNPQ = false;
5429 APInt Magic(Divisor.getBitWidth(), 0);
5430 unsigned PreShift = 0, PostShift = 0;
5431
5432 // Magic algorithm doesn't work for division by 1. We need to emit a select
5433 // at the end.
5434 // TODO: Use undef values for divisor of 1.
5435 if (!Divisor.isOne()) {
5436
5437 // UnsignedDivisionByConstantInfo doesn't work correctly if leading zeros
5438 // in the dividend exceeds the leading zeros for the divisor.
5439 UnsignedDivisionByConstantInfo magics =
5440 UnsignedDivisionByConstantInfo::get(
5441 D: Divisor, LeadingZeros: std::min(a: KnownLeadingZeros, b: Divisor.countl_zero()));
5442
5443 Magic = std::move(magics.Magic);
5444
5445 assert(magics.PreShift < Divisor.getBitWidth() &&
5446 "We shouldn't generate an undefined shift!");
5447 assert(magics.PostShift < Divisor.getBitWidth() &&
5448 "We shouldn't generate an undefined shift!");
5449 assert((!magics.IsAdd || magics.PreShift == 0) && "Unexpected pre-shift");
5450 PreShift = magics.PreShift;
5451 PostShift = magics.PostShift;
5452 SelNPQ = magics.IsAdd;
5453 }
5454
5455 PreShifts.push_back(
5456 Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: PreShift).getReg(Idx: 0));
5457 MagicFactors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Magic).getReg(Idx: 0));
5458 NPQFactors.push_back(
5459 Elt: MIB.buildConstant(Res: ScalarTy,
5460 Val: SelNPQ ? APInt::getOneBitSet(numBits: EltBits, BitNo: EltBits - 1)
5461 : APInt::getZero(numBits: EltBits))
5462 .getReg(Idx: 0));
5463 PostShifts.push_back(
5464 Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: PostShift).getReg(Idx: 0));
5465 UseNPQ |= SelNPQ;
5466 return true;
5467 };
5468
5469 // Collect the shifts/magic values from each element.
5470 bool Matched = matchUnaryPredicate(MRI, Reg: RHS, Match: BuildUDIVPattern);
5471 (void)Matched;
5472 assert(Matched && "Expected unary predicate match to succeed");
5473
5474 Register PreShift, PostShift, MagicFactor, NPQFactor;
5475 auto *RHSDef = getOpcodeDef<GBuildVector>(Reg: RHS, MRI);
5476 if (RHSDef) {
5477 PreShift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: PreShifts).getReg(Idx: 0);
5478 MagicFactor = MIB.buildBuildVector(Res: Ty, Ops: MagicFactors).getReg(Idx: 0);
5479 NPQFactor = MIB.buildBuildVector(Res: Ty, Ops: NPQFactors).getReg(Idx: 0);
5480 PostShift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: PostShifts).getReg(Idx: 0);
5481 } else {
5482 assert(MRI.getType(RHS).isScalar() &&
5483 "Non-build_vector operation should have been a scalar");
5484 PreShift = PreShifts[0];
5485 MagicFactor = MagicFactors[0];
5486 PostShift = PostShifts[0];
5487 }
5488
5489 Register Q = LHS;
5490 Q = MIB.buildLShr(Dst: Ty, Src0: Q, Src1: PreShift).getReg(Idx: 0);
5491
5492 // Multiply the numerator (operand 0) by the magic value.
5493 Q = MIB.buildUMulH(Dst: Ty, Src0: Q, Src1: MagicFactor).getReg(Idx: 0);
5494
5495 if (UseNPQ) {
5496 Register NPQ = MIB.buildSub(Dst: Ty, Src0: LHS, Src1: Q).getReg(Idx: 0);
5497
5498 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
5499 // G_UMULH to act as a SRL-by-1 for NPQ, else multiply by zero.
5500 if (Ty.isVector())
5501 NPQ = MIB.buildUMulH(Dst: Ty, Src0: NPQ, Src1: NPQFactor).getReg(Idx: 0);
5502 else
5503 NPQ = MIB.buildLShr(Dst: Ty, Src0: NPQ, Src1: MIB.buildConstant(Res: ShiftAmtTy, Val: 1)).getReg(Idx: 0);
5504
5505 Q = MIB.buildAdd(Dst: Ty, Src0: NPQ, Src1: Q).getReg(Idx: 0);
5506 }
5507
5508 Q = MIB.buildLShr(Dst: Ty, Src0: Q, Src1: PostShift).getReg(Idx: 0);
5509 auto One = MIB.buildConstant(Res: Ty, Val: 1);
5510 auto IsOne = MIB.buildICmp(
5511 Pred: CmpInst::Predicate::ICMP_EQ,
5512 Res: Ty.isScalar() ? LLT::integer(SizeInBits: 1) : Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1)),
5513 Op0: RHS, Op1: One);
5514 auto ret = MIB.buildSelect(Res: Ty, Tst: IsOne, Op0: LHS, Op1: Q);
5515
5516 if (Opcode == TargetOpcode::G_UREM) {
5517 auto Prod = MIB.buildMul(Dst: Ty, Src0: ret, Src1: RHS);
5518 return MIB.buildSub(Dst: Ty, Src0: LHS, Src1: Prod);
5519 }
5520 return ret;
5521}
5522
5523bool CombinerHelper::matchUDivOrURemByConst(MachineInstr &MI) const {
5524 unsigned Opcode = MI.getOpcode();
5525 assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
5526 Register Dst = MI.getOperand(i: 0).getReg();
5527 Register RHS = MI.getOperand(i: 2).getReg();
5528 LLT DstTy = MRI.getType(Reg: Dst);
5529
5530 auto &MF = *MI.getMF();
5531 AttributeList Attr = MF.getFunction().getAttributes();
5532 const auto &TLI = getTargetLowering();
5533 LLVMContext &Ctx = MF.getFunction().getContext();
5534 if (DstTy.getScalarSizeInBits() == 1 ||
5535 TLI.isIntDivCheap(VT: getApproximateEVTForLLT(Ty: DstTy, Ctx), Attr))
5536 return false;
5537
5538 // Don't do this for minsize because the instruction sequence is usually
5539 // larger.
5540 if (MF.getFunction().hasMinSize())
5541 return false;
5542
5543 if (Opcode == TargetOpcode::G_UDIV &&
5544 MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
5545 return matchUnaryPredicate(
5546 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5547 }
5548
5549 MachineInstr *RHSDef;
5550 if (!mi_match(R: RHS, MRI, P: m_MInstr(MI&: RHSDef)) ||
5551 !isConstantOrConstantVector(MI&: *RHSDef, MRI))
5552 return false;
5553
5554 // Don't do this if the types are not going to be legal.
5555 if (LI) {
5556 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_MUL, {DstTy, DstTy}}))
5557 return false;
5558 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_UMULH, {DstTy}}))
5559 return false;
5560 if (!isLegalOrBeforeLegalizer(
5561 Query: {TargetOpcode::G_ICMP,
5562 {DstTy.isVector() ? DstTy.changeElementSize(NewEltSize: 1) : LLT::scalar(SizeInBits: 1),
5563 DstTy}}))
5564 return false;
5565 if (Opcode == TargetOpcode::G_UREM &&
5566 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SUB, {DstTy, DstTy}}))
5567 return false;
5568 }
5569
5570 return matchUnaryPredicate(
5571 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5572}
5573
5574void CombinerHelper::applyUDivOrURemByConst(MachineInstr &MI) const {
5575 auto *NewMI = buildUDivOrURemUsingMul(MI);
5576 replaceSingleDefInstWithReg(MI, Replacement: NewMI->getOperand(i: 0).getReg());
5577}
5578
5579bool CombinerHelper::matchSDivOrSRemByConst(MachineInstr &MI) const {
5580 unsigned Opcode = MI.getOpcode();
5581 assert(Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM);
5582 Register Dst = MI.getOperand(i: 0).getReg();
5583 Register RHS = MI.getOperand(i: 2).getReg();
5584 LLT DstTy = MRI.getType(Reg: Dst);
5585 auto SizeInBits = DstTy.getScalarSizeInBits();
5586 LLT WideTy = DstTy.changeElementSize(NewEltSize: SizeInBits * 2);
5587
5588 auto &MF = *MI.getMF();
5589 AttributeList Attr = MF.getFunction().getAttributes();
5590 const auto &TLI = getTargetLowering();
5591 LLVMContext &Ctx = MF.getFunction().getContext();
5592 if (DstTy.getScalarSizeInBits() < 3 ||
5593 TLI.isIntDivCheap(VT: getApproximateEVTForLLT(Ty: DstTy, Ctx), Attr))
5594 return false;
5595
5596 // Don't do this for minsize because the instruction sequence is usually
5597 // larger.
5598 if (MF.getFunction().hasMinSize())
5599 return false;
5600
5601 // If the sdiv has an 'exact' flag we can use a simpler lowering.
5602 if (Opcode == TargetOpcode::G_SDIV &&
5603 MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
5604 return matchUnaryPredicate(
5605 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5606 }
5607
5608 MachineInstr *RHSDef;
5609 if (!mi_match(R: RHS, MRI, P: m_MInstr(MI&: RHSDef)) ||
5610 !isConstantOrConstantVector(MI&: *RHSDef, MRI))
5611 return false;
5612
5613 // Don't do this if the types are not going to be legal.
5614 if (LI) {
5615 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_MUL, {DstTy, DstTy}}))
5616 return false;
5617 if (!isLegal(Query: {TargetOpcode::G_SMULH, {DstTy}}) &&
5618 !isLegalOrHasWidenScalar(Query: {TargetOpcode::G_MUL, {WideTy, WideTy}}))
5619 return false;
5620 if (Opcode == TargetOpcode::G_SREM &&
5621 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SUB, {DstTy, DstTy}}))
5622 return false;
5623 }
5624
5625 return matchUnaryPredicate(
5626 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5627}
5628
5629void CombinerHelper::applySDivOrSRemByConst(MachineInstr &MI) const {
5630 auto *NewMI = buildSDivOrSRemUsingMul(MI);
5631 replaceSingleDefInstWithReg(MI, Replacement: NewMI->getOperand(i: 0).getReg());
5632}
5633
5634MachineInstr *CombinerHelper::buildSDivOrSRemUsingMul(MachineInstr &MI) const {
5635 unsigned Opcode = MI.getOpcode();
5636 assert(MI.getOpcode() == TargetOpcode::G_SDIV ||
5637 Opcode == TargetOpcode::G_SREM);
5638 auto &SDivorRem = cast<GenericMachineInstr>(Val&: MI);
5639 Register Dst = SDivorRem.getReg(Idx: 0);
5640 Register LHS = SDivorRem.getReg(Idx: 1);
5641 Register RHS = SDivorRem.getReg(Idx: 2);
5642 LLT Ty = MRI.getType(Reg: Dst);
5643 LLT ScalarTy = Ty.getScalarType();
5644 const unsigned EltBits = ScalarTy.getScalarSizeInBits();
5645 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5646 LLT ScalarShiftAmtTy = ShiftAmtTy.getScalarType();
5647 auto &MIB = Builder;
5648
5649 bool UseSRA = false;
5650 SmallVector<Register, 16> ExactShifts, ExactFactors;
5651
5652 auto *RHSDefInstr = cast<GenericMachineInstr>(Val: getDefIgnoringCopies(Reg: RHS, MRI));
5653 bool IsSplat = getIConstantSplatVal(MI: *RHSDefInstr, MRI).has_value();
5654
5655 auto BuildExactSDIVPattern = [&](const Constant *C) {
5656 // Don't recompute inverses for each splat element.
5657 if (IsSplat && !ExactFactors.empty()) {
5658 ExactShifts.push_back(Elt: ExactShifts[0]);
5659 ExactFactors.push_back(Elt: ExactFactors[0]);
5660 return true;
5661 }
5662
5663 auto *CI = cast<ConstantInt>(Val: C);
5664 APInt Divisor = CI->getValue();
5665 unsigned Shift = Divisor.countr_zero();
5666 if (Shift) {
5667 Divisor.ashrInPlace(ShiftAmt: Shift);
5668 UseSRA = true;
5669 }
5670
5671 // Calculate the multiplicative inverse modulo BW.
5672 // 2^W requires W + 1 bits, so we have to extend and then truncate.
5673 APInt Factor = Divisor.multiplicativeInverse();
5674 ExactShifts.push_back(Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: Shift).getReg(Idx: 0));
5675 ExactFactors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Factor).getReg(Idx: 0));
5676 return true;
5677 };
5678
5679 if (MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
5680 // Collect all magic values from the build vector.
5681 bool Matched = matchUnaryPredicate(MRI, Reg: RHS, Match: BuildExactSDIVPattern);
5682 (void)Matched;
5683 assert(Matched && "Expected unary predicate match to succeed");
5684
5685 Register Shift, Factor;
5686 if (Ty.isVector()) {
5687 Shift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: ExactShifts).getReg(Idx: 0);
5688 Factor = MIB.buildBuildVector(Res: Ty, Ops: ExactFactors).getReg(Idx: 0);
5689 } else {
5690 Shift = ExactShifts[0];
5691 Factor = ExactFactors[0];
5692 }
5693
5694 Register Res = LHS;
5695
5696 if (UseSRA)
5697 Res = MIB.buildAShr(Dst: Ty, Src0: Res, Src1: Shift, Flags: MachineInstr::IsExact).getReg(Idx: 0);
5698
5699 return MIB.buildMul(Dst: Ty, Src0: Res, Src1: Factor);
5700 }
5701
5702 SmallVector<Register, 16> MagicFactors, Factors, Shifts, ShiftMasks;
5703
5704 auto BuildSDIVPattern = [&](const Constant *C) {
5705 auto *CI = cast<ConstantInt>(Val: C);
5706 const APInt &Divisor = CI->getValue();
5707
5708 SignedDivisionByConstantInfo Magics =
5709 SignedDivisionByConstantInfo::get(D: Divisor);
5710 int NumeratorFactor = 0;
5711 int ShiftMask = -1;
5712
5713 if (Divisor.isOne() || Divisor.isAllOnes()) {
5714 // If d is +1/-1, we just multiply the numerator by +1/-1.
5715 NumeratorFactor = Divisor.getSExtValue();
5716 Magics.Magic = 0;
5717 Magics.ShiftAmount = 0;
5718 ShiftMask = 0;
5719 } else if (Divisor.isStrictlyPositive() && Magics.Magic.isNegative()) {
5720 // If d > 0 and m < 0, add the numerator.
5721 NumeratorFactor = 1;
5722 } else if (Divisor.isNegative() && Magics.Magic.isStrictlyPositive()) {
5723 // If d < 0 and m > 0, subtract the numerator.
5724 NumeratorFactor = -1;
5725 }
5726
5727 MagicFactors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Magics.Magic).getReg(Idx: 0));
5728 Factors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: NumeratorFactor).getReg(Idx: 0));
5729 Shifts.push_back(
5730 Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: Magics.ShiftAmount).getReg(Idx: 0));
5731 ShiftMasks.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: ShiftMask).getReg(Idx: 0));
5732
5733 return true;
5734 };
5735
5736 // Collect the shifts/magic values from each element.
5737 bool Matched = matchUnaryPredicate(MRI, Reg: RHS, Match: BuildSDIVPattern);
5738 (void)Matched;
5739 assert(Matched && "Expected unary predicate match to succeed");
5740
5741 Register MagicFactor, Factor, Shift, ShiftMask;
5742 auto *RHSDef = getOpcodeDef<GBuildVector>(Reg: RHS, MRI);
5743 if (RHSDef) {
5744 MagicFactor = MIB.buildBuildVector(Res: Ty, Ops: MagicFactors).getReg(Idx: 0);
5745 Factor = MIB.buildBuildVector(Res: Ty, Ops: Factors).getReg(Idx: 0);
5746 Shift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: Shifts).getReg(Idx: 0);
5747 ShiftMask = MIB.buildBuildVector(Res: Ty, Ops: ShiftMasks).getReg(Idx: 0);
5748 } else {
5749 assert(MRI.getType(RHS).isScalar() &&
5750 "Non-build_vector operation should have been a scalar");
5751 MagicFactor = MagicFactors[0];
5752 Factor = Factors[0];
5753 Shift = Shifts[0];
5754 ShiftMask = ShiftMasks[0];
5755 }
5756
5757 Register Q = LHS;
5758 Q = MIB.buildSMulH(Dst: Ty, Src0: LHS, Src1: MagicFactor).getReg(Idx: 0);
5759
5760 // (Optionally) Add/subtract the numerator using Factor.
5761 Factor = MIB.buildMul(Dst: Ty, Src0: LHS, Src1: Factor).getReg(Idx: 0);
5762 Q = MIB.buildAdd(Dst: Ty, Src0: Q, Src1: Factor).getReg(Idx: 0);
5763
5764 // Shift right algebraic by shift value.
5765 Q = MIB.buildAShr(Dst: Ty, Src0: Q, Src1: Shift).getReg(Idx: 0);
5766
5767 // Extract the sign bit, mask it and add it to the quotient.
5768 auto SignShift = MIB.buildConstant(Res: ShiftAmtTy, Val: EltBits - 1);
5769 auto T = MIB.buildLShr(Dst: Ty, Src0: Q, Src1: SignShift);
5770 T = MIB.buildAnd(Dst: Ty, Src0: T, Src1: ShiftMask);
5771 auto ret = MIB.buildAdd(Dst: Ty, Src0: Q, Src1: T);
5772
5773 if (Opcode == TargetOpcode::G_SREM) {
5774 auto Prod = MIB.buildMul(Dst: Ty, Src0: ret, Src1: RHS);
5775 return MIB.buildSub(Dst: Ty, Src0: LHS, Src1: Prod);
5776 }
5777 return ret;
5778}
5779
5780bool CombinerHelper::matchDivByPow2(MachineInstr &MI, bool IsSigned) const {
5781 assert((MI.getOpcode() == TargetOpcode::G_SDIV ||
5782 MI.getOpcode() == TargetOpcode::G_UDIV) &&
5783 "Expected SDIV or UDIV");
5784 auto &Div = cast<GenericMachineInstr>(Val&: MI);
5785 Register RHS = Div.getReg(Idx: 2);
5786 auto MatchPow2 = [&](const Constant *C) {
5787 auto *CI = dyn_cast<ConstantInt>(Val: C);
5788 return CI && (CI->getValue().isPowerOf2() ||
5789 (IsSigned && CI->getValue().isNegatedPowerOf2()));
5790 };
5791 return matchUnaryPredicate(MRI, Reg: RHS, Match: MatchPow2, /*AllowUndefs=*/false);
5792}
5793
5794void CombinerHelper::applySDivByPow2(MachineInstr &MI) const {
5795 assert(MI.getOpcode() == TargetOpcode::G_SDIV && "Expected SDIV");
5796 auto &SDiv = cast<GenericMachineInstr>(Val&: MI);
5797 Register Dst = SDiv.getReg(Idx: 0);
5798 Register LHS = SDiv.getReg(Idx: 1);
5799 Register RHS = SDiv.getReg(Idx: 2);
5800 LLT Ty = MRI.getType(Reg: Dst);
5801 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5802 LLT CCVT = Ty.isVector() ? LLT::vector(EC: Ty.getElementCount(), ScalarTy: LLT::integer(SizeInBits: 1))
5803 : LLT::integer(SizeInBits: 1);
5804
5805 // Effectively we want to lower G_SDIV %lhs, %rhs, where %rhs is a power of 2,
5806 // to the following version:
5807 //
5808 // %c1 = G_CTTZ %rhs
5809 // %inexact = G_SUB $bitwidth, %c1
5810 // %sign = %G_ASHR %lhs, $(bitwidth - 1)
5811 // %lshr = G_LSHR %sign, %inexact
5812 // %add = G_ADD %lhs, %lshr
5813 // %ashr = G_ASHR %add, %c1
5814 // %ashr = G_SELECT, %isoneorallones, %lhs, %ashr
5815 // %zero = G_CONSTANT $0
5816 // %neg = G_NEG %ashr
5817 // %isneg = G_ICMP SLT %rhs, %zero
5818 // %res = G_SELECT %isneg, %neg, %ashr
5819
5820 unsigned BitWidth = Ty.getScalarSizeInBits();
5821 auto Zero = Builder.buildConstant(Res: Ty, Val: 0);
5822
5823 auto Bits = Builder.buildConstant(Res: ShiftAmtTy, Val: BitWidth);
5824 auto C1 = Builder.buildCTTZ(Dst: ShiftAmtTy, Src0: RHS);
5825 auto Inexact = Builder.buildSub(Dst: ShiftAmtTy, Src0: Bits, Src1: C1);
5826 // Splat the sign bit into the register
5827 auto Sign = Builder.buildAShr(
5828 Dst: Ty, Src0: LHS, Src1: Builder.buildConstant(Res: ShiftAmtTy, Val: BitWidth - 1));
5829
5830 // Add (LHS < 0) ? abs2 - 1 : 0;
5831 auto LSrl = Builder.buildLShr(Dst: Ty, Src0: Sign, Src1: Inexact);
5832 auto Add = Builder.buildAdd(Dst: Ty, Src0: LHS, Src1: LSrl);
5833 auto AShr = Builder.buildAShr(Dst: Ty, Src0: Add, Src1: C1);
5834
5835 // Special case: (sdiv X, 1) -> X
5836 // Special Case: (sdiv X, -1) -> 0-X
5837 auto One = Builder.buildConstant(Res: Ty, Val: 1);
5838 auto MinusOne = Builder.buildConstant(Res: Ty, Val: -1);
5839 auto IsOne = Builder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: CCVT, Op0: RHS, Op1: One);
5840 auto IsMinusOne =
5841 Builder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: CCVT, Op0: RHS, Op1: MinusOne);
5842 auto IsOneOrMinusOne = Builder.buildOr(Dst: CCVT, Src0: IsOne, Src1: IsMinusOne);
5843 AShr = Builder.buildSelect(Res: Ty, Tst: IsOneOrMinusOne, Op0: LHS, Op1: AShr);
5844
5845 // If divided by a positive value, we're done. Otherwise, the result must be
5846 // negated.
5847 auto Neg = Builder.buildNeg(Dst: Ty, Src0: AShr);
5848 auto IsNeg = Builder.buildICmp(Pred: CmpInst::Predicate::ICMP_SLT, Res: CCVT, Op0: RHS, Op1: Zero);
5849 Builder.buildSelect(Res: MI.getOperand(i: 0).getReg(), Tst: IsNeg, Op0: Neg, Op1: AShr);
5850 MI.eraseFromParent();
5851}
5852
5853void CombinerHelper::applyUDivByPow2(MachineInstr &MI) const {
5854 assert(MI.getOpcode() == TargetOpcode::G_UDIV && "Expected UDIV");
5855 auto &UDiv = cast<GenericMachineInstr>(Val&: MI);
5856 Register Dst = UDiv.getReg(Idx: 0);
5857 Register LHS = UDiv.getReg(Idx: 1);
5858 Register RHS = UDiv.getReg(Idx: 2);
5859 LLT Ty = MRI.getType(Reg: Dst);
5860 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5861
5862 auto C1 = Builder.buildCTTZ(Dst: ShiftAmtTy, Src0: RHS);
5863 Builder.buildLShr(Dst: MI.getOperand(i: 0).getReg(), Src0: LHS, Src1: C1);
5864 MI.eraseFromParent();
5865}
5866
5867void CombinerHelper::applySimplifySRemByPow2(MachineInstr &MI) const {
5868 assert(MI.getOpcode() == TargetOpcode::G_SREM && "Expected SREM");
5869 auto &SRem = cast<GBinOp>(Val&: MI);
5870 Register Dst = SRem.getReg(Idx: 0);
5871 Register LHS = SRem.getLHSReg();
5872 Register RHS = SRem.getRHSReg();
5873 LLT Ty = MRI.getType(Reg: Dst);
5874 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5875
5876 // Effectively we want to lower G_SREM %lhs, %rhs, where %rhs is +/- a power
5877 // of 2, to the following branch-free bias-and-mask version:
5878 //
5879 // %abs = G_ABS %rhs
5880 // %mask = G_SUB %abs, 1
5881 // %sign = G_ASHR %lhs, $(bitwidth - 1)
5882 // %bias = G_AND %sign, %mask
5883 // %biased = G_ADD %lhs, %bias
5884 // %masked = G_AND %biased, %mask
5885 // %res = G_SUB %masked, %bias
5886 //
5887 // The bias adds (|%rhs| - 1) for negative %lhs, correcting rounding towards
5888 // zero (instead of towards -inf that a plain mask would give). Constant
5889 // divisors collapse %mask to a single G_CONSTANT via the CSEMIRBuilder folds
5890 // for G_ABS and G_SUB.
5891
5892 unsigned BitWidth = Ty.getScalarSizeInBits();
5893 auto AbsRHS = Builder.buildAbs(Dst: Ty, Src: RHS);
5894 auto Mask = Builder.buildSub(Dst: Ty, Src0: AbsRHS, Src1: Builder.buildConstant(Res: Ty, Val: 1));
5895 auto BWMinusOne = Builder.buildConstant(Res: ShiftAmtTy, Val: BitWidth - 1);
5896 auto Sign = Builder.buildAShr(Dst: Ty, Src0: LHS, Src1: BWMinusOne);
5897 auto Bias = Builder.buildAnd(Dst: Ty, Src0: Sign, Src1: Mask);
5898 auto Biased = Builder.buildAdd(Dst: Ty, Src0: LHS, Src1: Bias);
5899 auto Masked = Builder.buildAnd(Dst: Ty, Src0: Biased, Src1: Mask);
5900 Builder.buildSub(Dst, Src0: Masked, Src1: Bias);
5901 MI.eraseFromParent();
5902}
5903
5904bool CombinerHelper::matchUMulHToLShr(MachineInstr &MI) const {
5905 assert(MI.getOpcode() == TargetOpcode::G_UMULH);
5906 Register RHS = MI.getOperand(i: 2).getReg();
5907 Register Dst = MI.getOperand(i: 0).getReg();
5908 LLT Ty = MRI.getType(Reg: Dst);
5909 LLT RHSTy = MRI.getType(Reg: RHS);
5910 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5911 auto MatchPow2ExceptOne = [&](const Constant *C) {
5912 if (auto *CI = dyn_cast<ConstantInt>(Val: C))
5913 return CI->getValue().isPowerOf2() && !CI->getValue().isOne();
5914 return false;
5915 };
5916 if (!matchUnaryPredicate(MRI, Reg: RHS, Match: MatchPow2ExceptOne, AllowUndefs: false))
5917 return false;
5918 // We need to check both G_LSHR and G_CTLZ because the combine uses G_CTLZ to
5919 // get log base 2, and it is not always legal for on a target.
5920 return isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_LSHR, {Ty, ShiftAmtTy}}) &&
5921 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_CTLZ, {RHSTy, RHSTy}});
5922}
5923
5924void CombinerHelper::applyUMulHToLShr(MachineInstr &MI) const {
5925 Register LHS = MI.getOperand(i: 1).getReg();
5926 Register RHS = MI.getOperand(i: 2).getReg();
5927 Register Dst = MI.getOperand(i: 0).getReg();
5928 LLT Ty = MRI.getType(Reg: Dst);
5929 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5930 unsigned NumEltBits = Ty.getScalarSizeInBits();
5931
5932 auto LogBase2 = buildLogBase2(V: RHS, MIB&: Builder);
5933 auto ShiftAmt =
5934 Builder.buildSub(Dst: Ty, Src0: Builder.buildConstant(Res: Ty, Val: NumEltBits), Src1: LogBase2);
5935 auto Trunc = Builder.buildZExtOrTrunc(Res: ShiftAmtTy, Op: ShiftAmt);
5936 Builder.buildLShr(Dst, Src0: LHS, Src1: Trunc);
5937 MI.eraseFromParent();
5938}
5939
5940bool CombinerHelper::matchTruncSSatS(MachineInstr &MI,
5941 Register &MatchInfo) const {
5942 Register Dst = MI.getOperand(i: 0).getReg();
5943 Register Src = MI.getOperand(i: 1).getReg();
5944 LLT DstTy = MRI.getType(Reg: Dst);
5945 LLT SrcTy = MRI.getType(Reg: Src);
5946 unsigned NumDstBits = DstTy.getScalarSizeInBits();
5947 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
5948 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
5949
5950 if (!LI || !isLegalOrHasFewerElements(
5951 Query: {TargetOpcode::G_TRUNC_SSAT_S, {DstTy, SrcTy}}))
5952 return false;
5953
5954 APInt SignedMax = APInt::getSignedMaxValue(numBits: NumDstBits).sext(width: NumSrcBits);
5955 APInt SignedMin = APInt::getSignedMinValue(numBits: NumDstBits).sext(width: NumSrcBits);
5956 if (mi_match(
5957 R: Src, MRI,
5958 P: m_GSMin(L: m_GSMax(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: SignedMin)),
5959 R: m_SpecificICstOrSplat(RequestedValue: SignedMax))))
5960 return true;
5961 if (mi_match(
5962 R: Src, MRI,
5963 P: m_GSMax(L: m_GSMin(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: SignedMax)),
5964 R: m_SpecificICstOrSplat(RequestedValue: SignedMin))))
5965 return true;
5966
5967 // CVP in the midend will often transform trunc(smin(smax(..)) into
5968 // trunc nsw(smin(..)) as the smax against INT_MIN never saturates.
5969 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoSWrap) &&
5970 mi_match(R: Src, MRI,
5971 P: m_GSMin(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: SignedMax))))
5972 return true;
5973
5974 return false;
5975}
5976
5977void CombinerHelper::applyTruncSSatS(MachineInstr &MI,
5978 Register &MatchInfo) const {
5979 Register Dst = MI.getOperand(i: 0).getReg();
5980 Builder.buildTruncSSatS(Res: Dst, Op: MatchInfo);
5981 MI.eraseFromParent();
5982}
5983
5984bool CombinerHelper::matchTruncSSatU(MachineInstr &MI,
5985 Register &MatchInfo) const {
5986 Register Dst = MI.getOperand(i: 0).getReg();
5987 Register Src = MI.getOperand(i: 1).getReg();
5988 LLT DstTy = MRI.getType(Reg: Dst);
5989 LLT SrcTy = MRI.getType(Reg: Src);
5990 unsigned NumDstBits = DstTy.getScalarSizeInBits();
5991 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
5992 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
5993
5994 if (!LI || !isLegalOrHasFewerElements(
5995 Query: {TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))
5996 return false;
5997 APInt UnsignedMax = APInt::getMaxValue(numBits: NumDstBits).zext(width: NumSrcBits);
5998 return mi_match(R: Src, MRI,
5999 P: m_GSMin(L: m_GSMax(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: 0)),
6000 R: m_SpecificICstOrSplat(RequestedValue: UnsignedMax))) ||
6001 mi_match(R: Src, MRI,
6002 P: m_GSMax(L: m_GSMin(L: m_Reg(R&: MatchInfo),
6003 R: m_SpecificICstOrSplat(RequestedValue: UnsignedMax)),
6004 R: m_SpecificICstOrSplat(RequestedValue: 0))) ||
6005 mi_match(R: Src, MRI,
6006 P: m_GUMin(L: m_GSMax(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: 0)),
6007 R: m_SpecificICstOrSplat(RequestedValue: UnsignedMax)));
6008}
6009
6010void CombinerHelper::applyTruncSSatU(MachineInstr &MI,
6011 Register &MatchInfo) const {
6012 Register Dst = MI.getOperand(i: 0).getReg();
6013 Builder.buildTruncSSatU(Res: Dst, Op: MatchInfo);
6014 MI.eraseFromParent();
6015}
6016
6017bool CombinerHelper::matchTruncUSatU(MachineInstr &MI,
6018 MachineInstr &MinMI) const {
6019 Register Min = MinMI.getOperand(i: 2).getReg();
6020 Register Val = MinMI.getOperand(i: 1).getReg();
6021 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6022 LLT SrcTy = MRI.getType(Reg: Val);
6023 unsigned NumDstBits = DstTy.getScalarSizeInBits();
6024 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6025 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
6026
6027 if (!LI || !isLegalOrHasFewerElements(
6028 Query: {TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))
6029 return false;
6030 APInt UnsignedMax = APInt::getMaxValue(numBits: NumDstBits).zext(width: NumSrcBits);
6031 return mi_match(R: Min, MRI, P: m_SpecificICstOrSplat(RequestedValue: UnsignedMax)) &&
6032 !mi_match(R: Val, MRI, P: m_GSMax(L: m_Reg(), R: m_Reg()));
6033}
6034
6035bool CombinerHelper::matchTruncUSatUToFPTOUISat(MachineInstr &MI,
6036 MachineInstr &SrcMI) const {
6037 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6038 LLT SrcTy = MRI.getType(Reg: SrcMI.getOperand(i: 1).getReg());
6039
6040 return LI &&
6041 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FPTOUI_SAT, {DstTy, SrcTy}});
6042}
6043
6044bool CombinerHelper::matchFsubToFneg(MachineInstr &MI) const {
6045 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6046
6047 Register LHS = MI.getOperand(i: 1).getReg();
6048 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6049
6050 const auto LHSCst = Ty.isVector()
6051 ? getFConstantSplat(VReg: LHS, MRI, /* allowUndef */ AllowUndef: true)
6052 : getFConstantVRegValWithLookThrough(VReg: LHS, MRI);
6053 if (!LHSCst)
6054 return false;
6055
6056 // Skip fsub_to_fneg when both LHS and RHS are scalar constants (vector FP
6057 // binop constant folding is not supported yet), so constant_fold_fp_binop
6058 // folds these cases.
6059 if (!Ty.isVector() && getConstantFPVRegVal(VReg: LHS, MRI) &&
6060 getConstantFPVRegVal(VReg: MI.getOperand(i: 2).getReg(), MRI))
6061 return false;
6062
6063 // -0.0 is always allowed
6064 if (LHSCst->Value.isNegZero())
6065 return true;
6066
6067 // +0.0 is only allowed if nsz is set.
6068 if (LHSCst->Value.isPosZero())
6069 return MI.getFlag(Flag: MachineInstr::FmNsz);
6070
6071 return false;
6072}
6073
6074/// Checks if \p MI is TargetOpcode::G_FMUL and contractable either
6075/// due to global flags or MachineInstr flags.
6076static bool isContractableFMul(MachineInstr &MI, bool AllowFusionGlobally) {
6077 if (MI.getOpcode() != TargetOpcode::G_FMUL)
6078 return false;
6079 return AllowFusionGlobally || MI.getFlag(Flag: MachineInstr::MIFlag::FmContract);
6080}
6081
6082static bool hasMoreUses(const MachineInstr &MI0, const MachineInstr &MI1,
6083 const MachineRegisterInfo &MRI) {
6084 return std::distance(first: MRI.use_instr_nodbg_begin(RegNo: MI0.getOperand(i: 0).getReg()),
6085 last: MRI.use_instr_nodbg_end()) >
6086 std::distance(first: MRI.use_instr_nodbg_begin(RegNo: MI1.getOperand(i: 0).getReg()),
6087 last: MRI.use_instr_nodbg_end());
6088}
6089
6090bool CombinerHelper::canCombineFMadOrFMA(MachineInstr &MI,
6091 bool &AllowFusionGlobally,
6092 bool &HasFMAD, bool &Aggressive,
6093 bool CanReassociate) const {
6094
6095 auto *MF = MI.getMF();
6096 const auto &TLI = *MF->getSubtarget().getTargetLowering();
6097 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6098
6099 if (CanReassociate && !MI.getFlag(Flag: MachineInstr::MIFlag::FmReassoc))
6100 return false;
6101
6102 // Floating-point multiply-add with intermediate rounding.
6103 HasFMAD = (!isPreLegalize() && TLI.isFMADLegal(MI, Ty: DstType));
6104 // Floating-point multiply-add without intermediate rounding.
6105 bool HasFMA = TLI.isFMAFasterThanFMulAndFAdd(MF: *MF, DstType) &&
6106 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FMA, {DstType}});
6107 // No valid opcode, do not combine.
6108 if (!HasFMAD && !HasFMA)
6109 return false;
6110
6111 // FMAD (with intermediate rounding) is always safe to form; FMA requires the
6112 // contract fast-math flag.
6113 AllowFusionGlobally = HasFMAD;
6114 // If the addition is not contractable, do not combine.
6115 if (!AllowFusionGlobally && !MI.getFlag(Flag: MachineInstr::MIFlag::FmContract))
6116 return false;
6117
6118 Aggressive = TLI.enableAggressiveFMAFusion(Ty: DstType);
6119 return true;
6120}
6121
6122bool CombinerHelper::matchCombineFAddFMulToFMadOrFMA(
6123 MachineInstr &MI,
6124 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6125 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6126
6127 bool AllowFusionGlobally, HasFMAD, Aggressive;
6128 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6129 return false;
6130
6131 Register Op1 = MI.getOperand(i: 1).getReg();
6132 Register Op2 = MI.getOperand(i: 2).getReg();
6133 MachineInstr *Op1Def, *Op2Def;
6134 if (!mi_match(R: Op1, MRI, P: m_MInstr(MI&: Op1Def)) ||
6135 !mi_match(R: Op2, MRI, P: m_MInstr(MI&: Op2Def)))
6136 return false;
6137 DefinitionAndSourceRegister LHS = {.MI: Op1Def, .Reg: Op1};
6138 DefinitionAndSourceRegister RHS = {.MI: Op2Def, .Reg: Op2};
6139 unsigned PreferredFusedOpcode =
6140 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6141
6142 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
6143 // prefer to fold the multiply with fewer uses.
6144 if (Aggressive && isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6145 isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally)) {
6146 if (hasMoreUses(MI0: *LHS.MI, MI1: *RHS.MI, MRI))
6147 std::swap(a&: LHS, b&: RHS);
6148 }
6149
6150 // fold (fadd (fmul x, y), z) -> (fma x, y, z)
6151 if (isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6152 (Aggressive || MRI.hasOneNonDBGUse(RegNo: LHS.Reg))) {
6153 unsigned Flags = MI.getFlags() & LHS.MI->getFlags();
6154 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6155 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6156 SrcOps: {LHS.MI->getOperand(i: 1).getReg(),
6157 LHS.MI->getOperand(i: 2).getReg(), RHS.Reg},
6158 Flags);
6159 };
6160 return true;
6161 }
6162
6163 // fold (fadd x, (fmul y, z)) -> (fma y, z, x)
6164 if (isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally) &&
6165 (Aggressive || MRI.hasOneNonDBGUse(RegNo: RHS.Reg))) {
6166 unsigned Flags = MI.getFlags() & RHS.MI->getFlags();
6167 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6168 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6169 SrcOps: {RHS.MI->getOperand(i: 1).getReg(),
6170 RHS.MI->getOperand(i: 2).getReg(), LHS.Reg},
6171 Flags);
6172 };
6173 return true;
6174 }
6175
6176 return false;
6177}
6178
6179bool CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMA(
6180 MachineInstr &MI,
6181 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6182 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6183
6184 bool AllowFusionGlobally, HasFMAD, Aggressive;
6185 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6186 return false;
6187
6188 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
6189 Register Op1 = MI.getOperand(i: 1).getReg();
6190 Register Op2 = MI.getOperand(i: 2).getReg();
6191 MachineInstr *Op1Def, *Op2Def;
6192 if (!mi_match(R: Op1, MRI, P: m_MInstr(MI&: Op1Def)) ||
6193 !mi_match(R: Op2, MRI, P: m_MInstr(MI&: Op2Def)))
6194 return false;
6195 DefinitionAndSourceRegister LHS = {.MI: Op1Def, .Reg: Op1};
6196 DefinitionAndSourceRegister RHS = {.MI: Op2Def, .Reg: Op2};
6197 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6198
6199 unsigned PreferredFusedOpcode =
6200 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6201
6202 MachineInstr *LHSFpExtSrc;
6203 bool LHSContractable =
6204 mi_match(R: LHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: LHSFpExtSrc))) &&
6205 isContractableFMul(MI&: *LHSFpExtSrc, AllowFusionGlobally) &&
6206 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6207 SrcTy: MRI.getType(Reg: LHSFpExtSrc->getOperand(i: 1).getReg()));
6208 MachineInstr *RHSFpExtSrc;
6209 bool RHSContractable =
6210 mi_match(R: RHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: RHSFpExtSrc))) &&
6211 isContractableFMul(MI&: *RHSFpExtSrc, AllowFusionGlobally) &&
6212 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6213 SrcTy: MRI.getType(Reg: RHSFpExtSrc->getOperand(i: 1).getReg()));
6214
6215 // fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
6216 if (LHSContractable || RHSContractable) {
6217 // Ensure that the contractable fmul with the fewest uses (if both are
6218 // contractable) is the LHS operand.
6219 if (!LHSContractable ||
6220 (RHSContractable && hasMoreUses(MI0: *LHSFpExtSrc, MI1: *RHSFpExtSrc, MRI))) {
6221 std::swap(a&: LHS, b&: RHS);
6222 LHSFpExtSrc = RHSFpExtSrc;
6223 }
6224
6225 unsigned Flags = MI.getFlags() & LHSFpExtSrc->getFlags();
6226 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6227 auto FpExtX = B.buildFPExt(Res: DstType, Op: LHSFpExtSrc->getOperand(i: 1).getReg());
6228 auto FpExtY = B.buildFPExt(Res: DstType, Op: LHSFpExtSrc->getOperand(i: 2).getReg());
6229 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6230 SrcOps: {FpExtX.getReg(Idx: 0), FpExtY.getReg(Idx: 0), RHS.Reg}, Flags);
6231 };
6232 return true;
6233 }
6234
6235 return false;
6236}
6237
6238bool CombinerHelper::matchCombineFAddFMAFMulToFMadOrFMA(
6239 MachineInstr &MI,
6240 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6241 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6242
6243 bool AllowFusionGlobally, HasFMAD, Aggressive;
6244 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive, CanReassociate: true))
6245 return false;
6246
6247 Register Op1 = MI.getOperand(i: 1).getReg();
6248 Register Op2 = MI.getOperand(i: 2).getReg();
6249 MachineInstr *Op1Def, *Op2Def;
6250 if (!mi_match(R: Op1, MRI, P: m_MInstr(MI&: Op1Def)) ||
6251 !mi_match(R: Op2, MRI, P: m_MInstr(MI&: Op2Def)))
6252 return false;
6253 DefinitionAndSourceRegister LHS = {.MI: Op1Def, .Reg: Op1};
6254 DefinitionAndSourceRegister RHS = {.MI: Op2Def, .Reg: Op2};
6255 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6256
6257 unsigned PreferredFusedOpcode =
6258 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6259
6260 MachineInstr *FMA = nullptr;
6261 Register Z;
6262 // fold (fadd (fma x, y, (fmul u, v)), z) -> (fma x, y, (fma u, v, z))
6263 if (LHS.MI->getOpcode() == PreferredFusedOpcode &&
6264 mi_match(R: LHS.MI->getOperand(i: 3).getReg(), MRI,
6265 P: m_GFMul(L: m_Reg(), R: m_Reg())) &&
6266 MRI.hasOneNonDBGUse(RegNo: LHS.MI->getOperand(i: 0).getReg()) &&
6267 MRI.hasOneNonDBGUse(RegNo: LHS.MI->getOperand(i: 3).getReg())) {
6268 FMA = LHS.MI;
6269 Z = RHS.Reg;
6270 }
6271 // fold (fadd z, (fma x, y, (fmul u, v))) -> (fma x, y, (fma u, v, z))
6272 else if (RHS.MI->getOpcode() == PreferredFusedOpcode &&
6273 mi_match(R: RHS.MI->getOperand(i: 3).getReg(), MRI,
6274 P: m_GFMul(L: m_Reg(), R: m_Reg())) &&
6275 MRI.hasOneNonDBGUse(RegNo: RHS.MI->getOperand(i: 0).getReg()) &&
6276 MRI.hasOneNonDBGUse(RegNo: RHS.MI->getOperand(i: 3).getReg())) {
6277 Z = LHS.Reg;
6278 FMA = RHS.MI;
6279 }
6280
6281 if (FMA) {
6282 MachineInstr *FMulMI;
6283 if (!mi_match(R: FMA->getOperand(i: 3).getReg(), MRI, P: m_MInstr(MI&: FMulMI)))
6284 return false;
6285 Register X = FMA->getOperand(i: 1).getReg();
6286 Register Y = FMA->getOperand(i: 2).getReg();
6287 Register U = FMulMI->getOperand(i: 1).getReg();
6288 Register V = FMulMI->getOperand(i: 2).getReg();
6289 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6290 unsigned OuterFlags = MI.getFlags() & FMA->getFlags();
6291
6292 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6293 Register InnerFMA = MRI.createGenericVirtualRegister(Ty: DstTy);
6294 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {InnerFMA}, SrcOps: {U, V, Z}, Flags: InnerFlags);
6295 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6296 SrcOps: {X, Y, InnerFMA}, Flags: OuterFlags);
6297 };
6298 return true;
6299 }
6300
6301 return false;
6302}
6303
6304bool CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMAAggressive(
6305 MachineInstr &MI,
6306 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6307 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6308
6309 bool AllowFusionGlobally, HasFMAD, Aggressive;
6310 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6311 return false;
6312
6313 if (!Aggressive)
6314 return false;
6315
6316 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
6317 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6318 Register Op1 = MI.getOperand(i: 1).getReg();
6319 Register Op2 = MI.getOperand(i: 2).getReg();
6320 MachineInstr *Op1Def, *Op2Def;
6321 if (!mi_match(R: Op1, MRI, P: m_MInstr(MI&: Op1Def)) ||
6322 !mi_match(R: Op2, MRI, P: m_MInstr(MI&: Op2Def)))
6323 return false;
6324 DefinitionAndSourceRegister LHS = {.MI: Op1Def, .Reg: Op1};
6325 DefinitionAndSourceRegister RHS = {.MI: Op2Def, .Reg: Op2};
6326
6327 unsigned PreferredFusedOpcode =
6328 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6329
6330 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
6331 // prefer to fold the multiply with fewer uses.
6332 if (Aggressive && isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6333 isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally)) {
6334 if (hasMoreUses(MI0: *LHS.MI, MI1: *RHS.MI, MRI))
6335 std::swap(a&: LHS, b&: RHS);
6336 }
6337
6338 // Builds: (fma x, y, (fma (fpext u), (fpext v), z))
6339 auto buildMatchInfo = [=, &MI](Register U, Register V, Register Z, Register X,
6340 Register Y, unsigned InnerFlags,
6341 unsigned OuterFlags, MachineIRBuilder &B) {
6342 Register FpExtU = B.buildFPExt(Res: DstType, Op: U).getReg(Idx: 0);
6343 Register FpExtV = B.buildFPExt(Res: DstType, Op: V).getReg(Idx: 0);
6344 Register InnerFMA = B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {DstType},
6345 SrcOps: {FpExtU, FpExtV, Z}, Flags: InnerFlags)
6346 .getReg(Idx: 0);
6347 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6348 SrcOps: {X, Y, InnerFMA}, Flags: OuterFlags);
6349 };
6350
6351 MachineInstr *FMulMI, *FMAMI;
6352 // fold (fadd (fma x, y, (fpext (fmul u, v))), z)
6353 // -> (fma x, y, (fma (fpext u), (fpext v), z))
6354 if (LHS.MI->getOpcode() == PreferredFusedOpcode &&
6355 mi_match(R: LHS.MI->getOperand(i: 3).getReg(), MRI,
6356 P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6357 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6358 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6359 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
6360 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6361 unsigned OuterFlags = MI.getFlags() & LHS.MI->getFlags();
6362 MatchInfo = [=](MachineIRBuilder &B) {
6363 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6364 FMulMI->getOperand(i: 2).getReg(), RHS.Reg,
6365 LHS.MI->getOperand(i: 1).getReg(),
6366 LHS.MI->getOperand(i: 2).getReg(), InnerFlags, OuterFlags, B);
6367 };
6368 return true;
6369 }
6370
6371 // fold (fadd (fpext (fma x, y, (fmul u, v))), z)
6372 // -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z))
6373 // FIXME: This turns two single-precision and one double-precision
6374 // operation into two double-precision operations, which might not be
6375 // interesting for all targets, especially GPUs.
6376 if (mi_match(R: LHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMAMI))) &&
6377 FMAMI->getOpcode() == PreferredFusedOpcode) {
6378 MachineInstr *FMulMI;
6379 if (!mi_match(R: FMAMI->getOperand(i: 3).getReg(), MRI, P: m_MInstr(MI&: FMulMI)))
6380 return false;
6381 if (isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6382 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6383 SrcTy: MRI.getType(Reg: FMAMI->getOperand(i: 0).getReg()))) {
6384 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6385 unsigned OuterFlags = MI.getFlags() & FMAMI->getFlags();
6386 MatchInfo = [=](MachineIRBuilder &B) {
6387 Register X = FMAMI->getOperand(i: 1).getReg();
6388 Register Y = FMAMI->getOperand(i: 2).getReg();
6389 X = B.buildFPExt(Res: DstType, Op: X).getReg(Idx: 0);
6390 Y = B.buildFPExt(Res: DstType, Op: Y).getReg(Idx: 0);
6391 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6392 FMulMI->getOperand(i: 2).getReg(), RHS.Reg, X, Y,
6393 InnerFlags, OuterFlags, B);
6394 };
6395
6396 return true;
6397 }
6398 }
6399
6400 // fold (fadd z, (fma x, y, (fpext (fmul u, v)))
6401 // -> (fma x, y, (fma (fpext u), (fpext v), z))
6402 if (RHS.MI->getOpcode() == PreferredFusedOpcode &&
6403 mi_match(R: RHS.MI->getOperand(i: 3).getReg(), MRI,
6404 P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6405 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6406 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6407 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
6408 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6409 unsigned OuterFlags = MI.getFlags() & RHS.MI->getFlags();
6410 MatchInfo = [=](MachineIRBuilder &B) {
6411 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6412 FMulMI->getOperand(i: 2).getReg(), LHS.Reg,
6413 RHS.MI->getOperand(i: 1).getReg(),
6414 RHS.MI->getOperand(i: 2).getReg(), InnerFlags, OuterFlags, B);
6415 };
6416 return true;
6417 }
6418
6419 // fold (fadd z, (fpext (fma x, y, (fmul u, v)))
6420 // -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z))
6421 // FIXME: This turns two single-precision and one double-precision
6422 // operation into two double-precision operations, which might not be
6423 // interesting for all targets, especially GPUs.
6424 if (mi_match(R: RHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMAMI))) &&
6425 FMAMI->getOpcode() == PreferredFusedOpcode) {
6426 MachineInstr *FMulMI;
6427 if (!mi_match(R: FMAMI->getOperand(i: 3).getReg(), MRI, P: m_MInstr(MI&: FMulMI)))
6428 return false;
6429 if (isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6430 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6431 SrcTy: MRI.getType(Reg: FMAMI->getOperand(i: 0).getReg()))) {
6432 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6433 unsigned OuterFlags = MI.getFlags() & FMAMI->getFlags();
6434 MatchInfo = [=](MachineIRBuilder &B) {
6435 Register X = FMAMI->getOperand(i: 1).getReg();
6436 Register Y = FMAMI->getOperand(i: 2).getReg();
6437 X = B.buildFPExt(Res: DstType, Op: X).getReg(Idx: 0);
6438 Y = B.buildFPExt(Res: DstType, Op: Y).getReg(Idx: 0);
6439 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6440 FMulMI->getOperand(i: 2).getReg(), LHS.Reg, X, Y,
6441 InnerFlags, OuterFlags, B);
6442 };
6443 return true;
6444 }
6445 }
6446
6447 return false;
6448}
6449
6450bool CombinerHelper::matchCombineFSubFMulToFMadOrFMA(
6451 MachineInstr &MI,
6452 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6453 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6454
6455 bool AllowFusionGlobally, HasFMAD, Aggressive;
6456 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6457 return false;
6458
6459 Register Op1 = MI.getOperand(i: 1).getReg();
6460 Register Op2 = MI.getOperand(i: 2).getReg();
6461 MachineInstr *Op1Def, *Op2Def;
6462 if (!mi_match(R: Op1, MRI, P: m_MInstr(MI&: Op1Def)) ||
6463 !mi_match(R: Op2, MRI, P: m_MInstr(MI&: Op2Def)))
6464 return false;
6465 DefinitionAndSourceRegister LHS = {.MI: Op1Def, .Reg: Op1};
6466 DefinitionAndSourceRegister RHS = {.MI: Op2Def, .Reg: Op2};
6467 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6468
6469 // If we have two choices trying to fold (fsub (fmul u, v), (fmul x, y)),
6470 // prefer to fold the multiply with fewer uses.
6471 int FirstMulHasFewerUses = true;
6472 if (isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6473 isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally) &&
6474 hasMoreUses(MI0: *LHS.MI, MI1: *RHS.MI, MRI))
6475 FirstMulHasFewerUses = false;
6476
6477 unsigned PreferredFusedOpcode =
6478 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6479
6480 // fold (fsub (fmul x, y), z) -> (fma x, y, -z)
6481 if (FirstMulHasFewerUses &&
6482 (isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6483 (Aggressive || MRI.hasOneNonDBGUse(RegNo: LHS.Reg)))) {
6484 unsigned Flags = MI.getFlags() & LHS.MI->getFlags();
6485 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6486 Register NegZ = B.buildFNeg(Dst: DstTy, Src0: RHS.Reg).getReg(Idx: 0);
6487 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6488 SrcOps: {LHS.MI->getOperand(i: 1).getReg(),
6489 LHS.MI->getOperand(i: 2).getReg(), NegZ},
6490 Flags);
6491 };
6492 return true;
6493 }
6494 // fold (fsub x, (fmul y, z)) -> (fma -y, z, x)
6495 else if ((isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally) &&
6496 (Aggressive || MRI.hasOneNonDBGUse(RegNo: RHS.Reg)))) {
6497 unsigned Flags = MI.getFlags() & RHS.MI->getFlags();
6498 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6499 Register NegY =
6500 B.buildFNeg(Dst: DstTy, Src0: RHS.MI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6501 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6502 SrcOps: {NegY, RHS.MI->getOperand(i: 2).getReg(), LHS.Reg}, Flags);
6503 };
6504 return true;
6505 }
6506
6507 return false;
6508}
6509
6510bool CombinerHelper::matchCombineFSubFNegFMulToFMadOrFMA(
6511 MachineInstr &MI,
6512 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6513 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6514
6515 bool AllowFusionGlobally, HasFMAD, Aggressive;
6516 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6517 return false;
6518
6519 Register LHSReg = MI.getOperand(i: 1).getReg();
6520 Register RHSReg = MI.getOperand(i: 2).getReg();
6521 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6522
6523 unsigned PreferredFusedOpcode =
6524 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6525
6526 MachineInstr *FMulMI;
6527 // fold (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
6528 if (mi_match(R: LHSReg, MRI, P: m_GFNeg(Src: m_MInstr(MI&: FMulMI))) &&
6529 (Aggressive || (MRI.hasOneNonDBGUse(RegNo: LHSReg) &&
6530 MRI.hasOneNonDBGUse(RegNo: FMulMI->getOperand(i: 0).getReg()))) &&
6531 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally)) {
6532 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6533 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6534 Register NegX =
6535 B.buildFNeg(Dst: DstTy, Src0: FMulMI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6536 Register NegZ = B.buildFNeg(Dst: DstTy, Src0: RHSReg).getReg(Idx: 0);
6537 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6538 SrcOps: {NegX, FMulMI->getOperand(i: 2).getReg(), NegZ}, Flags);
6539 };
6540 return true;
6541 }
6542
6543 // fold (fsub x, (fneg (fmul, y, z))) -> (fma y, z, x)
6544 if (mi_match(R: RHSReg, MRI, P: m_GFNeg(Src: m_MInstr(MI&: FMulMI))) &&
6545 (Aggressive || (MRI.hasOneNonDBGUse(RegNo: RHSReg) &&
6546 MRI.hasOneNonDBGUse(RegNo: FMulMI->getOperand(i: 0).getReg()))) &&
6547 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally)) {
6548 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6549 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6550 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6551 SrcOps: {FMulMI->getOperand(i: 1).getReg(),
6552 FMulMI->getOperand(i: 2).getReg(), LHSReg},
6553 Flags);
6554 };
6555 return true;
6556 }
6557
6558 return false;
6559}
6560
6561bool CombinerHelper::matchCombineFSubFpExtFMulToFMadOrFMA(
6562 MachineInstr &MI,
6563 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6564 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6565
6566 bool AllowFusionGlobally, HasFMAD, Aggressive;
6567 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6568 return false;
6569
6570 Register LHSReg = MI.getOperand(i: 1).getReg();
6571 Register RHSReg = MI.getOperand(i: 2).getReg();
6572 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6573
6574 unsigned PreferredFusedOpcode =
6575 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6576
6577 MachineInstr *FMulMI;
6578 // fold (fsub (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), (fneg z))
6579 if (mi_match(R: LHSReg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6580 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6581 (Aggressive || MRI.hasOneNonDBGUse(RegNo: LHSReg))) {
6582 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6583 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6584 Register FpExtX =
6585 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6586 Register FpExtY =
6587 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 2).getReg()).getReg(Idx: 0);
6588 Register NegZ = B.buildFNeg(Dst: DstTy, Src0: RHSReg).getReg(Idx: 0);
6589 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6590 SrcOps: {FpExtX, FpExtY, NegZ}, Flags);
6591 };
6592 return true;
6593 }
6594
6595 // fold (fsub x, (fpext (fmul y, z))) -> (fma (fneg (fpext y)), (fpext z), x)
6596 if (mi_match(R: RHSReg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6597 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6598 (Aggressive || MRI.hasOneNonDBGUse(RegNo: RHSReg))) {
6599 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6600 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6601 Register FpExtY =
6602 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6603 Register NegY = B.buildFNeg(Dst: DstTy, Src0: FpExtY).getReg(Idx: 0);
6604 Register FpExtZ =
6605 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 2).getReg()).getReg(Idx: 0);
6606 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6607 SrcOps: {NegY, FpExtZ, LHSReg}, Flags);
6608 };
6609 return true;
6610 }
6611
6612 return false;
6613}
6614
6615bool CombinerHelper::matchCombineFSubFpExtFNegFMulToFMadOrFMA(
6616 MachineInstr &MI,
6617 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6618 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6619
6620 bool AllowFusionGlobally, HasFMAD, Aggressive;
6621 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6622 return false;
6623
6624 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
6625 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6626 Register LHSReg = MI.getOperand(i: 1).getReg();
6627 Register RHSReg = MI.getOperand(i: 2).getReg();
6628
6629 unsigned PreferredFusedOpcode =
6630 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6631
6632 auto buildMatchInfo = [=](Register Dst, Register X, Register Y, Register Z,
6633 unsigned Flags, MachineIRBuilder &B) {
6634 Register FpExtX = B.buildFPExt(Res: DstTy, Op: X).getReg(Idx: 0);
6635 Register FpExtY = B.buildFPExt(Res: DstTy, Op: Y).getReg(Idx: 0);
6636 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {Dst}, SrcOps: {FpExtX, FpExtY, Z}, Flags);
6637 };
6638
6639 MachineInstr *FMulMI;
6640 // fold (fsub (fpext (fneg (fmul x, y))), z) ->
6641 // (fneg (fma (fpext x), (fpext y), z))
6642 // fold (fsub (fneg (fpext (fmul x, y))), z) ->
6643 // (fneg (fma (fpext x), (fpext y), z))
6644 if ((mi_match(R: LHSReg, MRI, P: m_GFPExt(Src: m_GFNeg(Src: m_MInstr(MI&: FMulMI)))) ||
6645 mi_match(R: LHSReg, MRI, P: m_GFNeg(Src: m_GFPExt(Src: m_MInstr(MI&: FMulMI))))) &&
6646 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6647 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstTy,
6648 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
6649 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6650 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6651 Register FMAReg = MRI.createGenericVirtualRegister(Ty: DstTy);
6652 buildMatchInfo(FMAReg, FMulMI->getOperand(i: 1).getReg(),
6653 FMulMI->getOperand(i: 2).getReg(), RHSReg, Flags, B);
6654 B.buildFNeg(Dst: MI.getOperand(i: 0).getReg(), Src0: FMAReg);
6655 };
6656 return true;
6657 }
6658
6659 // fold (fsub x, (fpext (fneg (fmul y, z)))) -> (fma (fpext y), (fpext z), x)
6660 // fold (fsub x, (fneg (fpext (fmul y, z)))) -> (fma (fpext y), (fpext z), x)
6661 if ((mi_match(R: RHSReg, MRI, P: m_GFPExt(Src: m_GFNeg(Src: m_MInstr(MI&: FMulMI)))) ||
6662 mi_match(R: RHSReg, MRI, P: m_GFNeg(Src: m_GFPExt(Src: m_MInstr(MI&: FMulMI))))) &&
6663 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6664 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstTy,
6665 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
6666 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6667 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6668 buildMatchInfo(MI.getOperand(i: 0).getReg(), FMulMI->getOperand(i: 1).getReg(),
6669 FMulMI->getOperand(i: 2).getReg(), LHSReg, Flags, B);
6670 };
6671 return true;
6672 }
6673
6674 return false;
6675}
6676
6677// Combine multiple FDIVs with the same divisor into multiple FMULs by the
6678// reciprocal.
6679// E.g., (a / Y; b / Y;) -> (recip = 1.0 / Y; a * recip; b * recip)
6680bool CombinerHelper::matchRepeatedFPDivisor(
6681 MachineInstr &MI, SmallVector<MachineInstr *> &MatchInfo) const {
6682 assert(MI.getOpcode() == TargetOpcode::G_FDIV);
6683
6684 Register X = MI.getOperand(i: 1).getReg();
6685 Register Y = MI.getOperand(i: 2).getReg();
6686
6687 if (!MI.getFlag(Flag: MachineInstr::MIFlag::FmArcp))
6688 return false;
6689
6690 auto IsOne = [this](Register X) {
6691 auto N0CFP = isConstantOrConstantSplatVectorFP(Def: X, MRI);
6692 return N0CFP && (N0CFP->isOne() || N0CFP->isMinusOne());
6693 };
6694
6695 // Skip if current node is a reciprocal/fneg-reciprocal.
6696 if (IsOne(X))
6697 return false;
6698
6699 // Exit early if the target does not want this transform or if there can't
6700 // possibly be enough uses of the divisor to make the transform worthwhile.
6701 unsigned MinUses = getTargetLowering().combineRepeatedFPDivisors();
6702 if (!MinUses)
6703 return false;
6704
6705 // Find all FDIV users of the same divisor. For the moment we limit all
6706 // instructions to a single BB and use the first Instr in MatchInfo as the
6707 // dominating position.
6708 MatchInfo.push_back(Elt: &MI);
6709 for (auto &U : MRI.use_nodbg_instructions(Reg: Y)) {
6710 if (&U == &MI || U.getParent() != MI.getParent())
6711 continue;
6712 if (U.getOpcode() == TargetOpcode::G_FDIV &&
6713 U.getOperand(i: 2).getReg() == Y && U.getOperand(i: 1).getReg() != Y &&
6714 !IsOne(U.getOperand(i: 1).getReg())) {
6715 // This division is eligible for optimization only if global unsafe math
6716 // is enabled or if this division allows reciprocal formation.
6717 if (U.getFlag(Flag: MachineInstr::MIFlag::FmArcp)) {
6718 MatchInfo.push_back(Elt: &U);
6719 if (dominates(DefMI: U, UseMI: *MatchInfo[0]))
6720 std::swap(a&: MatchInfo[0], b&: MatchInfo.back());
6721 }
6722 }
6723 }
6724
6725 // Now that we have the actual number of divisor uses, make sure it meets
6726 // the minimum threshold specified by the target.
6727 return MatchInfo.size() >= MinUses;
6728}
6729
6730void CombinerHelper::applyRepeatedFPDivisor(
6731 SmallVector<MachineInstr *> &MatchInfo) const {
6732 // Generate the new div at the position of the first instruction, that we have
6733 // ensured will dominate all other instructions.
6734 Builder.setInsertPt(MBB&: *MatchInfo[0]->getParent(), II: MatchInfo[0]);
6735 LLT Ty = MRI.getType(Reg: MatchInfo[0]->getOperand(i: 0).getReg());
6736 auto Div = Builder.buildFDiv(Dst: Ty, Src0: Builder.buildFConstant(Res: Ty, Val: 1.0),
6737 Src1: MatchInfo[0]->getOperand(i: 2).getReg(),
6738 Flags: MatchInfo[0]->getFlags());
6739
6740 // Replace all found div's with fmul instructions.
6741 for (MachineInstr *MI : MatchInfo) {
6742 Builder.setInsertPt(MBB&: *MI->getParent(), II: MI);
6743 Builder.buildFMul(Dst: MI->getOperand(i: 0).getReg(), Src0: MI->getOperand(i: 1).getReg(),
6744 Src1: Div->getOperand(i: 0).getReg(), Flags: MI->getFlags());
6745 MI->eraseFromParent();
6746 }
6747}
6748
6749bool CombinerHelper::matchBuildVectorIdentityFold(MachineInstr &MI,
6750 Register &MatchInfo) const {
6751 // This combine folds the following patterns:
6752 //
6753 // G_BUILD_VECTOR_TRUNC (G_BITCAST(x), G_LSHR(G_BITCAST(x), k))
6754 // G_BUILD_VECTOR(G_TRUNC(G_BITCAST(x)), G_TRUNC(G_LSHR(G_BITCAST(x), k)))
6755 // into
6756 // x
6757 // if
6758 // k == sizeof(VecEltTy)/2
6759 // type(x) == type(dst)
6760 //
6761 // G_BUILD_VECTOR(G_TRUNC(G_BITCAST(x)), undef)
6762 // into
6763 // x
6764 // if
6765 // type(x) == type(dst)
6766
6767 LLT DstVecTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6768 LLT DstEltTy = DstVecTy.getElementType();
6769
6770 Register Lo, Hi;
6771
6772 if (mi_match(
6773 MI, MRI,
6774 P: m_GBuildVector(L: m_GTrunc(Src: m_GBitcast(Src: m_Reg(R&: Lo))), R: m_GImplicitDef()))) {
6775 MatchInfo = Lo;
6776 return MRI.getType(Reg: MatchInfo) == DstVecTy;
6777 }
6778
6779 std::optional<ValueAndVReg> ShiftAmount;
6780 const auto LoPattern = m_GBitcast(Src: m_Reg(R&: Lo));
6781 const auto HiPattern = m_GLShr(L: m_GBitcast(Src: m_Reg(R&: Hi)), R: m_GCst(ValReg&: ShiftAmount));
6782 if (mi_match(
6783 MI, MRI,
6784 P: m_any_of(preds: m_GBuildVectorTrunc(L: LoPattern, R: HiPattern),
6785 preds: m_GBuildVector(L: m_GTrunc(Src: LoPattern), R: m_GTrunc(Src: HiPattern))))) {
6786 if (Lo == Hi && ShiftAmount->Value == DstEltTy.getSizeInBits()) {
6787 MatchInfo = Lo;
6788 return MRI.getType(Reg: MatchInfo) == DstVecTy;
6789 }
6790 }
6791
6792 return false;
6793}
6794
6795bool CombinerHelper::matchTruncBuildVectorFold(MachineInstr &MI,
6796 Register &MatchInfo) const {
6797 // Replace (G_TRUNC (G_BITCAST (G_BUILD_VECTOR x, y)) with just x
6798 // if type(x) == type(G_TRUNC)
6799 if (!mi_match(R: MI.getOperand(i: 1).getReg(), MRI,
6800 P: m_GBitcast(Src: m_GBuildVector(L: m_Reg(R&: MatchInfo), R: m_Reg()))))
6801 return false;
6802
6803 return MRI.getType(Reg: MatchInfo) == MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6804}
6805
6806bool CombinerHelper::matchTruncLshrBuildVectorFold(MachineInstr &MI,
6807 Register &MatchInfo) const {
6808 // Replace (G_TRUNC (G_LSHR (G_BITCAST (G_BUILD_VECTOR x, y)), K)) with
6809 // y if K == size of vector element type
6810 std::optional<ValueAndVReg> ShiftAmt;
6811 if (!mi_match(R: MI.getOperand(i: 1).getReg(), MRI,
6812 P: m_GLShr(L: m_GBitcast(Src: m_GBuildVector(L: m_Reg(), R: m_Reg(R&: MatchInfo))),
6813 R: m_GCst(ValReg&: ShiftAmt))))
6814 return false;
6815
6816 LLT MatchTy = MRI.getType(Reg: MatchInfo);
6817 return ShiftAmt->Value.getZExtValue() == MatchTy.getSizeInBits() &&
6818 MatchTy == MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6819}
6820
6821unsigned CombinerHelper::getFPMinMaxOpcForSelect(
6822 CmpInst::Predicate Pred, LLT DstTy,
6823 SelectPatternNaNBehaviour VsNaNRetVal) const {
6824 assert(VsNaNRetVal != SelectPatternNaNBehaviour::NOT_APPLICABLE &&
6825 "Expected a NaN behaviour?");
6826 // Choose an opcode based off of legality or the behaviour when one of the
6827 // LHS/RHS may be NaN.
6828 switch (Pred) {
6829 default:
6830 return 0;
6831 case CmpInst::FCMP_UGT:
6832 case CmpInst::FCMP_UGE:
6833 case CmpInst::FCMP_OGT:
6834 case CmpInst::FCMP_OGE:
6835 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
6836 return TargetOpcode::G_FMAXNUM;
6837 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
6838 return TargetOpcode::G_FMAXIMUM;
6839 if (isLegal(Query: {TargetOpcode::G_FMAXNUM, {DstTy}}))
6840 return TargetOpcode::G_FMAXNUM;
6841 if (isLegal(Query: {TargetOpcode::G_FMAXIMUM, {DstTy}}))
6842 return TargetOpcode::G_FMAXIMUM;
6843 return 0;
6844 case CmpInst::FCMP_ULT:
6845 case CmpInst::FCMP_ULE:
6846 case CmpInst::FCMP_OLT:
6847 case CmpInst::FCMP_OLE:
6848 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
6849 return TargetOpcode::G_FMINNUM;
6850 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
6851 return TargetOpcode::G_FMINIMUM;
6852 if (isLegal(Query: {TargetOpcode::G_FMINNUM, {DstTy}}))
6853 return TargetOpcode::G_FMINNUM;
6854 if (!isLegal(Query: {TargetOpcode::G_FMINIMUM, {DstTy}}))
6855 return 0;
6856 return TargetOpcode::G_FMINIMUM;
6857 }
6858}
6859
6860CombinerHelper::SelectPatternNaNBehaviour
6861CombinerHelper::computeRetValAgainstNaN(Register LHS, Register RHS,
6862 bool IsOrderedComparison) const {
6863 bool LHSSafe = VT->isKnownNeverNaN(Val: LHS);
6864 bool RHSSafe = VT->isKnownNeverNaN(Val: RHS);
6865 // Completely unsafe.
6866 if (!LHSSafe && !RHSSafe)
6867 return SelectPatternNaNBehaviour::NOT_APPLICABLE;
6868 if (LHSSafe && RHSSafe)
6869 return SelectPatternNaNBehaviour::RETURNS_ANY;
6870 // An ordered comparison will return false when given a NaN, so it
6871 // returns the RHS.
6872 if (IsOrderedComparison)
6873 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_NAN
6874 : SelectPatternNaNBehaviour::RETURNS_OTHER;
6875 // An unordered comparison will return true when given a NaN, so it
6876 // returns the LHS.
6877 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_OTHER
6878 : SelectPatternNaNBehaviour::RETURNS_NAN;
6879}
6880
6881bool CombinerHelper::matchFPSelectToMinMax(Register Dst, Register Cond,
6882 Register TrueVal, Register FalseVal,
6883 BuildFnTy &MatchInfo) const {
6884 // Match: select (fcmp cond x, y) x, y
6885 // select (fcmp cond x, y) y, x
6886 // And turn it into fminnum/fmaxnum or fmin/fmax based off of the condition.
6887 LLT DstTy = MRI.getType(Reg: Dst);
6888 // Bail out early on pointers, since we'll never want to fold to a min/max.
6889 if (DstTy.isPointer())
6890 return false;
6891 // Match a floating point compare with a less-than/greater-than predicate.
6892 // TODO: Allow multiple users of the compare if they are all selects.
6893 CmpInst::Predicate Pred;
6894 Register CmpLHS, CmpRHS;
6895 if (!mi_match(R: Cond, MRI,
6896 P: m_OneNonDBGUse(
6897 SP: m_GFCmp(P: m_Pred(P&: Pred), L: m_Reg(R&: CmpLHS), R: m_Reg(R&: CmpRHS)))) ||
6898 CmpInst::isEquality(pred: Pred))
6899 return false;
6900 SelectPatternNaNBehaviour ResWithKnownNaNInfo =
6901 computeRetValAgainstNaN(LHS: CmpLHS, RHS: CmpRHS, IsOrderedComparison: CmpInst::isOrdered(predicate: Pred));
6902 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::NOT_APPLICABLE)
6903 return false;
6904 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
6905 std::swap(a&: CmpLHS, b&: CmpRHS);
6906 Pred = CmpInst::getSwappedPredicate(pred: Pred);
6907 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_NAN)
6908 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_OTHER;
6909 else if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_OTHER)
6910 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_NAN;
6911 }
6912 if (TrueVal != CmpLHS || FalseVal != CmpRHS)
6913 return false;
6914 // Decide what type of max/min this should be based off of the predicate.
6915 unsigned Opc = getFPMinMaxOpcForSelect(Pred, DstTy, VsNaNRetVal: ResWithKnownNaNInfo);
6916 if (!Opc || !isLegal(Query: {Opc, {DstTy}}))
6917 return false;
6918 // Comparisons between signed zero and zero may have different results...
6919 // unless we have fmaximum/fminimum. In that case, we know -0 < 0.
6920 if (Opc != TargetOpcode::G_FMAXIMUM && Opc != TargetOpcode::G_FMINIMUM) {
6921 // We don't know if a comparison between two 0s will give us a consistent
6922 // result. Be conservative and only proceed if at least one side is
6923 // non-zero.
6924 auto KnownNonZeroSide = getFConstantVRegValWithLookThrough(VReg: CmpLHS, MRI);
6925 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero()) {
6926 KnownNonZeroSide = getFConstantVRegValWithLookThrough(VReg: CmpRHS, MRI);
6927 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero())
6928 return false;
6929 }
6930 }
6931 MatchInfo = [=](MachineIRBuilder &B) {
6932 B.buildInstr(Opc, DstOps: {Dst}, SrcOps: {CmpLHS, CmpRHS});
6933 };
6934 return true;
6935}
6936
6937bool CombinerHelper::matchSimplifySelectToMinMax(MachineInstr &MI,
6938 BuildFnTy &MatchInfo) const {
6939 // TODO: Handle integer cases.
6940 assert(MI.getOpcode() == TargetOpcode::G_SELECT);
6941 // Condition may be fed by a truncated compare.
6942 Register Cond = MI.getOperand(i: 1).getReg();
6943 Register MaybeTrunc;
6944 if (mi_match(R: Cond, MRI, P: m_OneNonDBGUse(SP: m_GTrunc(Src: m_Reg(R&: MaybeTrunc)))))
6945 Cond = MaybeTrunc;
6946 Register Dst = MI.getOperand(i: 0).getReg();
6947 Register TrueVal = MI.getOperand(i: 2).getReg();
6948 Register FalseVal = MI.getOperand(i: 3).getReg();
6949 return matchFPSelectToMinMax(Dst, Cond, TrueVal, FalseVal, MatchInfo);
6950}
6951
6952bool CombinerHelper::matchRedundantBinOpInEquality(MachineInstr &MI,
6953 BuildFnTy &MatchInfo) const {
6954 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
6955 // (X + Y) == X --> Y == 0
6956 // (X + Y) != X --> Y != 0
6957 // (X - Y) == X --> Y == 0
6958 // (X - Y) != X --> Y != 0
6959 // (X ^ Y) == X --> Y == 0
6960 // (X ^ Y) != X --> Y != 0
6961 Register Dst = MI.getOperand(i: 0).getReg();
6962 CmpInst::Predicate Pred;
6963 Register X, Y, OpLHS, OpRHS;
6964 bool MatchedSub = mi_match(
6965 R: Dst, MRI,
6966 P: m_c_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: X), R: m_GSub(L: m_Reg(R&: OpLHS), R: m_Reg(R&: Y))));
6967 if (MatchedSub && X != OpLHS)
6968 return false;
6969 if (!MatchedSub) {
6970 if (!mi_match(R: Dst, MRI,
6971 P: m_c_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: X),
6972 R: m_any_of(preds: m_GAdd(L: m_Reg(R&: OpLHS), R: m_Reg(R&: OpRHS)),
6973 preds: m_GXor(L: m_Reg(R&: OpLHS), R: m_Reg(R&: OpRHS))))))
6974 return false;
6975 Y = X == OpLHS ? OpRHS : X == OpRHS ? OpLHS : Register();
6976 }
6977 MatchInfo = [=](MachineIRBuilder &B) {
6978 auto Zero = B.buildConstant(Res: MRI.getType(Reg: Y), Val: 0);
6979 B.buildICmp(Pred, Res: Dst, Op0: Y, Op1: Zero);
6980 };
6981 return CmpInst::isEquality(pred: Pred) && Y.isValid();
6982}
6983
6984/// Return the minimum useless shift amount that results in complete loss of the
6985/// source value. Return std::nullopt when it cannot determine a value.
6986static std::optional<unsigned>
6987getMinUselessShift(KnownBits ValueKB, unsigned Opcode,
6988 std::optional<int64_t> &Result) {
6989 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR ||
6990 Opcode == TargetOpcode::G_ASHR) &&
6991 "Expect G_SHL, G_LSHR or G_ASHR.");
6992 auto SignificantBits = 0;
6993 switch (Opcode) {
6994 case TargetOpcode::G_SHL:
6995 SignificantBits = ValueKB.countMinTrailingZeros();
6996 Result = 0;
6997 break;
6998 case TargetOpcode::G_LSHR:
6999 Result = 0;
7000 SignificantBits = ValueKB.countMinLeadingZeros();
7001 break;
7002 case TargetOpcode::G_ASHR:
7003 if (ValueKB.isNonNegative()) {
7004 SignificantBits = ValueKB.countMinLeadingZeros();
7005 Result = 0;
7006 } else if (ValueKB.isNegative()) {
7007 SignificantBits = ValueKB.countMinLeadingOnes();
7008 Result = -1;
7009 } else {
7010 // Cannot determine shift result.
7011 Result = std::nullopt;
7012 }
7013 break;
7014 default:
7015 break;
7016 }
7017 return ValueKB.getBitWidth() - SignificantBits;
7018}
7019
7020bool CombinerHelper::matchShiftsTooBig(
7021 MachineInstr &MI, std::optional<int64_t> &MatchInfo) const {
7022 Register ShiftVal = MI.getOperand(i: 1).getReg();
7023 Register ShiftReg = MI.getOperand(i: 2).getReg();
7024 LLT ResTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7025 auto IsShiftTooBig = [&](const Constant *C) {
7026 auto *CI = dyn_cast<ConstantInt>(Val: C);
7027 if (!CI)
7028 return false;
7029 if (CI->uge(Num: ResTy.getScalarSizeInBits())) {
7030 MatchInfo = std::nullopt;
7031 return true;
7032 }
7033 auto OptMaxUsefulShift = getMinUselessShift(ValueKB: VT->getKnownBits(R: ShiftVal),
7034 Opcode: MI.getOpcode(), Result&: MatchInfo);
7035 return OptMaxUsefulShift && CI->uge(Num: *OptMaxUsefulShift);
7036 };
7037 return matchUnaryPredicate(MRI, Reg: ShiftReg, Match: IsShiftTooBig);
7038}
7039
7040bool CombinerHelper::matchCommuteConstantToRHS(MachineInstr &MI) const {
7041 unsigned LHSOpndIdx = 1;
7042 unsigned RHSOpndIdx = 2;
7043 switch (MI.getOpcode()) {
7044 case TargetOpcode::G_UADDO:
7045 case TargetOpcode::G_SADDO:
7046 case TargetOpcode::G_UMULO:
7047 case TargetOpcode::G_SMULO:
7048 LHSOpndIdx = 2;
7049 RHSOpndIdx = 3;
7050 break;
7051 default:
7052 break;
7053 }
7054 Register LHS = MI.getOperand(i: LHSOpndIdx).getReg();
7055 Register RHS = MI.getOperand(i: RHSOpndIdx).getReg();
7056 MachineInstr *LHSDef, *RHSDef;
7057 if (!mi_match(R: LHS, MRI, P: m_MInstr(MI&: LHSDef)) ||
7058 !mi_match(R: RHS, MRI, P: m_MInstr(MI&: RHSDef)))
7059 return false;
7060
7061 if (!getIConstantVRegVal(VReg: LHS, MRI)) {
7062 // Skip commuting if LHS is not a constant. But, LHS may be a
7063 // G_CONSTANT_FOLD_BARRIER. If so we commute as long as we don't already
7064 // have a constant on the RHS.
7065 if (LHSDef->getOpcode() != TargetOpcode::G_CONSTANT_FOLD_BARRIER)
7066 return false;
7067 }
7068 // Commute as long as RHS is not a constant or G_CONSTANT_FOLD_BARRIER.
7069 return RHSDef->getOpcode() != TargetOpcode::G_CONSTANT_FOLD_BARRIER &&
7070 !getIConstantVRegVal(VReg: RHS, MRI);
7071}
7072
7073bool CombinerHelper::matchCommuteFPConstantToRHS(MachineInstr &MI) const {
7074 Register LHS = MI.getOperand(i: 1).getReg();
7075 Register RHS = MI.getOperand(i: 2).getReg();
7076 std::optional<FPValueAndVReg> ValAndVReg;
7077 if (!mi_match(R: LHS, MRI, P: m_GFCstOrSplat(FPValReg&: ValAndVReg)))
7078 return false;
7079 return !mi_match(R: RHS, MRI, P: m_GFCstOrSplat(FPValReg&: ValAndVReg));
7080}
7081
7082bool CombinerHelper::isOneOrOneSplat(Register Src, bool AllowUndefs) const {
7083 LLT SrcTy = MRI.getType(Reg: Src);
7084 if (SrcTy.isFixedVector())
7085 return isConstantSplatVector(Src, SplatValue: 1, AllowUndefs);
7086 if (SrcTy.isScalar()) {
7087 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Reg: Src, MRI) != nullptr)
7088 return true;
7089 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7090 return IConstant && IConstant->Value == 1;
7091 }
7092 return false; // scalable vector
7093}
7094
7095bool CombinerHelper::isZeroOrZeroSplat(Register Src, bool AllowUndefs) const {
7096 LLT SrcTy = MRI.getType(Reg: Src);
7097 if (SrcTy.isFixedVector())
7098 return isConstantSplatVector(Src, SplatValue: 0, AllowUndefs);
7099 if (SrcTy.isScalar()) {
7100 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Reg: Src, MRI) != nullptr)
7101 return true;
7102 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7103 return IConstant && IConstant->Value == 0;
7104 }
7105 return false; // scalable vector
7106}
7107
7108// Ignores COPYs during conformance checks.
7109// FIXME scalable vectors.
7110bool CombinerHelper::isConstantSplatVector(Register Src, int64_t SplatValue,
7111 bool AllowUndefs) const {
7112 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
7113 if (!BuildVector)
7114 return false;
7115 unsigned NumSources = BuildVector->getNumSources();
7116
7117 for (unsigned I = 0; I < NumSources; ++I) {
7118 GImplicitDef *ImplicitDef =
7119 getOpcodeDef<GImplicitDef>(Reg: BuildVector->getSourceReg(I), MRI);
7120 if (ImplicitDef && AllowUndefs)
7121 continue;
7122 if (ImplicitDef && !AllowUndefs)
7123 return false;
7124 std::optional<ValueAndVReg> IConstant =
7125 getIConstantVRegValWithLookThrough(VReg: BuildVector->getSourceReg(I), MRI);
7126 if (IConstant && IConstant->Value == SplatValue)
7127 continue;
7128 return false;
7129 }
7130 return true;
7131}
7132
7133// Ignores COPYs during lookups.
7134// FIXME scalable vectors
7135std::optional<APInt>
7136CombinerHelper::getConstantOrConstantSplatVector(Register Src) const {
7137 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7138 if (IConstant)
7139 return IConstant->Value;
7140
7141 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
7142 if (!BuildVector)
7143 return std::nullopt;
7144 unsigned NumSources = BuildVector->getNumSources();
7145
7146 std::optional<APInt> Value = std::nullopt;
7147 for (unsigned I = 0; I < NumSources; ++I) {
7148 std::optional<ValueAndVReg> IConstant =
7149 getIConstantVRegValWithLookThrough(VReg: BuildVector->getSourceReg(I), MRI);
7150 if (!IConstant)
7151 return std::nullopt;
7152 if (!Value)
7153 Value = IConstant->Value;
7154 else if (*Value != IConstant->Value)
7155 return std::nullopt;
7156 }
7157 return Value;
7158}
7159
7160// FIXME G_SPLAT_VECTOR
7161bool CombinerHelper::isConstantOrConstantVectorI(Register Src) const {
7162 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7163 if (IConstant)
7164 return true;
7165
7166 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
7167 if (!BuildVector)
7168 return false;
7169
7170 unsigned NumSources = BuildVector->getNumSources();
7171 for (unsigned I = 0; I < NumSources; ++I) {
7172 std::optional<ValueAndVReg> IConstant =
7173 getIConstantVRegValWithLookThrough(VReg: BuildVector->getSourceReg(I), MRI);
7174 if (!IConstant)
7175 return false;
7176 }
7177 return true;
7178}
7179
7180// TODO: use knownbits to determine zeros
7181bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select,
7182 BuildFnTy &MatchInfo) const {
7183 uint32_t Flags = Select->getFlags();
7184 Register Dest = Select->getReg(Idx: 0);
7185 Register Cond = Select->getCondReg();
7186 Register True = Select->getTrueReg();
7187 Register False = Select->getFalseReg();
7188 LLT CondTy = MRI.getType(Reg: Select->getCondReg());
7189 LLT TrueTy = MRI.getType(Reg: Select->getTrueReg());
7190
7191 // We only do this combine for scalar boolean conditions.
7192 if (CondTy != LLT::scalar(SizeInBits: 1))
7193 return false;
7194
7195 if (TrueTy.isPointer())
7196 return false;
7197
7198 // Both are scalars.
7199 std::optional<ValueAndVReg> TrueOpt =
7200 getIConstantVRegValWithLookThrough(VReg: True, MRI);
7201 std::optional<ValueAndVReg> FalseOpt =
7202 getIConstantVRegValWithLookThrough(VReg: False, MRI);
7203
7204 if (!TrueOpt || !FalseOpt)
7205 return false;
7206
7207 APInt TrueValue = TrueOpt->Value;
7208 APInt FalseValue = FalseOpt->Value;
7209
7210 // select Cond, 1, 0 --> zext (Cond)
7211 if (TrueValue.isOne() && FalseValue.isZero()) {
7212 MatchInfo = [=](MachineIRBuilder &B) {
7213 B.setInstrAndDebugLoc(*Select);
7214 B.buildZExtOrTrunc(Res: Dest, Op: Cond);
7215 };
7216 return true;
7217 }
7218
7219 // select Cond, -1, 0 --> sext (Cond)
7220 if (TrueValue.isAllOnes() && FalseValue.isZero()) {
7221 MatchInfo = [=](MachineIRBuilder &B) {
7222 B.setInstrAndDebugLoc(*Select);
7223 B.buildSExtOrTrunc(Res: Dest, Op: Cond);
7224 };
7225 return true;
7226 }
7227
7228 // select Cond, 0, 1 --> zext (!Cond)
7229 if (TrueValue.isZero() && FalseValue.isOne()) {
7230 MatchInfo = [=](MachineIRBuilder &B) {
7231 B.setInstrAndDebugLoc(*Select);
7232 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7233 B.buildNot(Dst: Inner, Src0: Cond);
7234 B.buildZExtOrTrunc(Res: Dest, Op: Inner);
7235 };
7236 return true;
7237 }
7238
7239 // select Cond, 0, -1 --> sext (!Cond)
7240 if (TrueValue.isZero() && FalseValue.isAllOnes()) {
7241 MatchInfo = [=](MachineIRBuilder &B) {
7242 B.setInstrAndDebugLoc(*Select);
7243 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7244 B.buildNot(Dst: Inner, Src0: Cond);
7245 B.buildSExtOrTrunc(Res: Dest, Op: Inner);
7246 };
7247 return true;
7248 }
7249
7250 // select Cond, C1, C1-1 --> add (zext Cond), C1-1
7251 if (TrueValue - 1 == FalseValue) {
7252 MatchInfo = [=](MachineIRBuilder &B) {
7253 B.setInstrAndDebugLoc(*Select);
7254 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7255 B.buildZExtOrTrunc(Res: Inner, Op: Cond);
7256 B.buildAdd(Dst: Dest, Src0: Inner, Src1: False);
7257 };
7258 return true;
7259 }
7260
7261 // select Cond, C1, C1+1 --> add (sext Cond), C1+1
7262 if (TrueValue + 1 == FalseValue) {
7263 MatchInfo = [=](MachineIRBuilder &B) {
7264 B.setInstrAndDebugLoc(*Select);
7265 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7266 B.buildSExtOrTrunc(Res: Inner, Op: Cond);
7267 B.buildAdd(Dst: Dest, Src0: Inner, Src1: False);
7268 };
7269 return true;
7270 }
7271
7272 // select Cond, Pow2, 0 --> (zext Cond) << log2(Pow2)
7273 if (TrueValue.isPowerOf2() && FalseValue.isZero()) {
7274 MatchInfo = [=](MachineIRBuilder &B) {
7275 B.setInstrAndDebugLoc(*Select);
7276 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7277 B.buildZExtOrTrunc(Res: Inner, Op: Cond);
7278 // The shift amount must be scalar.
7279 LLT ShiftTy = TrueTy.isVector() ? TrueTy.getElementType() : TrueTy;
7280 auto ShAmtC = B.buildConstant(Res: ShiftTy, Val: TrueValue.exactLogBase2());
7281 B.buildShl(Dst: Dest, Src0: Inner, Src1: ShAmtC, Flags);
7282 };
7283 return true;
7284 }
7285
7286 // select Cond, 0, Pow2 --> (zext (!Cond)) << log2(Pow2)
7287 if (FalseValue.isPowerOf2() && TrueValue.isZero()) {
7288 MatchInfo = [=](MachineIRBuilder &B) {
7289 B.setInstrAndDebugLoc(*Select);
7290 Register Not = MRI.createGenericVirtualRegister(Ty: CondTy);
7291 B.buildNot(Dst: Not, Src0: Cond);
7292 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7293 B.buildZExtOrTrunc(Res: Inner, Op: Not);
7294 // The shift amount must be scalar.
7295 LLT ShiftTy = TrueTy.isVector() ? TrueTy.getElementType() : TrueTy;
7296 auto ShAmtC = B.buildConstant(Res: ShiftTy, Val: FalseValue.exactLogBase2());
7297 B.buildShl(Dst: Dest, Src0: Inner, Src1: ShAmtC, Flags);
7298 };
7299 return true;
7300 }
7301
7302 // select Cond, -1, C --> or (sext Cond), C
7303 if (TrueValue.isAllOnes()) {
7304 MatchInfo = [=](MachineIRBuilder &B) {
7305 B.setInstrAndDebugLoc(*Select);
7306 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7307 B.buildSExtOrTrunc(Res: Inner, Op: Cond);
7308 B.buildOr(Dst: Dest, Src0: Inner, Src1: False, Flags);
7309 };
7310 return true;
7311 }
7312
7313 // select Cond, C, -1 --> or (sext (not Cond)), C
7314 if (FalseValue.isAllOnes()) {
7315 MatchInfo = [=](MachineIRBuilder &B) {
7316 B.setInstrAndDebugLoc(*Select);
7317 Register Not = MRI.createGenericVirtualRegister(Ty: CondTy);
7318 B.buildNot(Dst: Not, Src0: Cond);
7319 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7320 B.buildSExtOrTrunc(Res: Inner, Op: Not);
7321 B.buildOr(Dst: Dest, Src0: Inner, Src1: True, Flags);
7322 };
7323 return true;
7324 }
7325
7326 return false;
7327}
7328
7329// TODO: use knownbits to determine zeros
7330bool CombinerHelper::tryFoldBoolSelectToLogic(GSelect *Select,
7331 BuildFnTy &MatchInfo) const {
7332 uint32_t Flags = Select->getFlags();
7333 Register DstReg = Select->getReg(Idx: 0);
7334 Register Cond = Select->getCondReg();
7335 Register True = Select->getTrueReg();
7336 Register False = Select->getFalseReg();
7337 LLT CondTy = MRI.getType(Reg: Select->getCondReg());
7338 LLT TrueTy = MRI.getType(Reg: Select->getTrueReg());
7339
7340 // Boolean or fixed vector of booleans.
7341 if (CondTy.isScalableVector() ||
7342 (CondTy.isFixedVector() &&
7343 CondTy.getElementType().getScalarSizeInBits() != 1) ||
7344 CondTy.getScalarSizeInBits() != 1)
7345 return false;
7346
7347 if (CondTy != TrueTy)
7348 return false;
7349
7350 // select Cond, Cond, F --> or Cond, F
7351 // select Cond, 1, F --> or Cond, F
7352 if ((Cond == True) || isOneOrOneSplat(Src: True, /* AllowUndefs */ true)) {
7353 MatchInfo = [=](MachineIRBuilder &B) {
7354 B.setInstrAndDebugLoc(*Select);
7355 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7356 B.buildZExtOrTrunc(Res: Ext, Op: Cond);
7357 auto FreezeFalse = B.buildFreeze(Dst: TrueTy, Src: False);
7358 B.buildOr(Dst: DstReg, Src0: Ext, Src1: FreezeFalse, Flags);
7359 };
7360 return true;
7361 }
7362
7363 // select Cond, T, Cond --> and Cond, T
7364 // select Cond, T, 0 --> and Cond, T
7365 if ((Cond == False) || isZeroOrZeroSplat(Src: False, /* AllowUndefs */ true)) {
7366 MatchInfo = [=](MachineIRBuilder &B) {
7367 B.setInstrAndDebugLoc(*Select);
7368 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7369 B.buildZExtOrTrunc(Res: Ext, Op: Cond);
7370 auto FreezeTrue = B.buildFreeze(Dst: TrueTy, Src: True);
7371 B.buildAnd(Dst: DstReg, Src0: Ext, Src1: FreezeTrue);
7372 };
7373 return true;
7374 }
7375
7376 // select Cond, T, 1 --> or (not Cond), T
7377 if (isOneOrOneSplat(Src: False, /* AllowUndefs */ true)) {
7378 MatchInfo = [=](MachineIRBuilder &B) {
7379 B.setInstrAndDebugLoc(*Select);
7380 // First the not.
7381 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7382 B.buildNot(Dst: Inner, Src0: Cond);
7383 // Then an ext to match the destination register.
7384 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7385 B.buildZExtOrTrunc(Res: Ext, Op: Inner);
7386 auto FreezeTrue = B.buildFreeze(Dst: TrueTy, Src: True);
7387 B.buildOr(Dst: DstReg, Src0: Ext, Src1: FreezeTrue, Flags);
7388 };
7389 return true;
7390 }
7391
7392 // select Cond, 0, F --> and (not Cond), F
7393 if (isZeroOrZeroSplat(Src: True, /* AllowUndefs */ true)) {
7394 MatchInfo = [=](MachineIRBuilder &B) {
7395 B.setInstrAndDebugLoc(*Select);
7396 // First the not.
7397 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7398 B.buildNot(Dst: Inner, Src0: Cond);
7399 // Then an ext to match the destination register.
7400 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7401 B.buildZExtOrTrunc(Res: Ext, Op: Inner);
7402 auto FreezeFalse = B.buildFreeze(Dst: TrueTy, Src: False);
7403 B.buildAnd(Dst: DstReg, Src0: Ext, Src1: FreezeFalse);
7404 };
7405 return true;
7406 }
7407
7408 return false;
7409}
7410
7411bool CombinerHelper::matchSelectIMinMax(const MachineOperand &MO,
7412 BuildFnTy &MatchInfo) const {
7413 Register DstReg = MO.getReg();
7414 Register CondReg, True, False;
7415 if (!mi_match(R: DstReg, MRI,
7416 P: m_GISelect(Src0: m_Reg(R&: CondReg), Src1: m_Reg(R&: True), Src2: m_Reg(R&: False))))
7417 return false;
7418
7419 CmpInst::Predicate Pred;
7420 Register CmpLHS, CmpRHS;
7421 if (!mi_match(R: CondReg, MRI,
7422 P: m_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: CmpLHS), R: m_Reg(R&: CmpRHS))))
7423 return false;
7424
7425 LLT DstTy = MRI.getType(Reg: DstReg);
7426 if (DstTy.isPointerOrPointerVector())
7427 return false;
7428
7429 // We want to fold the icmp and replace the select.
7430 if (!MRI.hasOneNonDBGUse(RegNo: CondReg))
7431 return false;
7432
7433 // We need a larger or smaller predicate for
7434 // canonicalization.
7435 if (CmpInst::isEquality(pred: Pred))
7436 return false;
7437
7438 // We can swap CmpLHS and CmpRHS for higher hitrate.
7439 if (True == CmpRHS && False == CmpLHS) {
7440 std::swap(a&: CmpLHS, b&: CmpRHS);
7441 Pred = CmpInst::getSwappedPredicate(pred: Pred);
7442 }
7443
7444 // (icmp X, Y) ? X : Y -> integer minmax.
7445 // see matchSelectPattern in ValueTracking.
7446 // Legality between G_SELECT and integer minmax can differ.
7447 if (True != CmpLHS || False != CmpRHS)
7448 return false;
7449
7450 switch (Pred) {
7451 case ICmpInst::ICMP_UGT:
7452 case ICmpInst::ICMP_UGE: {
7453 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_UMAX, DstTy}))
7454 return false;
7455 MatchInfo = [=](MachineIRBuilder &B) { B.buildUMax(Dst: DstReg, Src0: True, Src1: False); };
7456 return true;
7457 }
7458 case ICmpInst::ICMP_SGT:
7459 case ICmpInst::ICMP_SGE: {
7460 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SMAX, DstTy}))
7461 return false;
7462 MatchInfo = [=](MachineIRBuilder &B) { B.buildSMax(Dst: DstReg, Src0: True, Src1: False); };
7463 return true;
7464 }
7465 case ICmpInst::ICMP_ULT:
7466 case ICmpInst::ICMP_ULE: {
7467 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_UMIN, DstTy}))
7468 return false;
7469 MatchInfo = [=](MachineIRBuilder &B) { B.buildUMin(Dst: DstReg, Src0: True, Src1: False); };
7470 return true;
7471 }
7472 case ICmpInst::ICMP_SLT:
7473 case ICmpInst::ICMP_SLE: {
7474 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SMIN, DstTy}))
7475 return false;
7476 MatchInfo = [=](MachineIRBuilder &B) { B.buildSMin(Dst: DstReg, Src0: True, Src1: False); };
7477 return true;
7478 }
7479 default:
7480 return false;
7481 }
7482}
7483
7484// (neg (min/max x, (neg x))) --> (max/min x, (neg x))
7485bool CombinerHelper::matchSimplifyNegMinMax(MachineInstr &MI,
7486 BuildFnTy &MatchInfo) const {
7487 assert(MI.getOpcode() == TargetOpcode::G_SUB);
7488 Register DestReg = MI.getOperand(i: 0).getReg();
7489 LLT DestTy = MRI.getType(Reg: DestReg);
7490
7491 Register X;
7492 Register Sub0;
7493 auto NegPattern = m_all_of(preds: m_Neg(Src: m_DeferredReg(R&: X)), preds: m_Reg(R&: Sub0));
7494 if (mi_match(R: DestReg, MRI,
7495 P: m_Neg(Src: m_OneUse(SP: m_any_of(preds: m_GSMin(L: m_Reg(R&: X), R: NegPattern),
7496 preds: m_GSMax(L: m_Reg(R&: X), R: NegPattern),
7497 preds: m_GUMin(L: m_Reg(R&: X), R: NegPattern),
7498 preds: m_GUMax(L: m_Reg(R&: X), R: NegPattern)))))) {
7499 MachineInstr *MinMaxMI;
7500 if (!mi_match(R: MI.getOperand(i: 2).getReg(), MRI, P: m_MInstr(MI&: MinMaxMI)))
7501 return false;
7502 unsigned NewOpc = getInverseGMinMaxOpcode(MinMaxOpc: MinMaxMI->getOpcode());
7503 if (isLegal(Query: {NewOpc, {DestTy}})) {
7504 MatchInfo = [=](MachineIRBuilder &B) {
7505 B.buildInstr(Opc: NewOpc, DstOps: {DestReg}, SrcOps: {X, Sub0});
7506 };
7507 return true;
7508 }
7509 }
7510
7511 return false;
7512}
7513
7514bool CombinerHelper::matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo) const {
7515 GSelect *Select = cast<GSelect>(Val: &MI);
7516
7517 if (tryFoldSelectOfConstants(Select, MatchInfo))
7518 return true;
7519
7520 if (tryFoldBoolSelectToLogic(Select, MatchInfo))
7521 return true;
7522
7523 return false;
7524}
7525
7526/// Fold (icmp Pred1 V1, C1) && (icmp Pred2 V2, C2)
7527/// or (icmp Pred1 V1, C1) || (icmp Pred2 V2, C2)
7528/// into a single comparison using range-based reasoning.
7529/// see InstCombinerImpl::foldAndOrOfICmpsUsingRanges.
7530bool CombinerHelper::tryFoldAndOrOrICmpsUsingRanges(
7531 GLogicalBinOp *Logic, BuildFnTy &MatchInfo) const {
7532 assert(Logic->getOpcode() != TargetOpcode::G_XOR && "unexpected xor");
7533 bool IsAnd = Logic->getOpcode() == TargetOpcode::G_AND;
7534 Register DstReg = Logic->getReg(Idx: 0);
7535 Register LHS = Logic->getLHSReg();
7536 Register RHS = Logic->getRHSReg();
7537 unsigned Flags = Logic->getFlags();
7538
7539 // We need an G_ICMP on the LHS register.
7540 GICmp *Cmp1 = getOpcodeDef<GICmp>(Reg: LHS, MRI);
7541 if (!Cmp1)
7542 return false;
7543
7544 // We need an G_ICMP on the RHS register.
7545 GICmp *Cmp2 = getOpcodeDef<GICmp>(Reg: RHS, MRI);
7546 if (!Cmp2)
7547 return false;
7548
7549 // We want to fold the icmps.
7550 if (!MRI.hasOneNonDBGUse(RegNo: Cmp1->getReg(Idx: 0)) ||
7551 !MRI.hasOneNonDBGUse(RegNo: Cmp2->getReg(Idx: 0)))
7552 return false;
7553
7554 APInt C1;
7555 APInt C2;
7556 std::optional<ValueAndVReg> MaybeC1 =
7557 getIConstantVRegValWithLookThrough(VReg: Cmp1->getRHSReg(), MRI);
7558 if (!MaybeC1)
7559 return false;
7560 C1 = MaybeC1->Value;
7561
7562 std::optional<ValueAndVReg> MaybeC2 =
7563 getIConstantVRegValWithLookThrough(VReg: Cmp2->getRHSReg(), MRI);
7564 if (!MaybeC2)
7565 return false;
7566 C2 = MaybeC2->Value;
7567
7568 Register R1 = Cmp1->getLHSReg();
7569 Register R2 = Cmp2->getLHSReg();
7570 CmpInst::Predicate Pred1 = Cmp1->getCond();
7571 CmpInst::Predicate Pred2 = Cmp2->getCond();
7572 LLT CmpTy = MRI.getType(Reg: Cmp1->getReg(Idx: 0));
7573 LLT CmpOperandTy = MRI.getType(Reg: R1);
7574
7575 if (CmpOperandTy.isPointer())
7576 return false;
7577
7578 // We build ands, adds, and constants of type CmpOperandTy.
7579 // They must be legal to build.
7580 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_AND, CmpOperandTy}) ||
7581 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ADD, CmpOperandTy}) ||
7582 !isConstantLegalOrBeforeLegalizer(Ty: CmpOperandTy))
7583 return false;
7584
7585 // Look through add of a constant offset on R1, R2, or both operands. This
7586 // allows us to interpret the R + C' < C'' range idiom into a proper range.
7587 std::optional<APInt> Offset1;
7588 std::optional<APInt> Offset2;
7589 if (R1 != R2) {
7590 if (GAdd *Add = getOpcodeDef<GAdd>(Reg: R1, MRI)) {
7591 std::optional<ValueAndVReg> MaybeOffset1 =
7592 getIConstantVRegValWithLookThrough(VReg: Add->getRHSReg(), MRI);
7593 if (MaybeOffset1) {
7594 R1 = Add->getLHSReg();
7595 Offset1 = MaybeOffset1->Value;
7596 }
7597 }
7598 if (GAdd *Add = getOpcodeDef<GAdd>(Reg: R2, MRI)) {
7599 std::optional<ValueAndVReg> MaybeOffset2 =
7600 getIConstantVRegValWithLookThrough(VReg: Add->getRHSReg(), MRI);
7601 if (MaybeOffset2) {
7602 R2 = Add->getLHSReg();
7603 Offset2 = MaybeOffset2->Value;
7604 }
7605 }
7606 }
7607
7608 if (R1 != R2)
7609 return false;
7610
7611 // We calculate the icmp ranges including maybe offsets.
7612 ConstantRange CR1 = ConstantRange::makeExactICmpRegion(
7613 Pred: IsAnd ? ICmpInst::getInversePredicate(pred: Pred1) : Pred1, Other: C1);
7614 if (Offset1)
7615 CR1 = CR1.subtract(CI: *Offset1);
7616
7617 ConstantRange CR2 = ConstantRange::makeExactICmpRegion(
7618 Pred: IsAnd ? ICmpInst::getInversePredicate(pred: Pred2) : Pred2, Other: C2);
7619 if (Offset2)
7620 CR2 = CR2.subtract(CI: *Offset2);
7621
7622 bool CreateMask = false;
7623 APInt LowerDiff;
7624 std::optional<ConstantRange> CR = CR1.exactUnionWith(CR: CR2);
7625 if (!CR) {
7626 // We need non-wrapping ranges.
7627 if (CR1.isWrappedSet() || CR2.isWrappedSet())
7628 return false;
7629
7630 // Check whether we have equal-size ranges that only differ by one bit.
7631 // In that case we can apply a mask to map one range onto the other.
7632 LowerDiff = CR1.getLower() ^ CR2.getLower();
7633 APInt UpperDiff = (CR1.getUpper() - 1) ^ (CR2.getUpper() - 1);
7634 APInt CR1Size = CR1.getUpper() - CR1.getLower();
7635 if (!LowerDiff.isPowerOf2() || LowerDiff != UpperDiff ||
7636 CR1Size != CR2.getUpper() - CR2.getLower())
7637 return false;
7638
7639 CR = CR1.getLower().ult(RHS: CR2.getLower()) ? CR1 : CR2;
7640 CreateMask = true;
7641 }
7642
7643 if (IsAnd)
7644 CR = CR->inverse();
7645
7646 CmpInst::Predicate NewPred;
7647 APInt NewC, Offset;
7648 CR->getEquivalentICmp(Pred&: NewPred, RHS&: NewC, Offset);
7649
7650 // We take the result type of one of the original icmps, CmpTy, for
7651 // the to be build icmp. The operand type, CmpOperandTy, is used for
7652 // the other instructions and constants to be build. The types of
7653 // the parameters and output are the same for add and and. CmpTy
7654 // and the type of DstReg might differ. That is why we zext or trunc
7655 // the icmp into the destination register.
7656
7657 MatchInfo = [=](MachineIRBuilder &B) {
7658 if (CreateMask && Offset != 0) {
7659 auto TildeLowerDiff = B.buildConstant(Res: CmpOperandTy, Val: ~LowerDiff);
7660 auto And = B.buildAnd(Dst: CmpOperandTy, Src0: R1, Src1: TildeLowerDiff); // the mask.
7661 auto OffsetC = B.buildConstant(Res: CmpOperandTy, Val: Offset);
7662 auto Add = B.buildAdd(Dst: CmpOperandTy, Src0: And, Src1: OffsetC, Flags);
7663 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
7664 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: Add, Op1: NewCon);
7665 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
7666 } else if (CreateMask && Offset == 0) {
7667 auto TildeLowerDiff = B.buildConstant(Res: CmpOperandTy, Val: ~LowerDiff);
7668 auto And = B.buildAnd(Dst: CmpOperandTy, Src0: R1, Src1: TildeLowerDiff); // the mask.
7669 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
7670 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: And, Op1: NewCon);
7671 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
7672 } else if (!CreateMask && Offset != 0) {
7673 auto OffsetC = B.buildConstant(Res: CmpOperandTy, Val: Offset);
7674 auto Add = B.buildAdd(Dst: CmpOperandTy, Src0: R1, Src1: OffsetC, Flags);
7675 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
7676 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: Add, Op1: NewCon);
7677 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
7678 } else if (!CreateMask && Offset == 0) {
7679 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
7680 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: R1, Op1: NewCon);
7681 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
7682 } else {
7683 llvm_unreachable("unexpected configuration of CreateMask and Offset");
7684 }
7685 };
7686 return true;
7687}
7688
7689bool CombinerHelper::tryFoldLogicOfFCmps(GLogicalBinOp *Logic,
7690 BuildFnTy &MatchInfo) const {
7691 assert(Logic->getOpcode() != TargetOpcode::G_XOR && "unexpecte xor");
7692 Register DestReg = Logic->getReg(Idx: 0);
7693 Register LHS = Logic->getLHSReg();
7694 Register RHS = Logic->getRHSReg();
7695 bool IsAnd = Logic->getOpcode() == TargetOpcode::G_AND;
7696
7697 // We need a compare on the LHS register.
7698 GFCmp *Cmp1 = getOpcodeDef<GFCmp>(Reg: LHS, MRI);
7699 if (!Cmp1)
7700 return false;
7701
7702 // We need a compare on the RHS register.
7703 GFCmp *Cmp2 = getOpcodeDef<GFCmp>(Reg: RHS, MRI);
7704 if (!Cmp2)
7705 return false;
7706
7707 LLT CmpTy = MRI.getType(Reg: Cmp1->getReg(Idx: 0));
7708 LLT CmpOperandTy = MRI.getType(Reg: Cmp1->getLHSReg());
7709
7710 // We build one fcmp, want to fold the fcmps, replace the logic op,
7711 // and the fcmps must have the same shape.
7712 if (!isLegalOrBeforeLegalizer(
7713 Query: {TargetOpcode::G_FCMP, {CmpTy, CmpOperandTy}}) ||
7714 !MRI.hasOneNonDBGUse(RegNo: Logic->getReg(Idx: 0)) ||
7715 !MRI.hasOneNonDBGUse(RegNo: Cmp1->getReg(Idx: 0)) ||
7716 !MRI.hasOneNonDBGUse(RegNo: Cmp2->getReg(Idx: 0)) ||
7717 MRI.getType(Reg: Cmp1->getLHSReg()) != MRI.getType(Reg: Cmp2->getLHSReg()))
7718 return false;
7719
7720 CmpInst::Predicate PredL = Cmp1->getCond();
7721 CmpInst::Predicate PredR = Cmp2->getCond();
7722 Register LHS0 = Cmp1->getLHSReg();
7723 Register LHS1 = Cmp1->getRHSReg();
7724 Register RHS0 = Cmp2->getLHSReg();
7725 Register RHS1 = Cmp2->getRHSReg();
7726
7727 if (LHS0 == RHS1 && LHS1 == RHS0) {
7728 // Swap RHS operands to match LHS.
7729 PredR = CmpInst::getSwappedPredicate(pred: PredR);
7730 std::swap(a&: RHS0, b&: RHS1);
7731 }
7732
7733 if (LHS0 == RHS0 && LHS1 == RHS1) {
7734 // We determine the new predicate.
7735 unsigned CmpCodeL = getFCmpCode(CC: PredL);
7736 unsigned CmpCodeR = getFCmpCode(CC: PredR);
7737 unsigned NewPred = IsAnd ? CmpCodeL & CmpCodeR : CmpCodeL | CmpCodeR;
7738 unsigned Flags = Cmp1->getFlags() | Cmp2->getFlags();
7739 MatchInfo = [=](MachineIRBuilder &B) {
7740 // The fcmp predicates fill the lower part of the enum.
7741 FCmpInst::Predicate Pred = static_cast<FCmpInst::Predicate>(NewPred);
7742 if (Pred == FCmpInst::FCMP_FALSE &&
7743 isConstantLegalOrBeforeLegalizer(Ty: CmpTy)) {
7744 auto False = B.buildConstant(Res: CmpTy, Val: 0);
7745 B.buildZExtOrTrunc(Res: DestReg, Op: False);
7746 } else if (Pred == FCmpInst::FCMP_TRUE &&
7747 isConstantLegalOrBeforeLegalizer(Ty: CmpTy)) {
7748 auto True =
7749 B.buildConstant(Res: CmpTy, Val: getICmpTrueVal(TLI: getTargetLowering(),
7750 IsVector: CmpTy.isVector() /*isVector*/,
7751 IsFP: true /*isFP*/));
7752 B.buildZExtOrTrunc(Res: DestReg, Op: True);
7753 } else { // We take the predicate without predicate optimizations.
7754 auto Cmp = B.buildFCmp(Pred, Res: CmpTy, Op0: LHS0, Op1: LHS1, Flags);
7755 B.buildZExtOrTrunc(Res: DestReg, Op: Cmp);
7756 }
7757 };
7758 return true;
7759 }
7760
7761 return false;
7762}
7763
7764bool CombinerHelper::matchAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const {
7765 GAnd *And = cast<GAnd>(Val: &MI);
7766
7767 if (tryFoldAndOrOrICmpsUsingRanges(Logic: And, MatchInfo))
7768 return true;
7769
7770 if (tryFoldLogicOfFCmps(Logic: And, MatchInfo))
7771 return true;
7772
7773 return false;
7774}
7775
7776bool CombinerHelper::matchOr(MachineInstr &MI, BuildFnTy &MatchInfo) const {
7777 GOr *Or = cast<GOr>(Val: &MI);
7778
7779 if (tryFoldAndOrOrICmpsUsingRanges(Logic: Or, MatchInfo))
7780 return true;
7781
7782 if (tryFoldLogicOfFCmps(Logic: Or, MatchInfo))
7783 return true;
7784
7785 return false;
7786}
7787
7788bool CombinerHelper::matchAddOverflow(MachineInstr &MI,
7789 BuildFnTy &MatchInfo) const {
7790 GAddCarryOut *Add = cast<GAddCarryOut>(Val: &MI);
7791
7792 // Addo has no flags
7793 Register Dst = Add->getReg(Idx: 0);
7794 Register Carry = Add->getReg(Idx: 1);
7795 Register LHS = Add->getLHSReg();
7796 Register RHS = Add->getRHSReg();
7797 bool IsSigned = Add->isSigned();
7798 LLT DstTy = MRI.getType(Reg: Dst);
7799 LLT CarryTy = MRI.getType(Reg: Carry);
7800
7801 // Canonicalize constant to RHS.
7802 if (isConstantOrConstantVectorI(Src: LHS) && !isConstantOrConstantVectorI(Src: RHS)) {
7803 if (IsSigned) {
7804 MatchInfo = [=](MachineIRBuilder &B) {
7805 B.buildSAddo(Res: Dst, CarryOut: Carry, Op0: RHS, Op1: LHS);
7806 };
7807 return true;
7808 }
7809 // !IsSigned
7810 MatchInfo = [=](MachineIRBuilder &B) {
7811 B.buildUAddo(Res: Dst, CarryOut: Carry, Op0: RHS, Op1: LHS);
7812 };
7813 return true;
7814 }
7815
7816 std::optional<APInt> MaybeLHS = getConstantOrConstantSplatVector(Src: LHS);
7817 std::optional<APInt> MaybeRHS = getConstantOrConstantSplatVector(Src: RHS);
7818
7819 // Fold addo(c1, c2) -> c3, carry.
7820 if (MaybeLHS && MaybeRHS && isConstantLegalOrBeforeLegalizer(Ty: DstTy) &&
7821 isConstantLegalOrBeforeLegalizer(Ty: CarryTy)) {
7822 bool Overflow;
7823 APInt Result = IsSigned ? MaybeLHS->sadd_ov(RHS: *MaybeRHS, Overflow)
7824 : MaybeLHS->uadd_ov(RHS: *MaybeRHS, Overflow);
7825 MatchInfo = [=](MachineIRBuilder &B) {
7826 B.buildConstant(Res: Dst, Val: Result);
7827 B.buildConstant(Res: Carry, Val: Overflow);
7828 };
7829 return true;
7830 }
7831
7832 // Fold (addo x, 0) -> x, no carry
7833 if (MaybeRHS && *MaybeRHS == 0 && isConstantLegalOrBeforeLegalizer(Ty: CarryTy)) {
7834 MatchInfo = [=](MachineIRBuilder &B) {
7835 B.buildCopy(Res: Dst, Op: LHS);
7836 B.buildConstant(Res: Carry, Val: 0);
7837 };
7838 return true;
7839 }
7840
7841 // Given 2 constant operands whose sum does not overflow:
7842 // uaddo (X +nuw C0), C1 -> uaddo X, C0 + C1
7843 // saddo (X +nsw C0), C1 -> saddo X, C0 + C1
7844 GAdd *AddLHS = getOpcodeDef<GAdd>(Reg: LHS, MRI);
7845 if (MaybeRHS && AddLHS && MRI.hasOneNonDBGUse(RegNo: Add->getReg(Idx: 0)) &&
7846 ((IsSigned && AddLHS->getFlag(Flag: MachineInstr::MIFlag::NoSWrap)) ||
7847 (!IsSigned && AddLHS->getFlag(Flag: MachineInstr::MIFlag::NoUWrap)))) {
7848 std::optional<APInt> MaybeAddRHS =
7849 getConstantOrConstantSplatVector(Src: AddLHS->getRHSReg());
7850 if (MaybeAddRHS) {
7851 bool Overflow;
7852 APInt NewC = IsSigned ? MaybeAddRHS->sadd_ov(RHS: *MaybeRHS, Overflow)
7853 : MaybeAddRHS->uadd_ov(RHS: *MaybeRHS, Overflow);
7854 if (!Overflow && isConstantLegalOrBeforeLegalizer(Ty: DstTy)) {
7855 if (IsSigned) {
7856 MatchInfo = [=](MachineIRBuilder &B) {
7857 auto ConstRHS = B.buildConstant(Res: DstTy, Val: NewC);
7858 B.buildSAddo(Res: Dst, CarryOut: Carry, Op0: AddLHS->getLHSReg(), Op1: ConstRHS);
7859 };
7860 return true;
7861 }
7862 // !IsSigned
7863 MatchInfo = [=](MachineIRBuilder &B) {
7864 auto ConstRHS = B.buildConstant(Res: DstTy, Val: NewC);
7865 B.buildUAddo(Res: Dst, CarryOut: Carry, Op0: AddLHS->getLHSReg(), Op1: ConstRHS);
7866 };
7867 return true;
7868 }
7869 }
7870 };
7871
7872 // We try to combine addo to non-overflowing add.
7873 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ADD, {DstTy}}) ||
7874 !isConstantLegalOrBeforeLegalizer(Ty: CarryTy))
7875 return false;
7876
7877 // We try to combine uaddo to non-overflowing add.
7878 if (!IsSigned) {
7879 ConstantRange CRLHS =
7880 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: LHS), /*IsSigned=*/false);
7881 ConstantRange CRRHS =
7882 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: RHS), /*IsSigned=*/false);
7883
7884 switch (CRLHS.unsignedAddMayOverflow(Other: CRRHS)) {
7885 case ConstantRange::OverflowResult::MayOverflow:
7886 return false;
7887 case ConstantRange::OverflowResult::NeverOverflows: {
7888 MatchInfo = [=](MachineIRBuilder &B) {
7889 B.buildAdd(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoUWrap);
7890 B.buildConstant(Res: Carry, Val: 0);
7891 };
7892 return true;
7893 }
7894 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
7895 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
7896 MatchInfo = [=](MachineIRBuilder &B) {
7897 B.buildAdd(Dst, Src0: LHS, Src1: RHS);
7898 B.buildConstant(Res: Carry, Val: 1);
7899 };
7900 return true;
7901 }
7902 }
7903 return false;
7904 }
7905
7906 // We try to combine saddo to non-overflowing add.
7907
7908 // If LHS and RHS each have at least two sign bits, then there is no signed
7909 // overflow.
7910 if (VT->computeNumSignBits(R: RHS) > 1 && VT->computeNumSignBits(R: LHS) > 1) {
7911 MatchInfo = [=](MachineIRBuilder &B) {
7912 B.buildAdd(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoSWrap);
7913 B.buildConstant(Res: Carry, Val: 0);
7914 };
7915 return true;
7916 }
7917
7918 ConstantRange CRLHS =
7919 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: LHS), /*IsSigned=*/true);
7920 ConstantRange CRRHS =
7921 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: RHS), /*IsSigned=*/true);
7922
7923 switch (CRLHS.signedAddMayOverflow(Other: CRRHS)) {
7924 case ConstantRange::OverflowResult::MayOverflow:
7925 return false;
7926 case ConstantRange::OverflowResult::NeverOverflows: {
7927 MatchInfo = [=](MachineIRBuilder &B) {
7928 B.buildAdd(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoSWrap);
7929 B.buildConstant(Res: Carry, Val: 0);
7930 };
7931 return true;
7932 }
7933 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
7934 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
7935 MatchInfo = [=](MachineIRBuilder &B) {
7936 B.buildAdd(Dst, Src0: LHS, Src1: RHS);
7937 B.buildConstant(Res: Carry, Val: 1);
7938 };
7939 return true;
7940 }
7941 }
7942
7943 return false;
7944}
7945
7946void CombinerHelper::applyBuildFnMO(const MachineOperand &MO,
7947 BuildFnTy &MatchInfo) const {
7948 MachineInstr *Root = getDefIgnoringCopies(Reg: MO.getReg(), MRI);
7949 MatchInfo(Builder);
7950 Root->eraseFromParent();
7951}
7952
7953bool CombinerHelper::matchFPowIExpansion(MachineInstr &MI,
7954 int64_t Exponent) const {
7955 bool OptForSize = MI.getMF()->getFunction().hasOptSize();
7956 return getTargetLowering().isBeneficialToExpandPowI(Exponent, OptForSize);
7957}
7958
7959void CombinerHelper::applyExpandFPowI(MachineInstr &MI,
7960 int64_t Exponent) const {
7961 auto [Dst, Base] = MI.getFirst2Regs();
7962 LLT Ty = MRI.getType(Reg: Dst);
7963 int64_t ExpVal = Exponent;
7964
7965 if (ExpVal == 0) {
7966 Builder.buildFConstant(Res: Dst, Val: 1.0);
7967 MI.removeFromParent();
7968 return;
7969 }
7970
7971 if (ExpVal < 0)
7972 ExpVal = -ExpVal;
7973
7974 // We use the simple binary decomposition method from SelectionDAG ExpandPowI
7975 // to generate the multiply sequence. There are more optimal ways to do this
7976 // (for example, powi(x,15) generates one more multiply than it should), but
7977 // this has the benefit of being both really simple and much better than a
7978 // libcall.
7979 std::optional<SrcOp> Res;
7980 SrcOp CurSquare = Base;
7981 while (ExpVal > 0) {
7982 if (ExpVal & 1) {
7983 if (!Res)
7984 Res = CurSquare;
7985 else
7986 Res = Builder.buildFMul(Dst: Ty, Src0: *Res, Src1: CurSquare);
7987 }
7988
7989 CurSquare = Builder.buildFMul(Dst: Ty, Src0: CurSquare, Src1: CurSquare);
7990 ExpVal >>= 1;
7991 }
7992
7993 // If the original exponent was negative, invert the result, producing
7994 // 1/(x*x*x).
7995 if (Exponent < 0)
7996 Res = Builder.buildFDiv(Dst: Ty, Src0: Builder.buildFConstant(Res: Ty, Val: 1.0), Src1: *Res,
7997 Flags: MI.getFlags());
7998
7999 Builder.buildCopy(Res: Dst, Op: *Res);
8000 MI.eraseFromParent();
8001}
8002
8003bool CombinerHelper::matchFoldAPlusC1MinusC2(const MachineInstr &MI,
8004 BuildFnTy &MatchInfo) const {
8005 // fold (A+C1)-C2 -> A+(C1-C2)
8006 const GSub *Sub = cast<GSub>(Val: &MI);
8007 Register A, C1Reg;
8008 if (!mi_match(R: Sub->getLHSReg(), MRI, P: m_GAdd(L: m_Reg(R&: A), R: m_Reg(R&: C1Reg))))
8009 return false;
8010
8011 if (!MRI.hasOneNonDBGUse(RegNo: Sub->getLHSReg()))
8012 return false;
8013
8014 APInt C2 = getIConstantFromReg(VReg: Sub->getRHSReg(), MRI);
8015 APInt C1 = getIConstantFromReg(VReg: C1Reg, MRI);
8016
8017 Register Dst = Sub->getReg(Idx: 0);
8018 LLT DstTy = MRI.getType(Reg: Dst);
8019
8020 MatchInfo = [=](MachineIRBuilder &B) {
8021 auto Const = B.buildConstant(Res: DstTy, Val: C1 - C2);
8022 B.buildAdd(Dst, Src0: A, Src1: Const);
8023 };
8024
8025 return true;
8026}
8027
8028bool CombinerHelper::matchFoldC2MinusAPlusC1(const MachineInstr &MI,
8029 BuildFnTy &MatchInfo) const {
8030 // fold C2-(A+C1) -> (C2-C1)-A
8031 const GSub *Sub = cast<GSub>(Val: &MI);
8032 Register A, C1Reg;
8033 if (!mi_match(R: Sub->getRHSReg(), MRI, P: m_GAdd(L: m_Reg(R&: A), R: m_Reg(R&: C1Reg))))
8034 return false;
8035
8036 if (!MRI.hasOneNonDBGUse(RegNo: Sub->getRHSReg()))
8037 return false;
8038
8039 APInt C2 = getIConstantFromReg(VReg: Sub->getLHSReg(), MRI);
8040 APInt C1 = getIConstantFromReg(VReg: C1Reg, MRI);
8041
8042 Register Dst = Sub->getReg(Idx: 0);
8043 LLT DstTy = MRI.getType(Reg: Dst);
8044
8045 MatchInfo = [=](MachineIRBuilder &B) {
8046 auto Const = B.buildConstant(Res: DstTy, Val: C2 - C1);
8047 B.buildSub(Dst, Src0: Const, Src1: A);
8048 };
8049
8050 return true;
8051}
8052
8053bool CombinerHelper::matchFoldAMinusC1MinusC2(const MachineInstr &MI,
8054 BuildFnTy &MatchInfo) const {
8055 // fold (A-C1)-C2 -> A-(C1+C2)
8056 const GSub *Sub1 = cast<GSub>(Val: &MI);
8057 Register A, C1Reg;
8058 if (!mi_match(R: Sub1->getLHSReg(), MRI, P: m_GSub(L: m_Reg(R&: A), R: m_Reg(R&: C1Reg))))
8059 return false;
8060
8061 if (!MRI.hasOneNonDBGUse(RegNo: Sub1->getLHSReg()))
8062 return false;
8063
8064 APInt C2 = getIConstantFromReg(VReg: Sub1->getRHSReg(), MRI);
8065 APInt C1 = getIConstantFromReg(VReg: C1Reg, MRI);
8066
8067 Register Dst = Sub1->getReg(Idx: 0);
8068 LLT DstTy = MRI.getType(Reg: Dst);
8069
8070 MatchInfo = [=](MachineIRBuilder &B) {
8071 auto Const = B.buildConstant(Res: DstTy, Val: C1 + C2);
8072 B.buildSub(Dst, Src0: A, Src1: Const);
8073 };
8074
8075 return true;
8076}
8077
8078bool CombinerHelper::matchFoldC1Minus2MinusC2(const MachineInstr &MI,
8079 BuildFnTy &MatchInfo) const {
8080 // fold (C1-A)-C2 -> (C1-C2)-A
8081 const GSub *Sub1 = cast<GSub>(Val: &MI);
8082 Register C1Reg, A;
8083 if (!mi_match(R: Sub1->getLHSReg(), MRI, P: m_GSub(L: m_Reg(R&: C1Reg), R: m_Reg(R&: A))))
8084 return false;
8085
8086 if (!MRI.hasOneNonDBGUse(RegNo: Sub1->getLHSReg()))
8087 return false;
8088
8089 APInt C2 = getIConstantFromReg(VReg: Sub1->getRHSReg(), MRI);
8090 APInt C1 = getIConstantFromReg(VReg: C1Reg, MRI);
8091
8092 Register Dst = Sub1->getReg(Idx: 0);
8093 LLT DstTy = MRI.getType(Reg: Dst);
8094
8095 MatchInfo = [=](MachineIRBuilder &B) {
8096 auto Const = B.buildConstant(Res: DstTy, Val: C1 - C2);
8097 B.buildSub(Dst, Src0: Const, Src1: A);
8098 };
8099
8100 return true;
8101}
8102
8103bool CombinerHelper::matchFoldAMinusC1PlusC2(const MachineInstr &MI,
8104 BuildFnTy &MatchInfo) const {
8105 // fold ((A-C1)+C2) -> (A+(C2-C1))
8106 const GAdd *Add = cast<GAdd>(Val: &MI);
8107 Register A, C1Reg;
8108 if (!mi_match(R: Add->getLHSReg(), MRI, P: m_GSub(L: m_Reg(R&: A), R: m_Reg(R&: C1Reg))))
8109 return false;
8110
8111 if (!MRI.hasOneNonDBGUse(RegNo: Add->getLHSReg()))
8112 return false;
8113
8114 APInt C2 = getIConstantFromReg(VReg: Add->getRHSReg(), MRI);
8115 APInt C1 = getIConstantFromReg(VReg: C1Reg, MRI);
8116
8117 Register Dst = Add->getReg(Idx: 0);
8118 LLT DstTy = MRI.getType(Reg: Dst);
8119
8120 MatchInfo = [=](MachineIRBuilder &B) {
8121 auto Const = B.buildConstant(Res: DstTy, Val: C2 - C1);
8122 B.buildAdd(Dst, Src0: A, Src1: Const);
8123 };
8124
8125 return true;
8126}
8127
8128bool CombinerHelper::matchUnmergeValuesAnyExtBuildVector(
8129 const MachineInstr &MI, BuildFnTy &MatchInfo) const {
8130 const GUnmerge *Unmerge = cast<GUnmerge>(Val: &MI);
8131
8132 if (!MRI.hasOneNonDBGUse(RegNo: Unmerge->getSourceReg()))
8133 return false;
8134
8135 LLT DstTy = MRI.getType(Reg: Unmerge->getReg(Idx: 0));
8136
8137 // $bv:_(<8 x s8>) = G_BUILD_VECTOR ....
8138 // $any:_(<8 x s16>) = G_ANYEXT $bv
8139 // $uv:_(<4 x s16>), $uv1:_(<4 x s16>) = G_UNMERGE_VALUES $any
8140 //
8141 // ->
8142 //
8143 // $any:_(s16) = G_ANYEXT $bv[0]
8144 // $any1:_(s16) = G_ANYEXT $bv[1]
8145 // $any2:_(s16) = G_ANYEXT $bv[2]
8146 // $any3:_(s16) = G_ANYEXT $bv[3]
8147 // $any4:_(s16) = G_ANYEXT $bv[4]
8148 // $any5:_(s16) = G_ANYEXT $bv[5]
8149 // $any6:_(s16) = G_ANYEXT $bv[6]
8150 // $any7:_(s16) = G_ANYEXT $bv[7]
8151 // $uv:_(<4 x s16>) = G_BUILD_VECTOR $any, $any1, $any2, $any3
8152 // $uv1:_(<4 x s16>) = G_BUILD_VECTOR $any4, $any5, $any6, $any7
8153
8154 // We want to unmerge into vectors.
8155 if (!DstTy.isFixedVector())
8156 return false;
8157
8158 Register AnySrcReg;
8159 if (!mi_match(R: Unmerge->getSourceReg(), MRI, P: m_GAnyExt(Src: m_Reg(R&: AnySrcReg))))
8160 return false;
8161
8162 GBuildVector *BV;
8163 if (mi_match(R: AnySrcReg, MRI, P: m_GBuildVector(Inst&: BV))) {
8164 // G_UNMERGE_VALUES G_ANYEXT G_BUILD_VECTOR
8165
8166 if (!MRI.hasOneNonDBGUse(RegNo: BV->getReg(Idx: 0)))
8167 return false;
8168
8169 // FIXME: check element types?
8170 if (BV->getNumSources() % Unmerge->getNumDefs() != 0)
8171 return false;
8172
8173 LLT BigBvTy = MRI.getType(Reg: BV->getReg(Idx: 0));
8174 LLT SmallBvTy = DstTy;
8175 LLT SmallBvElemenTy = SmallBvTy.getElementType();
8176
8177 if (!isLegalOrBeforeLegalizer(
8178 Query: {TargetOpcode::G_BUILD_VECTOR, {SmallBvTy, SmallBvElemenTy}}))
8179 return false;
8180
8181 // We check the legality of scalar anyext.
8182 if (!isLegalOrBeforeLegalizer(
8183 Query: {TargetOpcode::G_ANYEXT,
8184 {SmallBvElemenTy, BigBvTy.getElementType()}}))
8185 return false;
8186
8187 MatchInfo = [=](MachineIRBuilder &B) {
8188 // Build into each G_UNMERGE_VALUES def
8189 // a small build vector with anyext from the source build vector.
8190 for (unsigned I = 0; I < Unmerge->getNumDefs(); ++I) {
8191 SmallVector<Register> Ops;
8192 for (unsigned J = 0; J < SmallBvTy.getNumElements(); ++J) {
8193 Register SourceArray =
8194 BV->getSourceReg(I: I * SmallBvTy.getNumElements() + J);
8195 auto AnyExt = B.buildAnyExt(Res: SmallBvElemenTy, Op: SourceArray);
8196 Ops.push_back(Elt: AnyExt.getReg(Idx: 0));
8197 }
8198 B.buildBuildVector(Res: Unmerge->getOperand(i: I).getReg(), Ops);
8199 };
8200 };
8201 return true;
8202 };
8203
8204 return false;
8205}
8206
8207bool CombinerHelper::matchShuffleUndefRHS(MachineInstr &MI,
8208 BuildFnTy &MatchInfo) const {
8209
8210 bool Changed = false;
8211 auto &Shuffle = cast<GShuffleVector>(Val&: MI);
8212 ArrayRef<int> OrigMask = Shuffle.getMask();
8213 SmallVector<int, 16> NewMask;
8214 const LLT SrcTy = MRI.getType(Reg: Shuffle.getSrc1Reg());
8215 const unsigned NumSrcElems = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
8216 const unsigned NumDstElts = OrigMask.size();
8217 for (unsigned i = 0; i != NumDstElts; ++i) {
8218 int Idx = OrigMask[i];
8219 if (Idx >= (int)NumSrcElems) {
8220 Idx = -1;
8221 Changed = true;
8222 }
8223 NewMask.push_back(Elt: Idx);
8224 }
8225
8226 if (!Changed)
8227 return false;
8228
8229 MatchInfo = [&, NewMask = std::move(NewMask)](MachineIRBuilder &B) {
8230 B.buildShuffleVector(Res: MI.getOperand(i: 0), Src1: MI.getOperand(i: 1), Src2: MI.getOperand(i: 2),
8231 Mask: std::move(NewMask));
8232 };
8233
8234 return true;
8235}
8236
8237static void commuteMask(MutableArrayRef<int> Mask, const unsigned NumElems) {
8238 const unsigned MaskSize = Mask.size();
8239 for (unsigned I = 0; I < MaskSize; ++I) {
8240 int Idx = Mask[I];
8241 if (Idx < 0)
8242 continue;
8243
8244 if (Idx < (int)NumElems)
8245 Mask[I] = Idx + NumElems;
8246 else
8247 Mask[I] = Idx - NumElems;
8248 }
8249}
8250
8251bool CombinerHelper::matchShuffleDisjointMask(MachineInstr &MI,
8252 BuildFnTy &MatchInfo) const {
8253
8254 auto &Shuffle = cast<GShuffleVector>(Val&: MI);
8255 // If any of the two inputs is already undef, don't check the mask again to
8256 // prevent infinite loop
8257 if (getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: Shuffle.getSrc1Reg(), MRI))
8258 return false;
8259
8260 if (getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: Shuffle.getSrc2Reg(), MRI))
8261 return false;
8262
8263 const LLT DstTy = MRI.getType(Reg: Shuffle.getReg(Idx: 0));
8264 const LLT Src1Ty = MRI.getType(Reg: Shuffle.getSrc1Reg());
8265 if (!isLegalOrBeforeLegalizer(
8266 Query: {TargetOpcode::G_SHUFFLE_VECTOR, {DstTy, Src1Ty}}))
8267 return false;
8268
8269 ArrayRef<int> Mask = Shuffle.getMask();
8270 const unsigned NumSrcElems = Src1Ty.getNumElements();
8271
8272 bool TouchesSrc1 = false;
8273 bool TouchesSrc2 = false;
8274 const unsigned NumElems = Mask.size();
8275 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
8276 if (Mask[Idx] < 0)
8277 continue;
8278
8279 if (Mask[Idx] < (int)NumSrcElems)
8280 TouchesSrc1 = true;
8281 else
8282 TouchesSrc2 = true;
8283 }
8284
8285 if (TouchesSrc1 == TouchesSrc2)
8286 return false;
8287
8288 Register NewSrc1 = Shuffle.getSrc1Reg();
8289 SmallVector<int, 16> NewMask(Mask);
8290 if (TouchesSrc2) {
8291 NewSrc1 = Shuffle.getSrc2Reg();
8292 commuteMask(Mask: NewMask, NumElems: NumSrcElems);
8293 }
8294
8295 MatchInfo = [=, &Shuffle](MachineIRBuilder &B) {
8296 auto Undef = B.buildUndef(Res: Src1Ty);
8297 B.buildShuffleVector(Res: Shuffle.getReg(Idx: 0), Src1: NewSrc1, Src2: Undef, Mask: NewMask);
8298 };
8299
8300 return true;
8301}
8302
8303bool CombinerHelper::matchSuboCarryOut(const MachineInstr &MI,
8304 BuildFnTy &MatchInfo) const {
8305 const GSubCarryOut *Subo = cast<GSubCarryOut>(Val: &MI);
8306
8307 Register Dst = Subo->getReg(Idx: 0);
8308 Register LHS = Subo->getLHSReg();
8309 Register RHS = Subo->getRHSReg();
8310 Register Carry = Subo->getCarryOutReg();
8311 LLT DstTy = MRI.getType(Reg: Dst);
8312 LLT CarryTy = MRI.getType(Reg: Carry);
8313
8314 // Check legality before known bits.
8315 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SUB, {DstTy}}) ||
8316 !isConstantLegalOrBeforeLegalizer(Ty: CarryTy))
8317 return false;
8318
8319 ConstantRange KBLHS =
8320 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: LHS),
8321 /* IsSigned=*/Subo->isSigned());
8322 ConstantRange KBRHS =
8323 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: RHS),
8324 /* IsSigned=*/Subo->isSigned());
8325
8326 if (Subo->isSigned()) {
8327 // G_SSUBO
8328 switch (KBLHS.signedSubMayOverflow(Other: KBRHS)) {
8329 case ConstantRange::OverflowResult::MayOverflow:
8330 return false;
8331 case ConstantRange::OverflowResult::NeverOverflows: {
8332 MatchInfo = [=](MachineIRBuilder &B) {
8333 B.buildSub(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoSWrap);
8334 B.buildConstant(Res: Carry, Val: 0);
8335 };
8336 return true;
8337 }
8338 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
8339 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
8340 MatchInfo = [=](MachineIRBuilder &B) {
8341 B.buildSub(Dst, Src0: LHS, Src1: RHS);
8342 B.buildConstant(Res: Carry, Val: getICmpTrueVal(TLI: getTargetLowering(),
8343 /*isVector=*/IsVector: CarryTy.isVector(),
8344 /*isFP=*/IsFP: false));
8345 };
8346 return true;
8347 }
8348 }
8349 return false;
8350 }
8351
8352 // G_USUBO
8353 switch (KBLHS.unsignedSubMayOverflow(Other: KBRHS)) {
8354 case ConstantRange::OverflowResult::MayOverflow:
8355 return false;
8356 case ConstantRange::OverflowResult::NeverOverflows: {
8357 MatchInfo = [=](MachineIRBuilder &B) {
8358 B.buildSub(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoUWrap);
8359 B.buildConstant(Res: Carry, Val: 0);
8360 };
8361 return true;
8362 }
8363 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
8364 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
8365 MatchInfo = [=](MachineIRBuilder &B) {
8366 B.buildSub(Dst, Src0: LHS, Src1: RHS);
8367 B.buildConstant(Res: Carry, Val: getICmpTrueVal(TLI: getTargetLowering(),
8368 /*isVector=*/IsVector: CarryTy.isVector(),
8369 /*isFP=*/IsFP: false));
8370 };
8371 return true;
8372 }
8373 }
8374
8375 return false;
8376}
8377
8378// Fold (ctlz (xor x, (sra x, bitwidth-1))) -> (add (ctls x), 1).
8379// Fold (ctlz (or (shl (xor x, (sra x, bitwidth-1)), 1), 1) -> (ctls x)
8380bool CombinerHelper::matchCtls(MachineInstr &CtlzMI,
8381 BuildFnTy &MatchInfo) const {
8382 assert((CtlzMI.getOpcode() == TargetOpcode::G_CTLZ ||
8383 CtlzMI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_POISON) &&
8384 "Expected G_CTLZ variant");
8385
8386 const Register Dst = CtlzMI.getOperand(i: 0).getReg();
8387 Register Src = CtlzMI.getOperand(i: 1).getReg();
8388
8389 LLT Ty = MRI.getType(Reg: Dst);
8390 LLT SrcTy = MRI.getType(Reg: Src);
8391
8392 if (!(Ty.isValid() && Ty.isScalar()))
8393 return false;
8394
8395 if (!LI)
8396 return false;
8397
8398 SmallVector<LLT, 2> QueryTypes = {Ty, SrcTy};
8399 LegalityQuery Query(TargetOpcode::G_CTLS, QueryTypes);
8400
8401 switch (LI->getAction(Query).Action) {
8402 default:
8403 return false;
8404 case LegalizeActions::Legal:
8405 case LegalizeActions::Custom:
8406 case LegalizeActions::WidenScalar:
8407 break;
8408 }
8409
8410 // Src = or(shl(V, 1), 1) -> Src=V; NeedAdd = False
8411 Register V;
8412 bool NeedAdd = true;
8413 if (mi_match(R: Src, MRI,
8414 P: m_OneUse(SP: m_GOr(L: m_OneUse(SP: m_GShl(L: m_Reg(R&: V), R: m_SpecificICst(RequestedValue: 1))),
8415 R: m_SpecificICst(RequestedValue: 1))))) {
8416 NeedAdd = false;
8417 Src = V;
8418 }
8419
8420 unsigned BitWidth = Ty.getScalarSizeInBits();
8421
8422 Register X;
8423 if (!mi_match(R: Src, MRI,
8424 P: m_OneUse(SP: m_GXor(L: m_Reg(R&: X), R: m_OneUse(SP: m_GAShr(
8425 L: m_DeferredReg(R&: X),
8426 R: m_SpecificICst(RequestedValue: BitWidth - 1)))))))
8427 return false;
8428
8429 MatchInfo = [=](MachineIRBuilder &B) {
8430 if (!NeedAdd) {
8431 B.buildCTLS(Dst, Src0: X);
8432 return;
8433 }
8434
8435 auto Ctls = B.buildCTLS(Dst: Ty, Src0: X);
8436 auto One = B.buildConstant(Res: Ty, Val: 1);
8437
8438 B.buildAdd(Dst, Src0: Ctls, Src1: One);
8439 };
8440
8441 return true;
8442}
8443
8444bool CombinerHelper::isConstantNaN(Register Reg) const {
8445 const ConstantFP *C = getConstantFPVRegVal(VReg: Reg, MRI);
8446 return C && C->getValueAPF().isNaN();
8447}
8448
8449bool CombinerHelper::matchCountZeroToZeroPoison(MachineInstr &MI, unsigned Opc,
8450 unsigned ZeroPoisonOpc) const {
8451 const bool IsCtlzPair = Opc == TargetOpcode::G_CTLZ &&
8452 ZeroPoisonOpc == TargetOpcode::G_CTLZ_ZERO_POISON;
8453 const bool IsCttzPair = Opc == TargetOpcode::G_CTTZ &&
8454 ZeroPoisonOpc == TargetOpcode::G_CTTZ_ZERO_POISON;
8455 if (MI.getOpcode() != Opc || (!IsCtlzPair && !IsCttzPair))
8456 return false;
8457 if (!VT)
8458 return false;
8459
8460 Register Src = MI.getOperand(i: 1).getReg();
8461 if (!VT->isKnownNeverZero(R: Src))
8462 return false;
8463
8464 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
8465 LLT SrcTy = MRI.getType(Reg: Src);
8466 return isLegalOrBeforeLegalizer(Query: {ZeroPoisonOpc, {DstTy, SrcTy}});
8467}
8468
8469static APInt getDemandedBitsForLogicalOp(unsigned Opcode, const APInt &Demanded,
8470 const KnownBits &OtherKnown) {
8471 assert((Opcode == TargetOpcode::G_AND || Opcode == TargetOpcode::G_OR) &&
8472 "Expected G_AND or G_OR");
8473 return Demanded &
8474 ~(Opcode == TargetOpcode::G_AND ? OtherKnown.Zero : OtherKnown.One);
8475}
8476
8477Register CombinerHelper::simplifyMultipleUseDemandedBits(
8478 Register R, const APInt &DemandedBits, unsigned Depth) const {
8479 assert(R.isVirtual() && "Expected a virtual register");
8480 assert(!DemandedBits.isZero() && "Zero demand is handled by the caller");
8481 if (Depth >= MaxAnalysisRecursionDepth)
8482 return Register();
8483
8484 [[maybe_unused]] LLT Ty = MRI.getType(Reg: R);
8485 assert(Ty.isValid() && "Expected a typed register");
8486 assert(DemandedBits.getBitWidth() == Ty.getScalarSizeInBits() &&
8487 "DemandedBits width must match the register scalar type");
8488
8489 MachineInstr *DefMI = MRI.getVRegDef(Reg: R);
8490 assert(DefMI && "Expected a definition in generic SSA MIR");
8491 if (!isa<GAnd>(Val: DefMI) && !isa<GOr>(Val: DefMI))
8492 return Register();
8493
8494 unsigned Opcode = DefMI->getOpcode();
8495 auto *Logic = cast<GLogicalBinOp>(Val: DefMI);
8496 Register LHS = Logic->getLHSReg();
8497 Register RHS = Logic->getRHSReg();
8498 assert(MRI.getType(LHS) == Ty && MRI.getType(RHS) == Ty &&
8499 "Logical operands must match the result type");
8500
8501 auto LookThrough = [&](Register X, Register CReg) -> Register {
8502 std::optional<APInt> C = getConstantOrConstantSplatVector(Src: CReg);
8503 if (!C || C->getBitWidth() != DemandedBits.getBitWidth())
8504 return Register();
8505 bool BypassToX = Opcode == TargetOpcode::G_AND
8506 ? DemandedBits.isSubsetOf(RHS: *C)
8507 : DemandedBits.isSubsetOf(RHS: ~*C);
8508 if (BypassToX) {
8509 if (Register Deeper =
8510 simplifyMultipleUseDemandedBits(R: X, DemandedBits, Depth: Depth + 1))
8511 return Deeper;
8512 return X;
8513 }
8514 bool BypassToC = Opcode == TargetOpcode::G_AND
8515 ? DemandedBits.isSubsetOf(RHS: ~*C)
8516 : DemandedBits.isSubsetOf(RHS: *C);
8517 if (BypassToC)
8518 return CReg;
8519 return Register();
8520 };
8521
8522 if (Register Repl = LookThrough(LHS, RHS))
8523 return Repl;
8524 return LookThrough(RHS, LHS);
8525}
8526
8527bool CombinerHelper::simplifyDemandedBitsImpl(MachineInstr &MI, unsigned OpNo,
8528 const APInt &DemandedBits,
8529 KnownBits &Known, unsigned Depth,
8530 bool DoRewrite) const {
8531 Known = KnownBits(DemandedBits.getBitWidth());
8532 assert(MI.getOperand(OpNo).isReg() && !MI.getOperand(OpNo).isDef() &&
8533 "Expected a register use");
8534 Register OpReg = MI.getOperand(i: OpNo).getReg();
8535 assert(OpReg.isVirtual() && "Expected a virtual register");
8536 LLT OpTy = MRI.getType(Reg: OpReg);
8537 assert(OpTy.isValid() && "Expected a typed register");
8538 assert(DemandedBits.getBitWidth() == OpTy.getScalarSizeInBits() &&
8539 "DemandedBits width must match the operand scalar type");
8540
8541 MachineInstr *DefMI = MRI.getVRegDef(Reg: OpReg);
8542 assert(DefMI && "Expected a definition in generic SSA MIR");
8543 // Undemanded bits can still affect poison-generating flags in the user.
8544 auto DropPoisonFlags = [&]() {
8545 if (!DoRewrite ||
8546 !(MI.getFlags() & MachineInstr::getPoisonGeneratingFlags()))
8547 return;
8548 Observer.changingInstr(MI);
8549 MI.clearFlags(flags: MachineInstr::getPoisonGeneratingFlags());
8550 Observer.changedInstr(MI);
8551 };
8552 // Debug users can observe bits that are not demanded by other users.
8553 auto InvalidateDebugUses = [&](const APInt &Demanded) {
8554 if (DoRewrite && !Demanded.isAllOnes())
8555 MRI.markUsesInDebugValueAsUndef(Reg: OpReg);
8556 };
8557 auto Rewrite = [&](Register Repl) {
8558 if (!DoRewrite || Repl == OpReg)
8559 return Repl != OpReg;
8560 DropPoisonFlags();
8561 if (DefMI->getNumExplicitDefs() == 1 && MRI.hasOneNonDBGUse(RegNo: OpReg)) {
8562 InvalidateDebugUses(DemandedBits);
8563 replaceRegWith(MRI, FromReg: OpReg, ToReg: Repl);
8564 eraseInst(MI&: *DefMI);
8565 } else {
8566 replaceRegOpWith(MRI, FromRegOp&: MI.getOperand(i: OpNo), ToReg: Repl);
8567 }
8568 return true;
8569 };
8570
8571 if (DemandedBits.isZero()) {
8572 if (DefMI->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
8573 return false;
8574 if (!DoRewrite)
8575 return true;
8576 MachineBasicBlock &SaveMBB = Builder.getMBB();
8577 MachineBasicBlock::iterator SavePt = Builder.getInsertPt();
8578 DebugLoc SaveDL = Builder.getDL();
8579 Builder.setInstrAndDebugLoc(MI);
8580 Register Undef = Builder.buildUndef(Res: OpTy).getReg(Idx: 0);
8581 Builder.setInsertPt(MBB&: SaveMBB, II: SavePt);
8582 Builder.setDebugLoc(SaveDL);
8583 // The producer may have side effects; replace only this use.
8584 DropPoisonFlags();
8585 replaceRegOpWith(MRI, FromRegOp&: MI.getOperand(i: OpNo), ToReg: Undef);
8586 return true;
8587 }
8588
8589 if (!VT)
8590 return false;
8591
8592 unsigned BW = DemandedBits.getBitWidth();
8593 auto GiveUp = [&]() {
8594 APInt DemandedElts = OpTy.isFixedVector()
8595 ? APInt::getAllOnes(numBits: OpTy.getNumElements())
8596 : APInt(1, 1);
8597 Known = VT->getKnownBits(R: OpReg, DemandedElts, Depth);
8598 return false;
8599 };
8600
8601 if (Depth >= MaxAnalysisRecursionDepth || DefMI->getNumExplicitDefs() != 1)
8602 return GiveUp();
8603
8604 // Shared defs require full demand unless this use can be rerouted to an
8605 // existing value that agrees on its demanded bits.
8606 APInt Demanded = DemandedBits;
8607 if (!MRI.hasOneNonDBGUse(RegNo: OpReg) && !Demanded.isAllOnes()) {
8608 if (Register Repl =
8609 simplifyMultipleUseDemandedBits(R: OpReg, DemandedBits: Demanded, Depth)) {
8610 Known = VT->getKnownBits(R: Repl);
8611 if (Rewrite(Repl))
8612 return true;
8613 }
8614 Demanded = APInt::getAllOnes(numBits: BW);
8615 }
8616
8617 unsigned Opcode = DefMI->getOpcode();
8618 switch (Opcode) {
8619 case TargetOpcode::G_AND:
8620 case TargetOpcode::G_OR: {
8621 auto *Logic = cast<GLogicalBinOp>(Val: DefMI);
8622 Register LHS = Logic->getLHSReg();
8623 Register RHS = Logic->getRHSReg();
8624 assert(MRI.getType(LHS) == OpTy && MRI.getType(RHS) == OpTy &&
8625 "Logical operands must match the result type");
8626 auto SimplifyWithConst = [&](Register X,
8627 Register CReg) -> std::optional<Register> {
8628 std::optional<APInt> C = getConstantOrConstantSplatVector(Src: CReg);
8629 if (!C)
8630 return std::nullopt;
8631 assert(C->getBitWidth() == BW && "Constant width must match its type");
8632 // Check redundancy before rewriting X can change its known bits.
8633 if (Opcode == TargetOpcode::G_AND) {
8634 if (Demanded.isSubsetOf(RHS: *C | VT->getKnownBits(R: X).Zero))
8635 return X;
8636 if (Demanded.isSubsetOf(RHS: ~*C))
8637 return CReg;
8638 return std::nullopt;
8639 }
8640 if (Demanded.isSubsetOf(RHS: ~*C | VT->getKnownBits(R: X).One))
8641 return X;
8642 if (Demanded.isSubsetOf(RHS: *C))
8643 return CReg;
8644 return std::nullopt;
8645 };
8646
8647 // Compute known bits before a rewrite can erase the def.
8648 if (std::optional<Register> Repl = SimplifyWithConst(LHS, RHS)) {
8649 Known = VT->getKnownBits(R: *Repl);
8650 if (Rewrite(*Repl))
8651 return true;
8652 }
8653 if (std::optional<Register> Repl = SimplifyWithConst(RHS, LHS)) {
8654 Known = VT->getKnownBits(R: *Repl);
8655 if (Rewrite(*Repl))
8656 return true;
8657 }
8658
8659 KnownBits RHSKnown(BW);
8660 bool Changed = simplifyDemandedBitsImpl(MI&: *DefMI, /*OpNo=*/2, DemandedBits: Demanded,
8661 Known&: RHSKnown, Depth: Depth + 1, DoRewrite);
8662 APInt LHSDemand = getDemandedBitsForLogicalOp(Opcode, Demanded, OtherKnown: RHSKnown);
8663 KnownBits LHSKnown(BW);
8664 Changed |= simplifyDemandedBitsImpl(MI&: *DefMI, /*OpNo=*/1, DemandedBits: LHSDemand, Known&: LHSKnown,
8665 Depth: Depth + 1, DoRewrite);
8666 if (!Changed && MRI.hasOneNonDBGUse(RegNo: RHS) &&
8667 !getConstantOrConstantSplatVector(Src: RHS)) {
8668 // Reduced-demand LHS facts cannot justify reducing the RHS demand.
8669 APInt RHSDemand =
8670 LHSDemand == Demanded
8671 ? getDemandedBitsForLogicalOp(Opcode, Demanded, OtherKnown: LHSKnown)
8672 : getDemandedBitsForLogicalOp(
8673 Opcode, Demanded,
8674 OtherKnown: VT->getKnownBits(
8675 R: LHS,
8676 DemandedElts: OpTy.isFixedVector()
8677 ? APInt::getAllOnes(numBits: OpTy.getNumElements())
8678 : APInt(1, 1),
8679 Depth: Depth + 1));
8680 if (RHSDemand != Demanded) {
8681 // Keep full-demand RHS facts if the reduced-demand retry fails.
8682 KnownBits RetryKnown(BW);
8683 Changed = simplifyDemandedBitsImpl(MI&: *DefMI, /*OpNo=*/2, DemandedBits: RHSDemand,
8684 Known&: RetryKnown, Depth: Depth + 1, DoRewrite);
8685 }
8686 }
8687 if (Changed) {
8688 DropPoisonFlags();
8689 InvalidateDebugUses(Demanded);
8690 }
8691 Known = Opcode == TargetOpcode::G_AND ? LHSKnown & RHSKnown
8692 : LHSKnown | RHSKnown;
8693 return Changed;
8694 }
8695 default:
8696 return GiveUp();
8697 }
8698}
8699
8700bool CombinerHelper::simplifyDemandedBits(MachineInstr &MI, unsigned OpNo,
8701 const APInt &DemandedBits,
8702 KnownBits &Known,
8703 unsigned Depth) const {
8704 return simplifyDemandedBitsImpl(MI, OpNo, DemandedBits, Known, Depth,
8705 /*DoRewrite=*/true);
8706}
8707
8708bool CombinerHelper::matchSimplifyDemandedBits(MachineInstr &MI,
8709 BuildFnTy &MatchInfo) const {
8710 Register Dst = MI.getOperand(i: 0).getReg();
8711 LLT Ty = MRI.getType(Reg: Dst);
8712 assert(Ty.isValid() && "Expected a typed generic result");
8713 if (!VT || Ty.isVector())
8714 return false;
8715
8716 APInt RootDemand = APInt::getAllOnes(numBits: Ty.getScalarSizeInBits());
8717 auto Probe = [&](unsigned OpNo, const APInt &OpDemand, KnownBits &Known) {
8718 if (!simplifyDemandedBitsImpl(MI, OpNo, DemandedBits: OpDemand, Known, /*Depth=*/0,
8719 /*DoRewrite=*/false))
8720 return false;
8721
8722 MatchInfo = [this, &MI, OpNo, OpDemand](MachineIRBuilder &) {
8723 KnownBits K(OpDemand.getBitWidth());
8724 simplifyDemandedBits(MI, OpNo, DemandedBits: OpDemand, Known&: K);
8725 };
8726 return true;
8727 };
8728
8729 unsigned Opcode = MI.getOpcode();
8730 switch (Opcode) {
8731 default:
8732 return false;
8733 case TargetOpcode::G_AND:
8734 case TargetOpcode::G_OR: {
8735 // Let redundant_and/redundant_or fold the root before operand rewrites can
8736 // discard the known bits that make it redundant.
8737 KnownBits L = VT->getKnownBits(R: MI.getOperand(i: 1).getReg());
8738 KnownBits R = VT->getKnownBits(R: MI.getOperand(i: 2).getReg());
8739 bool RootRedundant = Opcode == TargetOpcode::G_AND
8740 ? RootDemand.isSubsetOf(RHS: L.Zero | R.One) ||
8741 RootDemand.isSubsetOf(RHS: R.Zero | L.One)
8742 : RootDemand.isSubsetOf(RHS: L.One | R.Zero) ||
8743 RootDemand.isSubsetOf(RHS: R.One | L.Zero);
8744 if (RootRedundant)
8745 return false;
8746
8747 KnownBits RHSKnown(RootDemand.getBitWidth());
8748 if (Probe(/*OpNo=*/2, RootDemand, RHSKnown))
8749 return true;
8750
8751 APInt LHSDemand = getDemandedBitsForLogicalOp(Opcode, Demanded: RootDemand, OtherKnown: RHSKnown);
8752
8753 KnownBits LHSKnown(RootDemand.getBitWidth());
8754 if (Probe(/*OpNo=*/1, LHSDemand, LHSKnown))
8755 return true;
8756
8757 Register RHS = MI.getOperand(i: 2).getReg();
8758 if (!MRI.hasOneNonDBGUse(RegNo: RHS) || getConstantOrConstantSplatVector(Src: RHS))
8759 return false;
8760
8761 APInt RHSDemand =
8762 LHSDemand == RootDemand
8763 ? getDemandedBitsForLogicalOp(Opcode, Demanded: RootDemand, OtherKnown: LHSKnown)
8764 : getDemandedBitsForLogicalOp(
8765 Opcode, Demanded: RootDemand,
8766 OtherKnown: VT->getKnownBits(R: MI.getOperand(i: 1).getReg()));
8767 KnownBits RetryKnown(RootDemand.getBitWidth());
8768 return RHSDemand != RootDemand && Probe(/*OpNo=*/2, RHSDemand, RetryKnown);
8769 }
8770 }
8771}
8772