1//===-- lib/CodeGen/GlobalISel/GICombinerHelper.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
9#include "llvm/ADT/APFloat.h"
10#include "llvm/ADT/STLExtras.h"
11#include "llvm/ADT/SetVector.h"
12#include "llvm/ADT/SmallBitVector.h"
13#include "llvm/Analysis/CmpInstAnalysis.h"
14#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
15#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
16#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
17#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
18#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
19#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
20#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
21#include "llvm/CodeGen/GlobalISel/Utils.h"
22#include "llvm/CodeGen/LowLevelTypeUtils.h"
23#include "llvm/CodeGen/MachineBasicBlock.h"
24#include "llvm/CodeGen/MachineDominators.h"
25#include "llvm/CodeGen/MachineInstr.h"
26#include "llvm/CodeGen/MachineMemOperand.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/Register.h"
29#include "llvm/CodeGen/RegisterBankInfo.h"
30#include "llvm/CodeGen/TargetInstrInfo.h"
31#include "llvm/CodeGen/TargetLowering.h"
32#include "llvm/CodeGen/TargetOpcodes.h"
33#include "llvm/IR/ConstantRange.h"
34#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/InstrTypes.h"
36#include "llvm/IR/PatternMatch.h"
37#include "llvm/Support/Casting.h"
38#include "llvm/Support/DivisionByConstantInfo.h"
39#include "llvm/Support/ErrorHandling.h"
40#include "llvm/Support/MathExtras.h"
41#include "llvm/Target/TargetMachine.h"
42#include <cmath>
43#include <optional>
44#include <tuple>
45
46#define DEBUG_TYPE "gi-combiner"
47
48using namespace llvm;
49using namespace MIPatternMatch;
50
51// Option to allow testing of the combiner while no targets know about indexed
52// addressing.
53static cl::opt<bool>
54 ForceLegalIndexing("force-legal-indexing", cl::Hidden, cl::init(Val: false),
55 cl::desc("Force all indexed operations to be "
56 "legal for the GlobalISel combiner"));
57
58CombinerHelper::CombinerHelper(GISelChangeObserver &Observer,
59 MachineIRBuilder &B, bool IsPreLegalize,
60 GISelValueTracking *VT,
61 MachineDominatorTree *MDT,
62 const LegalizerInfo *LI)
63 : Builder(B), MRI(Builder.getMF().getRegInfo()), Observer(Observer), VT(VT),
64 MDT(MDT), IsPreLegalize(IsPreLegalize), LI(LI),
65 TII(Builder.getMF().getSubtarget().getInstrInfo()),
66 RBI(Builder.getMF().getSubtarget().getRegBankInfo()),
67 TRI(Builder.getMF().getSubtarget().getRegisterInfo()) {
68 (void)this->VT;
69}
70
71const TargetLowering &CombinerHelper::getTargetLowering() const {
72 return *Builder.getMF().getSubtarget().getTargetLowering();
73}
74
75const MachineFunction &CombinerHelper::getMachineFunction() const {
76 return Builder.getMF();
77}
78
79const DataLayout &CombinerHelper::getDataLayout() const {
80 return getMachineFunction().getDataLayout();
81}
82
83LLVMContext &CombinerHelper::getContext() const { return Builder.getContext(); }
84
85/// \returns The little endian in-memory byte position of byte \p I in a
86/// \p ByteWidth bytes wide type.
87///
88/// E.g. Given a 4-byte type x, x[0] -> byte 0
89static unsigned littleEndianByteAt(const unsigned ByteWidth, const unsigned I) {
90 assert(I < ByteWidth && "I must be in [0, ByteWidth)");
91 return I;
92}
93
94/// Determines the LogBase2 value for a non-null input value using the
95/// transform: LogBase2(V) = (EltBits - 1) - ctlz(V).
96static Register buildLogBase2(Register V, MachineIRBuilder &MIB) {
97 auto &MRI = *MIB.getMRI();
98 LLT Ty = MRI.getType(Reg: V);
99 auto Ctlz = MIB.buildCTLZ(Dst: Ty, Src0: V);
100 auto Base = MIB.buildConstant(Res: Ty, Val: Ty.getScalarSizeInBits() - 1);
101 return MIB.buildSub(Dst: Ty, Src0: Base, Src1: Ctlz).getReg(Idx: 0);
102}
103
104/// \returns The big endian in-memory byte position of byte \p I in a
105/// \p ByteWidth bytes wide type.
106///
107/// E.g. Given a 4-byte type x, x[0] -> byte 3
108static unsigned bigEndianByteAt(const unsigned ByteWidth, const unsigned I) {
109 assert(I < ByteWidth && "I must be in [0, ByteWidth)");
110 return ByteWidth - I - 1;
111}
112
113/// Given a map from byte offsets in memory to indices in a load/store,
114/// determine if that map corresponds to a little or big endian byte pattern.
115///
116/// \param MemOffset2Idx maps memory offsets to address offsets.
117/// \param LowestIdx is the lowest index in \p MemOffset2Idx.
118///
119/// \returns true if the map corresponds to a big endian byte pattern, false if
120/// it corresponds to a little endian byte pattern, and std::nullopt otherwise.
121///
122/// E.g. given a 32-bit type x, and x[AddrOffset], the in-memory byte patterns
123/// are as follows:
124///
125/// AddrOffset Little endian Big endian
126/// 0 0 3
127/// 1 1 2
128/// 2 2 1
129/// 3 3 0
130static std::optional<bool>
131isBigEndian(const SmallDenseMap<int64_t, int64_t, 8> &MemOffset2Idx,
132 int64_t LowestIdx) {
133 // Need at least two byte positions to decide on endianness.
134 unsigned Width = MemOffset2Idx.size();
135 if (Width < 2)
136 return std::nullopt;
137 bool BigEndian = true, LittleEndian = true;
138 for (unsigned MemOffset = 0; MemOffset < Width; ++ MemOffset) {
139 auto MemOffsetAndIdx = MemOffset2Idx.find(Val: MemOffset);
140 if (MemOffsetAndIdx == MemOffset2Idx.end())
141 return std::nullopt;
142 const int64_t Idx = MemOffsetAndIdx->second - LowestIdx;
143 assert(Idx >= 0 && "Expected non-negative byte offset?");
144 LittleEndian &= Idx == littleEndianByteAt(ByteWidth: Width, I: MemOffset);
145 BigEndian &= Idx == bigEndianByteAt(ByteWidth: Width, I: MemOffset);
146 if (!BigEndian && !LittleEndian)
147 return std::nullopt;
148 }
149
150 assert((BigEndian != LittleEndian) &&
151 "Pattern cannot be both big and little endian!");
152 return BigEndian;
153}
154
155bool CombinerHelper::isPreLegalize() const { return IsPreLegalize; }
156
157bool CombinerHelper::isLegal(const LegalityQuery &Query) const {
158 assert(LI && "Must have LegalizerInfo to query isLegal!");
159 return LI->getAction(Query).Action == LegalizeActions::Legal;
160}
161
162bool CombinerHelper::isLegalOrBeforeLegalizer(
163 const LegalityQuery &Query) const {
164 return isPreLegalize() || isLegal(Query);
165}
166
167bool CombinerHelper::isLegalOrHasWidenScalar(const LegalityQuery &Query) const {
168 return isLegal(Query) ||
169 LI->getAction(Query).Action == LegalizeActions::WidenScalar;
170}
171
172bool CombinerHelper::isLegalOrHasFewerElements(
173 const LegalityQuery &Query) const {
174 LegalizeAction Action = LI->getAction(Query).Action;
175 return Action == LegalizeActions::Legal ||
176 Action == LegalizeActions::FewerElements;
177}
178
179bool CombinerHelper::isConstantLegalOrBeforeLegalizer(const LLT Ty) const {
180 if (!Ty.isVector())
181 return isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_CONSTANT, {Ty}});
182 // Vector constants are represented as a G_BUILD_VECTOR of scalar G_CONSTANTs.
183 if (isPreLegalize())
184 return true;
185 LLT EltTy = Ty.getElementType();
186 return isLegal(Query: {TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}}) &&
187 isLegal(Query: {TargetOpcode::G_CONSTANT, {EltTy}});
188}
189
190void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg,
191 Register ToReg) const {
192 Observer.changingAllUsesOfReg(MRI, Reg: FromReg);
193
194 if (MRI.constrainRegAttrs(Reg: ToReg, ConstrainingReg: FromReg))
195 MRI.replaceRegWith(FromReg, ToReg);
196 else
197 Builder.buildCopy(Res: FromReg, Op: ToReg);
198
199 Observer.finishedChangingAllUsesOfReg();
200}
201
202void CombinerHelper::replaceRegOpWith(MachineRegisterInfo &MRI,
203 MachineOperand &FromRegOp,
204 Register ToReg) const {
205 assert(FromRegOp.getParent() && "Expected an operand in an MI");
206 Observer.changingInstr(MI&: *FromRegOp.getParent());
207
208 FromRegOp.setReg(ToReg);
209
210 Observer.changedInstr(MI&: *FromRegOp.getParent());
211}
212
213void CombinerHelper::replaceOpcodeWith(MachineInstr &FromMI,
214 unsigned ToOpcode) const {
215 Observer.changingInstr(MI&: FromMI);
216
217 FromMI.setDesc(Builder.getTII().get(Opcode: ToOpcode));
218
219 Observer.changedInstr(MI&: FromMI);
220}
221
222const RegisterBank *CombinerHelper::getRegBank(Register Reg) const {
223 return RBI->getRegBank(Reg, MRI, TRI: *TRI);
224}
225
226void CombinerHelper::setRegBank(Register Reg,
227 const RegisterBank *RegBank) const {
228 if (RegBank)
229 MRI.setRegBank(Reg, RegBank: *RegBank);
230}
231
232bool CombinerHelper::tryCombineCopy(MachineInstr &MI) const {
233 if (matchCombineCopy(MI)) {
234 applyCombineCopy(MI);
235 return true;
236 }
237 return false;
238}
239bool CombinerHelper::matchCombineCopy(MachineInstr &MI) const {
240 if (MI.getOpcode() != TargetOpcode::COPY)
241 return false;
242 Register DstReg = MI.getOperand(i: 0).getReg();
243 Register SrcReg = MI.getOperand(i: 1).getReg();
244 return canReplaceReg(DstReg, SrcReg, MRI);
245}
246void CombinerHelper::applyCombineCopy(MachineInstr &MI) const {
247 Register DstReg = MI.getOperand(i: 0).getReg();
248 Register SrcReg = MI.getOperand(i: 1).getReg();
249 replaceRegWith(MRI, FromReg: DstReg, ToReg: SrcReg);
250 MI.eraseFromParent();
251}
252
253bool CombinerHelper::matchFreezeOfSingleMaybePoisonOperand(
254 MachineInstr &MI, BuildFnTy &MatchInfo) const {
255 assert(MI.getOpcode() == TargetOpcode::G_FREEZE && "Invalid instruction");
256
257 // Ported from InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating.
258 Register DstOp = MI.getOperand(i: 0).getReg();
259 Register OrigOp = MI.getOperand(i: 1).getReg();
260
261 if (!MRI.hasOneNonDBGUse(RegNo: OrigOp))
262 return false;
263
264 MachineInstr *OrigDef = MRI.getUniqueVRegDef(Reg: OrigOp);
265 // Even if only a single operand of the PHI is not guaranteed non-poison,
266 // moving freeze() backwards across a PHI can cause optimization issues for
267 // other users of that operand.
268 //
269 // Moving freeze() from one of the output registers of a G_UNMERGE_VALUES to
270 // the source register is unprofitable because it makes the freeze() more
271 // strict than is necessary (it would affect the whole register instead of
272 // just the subreg being frozen).
273 if (OrigDef->isPHI() || isa<GUnmerge>(Val: OrigDef))
274 return false;
275
276 if (canCreateUndefOrPoison(Reg: OrigOp, MRI,
277 /*ConsiderFlagsAndMetadata=*/false))
278 return false;
279
280 std::optional<MachineOperand> MaybePoisonOperand;
281 for (MachineOperand &Operand : OrigDef->uses()) {
282 if (!Operand.isReg())
283 return false;
284
285 if (isGuaranteedNotToBeUndefOrPoison(Reg: Operand.getReg(), MRI))
286 continue;
287
288 if (!MaybePoisonOperand)
289 MaybePoisonOperand = Operand;
290 else {
291 // We have more than one maybe-poison operand. Moving the freeze is
292 // unsafe.
293 return false;
294 }
295 }
296
297 // Eliminate freeze if all operands are guaranteed non-poison.
298 if (!MaybePoisonOperand) {
299 MatchInfo = [=](MachineIRBuilder &B) {
300 Observer.changingInstr(MI&: *OrigDef);
301 cast<GenericMachineInstr>(Val: OrigDef)->dropPoisonGeneratingFlags();
302 Observer.changedInstr(MI&: *OrigDef);
303 B.buildCopy(Res: DstOp, Op: OrigOp);
304 };
305 return true;
306 }
307
308 Register MaybePoisonOperandReg = MaybePoisonOperand->getReg();
309 LLT MaybePoisonOperandRegTy = MRI.getType(Reg: MaybePoisonOperandReg);
310
311 if (!isLegalOrBeforeLegalizer(
312 Query: {TargetOpcode::G_FREEZE, {MaybePoisonOperandRegTy}}))
313 return false;
314
315 MatchInfo = [=](MachineIRBuilder &B) mutable {
316 Observer.changingInstr(MI&: *OrigDef);
317 cast<GenericMachineInstr>(Val: OrigDef)->dropPoisonGeneratingFlags();
318 Observer.changedInstr(MI&: *OrigDef);
319 B.setInsertPt(MBB&: *OrigDef->getParent(), II: OrigDef->getIterator());
320 auto Freeze = B.buildFreeze(Dst: MaybePoisonOperandRegTy, Src: MaybePoisonOperandReg);
321 replaceRegOpWith(
322 MRI, FromRegOp&: *OrigDef->findRegisterUseOperand(Reg: MaybePoisonOperandReg, TRI),
323 ToReg: Freeze.getReg(Idx: 0));
324 replaceRegWith(MRI, FromReg: DstOp, ToReg: OrigOp);
325 };
326 return true;
327}
328
329bool CombinerHelper::matchCombineConcatVectors(
330 MachineInstr &MI, SmallVector<Register> &Ops) const {
331 assert(MI.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
332 "Invalid instruction");
333 bool IsUndef = true;
334 MachineInstr *Undef = nullptr;
335
336 // Walk over all the operands of concat vectors and check if they are
337 // build_vector themselves or undef.
338 // Then collect their operands in Ops.
339 for (const MachineOperand &MO : MI.uses()) {
340 Register Reg = MO.getReg();
341 MachineInstr *Def = MRI.getVRegDef(Reg);
342 assert(Def && "Operand not defined");
343 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
344 return false;
345 switch (Def->getOpcode()) {
346 case TargetOpcode::G_BUILD_VECTOR:
347 IsUndef = false;
348 // Remember the operands of the build_vector to fold
349 // them into the yet-to-build flattened concat vectors.
350 for (const MachineOperand &BuildVecMO : Def->uses())
351 Ops.push_back(Elt: BuildVecMO.getReg());
352 break;
353 case TargetOpcode::G_IMPLICIT_DEF: {
354 LLT OpType = MRI.getType(Reg);
355 // Keep one undef value for all the undef operands.
356 if (!Undef) {
357 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
358 Undef = Builder.buildUndef(Res: OpType.getScalarType());
359 }
360 assert(MRI.getType(Undef->getOperand(0).getReg()) ==
361 OpType.getScalarType() &&
362 "All undefs should have the same type");
363 // Break the undef vector in as many scalar elements as needed
364 // for the flattening.
365 for (unsigned EltIdx = 0, EltEnd = OpType.getNumElements();
366 EltIdx != EltEnd; ++EltIdx)
367 Ops.push_back(Elt: Undef->getOperand(i: 0).getReg());
368 break;
369 }
370 default:
371 return false;
372 }
373 }
374
375 // Check if the combine is illegal
376 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
377 if (!isLegalOrBeforeLegalizer(
378 Query: {TargetOpcode::G_BUILD_VECTOR, {DstTy, MRI.getType(Reg: Ops[0])}})) {
379 return false;
380 }
381
382 if (IsUndef)
383 Ops.clear();
384
385 return true;
386}
387void CombinerHelper::applyCombineConcatVectors(
388 MachineInstr &MI, SmallVector<Register> &Ops) const {
389 // We determined that the concat_vectors can be flatten.
390 // Generate the flattened build_vector.
391 Register DstReg = MI.getOperand(i: 0).getReg();
392 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
393 Register NewDstReg = MRI.cloneVirtualRegister(VReg: DstReg);
394
395 // Note: IsUndef is sort of redundant. We could have determine it by
396 // checking that at all Ops are undef. Alternatively, we could have
397 // generate a build_vector of undefs and rely on another combine to
398 // clean that up. For now, given we already gather this information
399 // in matchCombineConcatVectors, just save compile time and issue the
400 // right thing.
401 if (Ops.empty())
402 Builder.buildUndef(Res: NewDstReg);
403 else
404 Builder.buildBuildVector(Res: NewDstReg, Ops);
405 replaceRegWith(MRI, FromReg: DstReg, ToReg: NewDstReg);
406 MI.eraseFromParent();
407}
408
409bool CombinerHelper::matchCombineBuildVectorOfBitcast(
410 MachineInstr &MI, SmallVector<Register> &Ops) const {
411 auto &BV = cast<GBuildVector>(Val&: MI);
412
413 // Look at the first operand for a unmerge(bitcast) from a scalar type.
414 GUnmerge *Unmerge = getOpcodeDef<GUnmerge>(Reg: BV.getSourceReg(I: 0), MRI);
415 if (!Unmerge || Unmerge->getReg(Idx: 0) != BV.getSourceReg(I: 0))
416 return false;
417 MachineInstr *BC = MRI.getVRegDef(Reg: Unmerge->getSourceReg());
418 if (BC->getOpcode() != TargetOpcode::G_BITCAST)
419 return false;
420 LLT InputTy = MRI.getType(Reg: BC->getOperand(i: 1).getReg());
421 unsigned Factor = Unmerge->getNumDefs();
422 if (!InputTy.isScalar() || BV.getNumSources() % Factor != 0)
423 return false;
424
425 // Check if the build_vector is legal
426 LLT BVDstTy = LLT::fixed_vector(NumElements: BV.getNumSources() / Factor, ScalarTy: InputTy);
427 if (!isLegal(Query: {TargetOpcode::G_BUILD_VECTOR, {BVDstTy, InputTy}}))
428 return false;
429
430 // Check all other operands are bitcasts or undef.
431 for (unsigned Idx = 0; Idx < BV.getNumSources(); Idx += Factor) {
432 GUnmerge *Unmerge = getOpcodeDef<GUnmerge>(Reg: BV.getSourceReg(I: Idx), MRI);
433 if (!all_of(Range: iota_range<unsigned>(0, Factor, false), P: [&](unsigned J) {
434 MachineInstr *Src = MRI.getVRegDef(Reg: BV.getSourceReg(I: Idx + J));
435 if (Src->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
436 return true;
437 return Unmerge && BV.getSourceReg(I: Idx + J) == Unmerge->getReg(Idx: J);
438 }))
439 return false;
440 if (!Unmerge)
441 Ops.push_back(Elt: 0);
442 else {
443 MachineInstr *BC = MRI.getVRegDef(Reg: Unmerge->getSourceReg());
444 if (BC->getOpcode() != TargetOpcode::G_BITCAST ||
445 MRI.getType(Reg: BC->getOperand(i: 1).getReg()) != InputTy)
446 return false;
447 Ops.push_back(Elt: BC->getOperand(i: 1).getReg());
448 }
449 }
450
451 return true;
452}
453
454void CombinerHelper::applyCombineBuildVectorOfBitcast(
455 MachineInstr &MI, SmallVector<Register> &Ops) const {
456 LLT SrcTy = MRI.getType(Reg: Ops[0]);
457 // Build undef if any operations require it.
458 Register Undef = 0;
459 for (Register &Op : Ops) {
460 if (!Op) {
461 if (!Undef)
462 Undef = Builder.buildUndef(Res: SrcTy).getReg(Idx: 0);
463 Op = Undef;
464 }
465 }
466
467 LLT BVDstTy = LLT::fixed_vector(NumElements: Ops.size(), ScalarTy: SrcTy);
468 auto BV = Builder.buildBuildVector(Res: BVDstTy, Ops);
469 Builder.buildBitcast(Dst: MI.getOperand(i: 0).getReg(), Src: BV);
470 MI.eraseFromParent();
471}
472
473void CombinerHelper::applyCombineShuffleToBuildVector(MachineInstr &MI) const {
474 auto &Shuffle = cast<GShuffleVector>(Val&: MI);
475
476 Register SrcVec1 = Shuffle.getSrc1Reg();
477 Register SrcVec2 = Shuffle.getSrc2Reg();
478 LLT EltTy = MRI.getType(Reg: SrcVec1).getElementType();
479 int Width = MRI.getType(Reg: SrcVec1).getNumElements();
480
481 auto Unmerge1 = Builder.buildUnmerge(Res: EltTy, Op: SrcVec1);
482 auto Unmerge2 = Builder.buildUnmerge(Res: EltTy, Op: SrcVec2);
483
484 SmallVector<Register> Extracts;
485 // Select only applicable elements from unmerged values.
486 for (int Val : Shuffle.getMask()) {
487 if (Val == -1)
488 Extracts.push_back(Elt: Builder.buildUndef(Res: EltTy).getReg(Idx: 0));
489 else if (Val < Width)
490 Extracts.push_back(Elt: Unmerge1.getReg(Idx: Val));
491 else
492 Extracts.push_back(Elt: Unmerge2.getReg(Idx: Val - Width));
493 }
494 assert(Extracts.size() > 0 && "Expected at least one element in the shuffle");
495 if (Extracts.size() == 1)
496 Builder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: Extracts[0]);
497 else
498 Builder.buildBuildVector(Res: MI.getOperand(i: 0).getReg(), Ops: Extracts);
499 MI.eraseFromParent();
500}
501
502bool CombinerHelper::matchCombineShuffleConcat(
503 MachineInstr &MI, SmallVector<Register> &Ops) const {
504 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
505 auto ConcatMI1 =
506 dyn_cast<GConcatVectors>(Val: MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg()));
507 auto ConcatMI2 =
508 dyn_cast<GConcatVectors>(Val: MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg()));
509 if (!ConcatMI1 || !ConcatMI2)
510 return false;
511
512 // Check that the sources of the Concat instructions have the same type
513 if (MRI.getType(Reg: ConcatMI1->getSourceReg(I: 0)) !=
514 MRI.getType(Reg: ConcatMI2->getSourceReg(I: 0)))
515 return false;
516
517 LLT ConcatSrcTy = MRI.getType(Reg: ConcatMI1->getReg(Idx: 1));
518 LLT ShuffleSrcTy1 = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
519 unsigned ConcatSrcNumElt = ConcatSrcTy.getNumElements();
520 for (unsigned i = 0; i < Mask.size(); i += ConcatSrcNumElt) {
521 // Check if the index takes a whole source register from G_CONCAT_VECTORS
522 // Assumes that all Sources of G_CONCAT_VECTORS are the same type
523 if (Mask[i] == -1) {
524 for (unsigned j = 1; j < ConcatSrcNumElt; j++) {
525 if (i + j >= Mask.size())
526 return false;
527 if (Mask[i + j] != -1)
528 return false;
529 }
530 if (!isLegalOrBeforeLegalizer(
531 Query: {TargetOpcode::G_IMPLICIT_DEF, {ConcatSrcTy}}))
532 return false;
533 Ops.push_back(Elt: 0);
534 } else if (Mask[i] % ConcatSrcNumElt == 0) {
535 for (unsigned j = 1; j < ConcatSrcNumElt; j++) {
536 if (i + j >= Mask.size())
537 return false;
538 if (Mask[i + j] != Mask[i] + static_cast<int>(j))
539 return false;
540 }
541 // Retrieve the source register from its respective G_CONCAT_VECTORS
542 // instruction
543 if (Mask[i] < ShuffleSrcTy1.getNumElements()) {
544 Ops.push_back(Elt: ConcatMI1->getSourceReg(I: Mask[i] / ConcatSrcNumElt));
545 } else {
546 Ops.push_back(Elt: ConcatMI2->getSourceReg(I: Mask[i] / ConcatSrcNumElt -
547 ConcatMI1->getNumSources()));
548 }
549 } else {
550 return false;
551 }
552 }
553
554 if (!isLegalOrBeforeLegalizer(
555 Query: {TargetOpcode::G_CONCAT_VECTORS,
556 {MRI.getType(Reg: MI.getOperand(i: 0).getReg()), ConcatSrcTy}}))
557 return false;
558
559 return !Ops.empty();
560}
561
562void CombinerHelper::applyCombineShuffleConcat(
563 MachineInstr &MI, SmallVector<Register> &Ops) const {
564 LLT SrcTy;
565 for (Register &Reg : Ops) {
566 if (Reg != 0)
567 SrcTy = MRI.getType(Reg);
568 }
569 assert(SrcTy.isValid() && "Unexpected full undef vector in concat combine");
570
571 Register UndefReg = 0;
572
573 for (Register &Reg : Ops) {
574 if (Reg == 0) {
575 if (UndefReg == 0)
576 UndefReg = Builder.buildUndef(Res: SrcTy).getReg(Idx: 0);
577 Reg = UndefReg;
578 }
579 }
580
581 if (Ops.size() > 1)
582 Builder.buildConcatVectors(Res: MI.getOperand(i: 0).getReg(), Ops);
583 else
584 Builder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: Ops[0]);
585 MI.eraseFromParent();
586}
587
588bool CombinerHelper::matchCombineShuffleVector(
589 MachineInstr &MI, SmallVectorImpl<Register> &Ops) const {
590 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
591 "Invalid instruction kind");
592 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
593 Register Src1 = MI.getOperand(i: 1).getReg();
594 LLT SrcType = MRI.getType(Reg: Src1);
595
596 unsigned DstNumElts = DstType.getNumElements();
597 unsigned SrcNumElts = SrcType.getNumElements();
598
599 // If the resulting vector is smaller than the size of the source
600 // vectors being concatenated, we won't be able to replace the
601 // shuffle vector into a concat_vectors.
602 //
603 // Note: We may still be able to produce a concat_vectors fed by
604 // extract_vector_elt and so on. It is less clear that would
605 // be better though, so don't bother for now.
606 //
607 // If the destination is a scalar, the size of the sources doesn't
608 // matter. we will lower the shuffle to a plain copy. This will
609 // work only if the source and destination have the same size. But
610 // that's covered by the next condition.
611 //
612 // TODO: If the size between the source and destination don't match
613 // we could still emit an extract vector element in that case.
614 if (DstNumElts < 2 * SrcNumElts)
615 return false;
616
617 // Check that the shuffle mask can be broken evenly between the
618 // different sources.
619 if (DstNumElts % SrcNumElts != 0)
620 return false;
621
622 // Mask length is a multiple of the source vector length.
623 // Check if the shuffle is some kind of concatenation of the input
624 // vectors.
625 unsigned NumConcat = DstNumElts / SrcNumElts;
626 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
627 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
628 for (unsigned i = 0; i != DstNumElts; ++i) {
629 int Idx = Mask[i];
630 // Undef value.
631 if (Idx < 0)
632 continue;
633 // Ensure the indices in each SrcType sized piece are sequential and that
634 // the same source is used for the whole piece.
635 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
636 (ConcatSrcs[i / SrcNumElts] >= 0 &&
637 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts)))
638 return false;
639 // Remember which source this index came from.
640 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
641 }
642
643 // The shuffle is concatenating multiple vectors together.
644 // Collect the different operands for that.
645 Register UndefReg;
646 Register Src2 = MI.getOperand(i: 2).getReg();
647 for (auto Src : ConcatSrcs) {
648 if (Src < 0) {
649 if (!UndefReg) {
650 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
651 UndefReg = Builder.buildUndef(Res: SrcType).getReg(Idx: 0);
652 }
653 Ops.push_back(Elt: UndefReg);
654 } else if (Src == 0)
655 Ops.push_back(Elt: Src1);
656 else
657 Ops.push_back(Elt: Src2);
658 }
659 return true;
660}
661
662void CombinerHelper::applyCombineShuffleVector(MachineInstr &MI,
663 ArrayRef<Register> Ops) const {
664 Register DstReg = MI.getOperand(i: 0).getReg();
665 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
666 Register NewDstReg = MRI.cloneVirtualRegister(VReg: DstReg);
667
668 if (Ops.size() == 1)
669 Builder.buildCopy(Res: NewDstReg, Op: Ops[0]);
670 else
671 Builder.buildMergeLikeInstr(Res: NewDstReg, Ops);
672
673 replaceRegWith(MRI, FromReg: DstReg, ToReg: NewDstReg);
674 MI.eraseFromParent();
675}
676
677namespace {
678
679/// Select a preference between two uses. CurrentUse is the current preference
680/// while *ForCandidate is attributes of the candidate under consideration.
681PreferredTuple ChoosePreferredUse(MachineInstr &LoadMI,
682 PreferredTuple &CurrentUse,
683 const LLT TyForCandidate,
684 unsigned OpcodeForCandidate,
685 MachineInstr *MIForCandidate) {
686 if (!CurrentUse.Ty.isValid()) {
687 if (CurrentUse.ExtendOpcode == OpcodeForCandidate ||
688 CurrentUse.ExtendOpcode == TargetOpcode::G_ANYEXT)
689 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
690 return CurrentUse;
691 }
692
693 // We permit the extend to hoist through basic blocks but this is only
694 // sensible if the target has extending loads. If you end up lowering back
695 // into a load and extend during the legalizer then the end result is
696 // hoisting the extend up to the load.
697
698 // Prefer defined extensions to undefined extensions as these are more
699 // likely to reduce the number of instructions.
700 if (OpcodeForCandidate == TargetOpcode::G_ANYEXT &&
701 CurrentUse.ExtendOpcode != TargetOpcode::G_ANYEXT)
702 return CurrentUse;
703 else if (CurrentUse.ExtendOpcode == TargetOpcode::G_ANYEXT &&
704 OpcodeForCandidate != TargetOpcode::G_ANYEXT)
705 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
706
707 // Prefer sign extensions to zero extensions as sign-extensions tend to be
708 // more expensive. Don't do this if the load is already a zero-extend load
709 // though, otherwise we'll rewrite a zero-extend load into a sign-extend
710 // later.
711 if (!isa<GZExtLoad>(Val: LoadMI) && CurrentUse.Ty == TyForCandidate) {
712 if (CurrentUse.ExtendOpcode == TargetOpcode::G_SEXT &&
713 OpcodeForCandidate == TargetOpcode::G_ZEXT)
714 return CurrentUse;
715 else if (CurrentUse.ExtendOpcode == TargetOpcode::G_ZEXT &&
716 OpcodeForCandidate == TargetOpcode::G_SEXT)
717 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
718 }
719
720 // This is potentially target specific. We've chosen the largest type
721 // because G_TRUNC is usually free. One potential catch with this is that
722 // some targets have a reduced number of larger registers than smaller
723 // registers and this choice potentially increases the live-range for the
724 // larger value.
725 if (TyForCandidate.getSizeInBits() > CurrentUse.Ty.getSizeInBits()) {
726 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
727 }
728 return CurrentUse;
729}
730
731/// Find a suitable place to insert some instructions and insert them. This
732/// function accounts for special cases like inserting before a PHI node.
733/// The current strategy for inserting before PHI's is to duplicate the
734/// instructions for each predecessor. However, while that's ok for G_TRUNC
735/// on most targets since it generally requires no code, other targets/cases may
736/// want to try harder to find a dominating block.
737static void InsertInsnsWithoutSideEffectsBeforeUse(
738 MachineIRBuilder &Builder, MachineInstr &DefMI, MachineOperand &UseMO,
739 std::function<void(MachineBasicBlock *, MachineBasicBlock::iterator,
740 MachineOperand &UseMO)>
741 Inserter) {
742 MachineInstr &UseMI = *UseMO.getParent();
743
744 MachineBasicBlock *InsertBB = UseMI.getParent();
745
746 // If the use is a PHI then we want the predecessor block instead.
747 if (UseMI.isPHI()) {
748 MachineOperand *PredBB = std::next(x: &UseMO);
749 InsertBB = PredBB->getMBB();
750 }
751
752 // If the block is the same block as the def then we want to insert just after
753 // the def instead of at the start of the block.
754 if (InsertBB == DefMI.getParent()) {
755 MachineBasicBlock::iterator InsertPt = &DefMI;
756 Inserter(InsertBB, std::next(x: InsertPt), UseMO);
757 return;
758 }
759
760 // Otherwise we want the start of the BB
761 Inserter(InsertBB, InsertBB->getFirstNonPHI(), UseMO);
762}
763} // end anonymous namespace
764
765bool CombinerHelper::tryCombineExtendingLoads(MachineInstr &MI) const {
766 PreferredTuple Preferred;
767 if (matchCombineExtendingLoads(MI, MatchInfo&: Preferred)) {
768 applyCombineExtendingLoads(MI, MatchInfo&: Preferred);
769 return true;
770 }
771 return false;
772}
773
774static unsigned getExtLoadOpcForExtend(unsigned ExtOpc) {
775 unsigned CandidateLoadOpc;
776 switch (ExtOpc) {
777 case TargetOpcode::G_ANYEXT:
778 CandidateLoadOpc = TargetOpcode::G_LOAD;
779 break;
780 case TargetOpcode::G_SEXT:
781 CandidateLoadOpc = TargetOpcode::G_SEXTLOAD;
782 break;
783 case TargetOpcode::G_ZEXT:
784 CandidateLoadOpc = TargetOpcode::G_ZEXTLOAD;
785 break;
786 default:
787 llvm_unreachable("Unexpected extend opc");
788 }
789 return CandidateLoadOpc;
790}
791
792bool CombinerHelper::matchCombineExtendingLoads(
793 MachineInstr &MI, PreferredTuple &Preferred) const {
794 // We match the loads and follow the uses to the extend instead of matching
795 // the extends and following the def to the load. This is because the load
796 // must remain in the same position for correctness (unless we also add code
797 // to find a safe place to sink it) whereas the extend is freely movable.
798 // It also prevents us from duplicating the load for the volatile case or just
799 // for performance.
800 GAnyLoad *LoadMI = dyn_cast<GAnyLoad>(Val: &MI);
801 if (!LoadMI)
802 return false;
803
804 Register LoadReg = LoadMI->getDstReg();
805
806 LLT LoadValueTy = MRI.getType(Reg: LoadReg);
807 if (!LoadValueTy.isScalar())
808 return false;
809
810 // Most architectures are going to legalize <s8 loads into at least a 1 byte
811 // load, and the MMOs can only describe memory accesses in multiples of bytes.
812 // If we try to perform extload combining on those, we can end up with
813 // %a(s8) = extload %ptr (load 1 byte from %ptr)
814 // ... which is an illegal extload instruction.
815 if (LoadValueTy.getSizeInBits() < 8)
816 return false;
817
818 // For non power-of-2 types, they will very likely be legalized into multiple
819 // loads. Don't bother trying to match them into extending loads.
820 if (!llvm::has_single_bit<uint32_t>(Value: LoadValueTy.getSizeInBits()))
821 return false;
822
823 // Find the preferred type aside from the any-extends (unless it's the only
824 // one) and non-extending ops. We'll emit an extending load to that type and
825 // and emit a variant of (extend (trunc X)) for the others according to the
826 // relative type sizes. At the same time, pick an extend to use based on the
827 // extend involved in the chosen type.
828 unsigned PreferredOpcode =
829 isa<GLoad>(Val: &MI)
830 ? TargetOpcode::G_ANYEXT
831 : isa<GSExtLoad>(Val: &MI) ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
832 Preferred = {.Ty: LLT(), .ExtendOpcode: PreferredOpcode, .MI: nullptr};
833 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: LoadReg)) {
834 if (UseMI.getOpcode() == TargetOpcode::G_SEXT ||
835 UseMI.getOpcode() == TargetOpcode::G_ZEXT ||
836 (UseMI.getOpcode() == TargetOpcode::G_ANYEXT)) {
837 const auto &MMO = LoadMI->getMMO();
838 // Don't do anything for atomics.
839 if (MMO.isAtomic())
840 continue;
841 // Check for legality.
842 if (!isPreLegalize()) {
843 LegalityQuery::MemDesc MMDesc(MMO);
844 unsigned CandidateLoadOpc = getExtLoadOpcForExtend(ExtOpc: UseMI.getOpcode());
845 LLT UseTy = MRI.getType(Reg: UseMI.getOperand(i: 0).getReg());
846 LLT SrcTy = MRI.getType(Reg: LoadMI->getPointerReg());
847 if (LI->getAction(Query: {CandidateLoadOpc, {UseTy, SrcTy}, {MMDesc}})
848 .Action != LegalizeActions::Legal)
849 continue;
850 }
851 Preferred = ChoosePreferredUse(LoadMI&: MI, CurrentUse&: Preferred,
852 TyForCandidate: MRI.getType(Reg: UseMI.getOperand(i: 0).getReg()),
853 OpcodeForCandidate: UseMI.getOpcode(), MIForCandidate: &UseMI);
854 }
855 }
856
857 // There were no extends
858 if (!Preferred.MI)
859 return false;
860 // It should be impossible to chose an extend without selecting a different
861 // type since by definition the result of an extend is larger.
862 assert(Preferred.Ty != LoadValueTy && "Extending to same type?");
863
864 LLVM_DEBUG(dbgs() << "Preferred use is: " << *Preferred.MI);
865 return true;
866}
867
868void CombinerHelper::applyCombineExtendingLoads(
869 MachineInstr &MI, PreferredTuple &Preferred) const {
870 // Rewrite the load to the chosen extending load.
871 Register ChosenDstReg = Preferred.MI->getOperand(i: 0).getReg();
872
873 // Inserter to insert a truncate back to the original type at a given point
874 // with some basic CSE to limit truncate duplication to one per BB.
875 DenseMap<MachineBasicBlock *, MachineInstr *> EmittedInsns;
876 auto InsertTruncAt = [&](MachineBasicBlock *InsertIntoBB,
877 MachineBasicBlock::iterator InsertBefore,
878 MachineOperand &UseMO) {
879 MachineInstr *PreviouslyEmitted = EmittedInsns.lookup(Val: InsertIntoBB);
880 if (PreviouslyEmitted) {
881 Observer.changingInstr(MI&: *UseMO.getParent());
882 UseMO.setReg(PreviouslyEmitted->getOperand(i: 0).getReg());
883 Observer.changedInstr(MI&: *UseMO.getParent());
884 return;
885 }
886
887 Builder.setInsertPt(MBB&: *InsertIntoBB, II: InsertBefore);
888 Register NewDstReg = MRI.cloneVirtualRegister(VReg: MI.getOperand(i: 0).getReg());
889 MachineInstr *NewMI = Builder.buildTrunc(Res: NewDstReg, Op: ChosenDstReg);
890 EmittedInsns[InsertIntoBB] = NewMI;
891 replaceRegOpWith(MRI, FromRegOp&: UseMO, ToReg: NewDstReg);
892 };
893
894 Observer.changingInstr(MI);
895 unsigned LoadOpc = getExtLoadOpcForExtend(ExtOpc: Preferred.ExtendOpcode);
896 MI.setDesc(Builder.getTII().get(Opcode: LoadOpc));
897
898 // Rewrite all the uses to fix up the types.
899 auto &LoadValue = MI.getOperand(i: 0);
900 SmallVector<MachineOperand *, 4> Uses(
901 llvm::make_pointer_range(Range: MRI.use_operands(Reg: LoadValue.getReg())));
902
903 for (auto *UseMO : Uses) {
904 MachineInstr *UseMI = UseMO->getParent();
905
906 // If the extend is compatible with the preferred extend then we should fix
907 // up the type and extend so that it uses the preferred use.
908 if (UseMI->getOpcode() == Preferred.ExtendOpcode ||
909 UseMI->getOpcode() == TargetOpcode::G_ANYEXT) {
910 Register UseDstReg = UseMI->getOperand(i: 0).getReg();
911 MachineOperand &UseSrcMO = UseMI->getOperand(i: 1);
912 const LLT UseDstTy = MRI.getType(Reg: UseDstReg);
913 if (UseDstReg != ChosenDstReg) {
914 if (Preferred.Ty == UseDstTy) {
915 // If the use has the same type as the preferred use, then merge
916 // the vregs and erase the extend. For example:
917 // %1:_(s8) = G_LOAD ...
918 // %2:_(s32) = G_SEXT %1(s8)
919 // %3:_(s32) = G_ANYEXT %1(s8)
920 // ... = ... %3(s32)
921 // rewrites to:
922 // %2:_(s32) = G_SEXTLOAD ...
923 // ... = ... %2(s32)
924 replaceRegWith(MRI, FromReg: UseDstReg, ToReg: ChosenDstReg);
925 Observer.erasingInstr(MI&: *UseMO->getParent());
926 UseMO->getParent()->eraseFromParent();
927 } else if (Preferred.Ty.getSizeInBits() < UseDstTy.getSizeInBits()) {
928 // If the preferred size is smaller, then keep the extend but extend
929 // from the result of the extending load. For example:
930 // %1:_(s8) = G_LOAD ...
931 // %2:_(s32) = G_SEXT %1(s8)
932 // %3:_(s64) = G_ANYEXT %1(s8)
933 // ... = ... %3(s64)
934 /// rewrites to:
935 // %2:_(s32) = G_SEXTLOAD ...
936 // %3:_(s64) = G_ANYEXT %2:_(s32)
937 // ... = ... %3(s64)
938 replaceRegOpWith(MRI, FromRegOp&: UseSrcMO, ToReg: ChosenDstReg);
939 } else {
940 // If the preferred size is large, then insert a truncate. For
941 // example:
942 // %1:_(s8) = G_LOAD ...
943 // %2:_(s64) = G_SEXT %1(s8)
944 // %3:_(s32) = G_ZEXT %1(s8)
945 // ... = ... %3(s32)
946 /// rewrites to:
947 // %2:_(s64) = G_SEXTLOAD ...
948 // %4:_(s8) = G_TRUNC %2:_(s32)
949 // %3:_(s64) = G_ZEXT %2:_(s8)
950 // ... = ... %3(s64)
951 InsertInsnsWithoutSideEffectsBeforeUse(Builder, DefMI&: MI, UseMO&: *UseMO,
952 Inserter: InsertTruncAt);
953 }
954 continue;
955 }
956 // The use is (one of) the uses of the preferred use we chose earlier.
957 // We're going to update the load to def this value later so just erase
958 // the old extend.
959 Observer.erasingInstr(MI&: *UseMO->getParent());
960 UseMO->getParent()->eraseFromParent();
961 continue;
962 }
963
964 // The use isn't an extend. Truncate back to the type we originally loaded.
965 // This is free on many targets.
966 InsertInsnsWithoutSideEffectsBeforeUse(Builder, DefMI&: MI, UseMO&: *UseMO, Inserter: InsertTruncAt);
967 }
968
969 MI.getOperand(i: 0).setReg(ChosenDstReg);
970 Observer.changedInstr(MI);
971}
972
973bool CombinerHelper::matchCombineLoadWithAndMask(MachineInstr &MI,
974 BuildFnTy &MatchInfo) const {
975 assert(MI.getOpcode() == TargetOpcode::G_AND);
976
977 // If we have the following code:
978 // %mask = G_CONSTANT 255
979 // %ld = G_LOAD %ptr, (load s16)
980 // %and = G_AND %ld, %mask
981 //
982 // Try to fold it into
983 // %ld = G_ZEXTLOAD %ptr, (load s8)
984
985 Register Dst = MI.getOperand(i: 0).getReg();
986 if (MRI.getType(Reg: Dst).isVector())
987 return false;
988
989 auto MaybeMask =
990 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
991 if (!MaybeMask)
992 return false;
993
994 APInt MaskVal = MaybeMask->Value;
995
996 if (!MaskVal.isMask())
997 return false;
998
999 Register SrcReg = MI.getOperand(i: 1).getReg();
1000 // Don't use getOpcodeDef() here since intermediate instructions may have
1001 // multiple users.
1002 GAnyLoad *LoadMI = dyn_cast<GAnyLoad>(Val: MRI.getVRegDef(Reg: SrcReg));
1003 if (!LoadMI)
1004 return false;
1005
1006 Register LoadReg = LoadMI->getDstReg();
1007 LLT RegTy = MRI.getType(Reg: LoadReg);
1008 Register PtrReg = LoadMI->getPointerReg();
1009 unsigned RegSize = RegTy.getSizeInBits();
1010 unsigned LoadSizeBits = LoadMI->getMemSizeInBits().getValue();
1011 unsigned MaskSizeBits = MaskVal.countr_one();
1012
1013 if ((isa<GSExtLoad>(Val: LoadMI) || MaskSizeBits < LoadSizeBits) &&
1014 !MRI.hasOneNonDBGUse(RegNo: LoadReg))
1015 return false;
1016
1017 // The mask may not be larger than the in-memory type, as it might cover sign
1018 // extended bits
1019 if (MaskSizeBits > LoadSizeBits)
1020 return false;
1021
1022 // If the mask covers the whole destination register, there's nothing to
1023 // extend
1024 if (MaskSizeBits >= RegSize)
1025 return false;
1026
1027 // Most targets cannot deal with loads of size < 8 and need to re-legalize to
1028 // at least byte loads. Avoid creating such loads here
1029 if (MaskSizeBits < 8 || !isPowerOf2_32(Value: MaskSizeBits))
1030 return false;
1031
1032 const MachineMemOperand &MMO = LoadMI->getMMO();
1033 LegalityQuery::MemDesc MemDesc(MMO);
1034
1035 // Don't modify the memory access size if this is atomic/volatile, but we can
1036 // still adjust the opcode to indicate the high bit behavior.
1037 if (LoadMI->isSimple())
1038 MemDesc.MemoryTy = LLT::scalar(SizeInBits: MaskSizeBits);
1039 else if (LoadSizeBits > MaskSizeBits || LoadSizeBits == RegSize)
1040 return false;
1041
1042 // TODO: Could check if it's legal with the reduced or original memory size.
1043 if (!isLegalOrBeforeLegalizer(
1044 Query: {TargetOpcode::G_ZEXTLOAD, {RegTy, MRI.getType(Reg: PtrReg)}, {MemDesc}}))
1045 return false;
1046
1047 MatchInfo = [=](MachineIRBuilder &B) {
1048 B.setInstrAndDebugLoc(*LoadMI);
1049 auto &MF = B.getMF();
1050 auto PtrInfo = MMO.getPointerInfo();
1051 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Ty: MemDesc.MemoryTy);
1052 B.buildLoadInstr(Opcode: TargetOpcode::G_ZEXTLOAD, Res: Dst, Addr: PtrReg, MMO&: *NewMMO);
1053 replaceRegWith(MRI, FromReg: LoadReg, ToReg: Dst);
1054 LoadMI->eraseFromParent();
1055 };
1056 return true;
1057}
1058
1059bool CombinerHelper::isPredecessor(const MachineInstr &DefMI,
1060 const MachineInstr &UseMI) const {
1061 assert(!DefMI.isDebugInstr() && !UseMI.isDebugInstr() &&
1062 "shouldn't consider debug uses");
1063 assert(DefMI.getParent() == UseMI.getParent());
1064 if (&DefMI == &UseMI)
1065 return true;
1066 const MachineBasicBlock &MBB = *DefMI.getParent();
1067 auto DefOrUse = find_if(Range: MBB, P: [&DefMI, &UseMI](const MachineInstr &MI) {
1068 return &MI == &DefMI || &MI == &UseMI;
1069 });
1070 if (DefOrUse == MBB.end())
1071 llvm_unreachable("Block must contain both DefMI and UseMI!");
1072 return &*DefOrUse == &DefMI;
1073}
1074
1075bool CombinerHelper::dominates(const MachineInstr &DefMI,
1076 const MachineInstr &UseMI) const {
1077 assert(!DefMI.isDebugInstr() && !UseMI.isDebugInstr() &&
1078 "shouldn't consider debug uses");
1079 if (MDT)
1080 return MDT->dominates(A: &DefMI, B: &UseMI);
1081 else if (DefMI.getParent() != UseMI.getParent())
1082 return false;
1083
1084 return isPredecessor(DefMI, UseMI);
1085}
1086
1087bool CombinerHelper::matchSextTruncSextLoad(MachineInstr &MI) const {
1088 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1089 Register SrcReg = MI.getOperand(i: 1).getReg();
1090 Register LoadUser = SrcReg;
1091
1092 if (MRI.getType(Reg: SrcReg).isVector())
1093 return false;
1094
1095 Register TruncSrc;
1096 if (mi_match(R: SrcReg, MRI, P: m_GTrunc(Src: m_Reg(R&: TruncSrc))))
1097 LoadUser = TruncSrc;
1098
1099 uint64_t SizeInBits = MI.getOperand(i: 2).getImm();
1100 // If the source is a G_SEXTLOAD from the same bit width, then we don't
1101 // need any extend at all, just a truncate.
1102 if (auto *LoadMI = getOpcodeDef<GSExtLoad>(Reg: LoadUser, MRI)) {
1103 // If truncating more than the original extended value, abort.
1104 auto LoadSizeBits = LoadMI->getMemSizeInBits();
1105 if (TruncSrc &&
1106 MRI.getType(Reg: TruncSrc).getSizeInBits() < LoadSizeBits.getValue())
1107 return false;
1108 if (LoadSizeBits == SizeInBits)
1109 return true;
1110 }
1111 return false;
1112}
1113
1114void CombinerHelper::applySextTruncSextLoad(MachineInstr &MI) const {
1115 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1116 Builder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: MI.getOperand(i: 1).getReg());
1117 MI.eraseFromParent();
1118}
1119
1120bool CombinerHelper::matchSextInRegOfLoad(
1121 MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) const {
1122 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1123
1124 Register DstReg = MI.getOperand(i: 0).getReg();
1125 LLT RegTy = MRI.getType(Reg: DstReg);
1126
1127 // Only supports scalars for now.
1128 if (RegTy.isVector())
1129 return false;
1130
1131 Register SrcReg = MI.getOperand(i: 1).getReg();
1132 auto *LoadDef = dyn_cast<GLoad>(Val: MRI.getVRegDef(Reg: SrcReg));
1133 if (!LoadDef)
1134 return false;
1135
1136 uint64_t MemBits = LoadDef->getMemSizeInBits().getValue();
1137 uint64_t ExtFrom = MI.getOperand(i: 2).getImm();
1138
1139 if (MemBits > ExtFrom && !MRI.hasOneNonDBGUse(RegNo: SrcReg))
1140 return false;
1141
1142 // If the sign extend extends from a narrower width than the load's width,
1143 // then we can narrow the load width when we combine to a G_SEXTLOAD.
1144 // Avoid widening the load at all.
1145 unsigned NewSizeBits = std::min(a: ExtFrom, b: MemBits);
1146
1147 // Don't generate G_SEXTLOADs with a < 1 byte width.
1148 if (NewSizeBits < 8)
1149 return false;
1150 // Don't bother creating a non-power-2 sextload, it will likely be broken up
1151 // anyway for most targets.
1152 if (!isPowerOf2_32(Value: NewSizeBits))
1153 return false;
1154
1155 const MachineMemOperand &MMO = LoadDef->getMMO();
1156 LegalityQuery::MemDesc MMDesc(MMO);
1157
1158 // Don't modify the memory access size if this is atomic/volatile, but we can
1159 // still adjust the opcode to indicate the high bit behavior.
1160 if (LoadDef->isSimple())
1161 MMDesc.MemoryTy = LLT::scalar(SizeInBits: NewSizeBits);
1162 else if (MemBits > NewSizeBits || MemBits == RegTy.getSizeInBits())
1163 return false;
1164
1165 // TODO: Could check if it's legal with the reduced or original memory size.
1166 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SEXTLOAD,
1167 {MRI.getType(Reg: LoadDef->getDstReg()),
1168 MRI.getType(Reg: LoadDef->getPointerReg())},
1169 {MMDesc}}))
1170 return false;
1171
1172 MatchInfo = std::make_tuple(args: LoadDef->getDstReg(), args&: NewSizeBits);
1173 return true;
1174}
1175
1176void CombinerHelper::applySextInRegOfLoad(
1177 MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) const {
1178 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1179 Register LoadReg;
1180 unsigned ScalarSizeBits;
1181 std::tie(args&: LoadReg, args&: ScalarSizeBits) = MatchInfo;
1182 GLoad *LoadDef = cast<GLoad>(Val: MRI.getVRegDef(Reg: LoadReg));
1183
1184 // If we have the following:
1185 // %ld = G_LOAD %ptr, (load 2)
1186 // %ext = G_SEXT_INREG %ld, 8
1187 // ==>
1188 // %ld = G_SEXTLOAD %ptr (load 1)
1189
1190 auto &MMO = LoadDef->getMMO();
1191 Builder.setInstrAndDebugLoc(*LoadDef);
1192 auto &MF = Builder.getMF();
1193 auto PtrInfo = MMO.getPointerInfo();
1194 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Size: ScalarSizeBits / 8);
1195 Builder.buildLoadInstr(Opcode: TargetOpcode::G_SEXTLOAD, Res: MI.getOperand(i: 0).getReg(),
1196 Addr: LoadDef->getPointerReg(), MMO&: *NewMMO);
1197 replaceRegWith(MRI, FromReg: LoadReg, ToReg: MI.getOperand(i: 0).getReg());
1198 MI.eraseFromParent();
1199
1200 // Not all loads can be deleted, so make sure the old one is removed.
1201 LoadDef->eraseFromParent();
1202}
1203
1204/// Return true if 'MI' is a load or a store that may be fold it's address
1205/// operand into the load / store addressing mode.
1206static bool canFoldInAddressingMode(GLoadStore *MI, const TargetLowering &TLI,
1207 MachineRegisterInfo &MRI) {
1208 TargetLowering::AddrMode AM;
1209 auto *MF = MI->getMF();
1210 auto *Addr = getOpcodeDef<GPtrAdd>(Reg: MI->getPointerReg(), MRI);
1211 if (!Addr)
1212 return false;
1213
1214 AM.HasBaseReg = true;
1215 if (auto CstOff = getIConstantVRegVal(VReg: Addr->getOffsetReg(), MRI))
1216 AM.BaseOffs = CstOff->getSExtValue(); // [reg +/- imm]
1217 else
1218 AM.Scale = 1; // [reg +/- reg]
1219
1220 return TLI.isLegalAddressingMode(
1221 DL: MF->getDataLayout(), AM,
1222 Ty: getTypeForLLT(Ty: MI->getMMO().getMemoryType(),
1223 C&: MF->getFunction().getContext()),
1224 AddrSpace: MI->getMMO().getAddrSpace());
1225}
1226
1227static unsigned getIndexedOpc(unsigned LdStOpc) {
1228 switch (LdStOpc) {
1229 case TargetOpcode::G_LOAD:
1230 return TargetOpcode::G_INDEXED_LOAD;
1231 case TargetOpcode::G_STORE:
1232 return TargetOpcode::G_INDEXED_STORE;
1233 case TargetOpcode::G_ZEXTLOAD:
1234 return TargetOpcode::G_INDEXED_ZEXTLOAD;
1235 case TargetOpcode::G_SEXTLOAD:
1236 return TargetOpcode::G_INDEXED_SEXTLOAD;
1237 default:
1238 llvm_unreachable("Unexpected opcode");
1239 }
1240}
1241
1242bool CombinerHelper::isIndexedLoadStoreLegal(GLoadStore &LdSt) const {
1243 // Check for legality.
1244 LLT PtrTy = MRI.getType(Reg: LdSt.getPointerReg());
1245 LLT Ty = MRI.getType(Reg: LdSt.getReg(Idx: 0));
1246 LLT MemTy = LdSt.getMMO().getMemoryType();
1247 SmallVector<LegalityQuery::MemDesc, 2> MemDescrs(
1248 {{MemTy, MemTy.getSizeInBits().getKnownMinValue(),
1249 AtomicOrdering::NotAtomic, AtomicOrdering::NotAtomic}});
1250 unsigned IndexedOpc = getIndexedOpc(LdStOpc: LdSt.getOpcode());
1251 SmallVector<LLT> OpTys;
1252 if (IndexedOpc == TargetOpcode::G_INDEXED_STORE)
1253 OpTys = {PtrTy, Ty, Ty};
1254 else
1255 OpTys = {Ty, PtrTy}; // For G_INDEXED_LOAD, G_INDEXED_[SZ]EXTLOAD
1256
1257 LegalityQuery Q(IndexedOpc, OpTys, MemDescrs);
1258 return isLegal(Query: Q);
1259}
1260
1261static cl::opt<unsigned> PostIndexUseThreshold(
1262 "post-index-use-threshold", cl::Hidden, cl::init(Val: 32),
1263 cl::desc("Number of uses of a base pointer to check before it is no longer "
1264 "considered for post-indexing."));
1265
1266bool CombinerHelper::findPostIndexCandidate(GLoadStore &LdSt, Register &Addr,
1267 Register &Base, Register &Offset,
1268 bool &RematOffset) const {
1269 // We're looking for the following pattern, for either load or store:
1270 // %baseptr:_(p0) = ...
1271 // G_STORE %val(s64), %baseptr(p0)
1272 // %offset:_(s64) = G_CONSTANT i64 -256
1273 // %new_addr:_(p0) = G_PTR_ADD %baseptr, %offset(s64)
1274 const auto &TLI = getTargetLowering();
1275
1276 Register Ptr = LdSt.getPointerReg();
1277 // If the store is the only use, don't bother.
1278 if (MRI.hasOneNonDBGUse(RegNo: Ptr))
1279 return false;
1280
1281 if (!isIndexedLoadStoreLegal(LdSt))
1282 return false;
1283
1284 if (getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Ptr, MRI))
1285 return false;
1286
1287 MachineInstr *StoredValDef = getDefIgnoringCopies(Reg: LdSt.getReg(Idx: 0), MRI);
1288 auto *PtrDef = MRI.getVRegDef(Reg: Ptr);
1289
1290 unsigned NumUsesChecked = 0;
1291 for (auto &Use : MRI.use_nodbg_instructions(Reg: Ptr)) {
1292 if (++NumUsesChecked > PostIndexUseThreshold)
1293 return false; // Try to avoid exploding compile time.
1294
1295 auto *PtrAdd = dyn_cast<GPtrAdd>(Val: &Use);
1296 // The use itself might be dead. This can happen during combines if DCE
1297 // hasn't had a chance to run yet. Don't allow it to form an indexed op.
1298 if (!PtrAdd || MRI.use_nodbg_empty(RegNo: PtrAdd->getReg(Idx: 0)))
1299 continue;
1300
1301 // Check the user of this isn't the store, otherwise we'd be generate a
1302 // indexed store defining its own use.
1303 if (StoredValDef == &Use)
1304 continue;
1305
1306 Offset = PtrAdd->getOffsetReg();
1307 if (!ForceLegalIndexing &&
1308 !TLI.isIndexingLegal(MI&: LdSt, Base: PtrAdd->getBaseReg(), Offset,
1309 /*IsPre*/ false, MRI))
1310 continue;
1311
1312 // Make sure the offset calculation is before the potentially indexed op.
1313 MachineInstr *OffsetDef = MRI.getVRegDef(Reg: Offset);
1314 RematOffset = false;
1315 if (!dominates(DefMI: *OffsetDef, UseMI: LdSt)) {
1316 // If the offset however is just a G_CONSTANT, we can always just
1317 // rematerialize it where we need it.
1318 if (OffsetDef->getOpcode() != TargetOpcode::G_CONSTANT)
1319 continue;
1320 RematOffset = true;
1321 }
1322
1323 for (auto &BasePtrUse : MRI.use_nodbg_instructions(Reg: PtrAdd->getBaseReg())) {
1324 if (&BasePtrUse == PtrDef)
1325 continue;
1326
1327 // If the user is a later load/store that can be post-indexed, then don't
1328 // combine this one.
1329 auto *BasePtrLdSt = dyn_cast<GLoadStore>(Val: &BasePtrUse);
1330 if (BasePtrLdSt && BasePtrLdSt != &LdSt &&
1331 dominates(DefMI: LdSt, UseMI: *BasePtrLdSt) &&
1332 isIndexedLoadStoreLegal(LdSt&: *BasePtrLdSt))
1333 return false;
1334
1335 // Now we're looking for the key G_PTR_ADD instruction, which contains
1336 // the offset add that we want to fold.
1337 if (auto *BasePtrUseDef = dyn_cast<GPtrAdd>(Val: &BasePtrUse)) {
1338 Register PtrAddDefReg = BasePtrUseDef->getReg(Idx: 0);
1339 for (auto &BaseUseUse : MRI.use_nodbg_instructions(Reg: PtrAddDefReg)) {
1340 // If the use is in a different block, then we may produce worse code
1341 // due to the extra register pressure.
1342 if (BaseUseUse.getParent() != LdSt.getParent())
1343 return false;
1344
1345 if (auto *UseUseLdSt = dyn_cast<GLoadStore>(Val: &BaseUseUse))
1346 if (canFoldInAddressingMode(MI: UseUseLdSt, TLI, MRI))
1347 return false;
1348 }
1349 if (!dominates(DefMI: LdSt, UseMI: BasePtrUse))
1350 return false; // All use must be dominated by the load/store.
1351 }
1352 }
1353
1354 Addr = PtrAdd->getReg(Idx: 0);
1355 Base = PtrAdd->getBaseReg();
1356 return true;
1357 }
1358
1359 return false;
1360}
1361
1362bool CombinerHelper::findPreIndexCandidate(GLoadStore &LdSt, Register &Addr,
1363 Register &Base,
1364 Register &Offset) const {
1365 auto &MF = *LdSt.getParent()->getParent();
1366 const auto &TLI = *MF.getSubtarget().getTargetLowering();
1367
1368 Addr = LdSt.getPointerReg();
1369 if (!mi_match(R: Addr, MRI, P: m_GPtrAdd(L: m_Reg(R&: Base), R: m_Reg(R&: Offset))) ||
1370 MRI.hasOneNonDBGUse(RegNo: Addr))
1371 return false;
1372
1373 if (!ForceLegalIndexing &&
1374 !TLI.isIndexingLegal(MI&: LdSt, Base, Offset, /*IsPre*/ true, MRI))
1375 return false;
1376
1377 if (!isIndexedLoadStoreLegal(LdSt))
1378 return false;
1379
1380 MachineInstr *BaseDef = getDefIgnoringCopies(Reg: Base, MRI);
1381 if (BaseDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
1382 return false;
1383
1384 if (auto *St = dyn_cast<GStore>(Val: &LdSt)) {
1385 // Would require a copy.
1386 if (Base == St->getValueReg())
1387 return false;
1388
1389 // We're expecting one use of Addr in MI, but it could also be the
1390 // value stored, which isn't actually dominated by the instruction.
1391 if (St->getValueReg() == Addr)
1392 return false;
1393 }
1394
1395 // Avoid increasing cross-block register pressure.
1396 for (auto &AddrUse : MRI.use_nodbg_instructions(Reg: Addr))
1397 if (AddrUse.getParent() != LdSt.getParent())
1398 return false;
1399
1400 // FIXME: check whether all uses of the base pointer are constant PtrAdds.
1401 // That might allow us to end base's liveness here by adjusting the constant.
1402 bool RealUse = false;
1403 for (auto &AddrUse : MRI.use_nodbg_instructions(Reg: Addr)) {
1404 if (!dominates(DefMI: LdSt, UseMI: AddrUse))
1405 return false; // All use must be dominated by the load/store.
1406
1407 // If Ptr may be folded in addressing mode of other use, then it's
1408 // not profitable to do this transformation.
1409 if (auto *UseLdSt = dyn_cast<GLoadStore>(Val: &AddrUse)) {
1410 if (!canFoldInAddressingMode(MI: UseLdSt, TLI, MRI))
1411 RealUse = true;
1412 } else {
1413 RealUse = true;
1414 }
1415 }
1416 return RealUse;
1417}
1418
1419bool CombinerHelper::matchCombineExtractedVectorLoad(
1420 MachineInstr &MI, BuildFnTy &MatchInfo) const {
1421 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
1422
1423 // Check if there is a load that defines the vector being extracted from.
1424 auto *LoadMI = getOpcodeDef<GLoad>(Reg: MI.getOperand(i: 1).getReg(), MRI);
1425 if (!LoadMI)
1426 return false;
1427
1428 Register Vector = MI.getOperand(i: 1).getReg();
1429 LLT VecEltTy = MRI.getType(Reg: Vector).getElementType();
1430
1431 assert(MRI.getType(MI.getOperand(0).getReg()) == VecEltTy);
1432
1433 // Checking whether we should reduce the load width.
1434 if (!MRI.hasOneNonDBGUse(RegNo: Vector))
1435 return false;
1436
1437 // Check if the defining load is simple.
1438 if (!LoadMI->isSimple())
1439 return false;
1440
1441 // If the vector element type is not a multiple of a byte then we are unable
1442 // to correctly compute an address to load only the extracted element as a
1443 // scalar.
1444 if (!VecEltTy.isByteSized())
1445 return false;
1446
1447 // Check for load fold barriers between the extraction and the load.
1448 if (MI.getParent() != LoadMI->getParent())
1449 return false;
1450 const unsigned MaxIter = 20;
1451 unsigned Iter = 0;
1452 for (auto II = LoadMI->getIterator(), IE = MI.getIterator(); II != IE; ++II) {
1453 if (II->isLoadFoldBarrier())
1454 return false;
1455 if (Iter++ == MaxIter)
1456 return false;
1457 }
1458
1459 // Check if the new load that we are going to create is legal
1460 // if we are in the post-legalization phase.
1461 MachineMemOperand MMO = LoadMI->getMMO();
1462 Align Alignment = MMO.getAlign();
1463 MachinePointerInfo PtrInfo;
1464 uint64_t Offset;
1465
1466 // Finding the appropriate PtrInfo if offset is a known constant.
1467 // This is required to create the memory operand for the narrowed load.
1468 // This machine memory operand object helps us infer about legality
1469 // before we proceed to combine the instruction.
1470 if (auto CVal = getIConstantVRegVal(VReg: Vector, MRI)) {
1471 int Elt = CVal->getZExtValue();
1472 // FIXME: should be (ABI size)*Elt.
1473 Offset = VecEltTy.getSizeInBits() * Elt / 8;
1474 PtrInfo = MMO.getPointerInfo().getWithOffset(O: Offset);
1475 } else {
1476 // Discard the pointer info except the address space because the memory
1477 // operand can't represent this new access since the offset is variable.
1478 Offset = VecEltTy.getSizeInBits() / 8;
1479 PtrInfo = MachinePointerInfo(MMO.getPointerInfo().getAddrSpace());
1480 }
1481
1482 Alignment = commonAlignment(A: Alignment, Offset);
1483
1484 Register VecPtr = LoadMI->getPointerReg();
1485 LLT PtrTy = MRI.getType(Reg: VecPtr);
1486
1487 MachineFunction &MF = *MI.getMF();
1488 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Ty: VecEltTy);
1489
1490 LegalityQuery::MemDesc MMDesc(*NewMMO);
1491
1492 if (!isLegalOrBeforeLegalizer(
1493 Query: {TargetOpcode::G_LOAD, {VecEltTy, PtrTy}, {MMDesc}}))
1494 return false;
1495
1496 // Load must be allowed and fast on the target.
1497 LLVMContext &C = MF.getFunction().getContext();
1498 auto &DL = MF.getDataLayout();
1499 unsigned Fast = 0;
1500 if (!getTargetLowering().allowsMemoryAccess(Context&: C, DL, Ty: VecEltTy, MMO: *NewMMO,
1501 Fast: &Fast) ||
1502 !Fast)
1503 return false;
1504
1505 Register Result = MI.getOperand(i: 0).getReg();
1506 Register Index = MI.getOperand(i: 2).getReg();
1507
1508 MatchInfo = [=](MachineIRBuilder &B) {
1509 GISelObserverWrapper DummyObserver;
1510 LegalizerHelper Helper(B.getMF(), DummyObserver, B);
1511 //// Get pointer to the vector element.
1512 Register finalPtr = Helper.getVectorElementPointer(
1513 VecPtr: LoadMI->getPointerReg(), VecTy: MRI.getType(Reg: LoadMI->getOperand(i: 0).getReg()),
1514 Index);
1515 // New G_LOAD instruction.
1516 B.buildLoad(Res: Result, Addr: finalPtr, PtrInfo, Alignment);
1517 // Remove original GLOAD instruction.
1518 LoadMI->eraseFromParent();
1519 };
1520
1521 return true;
1522}
1523
1524bool CombinerHelper::matchCombineIndexedLoadStore(
1525 MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) const {
1526 auto &LdSt = cast<GLoadStore>(Val&: MI);
1527
1528 if (LdSt.isAtomic())
1529 return false;
1530
1531 MatchInfo.IsPre = findPreIndexCandidate(LdSt, Addr&: MatchInfo.Addr, Base&: MatchInfo.Base,
1532 Offset&: MatchInfo.Offset);
1533 if (!MatchInfo.IsPre &&
1534 !findPostIndexCandidate(LdSt, Addr&: MatchInfo.Addr, Base&: MatchInfo.Base,
1535 Offset&: MatchInfo.Offset, RematOffset&: MatchInfo.RematOffset))
1536 return false;
1537
1538 return true;
1539}
1540
1541void CombinerHelper::applyCombineIndexedLoadStore(
1542 MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) const {
1543 MachineInstr &AddrDef = *MRI.getUniqueVRegDef(Reg: MatchInfo.Addr);
1544 unsigned Opcode = MI.getOpcode();
1545 bool IsStore = Opcode == TargetOpcode::G_STORE;
1546 unsigned NewOpcode = getIndexedOpc(LdStOpc: Opcode);
1547
1548 // If the offset constant didn't happen to dominate the load/store, we can
1549 // just clone it as needed.
1550 if (MatchInfo.RematOffset) {
1551 auto *OldCst = MRI.getVRegDef(Reg: MatchInfo.Offset);
1552 auto NewCst = Builder.buildConstant(Res: MRI.getType(Reg: MatchInfo.Offset),
1553 Val: *OldCst->getOperand(i: 1).getCImm());
1554 MatchInfo.Offset = NewCst.getReg(Idx: 0);
1555 }
1556
1557 auto MIB = Builder.buildInstr(Opcode: NewOpcode);
1558 if (IsStore) {
1559 MIB.addDef(RegNo: MatchInfo.Addr);
1560 MIB.addUse(RegNo: MI.getOperand(i: 0).getReg());
1561 } else {
1562 MIB.addDef(RegNo: MI.getOperand(i: 0).getReg());
1563 MIB.addDef(RegNo: MatchInfo.Addr);
1564 }
1565
1566 MIB.addUse(RegNo: MatchInfo.Base);
1567 MIB.addUse(RegNo: MatchInfo.Offset);
1568 MIB.addImm(Val: MatchInfo.IsPre);
1569 MIB->cloneMemRefs(MF&: *MI.getMF(), MI);
1570 MI.eraseFromParent();
1571 AddrDef.eraseFromParent();
1572
1573 LLVM_DEBUG(dbgs() << " Combinined to indexed operation");
1574}
1575
1576bool CombinerHelper::matchCombineDivRem(MachineInstr &MI,
1577 MachineInstr *&OtherMI) const {
1578 unsigned Opcode = MI.getOpcode();
1579 bool IsDiv, IsSigned;
1580
1581 switch (Opcode) {
1582 default:
1583 llvm_unreachable("Unexpected opcode!");
1584 case TargetOpcode::G_SDIV:
1585 case TargetOpcode::G_UDIV: {
1586 IsDiv = true;
1587 IsSigned = Opcode == TargetOpcode::G_SDIV;
1588 break;
1589 }
1590 case TargetOpcode::G_SREM:
1591 case TargetOpcode::G_UREM: {
1592 IsDiv = false;
1593 IsSigned = Opcode == TargetOpcode::G_SREM;
1594 break;
1595 }
1596 }
1597
1598 Register Src1 = MI.getOperand(i: 1).getReg();
1599 unsigned DivOpcode, RemOpcode, DivremOpcode;
1600 if (IsSigned) {
1601 DivOpcode = TargetOpcode::G_SDIV;
1602 RemOpcode = TargetOpcode::G_SREM;
1603 DivremOpcode = TargetOpcode::G_SDIVREM;
1604 } else {
1605 DivOpcode = TargetOpcode::G_UDIV;
1606 RemOpcode = TargetOpcode::G_UREM;
1607 DivremOpcode = TargetOpcode::G_UDIVREM;
1608 }
1609
1610 if (!isLegalOrBeforeLegalizer(Query: {DivremOpcode, {MRI.getType(Reg: Src1)}}))
1611 return false;
1612
1613 // Combine:
1614 // %div:_ = G_[SU]DIV %src1:_, %src2:_
1615 // %rem:_ = G_[SU]REM %src1:_, %src2:_
1616 // into:
1617 // %div:_, %rem:_ = G_[SU]DIVREM %src1:_, %src2:_
1618
1619 // Combine:
1620 // %rem:_ = G_[SU]REM %src1:_, %src2:_
1621 // %div:_ = G_[SU]DIV %src1:_, %src2:_
1622 // into:
1623 // %div:_, %rem:_ = G_[SU]DIVREM %src1:_, %src2:_
1624
1625 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: Src1)) {
1626 if (MI.getParent() == UseMI.getParent() &&
1627 ((IsDiv && UseMI.getOpcode() == RemOpcode) ||
1628 (!IsDiv && UseMI.getOpcode() == DivOpcode)) &&
1629 matchEqualDefs(MOP1: MI.getOperand(i: 2), MOP2: UseMI.getOperand(i: 2)) &&
1630 matchEqualDefs(MOP1: MI.getOperand(i: 1), MOP2: UseMI.getOperand(i: 1))) {
1631 OtherMI = &UseMI;
1632 return true;
1633 }
1634 }
1635
1636 return false;
1637}
1638
1639void CombinerHelper::applyCombineDivRem(MachineInstr &MI,
1640 MachineInstr *&OtherMI) const {
1641 unsigned Opcode = MI.getOpcode();
1642 assert(OtherMI && "OtherMI shouldn't be empty.");
1643
1644 Register DestDivReg, DestRemReg;
1645 if (Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_UDIV) {
1646 DestDivReg = MI.getOperand(i: 0).getReg();
1647 DestRemReg = OtherMI->getOperand(i: 0).getReg();
1648 } else {
1649 DestDivReg = OtherMI->getOperand(i: 0).getReg();
1650 DestRemReg = MI.getOperand(i: 0).getReg();
1651 }
1652
1653 bool IsSigned =
1654 Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM;
1655
1656 // Check which instruction is first in the block so we don't break def-use
1657 // deps by "moving" the instruction incorrectly. Also keep track of which
1658 // instruction is first so we pick it's operands, avoiding use-before-def
1659 // bugs.
1660 MachineInstr *FirstInst = dominates(DefMI: MI, UseMI: *OtherMI) ? &MI : OtherMI;
1661 Builder.setInstrAndDebugLoc(*FirstInst);
1662
1663 Builder.buildInstr(Opc: IsSigned ? TargetOpcode::G_SDIVREM
1664 : TargetOpcode::G_UDIVREM,
1665 DstOps: {DestDivReg, DestRemReg},
1666 SrcOps: { FirstInst->getOperand(i: 1), FirstInst->getOperand(i: 2) });
1667 MI.eraseFromParent();
1668 OtherMI->eraseFromParent();
1669}
1670
1671bool CombinerHelper::matchOptBrCondByInvertingCond(
1672 MachineInstr &MI, MachineInstr *&BrCond) const {
1673 assert(MI.getOpcode() == TargetOpcode::G_BR);
1674
1675 // Try to match the following:
1676 // bb1:
1677 // G_BRCOND %c1, %bb2
1678 // G_BR %bb3
1679 // bb2:
1680 // ...
1681 // bb3:
1682
1683 // The above pattern does not have a fall through to the successor bb2, always
1684 // resulting in a branch no matter which path is taken. Here we try to find
1685 // and replace that pattern with conditional branch to bb3 and otherwise
1686 // fallthrough to bb2. This is generally better for branch predictors.
1687
1688 MachineBasicBlock *MBB = MI.getParent();
1689 MachineBasicBlock::iterator BrIt(MI);
1690 if (BrIt == MBB->begin())
1691 return false;
1692 assert(std::next(BrIt) == MBB->end() && "expected G_BR to be a terminator");
1693
1694 BrCond = &*std::prev(x: BrIt);
1695 if (BrCond->getOpcode() != TargetOpcode::G_BRCOND)
1696 return false;
1697
1698 // Check that the next block is the conditional branch target. Also make sure
1699 // that it isn't the same as the G_BR's target (otherwise, this will loop.)
1700 MachineBasicBlock *BrCondTarget = BrCond->getOperand(i: 1).getMBB();
1701 return BrCondTarget != MI.getOperand(i: 0).getMBB() &&
1702 MBB->isLayoutSuccessor(MBB: BrCondTarget);
1703}
1704
1705void CombinerHelper::applyOptBrCondByInvertingCond(
1706 MachineInstr &MI, MachineInstr *&BrCond) const {
1707 MachineBasicBlock *BrTarget = MI.getOperand(i: 0).getMBB();
1708 Builder.setInstrAndDebugLoc(*BrCond);
1709 LLT Ty = MRI.getType(Reg: BrCond->getOperand(i: 0).getReg());
1710 // FIXME: Does int/fp matter for this? If so, we might need to restrict
1711 // this to i1 only since we might not know for sure what kind of
1712 // compare generated the condition value.
1713 auto True = Builder.buildConstant(
1714 Res: Ty, Val: getICmpTrueVal(TLI: getTargetLowering(), IsVector: false, IsFP: false));
1715 auto Xor = Builder.buildXor(Dst: Ty, Src0: BrCond->getOperand(i: 0), Src1: True);
1716
1717 auto *FallthroughBB = BrCond->getOperand(i: 1).getMBB();
1718 Observer.changingInstr(MI);
1719 MI.getOperand(i: 0).setMBB(FallthroughBB);
1720 Observer.changedInstr(MI);
1721
1722 // Change the conditional branch to use the inverted condition and
1723 // new target block.
1724 Observer.changingInstr(MI&: *BrCond);
1725 BrCond->getOperand(i: 0).setReg(Xor.getReg(Idx: 0));
1726 BrCond->getOperand(i: 1).setMBB(BrTarget);
1727 Observer.changedInstr(MI&: *BrCond);
1728}
1729
1730bool CombinerHelper::matchCombineMemCpyFamily(
1731 MachineInstr &MI, MemCpyFamilyLoweringInfo &MatchInfo,
1732 unsigned MaxLen) const {
1733 auto &[Dst, Src, KnownLen, Alignment, DstAlignCanChange, MemOps] = MatchInfo;
1734 return canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, Alignment,
1735 DstAlignCanChange, MemOps);
1736}
1737
1738void CombinerHelper::applyCombineMemCpyFamily(
1739 MachineInstr &MI, MemCpyFamilyLoweringInfo &MatchInfo) const {
1740 auto &[Dst, Src, KnownLen, Alignment, DstAlignCanChange, MemOps] = MatchInfo;
1741 MachineIRBuilder HelperBuilder(MI);
1742 GISelObserverWrapper DummyObserver;
1743 LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);
1744 bool Changed = Helper.lowerMemCpyFamily(MI, Dst, Src, KnownLen, Alignment,
1745 DstAlignCanChange, MemOps) ==
1746 LegalizerHelper::LegalizeResult::Legalized;
1747 assert(Changed && "expected memcpy-family instruction to lower");
1748 (void)Changed;
1749}
1750
1751bool CombinerHelper::tryCombineMemCpyFamily(MachineInstr &MI,
1752 unsigned MaxLen) const {
1753 MachineIRBuilder HelperBuilder(MI);
1754 GISelObserverWrapper DummyObserver;
1755 LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);
1756 return Helper.lowerMemCpyFamily(MI, MaxLen) ==
1757 LegalizerHelper::LegalizeResult::Legalized;
1758}
1759
1760static APFloat constantFoldFpUnary(const MachineInstr &MI,
1761 const MachineRegisterInfo &MRI,
1762 const APFloat &Val) {
1763 APFloat Result(Val);
1764 switch (MI.getOpcode()) {
1765 default:
1766 llvm_unreachable("Unexpected opcode!");
1767 case TargetOpcode::G_FNEG: {
1768 Result.changeSign();
1769 return Result;
1770 }
1771 case TargetOpcode::G_FABS: {
1772 Result.clearSign();
1773 return Result;
1774 }
1775 case TargetOpcode::G_FCEIL:
1776 Result.roundToIntegral(RM: APFloat::rmTowardPositive);
1777 return Result;
1778 case TargetOpcode::G_FFLOOR:
1779 Result.roundToIntegral(RM: APFloat::rmTowardNegative);
1780 return Result;
1781 case TargetOpcode::G_INTRINSIC_TRUNC:
1782 Result.roundToIntegral(RM: APFloat::rmTowardZero);
1783 return Result;
1784 case TargetOpcode::G_INTRINSIC_ROUND:
1785 Result.roundToIntegral(RM: APFloat::rmNearestTiesToAway);
1786 return Result;
1787 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
1788 Result.roundToIntegral(RM: APFloat::rmNearestTiesToEven);
1789 return Result;
1790 case TargetOpcode::G_FRINT:
1791 case TargetOpcode::G_FNEARBYINT:
1792 // Use default rounding mode (round to nearest, ties to even)
1793 Result.roundToIntegral(RM: APFloat::rmNearestTiesToEven);
1794 return Result;
1795 case TargetOpcode::G_FPEXT:
1796 case TargetOpcode::G_FPTRUNC: {
1797 bool Unused;
1798 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1799 Result.convert(ToSemantics: getFltSemanticForLLT(Ty: DstTy), RM: APFloat::rmNearestTiesToEven,
1800 losesInfo: &Unused);
1801 return Result;
1802 }
1803 case TargetOpcode::G_FSQRT: {
1804 bool Unused;
1805 Result.convert(ToSemantics: APFloat::IEEEdouble(), RM: APFloat::rmNearestTiesToEven,
1806 losesInfo: &Unused);
1807 Result = APFloat(sqrt(x: Result.convertToDouble()));
1808 break;
1809 }
1810 case TargetOpcode::G_FLOG2: {
1811 bool Unused;
1812 Result.convert(ToSemantics: APFloat::IEEEdouble(), RM: APFloat::rmNearestTiesToEven,
1813 losesInfo: &Unused);
1814 Result = APFloat(log2(x: Result.convertToDouble()));
1815 break;
1816 }
1817 }
1818 // Convert `APFloat` to appropriate IEEE type depending on `DstTy`. Otherwise,
1819 // `buildFConstant` will assert on size mismatch. Only `G_FSQRT`, and
1820 // `G_FLOG2` reach here.
1821 bool Unused;
1822 Result.convert(ToSemantics: Val.getSemantics(), RM: APFloat::rmNearestTiesToEven, losesInfo: &Unused);
1823 return Result;
1824}
1825
1826void CombinerHelper::applyCombineConstantFoldFpUnary(
1827 MachineInstr &MI, const ConstantFP *Cst) const {
1828 APFloat Folded = constantFoldFpUnary(MI, MRI, Val: Cst->getValue());
1829 const ConstantFP *NewCst = ConstantFP::get(Context&: Builder.getContext(), V: Folded);
1830 Builder.buildFConstant(Res: MI.getOperand(i: 0), Val: *NewCst);
1831 MI.eraseFromParent();
1832}
1833
1834bool CombinerHelper::matchPtrAddImmedChain(MachineInstr &MI,
1835 PtrAddChain &MatchInfo) const {
1836 // We're trying to match the following pattern:
1837 // %t1 = G_PTR_ADD %base, G_CONSTANT imm1
1838 // %root = G_PTR_ADD %t1, G_CONSTANT imm2
1839 // -->
1840 // %root = G_PTR_ADD %base, G_CONSTANT (imm1 + imm2)
1841
1842 if (MI.getOpcode() != TargetOpcode::G_PTR_ADD)
1843 return false;
1844
1845 Register Add2 = MI.getOperand(i: 1).getReg();
1846 Register Imm1 = MI.getOperand(i: 2).getReg();
1847 auto MaybeImmVal = getIConstantVRegValWithLookThrough(VReg: Imm1, MRI);
1848 if (!MaybeImmVal)
1849 return false;
1850
1851 MachineInstr *Add2Def = MRI.getVRegDef(Reg: Add2);
1852 if (!Add2Def || Add2Def->getOpcode() != TargetOpcode::G_PTR_ADD)
1853 return false;
1854
1855 Register Base = Add2Def->getOperand(i: 1).getReg();
1856 Register Imm2 = Add2Def->getOperand(i: 2).getReg();
1857 auto MaybeImm2Val = getIConstantVRegValWithLookThrough(VReg: Imm2, MRI);
1858 if (!MaybeImm2Val)
1859 return false;
1860
1861 // Check if the new combined immediate forms an illegal addressing mode.
1862 // Do not combine if it was legal before but would get illegal.
1863 // To do so, we need to find a load/store user of the pointer to get
1864 // the access type.
1865 Type *AccessTy = nullptr;
1866 auto &MF = *MI.getMF();
1867 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg())) {
1868 if (auto *LdSt = dyn_cast<GLoadStore>(Val: &UseMI)) {
1869 AccessTy = getTypeForLLT(Ty: MRI.getType(Reg: LdSt->getReg(Idx: 0)),
1870 C&: MF.getFunction().getContext());
1871 break;
1872 }
1873 }
1874 TargetLoweringBase::AddrMode AMNew;
1875 APInt CombinedImm = MaybeImmVal->Value + MaybeImm2Val->Value;
1876 AMNew.BaseOffs = CombinedImm.getSExtValue();
1877 if (AccessTy) {
1878 AMNew.HasBaseReg = true;
1879 TargetLoweringBase::AddrMode AMOld;
1880 AMOld.BaseOffs = MaybeImmVal->Value.getSExtValue();
1881 AMOld.HasBaseReg = true;
1882 unsigned AS = MRI.getType(Reg: Add2).getAddressSpace();
1883 const auto &TLI = *MF.getSubtarget().getTargetLowering();
1884 if (TLI.isLegalAddressingMode(DL: MF.getDataLayout(), AM: AMOld, Ty: AccessTy, AddrSpace: AS) &&
1885 !TLI.isLegalAddressingMode(DL: MF.getDataLayout(), AM: AMNew, Ty: AccessTy, AddrSpace: AS))
1886 return false;
1887 }
1888
1889 // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
1890 // inbounds, reaching the same result in one G_PTR_ADD is also inbounds.
1891 // The nusw constraints are satisfied because imm1+imm2 cannot exceed the
1892 // largest signed integer that fits into the index type, which is the maximum
1893 // size of allocated objects according to the IR Language Reference.
1894 unsigned PtrAddFlags = MI.getFlags();
1895 unsigned LHSPtrAddFlags = Add2Def->getFlags();
1896 bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
1897 bool IsInBounds =
1898 PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::InBounds;
1899 unsigned Flags = 0;
1900 if (IsNoUWrap)
1901 Flags |= MachineInstr::MIFlag::NoUWrap;
1902 if (IsInBounds) {
1903 Flags |= MachineInstr::MIFlag::InBounds;
1904 Flags |= MachineInstr::MIFlag::NoUSWrap;
1905 }
1906
1907 // Pass the combined immediate to the apply function.
1908 MatchInfo.Imm = AMNew.BaseOffs;
1909 MatchInfo.Base = Base;
1910 MatchInfo.Bank = getRegBank(Reg: Imm2);
1911 MatchInfo.Flags = Flags;
1912 return true;
1913}
1914
1915void CombinerHelper::applyPtrAddImmedChain(MachineInstr &MI,
1916 PtrAddChain &MatchInfo) const {
1917 assert(MI.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD");
1918 MachineIRBuilder MIB(MI);
1919 LLT OffsetTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
1920 auto NewOffset = MIB.buildConstant(Res: OffsetTy, Val: MatchInfo.Imm);
1921 setRegBank(Reg: NewOffset.getReg(Idx: 0), RegBank: MatchInfo.Bank);
1922 Observer.changingInstr(MI);
1923 MI.getOperand(i: 1).setReg(MatchInfo.Base);
1924 MI.getOperand(i: 2).setReg(NewOffset.getReg(Idx: 0));
1925 MI.setFlags(MatchInfo.Flags);
1926 Observer.changedInstr(MI);
1927}
1928
1929bool CombinerHelper::matchShiftImmedChain(MachineInstr &MI,
1930 RegisterImmPair &MatchInfo) const {
1931 // We're trying to match the following pattern with any of
1932 // G_SHL/G_ASHR/G_LSHR/G_SSHLSAT/G_USHLSAT shift instructions:
1933 // %t1 = SHIFT %base, G_CONSTANT imm1
1934 // %root = SHIFT %t1, G_CONSTANT imm2
1935 // -->
1936 // %root = SHIFT %base, G_CONSTANT (imm1 + imm2)
1937
1938 unsigned Opcode = MI.getOpcode();
1939 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1940 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1941 Opcode == TargetOpcode::G_USHLSAT) &&
1942 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1943
1944 Register Shl2 = MI.getOperand(i: 1).getReg();
1945 Register Imm1 = MI.getOperand(i: 2).getReg();
1946 auto MaybeImmVal = getIConstantVRegValWithLookThrough(VReg: Imm1, MRI);
1947 if (!MaybeImmVal)
1948 return false;
1949
1950 MachineInstr *Shl2Def = MRI.getUniqueVRegDef(Reg: Shl2);
1951 if (Shl2Def->getOpcode() != Opcode)
1952 return false;
1953
1954 Register Base = Shl2Def->getOperand(i: 1).getReg();
1955 Register Imm2 = Shl2Def->getOperand(i: 2).getReg();
1956 auto MaybeImm2Val = getIConstantVRegValWithLookThrough(VReg: Imm2, MRI);
1957 if (!MaybeImm2Val)
1958 return false;
1959
1960 // Pass the combined immediate to the apply function.
1961 MatchInfo.Imm =
1962 (MaybeImmVal->Value.getZExtValue() + MaybeImm2Val->Value).getZExtValue();
1963 MatchInfo.Reg = Base;
1964
1965 // There is no simple replacement for a saturating unsigned left shift that
1966 // exceeds the scalar size.
1967 if (Opcode == TargetOpcode::G_USHLSAT &&
1968 MatchInfo.Imm >= MRI.getType(Reg: Shl2).getScalarSizeInBits())
1969 return false;
1970
1971 return true;
1972}
1973
1974void CombinerHelper::applyShiftImmedChain(MachineInstr &MI,
1975 RegisterImmPair &MatchInfo) const {
1976 unsigned Opcode = MI.getOpcode();
1977 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1978 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1979 Opcode == TargetOpcode::G_USHLSAT) &&
1980 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1981
1982 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1983 unsigned const ScalarSizeInBits = Ty.getScalarSizeInBits();
1984 auto Imm = MatchInfo.Imm;
1985
1986 if (Imm >= ScalarSizeInBits) {
1987 // Any logical shift that exceeds scalar size will produce zero.
1988 if (Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR) {
1989 Builder.buildConstant(Res: MI.getOperand(i: 0), Val: 0);
1990 MI.eraseFromParent();
1991 return;
1992 }
1993 // Arithmetic shift and saturating signed left shift have no effect beyond
1994 // scalar size.
1995 Imm = ScalarSizeInBits - 1;
1996 }
1997
1998 LLT ImmTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
1999 Register NewImm = Builder.buildConstant(Res: ImmTy, Val: Imm).getReg(Idx: 0);
2000 Observer.changingInstr(MI);
2001 MI.getOperand(i: 1).setReg(MatchInfo.Reg);
2002 MI.getOperand(i: 2).setReg(NewImm);
2003 Observer.changedInstr(MI);
2004}
2005
2006bool CombinerHelper::matchShiftOfShiftedLogic(
2007 MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo) const {
2008 // We're trying to match the following pattern with any of
2009 // G_SHL/G_ASHR/G_LSHR/G_USHLSAT/G_SSHLSAT shift instructions in combination
2010 // with any of G_AND/G_OR/G_XOR logic instructions.
2011 // %t1 = SHIFT %X, G_CONSTANT C0
2012 // %t2 = LOGIC %t1, %Y
2013 // %root = SHIFT %t2, G_CONSTANT C1
2014 // -->
2015 // %t3 = SHIFT %X, G_CONSTANT (C0+C1)
2016 // %t4 = SHIFT %Y, G_CONSTANT C1
2017 // %root = LOGIC %t3, %t4
2018 unsigned ShiftOpcode = MI.getOpcode();
2019 assert((ShiftOpcode == TargetOpcode::G_SHL ||
2020 ShiftOpcode == TargetOpcode::G_ASHR ||
2021 ShiftOpcode == TargetOpcode::G_LSHR ||
2022 ShiftOpcode == TargetOpcode::G_USHLSAT ||
2023 ShiftOpcode == TargetOpcode::G_SSHLSAT) &&
2024 "Expected G_SHL, G_ASHR, G_LSHR, G_USHLSAT and G_SSHLSAT");
2025
2026 // Match a one-use bitwise logic op.
2027 Register LogicDest = MI.getOperand(i: 1).getReg();
2028 if (!MRI.hasOneNonDBGUse(RegNo: LogicDest))
2029 return false;
2030
2031 MachineInstr *LogicMI = MRI.getUniqueVRegDef(Reg: LogicDest);
2032 unsigned LogicOpcode = LogicMI->getOpcode();
2033 if (LogicOpcode != TargetOpcode::G_AND && LogicOpcode != TargetOpcode::G_OR &&
2034 LogicOpcode != TargetOpcode::G_XOR)
2035 return false;
2036
2037 // Find a matching one-use shift by constant.
2038 const Register C1 = MI.getOperand(i: 2).getReg();
2039 auto MaybeImmVal = getIConstantVRegValWithLookThrough(VReg: C1, MRI);
2040 if (!MaybeImmVal || MaybeImmVal->Value == 0)
2041 return false;
2042
2043 const uint64_t C1Val = MaybeImmVal->Value.getZExtValue();
2044
2045 auto matchFirstShift = [&](const MachineInstr *MI, uint64_t &ShiftVal) {
2046 // Shift should match previous one and should be a one-use.
2047 if (MI->getOpcode() != ShiftOpcode ||
2048 !MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
2049 return false;
2050
2051 // Must be a constant.
2052 auto MaybeImmVal =
2053 getIConstantVRegValWithLookThrough(VReg: MI->getOperand(i: 2).getReg(), MRI);
2054 if (!MaybeImmVal)
2055 return false;
2056
2057 ShiftVal = MaybeImmVal->Value.getSExtValue();
2058 return true;
2059 };
2060
2061 // Logic ops are commutative, so check each operand for a match.
2062 Register LogicMIReg1 = LogicMI->getOperand(i: 1).getReg();
2063 MachineInstr *LogicMIOp1 = MRI.getUniqueVRegDef(Reg: LogicMIReg1);
2064 Register LogicMIReg2 = LogicMI->getOperand(i: 2).getReg();
2065 MachineInstr *LogicMIOp2 = MRI.getUniqueVRegDef(Reg: LogicMIReg2);
2066 uint64_t C0Val;
2067
2068 if (matchFirstShift(LogicMIOp1, C0Val)) {
2069 MatchInfo.LogicNonShiftReg = LogicMIReg2;
2070 MatchInfo.Shift2 = LogicMIOp1;
2071 } else if (matchFirstShift(LogicMIOp2, C0Val)) {
2072 MatchInfo.LogicNonShiftReg = LogicMIReg1;
2073 MatchInfo.Shift2 = LogicMIOp2;
2074 } else
2075 return false;
2076
2077 MatchInfo.ValSum = C0Val + C1Val;
2078
2079 // The fold is not valid if the sum of the shift values exceeds bitwidth.
2080 if (MatchInfo.ValSum >= MRI.getType(Reg: LogicDest).getScalarSizeInBits())
2081 return false;
2082
2083 MatchInfo.Logic = LogicMI;
2084 return true;
2085}
2086
2087void CombinerHelper::applyShiftOfShiftedLogic(
2088 MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo) const {
2089 unsigned Opcode = MI.getOpcode();
2090 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
2091 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_USHLSAT ||
2092 Opcode == TargetOpcode::G_SSHLSAT) &&
2093 "Expected G_SHL, G_ASHR, G_LSHR, G_USHLSAT and G_SSHLSAT");
2094
2095 LLT ShlType = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
2096 LLT DestType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2097
2098 Register Const = Builder.buildConstant(Res: ShlType, Val: MatchInfo.ValSum).getReg(Idx: 0);
2099
2100 Register Shift1Base = MatchInfo.Shift2->getOperand(i: 1).getReg();
2101 Register Shift1 =
2102 Builder.buildInstr(Opc: Opcode, DstOps: {DestType}, SrcOps: {Shift1Base, Const}).getReg(Idx: 0);
2103
2104 // If LogicNonShiftReg is the same to Shift1Base, and shift1 const is the same
2105 // to MatchInfo.Shift2 const, CSEMIRBuilder will reuse the old shift1 when
2106 // build shift2. So, if we erase MatchInfo.Shift2 at the end, actually we
2107 // remove old shift1. And it will cause crash later. So erase it earlier to
2108 // avoid the crash.
2109 MatchInfo.Shift2->eraseFromParent();
2110
2111 Register Shift2Const = MI.getOperand(i: 2).getReg();
2112 Register Shift2 = Builder
2113 .buildInstr(Opc: Opcode, DstOps: {DestType},
2114 SrcOps: {MatchInfo.LogicNonShiftReg, Shift2Const})
2115 .getReg(Idx: 0);
2116
2117 Register Dest = MI.getOperand(i: 0).getReg();
2118 Builder.buildInstr(Opc: MatchInfo.Logic->getOpcode(), DstOps: {Dest}, SrcOps: {Shift1, Shift2});
2119
2120 // This was one use so it's safe to remove it.
2121 MatchInfo.Logic->eraseFromParent();
2122
2123 MI.eraseFromParent();
2124}
2125
2126bool CombinerHelper::matchCommuteShift(MachineInstr &MI,
2127 BuildFnTy &MatchInfo) const {
2128 assert(MI.getOpcode() == TargetOpcode::G_SHL && "Expected G_SHL");
2129 // Combine (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
2130 // Combine (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
2131 auto &Shl = cast<GenericMachineInstr>(Val&: MI);
2132 Register DstReg = Shl.getReg(Idx: 0);
2133 Register SrcReg = Shl.getReg(Idx: 1);
2134 Register ShiftReg = Shl.getReg(Idx: 2);
2135 Register X, C1;
2136
2137 if (!getTargetLowering().isDesirableToCommuteWithShift(MI, IsAfterLegal: !isPreLegalize()))
2138 return false;
2139
2140 if (!mi_match(R: SrcReg, MRI,
2141 P: m_OneNonDBGUse(SP: m_any_of(preds: m_GAdd(L: m_Reg(R&: X), R: m_Reg(R&: C1)),
2142 preds: m_GOr(L: m_Reg(R&: X), R: m_Reg(R&: C1))))))
2143 return false;
2144
2145 APInt C1Val, C2Val;
2146 if (!mi_match(R: C1, MRI, P: m_ICstOrSplat(Cst&: C1Val)) ||
2147 !mi_match(R: ShiftReg, MRI, P: m_ICstOrSplat(Cst&: C2Val)))
2148 return false;
2149
2150 auto *SrcDef = MRI.getVRegDef(Reg: SrcReg);
2151 assert((SrcDef->getOpcode() == TargetOpcode::G_ADD ||
2152 SrcDef->getOpcode() == TargetOpcode::G_OR) && "Unexpected op");
2153 LLT SrcTy = MRI.getType(Reg: SrcReg);
2154 MatchInfo = [=](MachineIRBuilder &B) {
2155 auto S1 = B.buildShl(Dst: SrcTy, Src0: X, Src1: ShiftReg);
2156 auto S2 = B.buildShl(Dst: SrcTy, Src0: C1, Src1: ShiftReg);
2157 B.buildInstr(Opc: SrcDef->getOpcode(), DstOps: {DstReg}, SrcOps: {S1, S2});
2158 };
2159 return true;
2160}
2161
2162bool CombinerHelper::matchLshrOfTruncOfLshr(MachineInstr &MI,
2163 LshrOfTruncOfLshr &MatchInfo,
2164 MachineInstr &ShiftMI) const {
2165 assert(MI.getOpcode() == TargetOpcode::G_LSHR && "Expected a G_LSHR");
2166
2167 Register N0 = MI.getOperand(i: 1).getReg();
2168 Register N1 = MI.getOperand(i: 2).getReg();
2169 unsigned OpSizeInBits = MRI.getType(Reg: N0).getScalarSizeInBits();
2170
2171 APInt N1C, N001C;
2172 if (!mi_match(R: N1, MRI, P: m_ICstOrSplat(Cst&: N1C)))
2173 return false;
2174 auto N001 = ShiftMI.getOperand(i: 2).getReg();
2175 if (!mi_match(R: N001, MRI, P: m_ICstOrSplat(Cst&: N001C)))
2176 return false;
2177
2178 if (N001C.getBitWidth() > N1C.getBitWidth())
2179 N1C = N1C.zext(width: N001C.getBitWidth());
2180 else
2181 N001C = N001C.zext(width: N1C.getBitWidth());
2182
2183 Register InnerShift = ShiftMI.getOperand(i: 0).getReg();
2184 LLT InnerShiftTy = MRI.getType(Reg: InnerShift);
2185 uint64_t InnerShiftSize = InnerShiftTy.getScalarSizeInBits();
2186 if ((N1C + N001C).ult(RHS: InnerShiftSize)) {
2187 MatchInfo.Src = ShiftMI.getOperand(i: 1).getReg();
2188 MatchInfo.ShiftAmt = N1C + N001C;
2189 MatchInfo.ShiftAmtTy = MRI.getType(Reg: N001);
2190 MatchInfo.InnerShiftTy = InnerShiftTy;
2191
2192 if ((N001C + OpSizeInBits) == InnerShiftSize)
2193 return true;
2194 if (MRI.hasOneUse(RegNo: N0) && MRI.hasOneUse(RegNo: InnerShift)) {
2195 MatchInfo.Mask = true;
2196 MatchInfo.MaskVal = APInt(N1C.getBitWidth(), OpSizeInBits) - N1C;
2197 return true;
2198 }
2199 }
2200 return false;
2201}
2202
2203void CombinerHelper::applyLshrOfTruncOfLshr(
2204 MachineInstr &MI, LshrOfTruncOfLshr &MatchInfo) const {
2205 assert(MI.getOpcode() == TargetOpcode::G_LSHR && "Expected a G_LSHR");
2206
2207 Register Dst = MI.getOperand(i: 0).getReg();
2208 auto ShiftAmt =
2209 Builder.buildConstant(Res: MatchInfo.ShiftAmtTy, Val: MatchInfo.ShiftAmt);
2210 auto Shift =
2211 Builder.buildLShr(Dst: MatchInfo.InnerShiftTy, Src0: MatchInfo.Src, Src1: ShiftAmt);
2212 if (MatchInfo.Mask == true) {
2213 APInt MaskVal =
2214 APInt::getLowBitsSet(numBits: MatchInfo.InnerShiftTy.getScalarSizeInBits(),
2215 loBitsSet: MatchInfo.MaskVal.getZExtValue());
2216 auto Mask = Builder.buildConstant(Res: MatchInfo.InnerShiftTy, Val: MaskVal);
2217 auto And = Builder.buildAnd(Dst: MatchInfo.InnerShiftTy, Src0: Shift, Src1: Mask);
2218 Builder.buildTrunc(Res: Dst, Op: And);
2219 } else
2220 Builder.buildTrunc(Res: Dst, Op: Shift);
2221 MI.eraseFromParent();
2222}
2223
2224bool CombinerHelper::matchCombineMulToShl(MachineInstr &MI,
2225 unsigned &ShiftVal) const {
2226 assert(MI.getOpcode() == TargetOpcode::G_MUL && "Expected a G_MUL");
2227 auto MaybeImmVal =
2228 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
2229 if (!MaybeImmVal)
2230 return false;
2231
2232 ShiftVal = MaybeImmVal->Value.exactLogBase2();
2233 return (static_cast<int32_t>(ShiftVal) != -1);
2234}
2235
2236void CombinerHelper::applyCombineMulToShl(MachineInstr &MI,
2237 unsigned &ShiftVal) const {
2238 assert(MI.getOpcode() == TargetOpcode::G_MUL && "Expected a G_MUL");
2239 MachineIRBuilder MIB(MI);
2240 LLT ShiftTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2241 auto ShiftCst = MIB.buildConstant(Res: ShiftTy, Val: ShiftVal);
2242 Observer.changingInstr(MI);
2243 MI.setDesc(MIB.getTII().get(Opcode: TargetOpcode::G_SHL));
2244 MI.getOperand(i: 2).setReg(ShiftCst.getReg(Idx: 0));
2245 if (ShiftVal == ShiftTy.getScalarSizeInBits() - 1)
2246 MI.clearFlag(Flag: MachineInstr::MIFlag::NoSWrap);
2247 Observer.changedInstr(MI);
2248}
2249
2250bool CombinerHelper::matchCombineSubToAdd(MachineInstr &MI,
2251 BuildFnTy &MatchInfo) const {
2252 GSub &Sub = cast<GSub>(Val&: MI);
2253
2254 LLT Ty = MRI.getType(Reg: Sub.getReg(Idx: 0));
2255
2256 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ADD, {Ty}}))
2257 return false;
2258
2259 if (!isConstantLegalOrBeforeLegalizer(Ty))
2260 return false;
2261
2262 APInt Imm = getIConstantFromReg(VReg: Sub.getRHSReg(), MRI);
2263
2264 MatchInfo = [=, &MI](MachineIRBuilder &B) {
2265 auto NegCst = B.buildConstant(Res: Ty, Val: -Imm);
2266 Observer.changingInstr(MI);
2267 MI.setDesc(B.getTII().get(Opcode: TargetOpcode::G_ADD));
2268 MI.getOperand(i: 2).setReg(NegCst.getReg(Idx: 0));
2269 MI.clearFlag(Flag: MachineInstr::MIFlag::NoUWrap);
2270 if (Imm.isMinSignedValue())
2271 MI.clearFlags(flags: MachineInstr::MIFlag::NoSWrap);
2272 Observer.changedInstr(MI);
2273 };
2274 return true;
2275}
2276
2277// shl ([sza]ext x), y => zext (shl x, y), if shift does not overflow source
2278bool CombinerHelper::matchCombineShlOfExtend(MachineInstr &MI,
2279 RegisterImmPair &MatchData) const {
2280 assert(MI.getOpcode() == TargetOpcode::G_SHL && VT);
2281 if (!getTargetLowering().isDesirableToPullExtFromShl(MI))
2282 return false;
2283
2284 Register LHS = MI.getOperand(i: 1).getReg();
2285
2286 Register ExtSrc;
2287 if (!mi_match(R: LHS, MRI, P: m_GAnyExt(Src: m_Reg(R&: ExtSrc))) &&
2288 !mi_match(R: LHS, MRI, P: m_GZExt(Src: m_Reg(R&: ExtSrc))) &&
2289 !mi_match(R: LHS, MRI, P: m_GSExt(Src: m_Reg(R&: ExtSrc))))
2290 return false;
2291
2292 Register RHS = MI.getOperand(i: 2).getReg();
2293 auto MaybeShiftAmtVal = isConstantOrConstantSplatVector(Def: RHS, MRI);
2294 if (!MaybeShiftAmtVal)
2295 return false;
2296
2297 if (LI) {
2298 LLT SrcTy = MRI.getType(Reg: ExtSrc);
2299
2300 // We only really care about the legality with the shifted value. We can
2301 // pick any type the constant shift amount, so ask the target what to
2302 // use. Otherwise we would have to guess and hope it is reported as legal.
2303 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: SrcTy);
2304 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SHL, {SrcTy, ShiftAmtTy}}))
2305 return false;
2306 }
2307
2308 int64_t ShiftAmt = MaybeShiftAmtVal->getSExtValue();
2309 MatchData.Reg = ExtSrc;
2310 MatchData.Imm = ShiftAmt;
2311
2312 unsigned MinLeadingZeros = VT->getKnownZeroes(R: ExtSrc).countl_one();
2313 unsigned SrcTySize = MRI.getType(Reg: ExtSrc).getScalarSizeInBits();
2314 return MinLeadingZeros >= ShiftAmt && ShiftAmt < SrcTySize;
2315}
2316
2317void CombinerHelper::applyCombineShlOfExtend(
2318 MachineInstr &MI, const RegisterImmPair &MatchData) const {
2319 Register ExtSrcReg = MatchData.Reg;
2320 int64_t ShiftAmtVal = MatchData.Imm;
2321
2322 LLT ExtSrcTy = MRI.getType(Reg: ExtSrcReg);
2323 auto ShiftAmt = Builder.buildConstant(Res: ExtSrcTy, Val: ShiftAmtVal);
2324 auto NarrowShift =
2325 Builder.buildShl(Dst: ExtSrcTy, Src0: ExtSrcReg, Src1: ShiftAmt, Flags: MI.getFlags());
2326 Builder.buildZExt(Res: MI.getOperand(i: 0), Op: NarrowShift);
2327 MI.eraseFromParent();
2328}
2329
2330bool CombinerHelper::matchCombineMergeUnmerge(MachineInstr &MI,
2331 Register &MatchInfo) const {
2332 GMerge &Merge = cast<GMerge>(Val&: MI);
2333 SmallVector<Register, 16> MergedValues;
2334 for (unsigned I = 0; I < Merge.getNumSources(); ++I)
2335 MergedValues.emplace_back(Args: Merge.getSourceReg(I));
2336
2337 auto *Unmerge = getOpcodeDef<GUnmerge>(Reg: MergedValues[0], MRI);
2338 if (!Unmerge || Unmerge->getNumDefs() != Merge.getNumSources())
2339 return false;
2340
2341 for (unsigned I = 0; I < MergedValues.size(); ++I)
2342 if (MergedValues[I] != Unmerge->getReg(Idx: I))
2343 return false;
2344
2345 MatchInfo = Unmerge->getSourceReg();
2346 return true;
2347}
2348
2349static Register peekThroughBitcast(Register Reg,
2350 const MachineRegisterInfo &MRI) {
2351 while (mi_match(R: Reg, MRI, P: m_GBitcast(Src: m_Reg(R&: Reg))))
2352 ;
2353
2354 return Reg;
2355}
2356
2357bool CombinerHelper::matchCombineUnmergeMergeToPlainValues(
2358 MachineInstr &MI, SmallVectorImpl<Register> &Operands) const {
2359 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2360 "Expected an unmerge");
2361 auto &Unmerge = cast<GUnmerge>(Val&: MI);
2362 Register SrcReg = peekThroughBitcast(Reg: Unmerge.getSourceReg(), MRI);
2363
2364 auto *SrcInstr = getOpcodeDef<GMergeLikeInstr>(Reg: SrcReg, MRI);
2365 if (!SrcInstr)
2366 return false;
2367
2368 // Check the source type of the merge.
2369 LLT SrcMergeTy = MRI.getType(Reg: SrcInstr->getSourceReg(I: 0));
2370 LLT Dst0Ty = MRI.getType(Reg: Unmerge.getReg(Idx: 0));
2371 bool SameSize = Dst0Ty.getSizeInBits() == SrcMergeTy.getSizeInBits();
2372 if (SrcMergeTy != Dst0Ty && !SameSize)
2373 return false;
2374 // They are the same now (modulo a bitcast).
2375 // We can collect all the src registers.
2376 for (unsigned Idx = 0; Idx < SrcInstr->getNumSources(); ++Idx)
2377 Operands.push_back(Elt: SrcInstr->getSourceReg(I: Idx));
2378 return true;
2379}
2380
2381void CombinerHelper::applyCombineUnmergeMergeToPlainValues(
2382 MachineInstr &MI, SmallVectorImpl<Register> &Operands) const {
2383 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2384 "Expected an unmerge");
2385 assert((MI.getNumOperands() - 1 == Operands.size()) &&
2386 "Not enough operands to replace all defs");
2387 unsigned NumElems = MI.getNumOperands() - 1;
2388
2389 LLT SrcTy = MRI.getType(Reg: Operands[0]);
2390 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2391 bool CanReuseInputDirectly = DstTy == SrcTy;
2392 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2393 Register DstReg = MI.getOperand(i: Idx).getReg();
2394 Register SrcReg = Operands[Idx];
2395
2396 // This combine may run after RegBankSelect, so we need to be aware of
2397 // register banks.
2398 const auto &DstCB = MRI.getRegClassOrRegBank(Reg: DstReg);
2399 if (!DstCB.isNull() && DstCB != MRI.getRegClassOrRegBank(Reg: SrcReg)) {
2400 SrcReg = Builder.buildCopy(Res: MRI.getType(Reg: SrcReg), Op: SrcReg).getReg(Idx: 0);
2401 MRI.setRegClassOrRegBank(Reg: SrcReg, RCOrRB: DstCB);
2402 }
2403
2404 if (CanReuseInputDirectly)
2405 replaceRegWith(MRI, FromReg: DstReg, ToReg: SrcReg);
2406 else
2407 Builder.buildCast(Dst: DstReg, Src: SrcReg);
2408 }
2409 MI.eraseFromParent();
2410}
2411
2412bool CombinerHelper::matchCombineUnmergeConstant(
2413 MachineInstr &MI, SmallVectorImpl<APInt> &Csts) const {
2414 unsigned SrcIdx = MI.getNumOperands() - 1;
2415 Register SrcReg = MI.getOperand(i: SrcIdx).getReg();
2416 MachineInstr *SrcInstr = MRI.getVRegDef(Reg: SrcReg);
2417 if (SrcInstr->getOpcode() != TargetOpcode::G_CONSTANT &&
2418 SrcInstr->getOpcode() != TargetOpcode::G_FCONSTANT)
2419 return false;
2420 // Break down the big constant in smaller ones.
2421 const MachineOperand &CstVal = SrcInstr->getOperand(i: 1);
2422 APInt Val = SrcInstr->getOpcode() == TargetOpcode::G_CONSTANT
2423 ? CstVal.getCImm()->getValue()
2424 : CstVal.getFPImm()->getValueAPF().bitcastToAPInt();
2425
2426 LLT Dst0Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2427 unsigned ShiftAmt = Dst0Ty.getSizeInBits();
2428 // Unmerge a constant.
2429 for (unsigned Idx = 0; Idx != SrcIdx; ++Idx) {
2430 Csts.emplace_back(Args: Val.trunc(width: ShiftAmt));
2431 Val = Val.lshr(shiftAmt: ShiftAmt);
2432 }
2433
2434 return true;
2435}
2436
2437void CombinerHelper::applyCombineUnmergeConstant(
2438 MachineInstr &MI, SmallVectorImpl<APInt> &Csts) const {
2439 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2440 "Expected an unmerge");
2441 assert((MI.getNumOperands() - 1 == Csts.size()) &&
2442 "Not enough operands to replace all defs");
2443 unsigned NumElems = MI.getNumOperands() - 1;
2444 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2445 Register DstReg = MI.getOperand(i: Idx).getReg();
2446 Builder.buildConstant(Res: DstReg, Val: Csts[Idx]);
2447 }
2448
2449 MI.eraseFromParent();
2450}
2451
2452bool CombinerHelper::matchCombineUnmergeUndef(
2453 MachineInstr &MI,
2454 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
2455 unsigned SrcIdx = MI.getNumOperands() - 1;
2456 Register SrcReg = MI.getOperand(i: SrcIdx).getReg();
2457 MatchInfo = [&MI](MachineIRBuilder &B) {
2458 unsigned NumElems = MI.getNumOperands() - 1;
2459 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2460 Register DstReg = MI.getOperand(i: Idx).getReg();
2461 B.buildUndef(Res: DstReg);
2462 }
2463 };
2464 return isa<GImplicitDef>(Val: MRI.getVRegDef(Reg: SrcReg));
2465}
2466
2467bool CombinerHelper::matchCombineUnmergeWithDeadLanesToTrunc(
2468 MachineInstr &MI) const {
2469 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2470 "Expected an unmerge");
2471 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isScalar() ||
2472 !MRI.getType(Reg: MI.getOperand(i: MI.getNumDefs()).getReg()).isScalar())
2473 return false;
2474 // Check that all the lanes are dead except the first one.
2475 for (unsigned Idx = 1, EndIdx = MI.getNumDefs(); Idx != EndIdx; ++Idx) {
2476 if (!MRI.use_nodbg_empty(RegNo: MI.getOperand(i: Idx).getReg()))
2477 return false;
2478 }
2479 return true;
2480}
2481
2482void CombinerHelper::applyCombineUnmergeWithDeadLanesToTrunc(
2483 MachineInstr &MI) const {
2484 Register SrcReg = MI.getOperand(i: MI.getNumDefs()).getReg();
2485 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2486 Builder.buildTrunc(Res: Dst0Reg, Op: SrcReg);
2487 MI.eraseFromParent();
2488}
2489
2490bool CombinerHelper::matchCombineUnmergeZExtToZExt(MachineInstr &MI) const {
2491 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2492 "Expected an unmerge");
2493 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2494 LLT Dst0Ty = MRI.getType(Reg: Dst0Reg);
2495 // G_ZEXT on vector applies to each lane, so it will
2496 // affect all destinations. Therefore we won't be able
2497 // to simplify the unmerge to just the first definition.
2498 if (Dst0Ty.isVector())
2499 return false;
2500 Register SrcReg = MI.getOperand(i: MI.getNumDefs()).getReg();
2501 LLT SrcTy = MRI.getType(Reg: SrcReg);
2502 if (SrcTy.isVector())
2503 return false;
2504
2505 Register ZExtSrcReg;
2506 if (!mi_match(R: SrcReg, MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg))))
2507 return false;
2508
2509 // Finally we can replace the first definition with
2510 // a zext of the source if the definition is big enough to hold
2511 // all of ZExtSrc bits.
2512 LLT ZExtSrcTy = MRI.getType(Reg: ZExtSrcReg);
2513 return ZExtSrcTy.getSizeInBits() <= Dst0Ty.getSizeInBits();
2514}
2515
2516void CombinerHelper::applyCombineUnmergeZExtToZExt(MachineInstr &MI) const {
2517 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2518 "Expected an unmerge");
2519
2520 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2521
2522 MachineInstr *ZExtInstr =
2523 MRI.getVRegDef(Reg: MI.getOperand(i: MI.getNumDefs()).getReg());
2524 assert(ZExtInstr && ZExtInstr->getOpcode() == TargetOpcode::G_ZEXT &&
2525 "Expecting a G_ZEXT");
2526
2527 Register ZExtSrcReg = ZExtInstr->getOperand(i: 1).getReg();
2528 LLT Dst0Ty = MRI.getType(Reg: Dst0Reg);
2529 LLT ZExtSrcTy = MRI.getType(Reg: ZExtSrcReg);
2530
2531 if (Dst0Ty.getSizeInBits() > ZExtSrcTy.getSizeInBits()) {
2532 Builder.buildZExt(Res: Dst0Reg, Op: ZExtSrcReg);
2533 } else {
2534 assert(Dst0Ty.getSizeInBits() == ZExtSrcTy.getSizeInBits() &&
2535 "ZExt src doesn't fit in destination");
2536 replaceRegWith(MRI, FromReg: Dst0Reg, ToReg: ZExtSrcReg);
2537 }
2538
2539 Register ZeroReg;
2540 for (unsigned Idx = 1, EndIdx = MI.getNumDefs(); Idx != EndIdx; ++Idx) {
2541 if (!ZeroReg)
2542 ZeroReg = Builder.buildConstant(Res: Dst0Ty, Val: 0).getReg(Idx: 0);
2543 replaceRegWith(MRI, FromReg: MI.getOperand(i: Idx).getReg(), ToReg: ZeroReg);
2544 }
2545 MI.eraseFromParent();
2546}
2547
2548bool CombinerHelper::matchCombineShiftToUnmerge(MachineInstr &MI,
2549 unsigned TargetShiftSize,
2550 unsigned &ShiftVal) const {
2551 assert((MI.getOpcode() == TargetOpcode::G_SHL ||
2552 MI.getOpcode() == TargetOpcode::G_LSHR ||
2553 MI.getOpcode() == TargetOpcode::G_ASHR) && "Expected a shift");
2554
2555 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2556 if (Ty.isVector()) // TODO:
2557 return false;
2558
2559 // Don't narrow further than the requested size.
2560 unsigned Size = Ty.getSizeInBits();
2561 if (Size <= TargetShiftSize)
2562 return false;
2563
2564 auto MaybeImmVal =
2565 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
2566 if (!MaybeImmVal)
2567 return false;
2568
2569 ShiftVal = MaybeImmVal->Value.getSExtValue();
2570 return ShiftVal >= Size / 2 && ShiftVal < Size;
2571}
2572
2573void CombinerHelper::applyCombineShiftToUnmerge(
2574 MachineInstr &MI, const unsigned &ShiftVal) const {
2575 Register DstReg = MI.getOperand(i: 0).getReg();
2576 Register SrcReg = MI.getOperand(i: 1).getReg();
2577 LLT Ty = MRI.getType(Reg: SrcReg);
2578 unsigned Size = Ty.getSizeInBits();
2579 unsigned HalfSize = Size / 2;
2580 assert(ShiftVal >= HalfSize);
2581
2582 LLT HalfTy = Ty.changeElementSize(NewEltSize: HalfSize);
2583
2584 auto Unmerge = Builder.buildUnmerge(Res: HalfTy, Op: SrcReg);
2585 unsigned NarrowShiftAmt = ShiftVal - HalfSize;
2586
2587 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
2588 Register Narrowed = Unmerge.getReg(Idx: 1);
2589
2590 // dst = G_LSHR s64:x, C for C >= 32
2591 // =>
2592 // lo, hi = G_UNMERGE_VALUES x
2593 // dst = G_MERGE_VALUES (G_LSHR hi, C - 32), 0
2594
2595 if (NarrowShiftAmt != 0) {
2596 Narrowed = Builder.buildLShr(Dst: HalfTy, Src0: Narrowed,
2597 Src1: Builder.buildConstant(Res: HalfTy, Val: NarrowShiftAmt)).getReg(Idx: 0);
2598 }
2599
2600 auto Zero = Builder.buildConstant(Res: HalfTy, Val: 0);
2601 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Narrowed, Zero});
2602 } else if (MI.getOpcode() == TargetOpcode::G_SHL) {
2603 Register Narrowed = Unmerge.getReg(Idx: 0);
2604 // dst = G_SHL s64:x, C for C >= 32
2605 // =>
2606 // lo, hi = G_UNMERGE_VALUES x
2607 // dst = G_MERGE_VALUES 0, (G_SHL hi, C - 32)
2608 if (NarrowShiftAmt != 0) {
2609 Narrowed = Builder.buildShl(Dst: HalfTy, Src0: Narrowed,
2610 Src1: Builder.buildConstant(Res: HalfTy, Val: NarrowShiftAmt)).getReg(Idx: 0);
2611 }
2612
2613 auto Zero = Builder.buildConstant(Res: HalfTy, Val: 0);
2614 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Zero, Narrowed});
2615 } else {
2616 assert(MI.getOpcode() == TargetOpcode::G_ASHR);
2617 auto Hi = Builder.buildAShr(
2618 Dst: HalfTy, Src0: Unmerge.getReg(Idx: 1),
2619 Src1: Builder.buildConstant(Res: HalfTy, Val: HalfSize - 1));
2620
2621 if (ShiftVal == HalfSize) {
2622 // (G_ASHR i64:x, 32) ->
2623 // G_MERGE_VALUES hi_32(x), (G_ASHR hi_32(x), 31)
2624 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Unmerge.getReg(Idx: 1), Hi});
2625 } else if (ShiftVal == Size - 1) {
2626 // Don't need a second shift.
2627 // (G_ASHR i64:x, 63) ->
2628 // %narrowed = (G_ASHR hi_32(x), 31)
2629 // G_MERGE_VALUES %narrowed, %narrowed
2630 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Hi, Hi});
2631 } else {
2632 auto Lo = Builder.buildAShr(
2633 Dst: HalfTy, Src0: Unmerge.getReg(Idx: 1),
2634 Src1: Builder.buildConstant(Res: HalfTy, Val: ShiftVal - HalfSize));
2635
2636 // (G_ASHR i64:x, C) ->, for C >= 32
2637 // G_MERGE_VALUES (G_ASHR hi_32(x), C - 32), (G_ASHR hi_32(x), 31)
2638 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Lo, Hi});
2639 }
2640 }
2641
2642 MI.eraseFromParent();
2643}
2644
2645bool CombinerHelper::tryCombineShiftToUnmerge(
2646 MachineInstr &MI, unsigned TargetShiftAmount) const {
2647 unsigned ShiftAmt;
2648 if (matchCombineShiftToUnmerge(MI, TargetShiftSize: TargetShiftAmount, ShiftVal&: ShiftAmt)) {
2649 applyCombineShiftToUnmerge(MI, ShiftVal: ShiftAmt);
2650 return true;
2651 }
2652
2653 return false;
2654}
2655
2656bool CombinerHelper::matchCombineI2PToP2I(MachineInstr &MI,
2657 Register &Reg) const {
2658 assert(MI.getOpcode() == TargetOpcode::G_INTTOPTR && "Expected a G_INTTOPTR");
2659 Register DstReg = MI.getOperand(i: 0).getReg();
2660 LLT DstTy = MRI.getType(Reg: DstReg);
2661 Register SrcReg = MI.getOperand(i: 1).getReg();
2662 return mi_match(R: SrcReg, MRI,
2663 P: m_GPtrToInt(Src: m_all_of(preds: m_SpecificType(Ty: DstTy), preds: m_Reg(R&: Reg))));
2664}
2665
2666void CombinerHelper::applyCombineI2PToP2I(MachineInstr &MI,
2667 Register &Reg) const {
2668 assert(MI.getOpcode() == TargetOpcode::G_INTTOPTR && "Expected a G_INTTOPTR");
2669 Register DstReg = MI.getOperand(i: 0).getReg();
2670 Builder.buildCopy(Res: DstReg, Op: Reg);
2671 MI.eraseFromParent();
2672}
2673
2674void CombinerHelper::applyCombineP2IToI2P(MachineInstr &MI,
2675 Register &Reg) const {
2676 assert(MI.getOpcode() == TargetOpcode::G_PTRTOINT && "Expected a G_PTRTOINT");
2677 Register DstReg = MI.getOperand(i: 0).getReg();
2678 Builder.buildZExtOrTrunc(Res: DstReg, Op: Reg);
2679 MI.eraseFromParent();
2680}
2681
2682bool CombinerHelper::matchCombineAddP2IToPtrAdd(
2683 MachineInstr &MI, std::pair<Register, bool> &PtrReg) const {
2684 assert(MI.getOpcode() == TargetOpcode::G_ADD);
2685 Register LHS = MI.getOperand(i: 1).getReg();
2686 Register RHS = MI.getOperand(i: 2).getReg();
2687 LLT IntTy = MRI.getType(Reg: LHS);
2688
2689 // G_PTR_ADD always has the pointer in the LHS, so we may need to commute the
2690 // instruction.
2691 PtrReg.second = false;
2692 for (Register SrcReg : {LHS, RHS}) {
2693 if (mi_match(R: SrcReg, MRI, P: m_GPtrToInt(Src: m_Reg(R&: PtrReg.first)))) {
2694 // Don't handle cases where the integer is implicitly converted to the
2695 // pointer width.
2696 LLT PtrTy = MRI.getType(Reg: PtrReg.first);
2697 if (PtrTy.getScalarSizeInBits() == IntTy.getScalarSizeInBits())
2698 return true;
2699 }
2700
2701 PtrReg.second = true;
2702 }
2703
2704 return false;
2705}
2706
2707void CombinerHelper::applyCombineAddP2IToPtrAdd(
2708 MachineInstr &MI, std::pair<Register, bool> &PtrReg) const {
2709 Register Dst = MI.getOperand(i: 0).getReg();
2710 Register LHS = MI.getOperand(i: 1).getReg();
2711 Register RHS = MI.getOperand(i: 2).getReg();
2712
2713 const bool DoCommute = PtrReg.second;
2714 if (DoCommute)
2715 std::swap(a&: LHS, b&: RHS);
2716 LHS = PtrReg.first;
2717
2718 LLT PtrTy = MRI.getType(Reg: LHS);
2719
2720 auto PtrAdd = Builder.buildPtrAdd(Res: PtrTy, Op0: LHS, Op1: RHS);
2721 Builder.buildPtrToInt(Dst, Src: PtrAdd);
2722 MI.eraseFromParent();
2723}
2724
2725bool CombinerHelper::matchCombineConstPtrAddToI2P(MachineInstr &MI,
2726 APInt &NewCst) const {
2727 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
2728 Register LHS = PtrAdd.getBaseReg();
2729 Register RHS = PtrAdd.getOffsetReg();
2730 MachineRegisterInfo &MRI = Builder.getMF().getRegInfo();
2731
2732 if (auto RHSCst = getIConstantVRegVal(VReg: RHS, MRI)) {
2733 APInt Cst;
2734 if (mi_match(R: LHS, MRI, P: m_GIntToPtr(Src: m_ICst(Cst)))) {
2735 auto DstTy = MRI.getType(Reg: PtrAdd.getReg(Idx: 0));
2736 // G_INTTOPTR uses zero-extension
2737 NewCst = Cst.zextOrTrunc(width: DstTy.getSizeInBits());
2738 NewCst += RHSCst->sextOrTrunc(width: DstTy.getSizeInBits());
2739 return true;
2740 }
2741 }
2742
2743 return false;
2744}
2745
2746void CombinerHelper::applyCombineConstPtrAddToI2P(MachineInstr &MI,
2747 APInt &NewCst) const {
2748 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
2749 Register Dst = PtrAdd.getReg(Idx: 0);
2750
2751 Builder.buildConstant(Res: Dst, Val: NewCst);
2752 PtrAdd.eraseFromParent();
2753}
2754
2755bool CombinerHelper::matchCombineAnyExtTrunc(MachineInstr &MI,
2756 Register &Reg) const {
2757 assert(MI.getOpcode() == TargetOpcode::G_ANYEXT && "Expected a G_ANYEXT");
2758 Register DstReg = MI.getOperand(i: 0).getReg();
2759 Register SrcReg = MI.getOperand(i: 1).getReg();
2760 Register OriginalSrcReg = getSrcRegIgnoringCopies(Reg: SrcReg, MRI);
2761 if (OriginalSrcReg.isValid())
2762 SrcReg = OriginalSrcReg;
2763 LLT DstTy = MRI.getType(Reg: DstReg);
2764 return mi_match(R: SrcReg, MRI,
2765 P: m_GTrunc(Src: m_all_of(preds: m_Reg(R&: Reg), preds: m_SpecificType(Ty: DstTy)))) &&
2766 canReplaceReg(DstReg, SrcReg: Reg, MRI);
2767}
2768
2769bool CombinerHelper::matchCombineZextTrunc(MachineInstr &MI,
2770 Register &Reg) const {
2771 assert(MI.getOpcode() == TargetOpcode::G_ZEXT && "Expected a G_ZEXT");
2772 Register DstReg = MI.getOperand(i: 0).getReg();
2773 Register SrcReg = MI.getOperand(i: 1).getReg();
2774 LLT DstTy = MRI.getType(Reg: DstReg);
2775 if (mi_match(R: SrcReg, MRI,
2776 P: m_GTrunc(Src: m_all_of(preds: m_Reg(R&: Reg), preds: m_SpecificType(Ty: DstTy)))) &&
2777 canReplaceReg(DstReg, SrcReg: Reg, MRI)) {
2778 unsigned DstSize = DstTy.getScalarSizeInBits();
2779 unsigned SrcSize = MRI.getType(Reg: SrcReg).getScalarSizeInBits();
2780 return VT->getKnownBits(R: Reg).countMinLeadingZeros() >= DstSize - SrcSize;
2781 }
2782 return false;
2783}
2784
2785static LLT getMidVTForTruncRightShiftCombine(LLT ShiftTy, LLT TruncTy) {
2786 const unsigned ShiftSize = ShiftTy.getScalarSizeInBits();
2787 const unsigned TruncSize = TruncTy.getScalarSizeInBits();
2788
2789 // ShiftTy > 32 > TruncTy -> 32
2790 if (ShiftSize > 32 && TruncSize < 32)
2791 return ShiftTy.changeElementSize(NewEltSize: 32);
2792
2793 // TODO: We could also reduce to 16 bits, but that's more target-dependent.
2794 // Some targets like it, some don't, some only like it under certain
2795 // conditions/processor versions, etc.
2796 // A TL hook might be needed for this.
2797
2798 // Don't combine
2799 return ShiftTy;
2800}
2801
2802bool CombinerHelper::matchCombineTruncOfShift(
2803 MachineInstr &MI, std::pair<MachineInstr *, LLT> &MatchInfo) const {
2804 assert(MI.getOpcode() == TargetOpcode::G_TRUNC && "Expected a G_TRUNC");
2805 Register DstReg = MI.getOperand(i: 0).getReg();
2806 Register SrcReg = MI.getOperand(i: 1).getReg();
2807
2808 if (!MRI.hasOneNonDBGUse(RegNo: SrcReg))
2809 return false;
2810
2811 LLT SrcTy = MRI.getType(Reg: SrcReg);
2812 LLT DstTy = MRI.getType(Reg: DstReg);
2813
2814 MachineInstr *SrcMI = getDefIgnoringCopies(Reg: SrcReg, MRI);
2815 const auto &TL = getTargetLowering();
2816
2817 LLT NewShiftTy;
2818 switch (SrcMI->getOpcode()) {
2819 default:
2820 return false;
2821 case TargetOpcode::G_SHL: {
2822 NewShiftTy = DstTy;
2823
2824 // Make sure new shift amount is legal.
2825 KnownBits Known = VT->getKnownBits(R: SrcMI->getOperand(i: 2).getReg());
2826 if (Known.getMaxValue().uge(RHS: NewShiftTy.getScalarSizeInBits()))
2827 return false;
2828 break;
2829 }
2830 case TargetOpcode::G_LSHR:
2831 case TargetOpcode::G_ASHR: {
2832 // For right shifts, we conservatively do not do the transform if the TRUNC
2833 // has any STORE users. The reason is that if we change the type of the
2834 // shift, we may break the truncstore combine.
2835 //
2836 // TODO: Fix truncstore combine to handle (trunc(lshr (trunc x), k)).
2837 for (auto &User : MRI.use_instructions(Reg: DstReg))
2838 if (User.getOpcode() == TargetOpcode::G_STORE)
2839 return false;
2840
2841 NewShiftTy = getMidVTForTruncRightShiftCombine(ShiftTy: SrcTy, TruncTy: DstTy);
2842 if (NewShiftTy == SrcTy)
2843 return false;
2844
2845 // Make sure we won't lose information by truncating the high bits.
2846 KnownBits Known = VT->getKnownBits(R: SrcMI->getOperand(i: 2).getReg());
2847 if (Known.getMaxValue().ugt(RHS: NewShiftTy.getScalarSizeInBits() -
2848 DstTy.getScalarSizeInBits()))
2849 return false;
2850 break;
2851 }
2852 }
2853
2854 if (!isLegalOrBeforeLegalizer(
2855 Query: {SrcMI->getOpcode(),
2856 {NewShiftTy, TL.getPreferredShiftAmountTy(ShiftValueTy: NewShiftTy)}}))
2857 return false;
2858
2859 MatchInfo = std::make_pair(x&: SrcMI, y&: NewShiftTy);
2860 return true;
2861}
2862
2863void CombinerHelper::applyCombineTruncOfShift(
2864 MachineInstr &MI, std::pair<MachineInstr *, LLT> &MatchInfo) const {
2865 MachineInstr *ShiftMI = MatchInfo.first;
2866 LLT NewShiftTy = MatchInfo.second;
2867
2868 Register Dst = MI.getOperand(i: 0).getReg();
2869 LLT DstTy = MRI.getType(Reg: Dst);
2870
2871 Register ShiftAmt = ShiftMI->getOperand(i: 2).getReg();
2872 Register ShiftSrc = ShiftMI->getOperand(i: 1).getReg();
2873 ShiftSrc = Builder.buildTrunc(Res: NewShiftTy, Op: ShiftSrc).getReg(Idx: 0);
2874
2875 Register NewShift =
2876 Builder
2877 .buildInstr(Opc: ShiftMI->getOpcode(), DstOps: {NewShiftTy}, SrcOps: {ShiftSrc, ShiftAmt})
2878 .getReg(Idx: 0);
2879
2880 if (NewShiftTy == DstTy)
2881 replaceRegWith(MRI, FromReg: Dst, ToReg: NewShift);
2882 else
2883 Builder.buildTrunc(Res: Dst, Op: NewShift);
2884
2885 eraseInst(MI);
2886}
2887
2888bool CombinerHelper::matchAnyExplicitUseIsUndef(MachineInstr &MI) const {
2889 return any_of(Range: MI.explicit_uses(), P: [this](const MachineOperand &MO) {
2890 return MO.isReg() &&
2891 getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: MO.getReg(), MRI);
2892 });
2893}
2894
2895bool CombinerHelper::matchAllExplicitUsesAreUndef(MachineInstr &MI) const {
2896 return all_of(Range: MI.explicit_uses(), P: [this](const MachineOperand &MO) {
2897 return !MO.isReg() ||
2898 getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: MO.getReg(), MRI);
2899 });
2900}
2901
2902bool CombinerHelper::matchUndefShuffleVectorMask(MachineInstr &MI) const {
2903 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
2904 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
2905 return all_of(Range&: Mask, P: [](int Elt) { return Elt < 0; });
2906}
2907
2908bool CombinerHelper::matchUndefStore(MachineInstr &MI) const {
2909 assert(MI.getOpcode() == TargetOpcode::G_STORE);
2910 return getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: MI.getOperand(i: 0).getReg(),
2911 MRI);
2912}
2913
2914bool CombinerHelper::matchUndefSelectCmp(MachineInstr &MI) const {
2915 assert(MI.getOpcode() == TargetOpcode::G_SELECT);
2916 return getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: MI.getOperand(i: 1).getReg(),
2917 MRI);
2918}
2919
2920bool CombinerHelper::matchInsertExtractVecEltOutOfBounds(
2921 MachineInstr &MI) const {
2922 assert((MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT ||
2923 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT) &&
2924 "Expected an insert/extract element op");
2925 LLT VecTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
2926 if (VecTy.isScalableVector())
2927 return false;
2928
2929 unsigned IdxIdx =
2930 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
2931 auto Idx = getIConstantVRegVal(VReg: MI.getOperand(i: IdxIdx).getReg(), MRI);
2932 if (!Idx)
2933 return false;
2934 return Idx->getZExtValue() >= VecTy.getNumElements();
2935}
2936
2937bool CombinerHelper::matchConstantSelectCmp(MachineInstr &MI,
2938 unsigned &OpIdx) const {
2939 GSelect &SelMI = cast<GSelect>(Val&: MI);
2940 auto Cst = isConstantOrConstantSplatVector(Def: SelMI.getCondReg(), MRI);
2941 if (!Cst)
2942 return false;
2943 OpIdx = Cst->isZero() ? 3 : 2;
2944 return true;
2945}
2946
2947void CombinerHelper::eraseInst(MachineInstr &MI) const { MI.eraseFromParent(); }
2948
2949bool CombinerHelper::matchEqualDefs(const MachineOperand &MOP1,
2950 const MachineOperand &MOP2) const {
2951 if (!MOP1.isReg() || !MOP2.isReg())
2952 return false;
2953 auto InstAndDef1 = getDefSrcRegIgnoringCopies(Reg: MOP1.getReg(), MRI);
2954 if (!InstAndDef1)
2955 return false;
2956 auto InstAndDef2 = getDefSrcRegIgnoringCopies(Reg: MOP2.getReg(), MRI);
2957 if (!InstAndDef2)
2958 return false;
2959 MachineInstr *I1 = InstAndDef1->MI;
2960 MachineInstr *I2 = InstAndDef2->MI;
2961
2962 // Handle a case like this:
2963 //
2964 // %0:_(s64), %1:_(s64) = G_UNMERGE_VALUES %2:_(<2 x s64>)
2965 //
2966 // Even though %0 and %1 are produced by the same instruction they are not
2967 // the same values.
2968 if (I1 == I2)
2969 return MOP1.getReg() == MOP2.getReg();
2970
2971 // If we have an instruction which loads or stores, we can't guarantee that
2972 // it is identical.
2973 //
2974 // For example, we may have
2975 //
2976 // %x1 = G_LOAD %addr (load N from @somewhere)
2977 // ...
2978 // call @foo
2979 // ...
2980 // %x2 = G_LOAD %addr (load N from @somewhere)
2981 // ...
2982 // %or = G_OR %x1, %x2
2983 //
2984 // It's possible that @foo will modify whatever lives at the address we're
2985 // loading from. To be safe, let's just assume that all loads and stores
2986 // are different (unless we have something which is guaranteed to not
2987 // change.)
2988 if (I1->mayLoadOrStore() && !I1->isDereferenceableInvariantLoad())
2989 return false;
2990
2991 // If both instructions are loads or stores, they are equal only if both
2992 // are dereferenceable invariant loads with the same number of bits.
2993 if (I1->mayLoadOrStore() && I2->mayLoadOrStore()) {
2994 GLoadStore *LS1 = dyn_cast<GLoadStore>(Val: I1);
2995 GLoadStore *LS2 = dyn_cast<GLoadStore>(Val: I2);
2996 if (!LS1 || !LS2)
2997 return false;
2998
2999 if (!I2->isDereferenceableInvariantLoad() ||
3000 (LS1->getMemSizeInBits() != LS2->getMemSizeInBits()))
3001 return false;
3002 }
3003
3004 // Check for physical registers on the instructions first to avoid cases
3005 // like this:
3006 //
3007 // %a = COPY $physreg
3008 // ...
3009 // SOMETHING implicit-def $physreg
3010 // ...
3011 // %b = COPY $physreg
3012 //
3013 // These copies are not equivalent.
3014 if (any_of(Range: I1->uses(), P: [](const MachineOperand &MO) {
3015 return MO.isReg() && MO.getReg().isPhysical();
3016 })) {
3017 // Check if we have a case like this:
3018 //
3019 // %a = COPY $physreg
3020 // %b = COPY %a
3021 //
3022 // In this case, I1 and I2 will both be equal to %a = COPY $physreg.
3023 // From that, we know that they must have the same value, since they must
3024 // have come from the same COPY.
3025 return I1->isIdenticalTo(Other: *I2);
3026 }
3027
3028 // We don't have any physical registers, so we don't necessarily need the
3029 // same vreg defs.
3030 //
3031 // On the off-chance that there's some target instruction feeding into the
3032 // instruction, let's use produceSameValue instead of isIdenticalTo.
3033 if (Builder.getTII().produceSameValue(MI0: *I1, MI1: *I2, MRI: &MRI)) {
3034 // Handle instructions with multiple defs that produce same values. Values
3035 // are same for operands with same index.
3036 // %0:_(s8), %1:_(s8), %2:_(s8), %3:_(s8) = G_UNMERGE_VALUES %4:_(<4 x s8>)
3037 // %5:_(s8), %6:_(s8), %7:_(s8), %8:_(s8) = G_UNMERGE_VALUES %4:_(<4 x s8>)
3038 // I1 and I2 are different instructions but produce same values,
3039 // %1 and %6 are same, %1 and %7 are not the same value.
3040 return I1->findRegisterDefOperandIdx(Reg: InstAndDef1->Reg, /*TRI=*/nullptr) ==
3041 I2->findRegisterDefOperandIdx(Reg: InstAndDef2->Reg, /*TRI=*/nullptr);
3042 }
3043 return false;
3044}
3045
3046bool CombinerHelper::matchConstantOp(const MachineOperand &MOP,
3047 int64_t C) const {
3048 if (!MOP.isReg())
3049 return false;
3050 auto MaybeCst = isConstantOrConstantSplatVector(Def: MOP.getReg(), MRI);
3051 return MaybeCst && MaybeCst->getBitWidth() <= 64 &&
3052 MaybeCst->getSExtValue() == C;
3053}
3054
3055bool CombinerHelper::matchConstantFPOp(const MachineOperand &MOP,
3056 double C) const {
3057 if (!MOP.isReg())
3058 return false;
3059 std::optional<FPValueAndVReg> MaybeCst;
3060 if (!mi_match(R: MOP.getReg(), MRI, P: m_GFCstOrSplat(FPValReg&: MaybeCst)))
3061 return false;
3062
3063 return MaybeCst->Value.isExactlyValue(V: C);
3064}
3065
3066void CombinerHelper::replaceSingleDefInstWithOperand(MachineInstr &MI,
3067 unsigned OpIdx) const {
3068 assert(MI.getNumExplicitDefs() == 1 && "Expected one explicit def?");
3069 Register OldReg = MI.getOperand(i: 0).getReg();
3070 Register Replacement = MI.getOperand(i: OpIdx).getReg();
3071 assert(canReplaceReg(OldReg, Replacement, MRI) && "Cannot replace register?");
3072 replaceRegWith(MRI, FromReg: OldReg, ToReg: Replacement);
3073 MI.eraseFromParent();
3074}
3075
3076void CombinerHelper::replaceSingleDefInstWithReg(MachineInstr &MI,
3077 Register Replacement) const {
3078 assert(MI.getNumExplicitDefs() == 1 && "Expected one explicit def?");
3079 Register OldReg = MI.getOperand(i: 0).getReg();
3080 assert(canReplaceReg(OldReg, Replacement, MRI) && "Cannot replace register?");
3081 replaceRegWith(MRI, FromReg: OldReg, ToReg: Replacement);
3082 MI.eraseFromParent();
3083}
3084
3085bool CombinerHelper::matchConstantLargerBitWidth(MachineInstr &MI,
3086 unsigned ConstIdx) const {
3087 Register ConstReg = MI.getOperand(i: ConstIdx).getReg();
3088 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3089
3090 // Get the shift amount
3091 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstReg, MRI);
3092 if (!VRegAndVal)
3093 return false;
3094
3095 // Return true of shift amount >= Bitwidth
3096 return (VRegAndVal->Value.uge(RHS: DstTy.getSizeInBits()));
3097}
3098
3099void CombinerHelper::applyFunnelShiftConstantModulo(MachineInstr &MI) const {
3100 assert((MI.getOpcode() == TargetOpcode::G_FSHL ||
3101 MI.getOpcode() == TargetOpcode::G_FSHR) &&
3102 "This is not a funnel shift operation");
3103
3104 Register ConstReg = MI.getOperand(i: 3).getReg();
3105 LLT ConstTy = MRI.getType(Reg: ConstReg);
3106 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3107
3108 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstReg, MRI);
3109 assert((VRegAndVal) && "Value is not a constant");
3110
3111 // Calculate the new Shift Amount = Old Shift Amount % BitWidth
3112 APInt NewConst = VRegAndVal->Value.urem(
3113 RHS: APInt(ConstTy.getSizeInBits(), DstTy.getScalarSizeInBits()));
3114
3115 auto NewConstInstr = Builder.buildConstant(Res: ConstTy, Val: NewConst.getZExtValue());
3116 Builder.buildInstr(
3117 Opc: MI.getOpcode(), DstOps: {MI.getOperand(i: 0)},
3118 SrcOps: {MI.getOperand(i: 1), MI.getOperand(i: 2), NewConstInstr.getReg(Idx: 0)});
3119
3120 MI.eraseFromParent();
3121}
3122
3123bool CombinerHelper::matchSelectSameVal(MachineInstr &MI) const {
3124 assert(MI.getOpcode() == TargetOpcode::G_SELECT);
3125 // Match (cond ? x : x)
3126 return matchEqualDefs(MOP1: MI.getOperand(i: 2), MOP2: MI.getOperand(i: 3)) &&
3127 canReplaceReg(DstReg: MI.getOperand(i: 0).getReg(), SrcReg: MI.getOperand(i: 2).getReg(),
3128 MRI);
3129}
3130
3131bool CombinerHelper::matchBinOpSameVal(MachineInstr &MI) const {
3132 return matchEqualDefs(MOP1: MI.getOperand(i: 1), MOP2: MI.getOperand(i: 2)) &&
3133 canReplaceReg(DstReg: MI.getOperand(i: 0).getReg(), SrcReg: MI.getOperand(i: 1).getReg(),
3134 MRI);
3135}
3136
3137bool CombinerHelper::matchOperandIsUndef(MachineInstr &MI,
3138 unsigned OpIdx) const {
3139 MachineOperand &MO = MI.getOperand(i: OpIdx);
3140 return MO.isReg() &&
3141 getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: MO.getReg(), MRI);
3142}
3143
3144bool CombinerHelper::matchOperandIsKnownToBeAPowerOfTwo(
3145 const MachineOperand &MO, bool OrNegative) const {
3146 return isKnownToBeAPowerOfTwo(Val: MO.getReg(), MRI, ValueTracking: VT, OrNegative);
3147}
3148
3149void CombinerHelper::replaceInstWithFConstant(MachineInstr &MI,
3150 double C) const {
3151 assert(MI.getNumDefs() == 1 && "Expected only one def?");
3152 Builder.buildFConstant(Res: MI.getOperand(i: 0), Val: C);
3153 MI.eraseFromParent();
3154}
3155
3156void CombinerHelper::replaceInstWithConstant(MachineInstr &MI,
3157 int64_t C) const {
3158 assert(MI.getNumDefs() == 1 && "Expected only one def?");
3159 Builder.buildConstant(Res: MI.getOperand(i: 0), Val: C);
3160 MI.eraseFromParent();
3161}
3162
3163void CombinerHelper::replaceInstWithConstant(MachineInstr &MI, APInt C) const {
3164 assert(MI.getNumDefs() == 1 && "Expected only one def?");
3165 Builder.buildConstant(Res: MI.getOperand(i: 0), Val: C);
3166 MI.eraseFromParent();
3167}
3168
3169void CombinerHelper::replaceInstWithFConstant(MachineInstr &MI,
3170 ConstantFP *CFP) const {
3171 assert(MI.getNumDefs() == 1 && "Expected only one def?");
3172 Builder.buildFConstant(Res: MI.getOperand(i: 0), Val: CFP->getValueAPF());
3173 MI.eraseFromParent();
3174}
3175
3176void CombinerHelper::replaceInstWithUndef(MachineInstr &MI) const {
3177 assert(MI.getNumDefs() == 1 && "Expected only one def?");
3178 Builder.buildUndef(Res: MI.getOperand(i: 0));
3179 MI.eraseFromParent();
3180}
3181
3182bool CombinerHelper::matchSimplifyAddToSub(
3183 MachineInstr &MI, std::tuple<Register, Register> &MatchInfo) const {
3184 Register LHS = MI.getOperand(i: 1).getReg();
3185 Register RHS = MI.getOperand(i: 2).getReg();
3186 Register &NewLHS = std::get<0>(t&: MatchInfo);
3187 Register &NewRHS = std::get<1>(t&: MatchInfo);
3188
3189 // Helper lambda to check for opportunities for
3190 // ((0-A) + B) -> B - A
3191 // (A + (0-B)) -> A - B
3192 auto CheckFold = [&](Register &MaybeSub, Register &MaybeNewLHS) {
3193 if (!mi_match(R: MaybeSub, MRI, P: m_Neg(Src: m_Reg(R&: NewRHS))))
3194 return false;
3195 NewLHS = MaybeNewLHS;
3196 return true;
3197 };
3198
3199 return CheckFold(LHS, RHS) || CheckFold(RHS, LHS);
3200}
3201
3202bool CombinerHelper::matchCombineInsertVecElts(
3203 MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo) const {
3204 assert(MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT &&
3205 "Invalid opcode");
3206 Register DstReg = MI.getOperand(i: 0).getReg();
3207 LLT DstTy = MRI.getType(Reg: DstReg);
3208 assert(DstTy.isVector() && "Invalid G_INSERT_VECTOR_ELT?");
3209
3210 if (DstTy.isScalableVector())
3211 return false;
3212
3213 unsigned NumElts = DstTy.getNumElements();
3214 // If this MI is part of a sequence of insert_vec_elts, then
3215 // don't do the combine in the middle of the sequence.
3216 if (MRI.hasOneUse(RegNo: DstReg) && MRI.use_instr_begin(RegNo: DstReg)->getOpcode() ==
3217 TargetOpcode::G_INSERT_VECTOR_ELT)
3218 return false;
3219 MachineInstr *CurrInst = &MI;
3220 MachineInstr *TmpInst;
3221 int64_t IntImm;
3222 Register TmpReg;
3223 MatchInfo.resize(N: NumElts);
3224 while (mi_match(
3225 MI&: *CurrInst, MRI,
3226 P: m_GInsertVecElt(Src0: m_MInstr(MI&: TmpInst), Src1: m_Reg(R&: TmpReg), Src2: m_ICst(Cst&: IntImm)))) {
3227 if (IntImm >= NumElts || IntImm < 0)
3228 return false;
3229 if (!MatchInfo[IntImm])
3230 MatchInfo[IntImm] = TmpReg;
3231 CurrInst = TmpInst;
3232 }
3233 // Variable index.
3234 if (CurrInst->getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
3235 return false;
3236 if (TmpInst->getOpcode() == TargetOpcode::G_BUILD_VECTOR) {
3237 for (unsigned I = 1; I < TmpInst->getNumOperands(); ++I) {
3238 if (!MatchInfo[I - 1].isValid())
3239 MatchInfo[I - 1] = TmpInst->getOperand(i: I).getReg();
3240 }
3241 return true;
3242 }
3243 // If we didn't end in a G_IMPLICIT_DEF and the source is not fully
3244 // overwritten, bail out.
3245 return TmpInst->getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
3246 all_of(Range&: MatchInfo, P: [](Register Reg) { return !!Reg; });
3247}
3248
3249void CombinerHelper::applyCombineInsertVecElts(
3250 MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo) const {
3251 Register UndefReg;
3252 auto GetUndef = [&]() {
3253 if (UndefReg)
3254 return UndefReg;
3255 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3256 UndefReg = Builder.buildUndef(Res: DstTy.getScalarType()).getReg(Idx: 0);
3257 return UndefReg;
3258 };
3259 for (Register &Reg : MatchInfo) {
3260 if (!Reg)
3261 Reg = GetUndef();
3262 }
3263 Builder.buildBuildVector(Res: MI.getOperand(i: 0).getReg(), Ops: MatchInfo);
3264 MI.eraseFromParent();
3265}
3266
3267void CombinerHelper::applySimplifyAddToSub(
3268 MachineInstr &MI, std::tuple<Register, Register> &MatchInfo) const {
3269 Register SubLHS, SubRHS;
3270 std::tie(args&: SubLHS, args&: SubRHS) = MatchInfo;
3271 Builder.buildSub(Dst: MI.getOperand(i: 0).getReg(), Src0: SubLHS, Src1: SubRHS);
3272 MI.eraseFromParent();
3273}
3274
3275bool CombinerHelper::matchBinopWithNegInner(Register MInner, Register Other,
3276 unsigned RootOpc, Register Dst,
3277 LLT Ty,
3278 BuildFnTy &MatchInfo) const {
3279 /// Helper function for matchBinopWithNeg: tries to match one commuted form
3280 /// of `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`.
3281 MachineInstr *InnerDef = MRI.getVRegDef(Reg: MInner);
3282 if (!InnerDef)
3283 return false;
3284
3285 unsigned InnerOpc = InnerDef->getOpcode();
3286 if (InnerOpc != TargetOpcode::G_ADD && InnerOpc != TargetOpcode::G_SUB)
3287 return false;
3288
3289 if (!MRI.hasOneNonDBGUse(RegNo: MInner))
3290 return false;
3291
3292 Register InnerLHS = InnerDef->getOperand(i: 1).getReg();
3293 Register InnerRHS = InnerDef->getOperand(i: 2).getReg();
3294 Register NotSrc;
3295 Register B, C;
3296
3297 // Check if either operand is ~b
3298 auto TryMatch = [&](Register MaybeNot, Register Other) {
3299 if (mi_match(R: MaybeNot, MRI, P: m_Not(Src: m_Reg(R&: NotSrc)))) {
3300 if (!MRI.hasOneNonDBGUse(RegNo: MaybeNot))
3301 return false;
3302 B = NotSrc;
3303 C = Other;
3304 return true;
3305 }
3306 return false;
3307 };
3308
3309 // For SUB, the not must be the LHS. For ADD, it can be either operand.
3310 if (!TryMatch(InnerLHS, InnerRHS) &&
3311 !(InnerOpc == TargetOpcode::G_ADD && TryMatch(InnerRHS, InnerLHS)))
3312 return false;
3313
3314 // Flip add/sub
3315 unsigned FlippedOpc = (InnerOpc == TargetOpcode::G_ADD) ? TargetOpcode::G_SUB
3316 : TargetOpcode::G_ADD;
3317
3318 Register A = Other;
3319 MatchInfo = [=](MachineIRBuilder &Builder) {
3320 auto NewInner = Builder.buildInstr(Opc: FlippedOpc, DstOps: {Ty}, SrcOps: {B, C});
3321 auto NewNot = Builder.buildNot(Dst: Ty, Src0: NewInner);
3322 Builder.buildInstr(Opc: RootOpc, DstOps: {Dst}, SrcOps: {A, NewNot});
3323 };
3324 return true;
3325}
3326
3327bool CombinerHelper::matchBinopWithNeg(MachineInstr &MI,
3328 BuildFnTy &MatchInfo) const {
3329 // Fold `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`
3330 // Root MI is one of G_AND, G_OR, G_XOR.
3331 // We also look for commuted forms of operations. Pattern shouldn't apply
3332 // if there are multiple reasons of inner operations.
3333
3334 unsigned RootOpc = MI.getOpcode();
3335 Register Dst = MI.getOperand(i: 0).getReg();
3336 LLT Ty = MRI.getType(Reg: Dst);
3337
3338 Register LHS = MI.getOperand(i: 1).getReg();
3339 Register RHS = MI.getOperand(i: 2).getReg();
3340 // Check the commuted and uncommuted forms of the operation.
3341 return matchBinopWithNegInner(MInner: LHS, Other: RHS, RootOpc, Dst, Ty, MatchInfo) ||
3342 matchBinopWithNegInner(MInner: RHS, Other: LHS, RootOpc, Dst, Ty, MatchInfo);
3343}
3344
3345bool CombinerHelper::matchHoistLogicOpWithSameOpcodeHands(
3346 MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) const {
3347 // Matches: logic (hand x, ...), (hand y, ...) -> hand (logic x, y), ...
3348 //
3349 // Creates the new hand + logic instruction (but does not insert them.)
3350 //
3351 // On success, MatchInfo is populated with the new instructions. These are
3352 // inserted in applyHoistLogicOpWithSameOpcodeHands.
3353 unsigned LogicOpcode = MI.getOpcode();
3354 assert(LogicOpcode == TargetOpcode::G_AND ||
3355 LogicOpcode == TargetOpcode::G_OR ||
3356 LogicOpcode == TargetOpcode::G_XOR);
3357 MachineIRBuilder MIB(MI);
3358 Register Dst = MI.getOperand(i: 0).getReg();
3359 Register LHSReg = MI.getOperand(i: 1).getReg();
3360 Register RHSReg = MI.getOperand(i: 2).getReg();
3361
3362 // Don't recompute anything.
3363 if (!MRI.hasOneNonDBGUse(RegNo: LHSReg) || !MRI.hasOneNonDBGUse(RegNo: RHSReg))
3364 return false;
3365
3366 // Make sure we have (hand x, ...), (hand y, ...)
3367 MachineInstr *LeftHandInst = getDefIgnoringCopies(Reg: LHSReg, MRI);
3368 MachineInstr *RightHandInst = getDefIgnoringCopies(Reg: RHSReg, MRI);
3369 if (!LeftHandInst || !RightHandInst)
3370 return false;
3371 unsigned HandOpcode = LeftHandInst->getOpcode();
3372 if (HandOpcode != RightHandInst->getOpcode())
3373 return false;
3374 if (LeftHandInst->getNumOperands() < 2 ||
3375 !LeftHandInst->getOperand(i: 1).isReg() ||
3376 RightHandInst->getNumOperands() < 2 ||
3377 !RightHandInst->getOperand(i: 1).isReg())
3378 return false;
3379
3380 // Make sure the types match up, and if we're doing this post-legalization,
3381 // we end up with legal types.
3382 Register X = LeftHandInst->getOperand(i: 1).getReg();
3383 Register Y = RightHandInst->getOperand(i: 1).getReg();
3384 LLT XTy = MRI.getType(Reg: X);
3385 LLT YTy = MRI.getType(Reg: Y);
3386 if (!XTy.isValid() || XTy != YTy)
3387 return false;
3388
3389 // Optional extra source register.
3390 Register ExtraHandOpSrcReg;
3391 switch (HandOpcode) {
3392 default:
3393 return false;
3394 case TargetOpcode::G_ANYEXT:
3395 case TargetOpcode::G_SEXT:
3396 case TargetOpcode::G_ZEXT: {
3397 // Match: logic (ext X), (ext Y) --> ext (logic X, Y)
3398 break;
3399 }
3400 case TargetOpcode::G_TRUNC: {
3401 // Match: logic (trunc X), (trunc Y) -> trunc (logic X, Y)
3402 const MachineFunction *MF = MI.getMF();
3403 LLVMContext &Ctx = MF->getFunction().getContext();
3404
3405 LLT DstTy = MRI.getType(Reg: Dst);
3406 const TargetLowering &TLI = getTargetLowering();
3407
3408 // Be extra careful sinking truncate. If it's free, there's no benefit in
3409 // widening a binop.
3410 if (TLI.isZExtFree(FromTy: DstTy, ToTy: XTy, Ctx) && TLI.isTruncateFree(FromTy: XTy, ToTy: DstTy, Ctx))
3411 return false;
3412 break;
3413 }
3414 case TargetOpcode::G_AND:
3415 case TargetOpcode::G_ASHR:
3416 case TargetOpcode::G_LSHR:
3417 case TargetOpcode::G_SHL: {
3418 // Match: logic (binop x, z), (binop y, z) -> binop (logic x, y), z
3419 MachineOperand &ZOp = LeftHandInst->getOperand(i: 2);
3420 if (!matchEqualDefs(MOP1: ZOp, MOP2: RightHandInst->getOperand(i: 2)))
3421 return false;
3422 ExtraHandOpSrcReg = ZOp.getReg();
3423 break;
3424 }
3425 }
3426
3427 if (!isLegalOrBeforeLegalizer(Query: {LogicOpcode, {XTy, YTy}}))
3428 return false;
3429
3430 // Record the steps to build the new instructions.
3431 //
3432 // Steps to build (logic x, y)
3433 auto NewLogicDst = MRI.createGenericVirtualRegister(Ty: XTy);
3434 OperandBuildSteps LogicBuildSteps = {
3435 [=](MachineInstrBuilder &MIB) { MIB.addDef(RegNo: NewLogicDst); },
3436 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: X); },
3437 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Y); }};
3438 InstructionBuildSteps LogicSteps(LogicOpcode, LogicBuildSteps);
3439
3440 // Steps to build hand (logic x, y), ...z
3441 OperandBuildSteps HandBuildSteps = {
3442 [=](MachineInstrBuilder &MIB) { MIB.addDef(RegNo: Dst); },
3443 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: NewLogicDst); }};
3444 if (ExtraHandOpSrcReg.isValid())
3445 HandBuildSteps.push_back(
3446 Elt: [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ExtraHandOpSrcReg); });
3447 InstructionBuildSteps HandSteps(HandOpcode, HandBuildSteps);
3448
3449 MatchInfo = InstructionStepsMatchInfo({LogicSteps, HandSteps});
3450 return true;
3451}
3452
3453void CombinerHelper::applyBuildInstructionSteps(
3454 MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) const {
3455 assert(MatchInfo.InstrsToBuild.size() &&
3456 "Expected at least one instr to build?");
3457 for (auto &InstrToBuild : MatchInfo.InstrsToBuild) {
3458 assert(InstrToBuild.Opcode && "Expected a valid opcode?");
3459 assert(InstrToBuild.OperandFns.size() && "Expected at least one operand?");
3460 MachineInstrBuilder Instr = Builder.buildInstr(Opcode: InstrToBuild.Opcode);
3461 for (auto &OperandFn : InstrToBuild.OperandFns)
3462 OperandFn(Instr);
3463 }
3464 MI.eraseFromParent();
3465}
3466
3467bool CombinerHelper::matchAshrShlToSextInreg(
3468 MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {
3469 assert(MI.getOpcode() == TargetOpcode::G_ASHR);
3470 int64_t ShlCst, AshrCst;
3471 Register Src;
3472 if (!mi_match(R: MI.getOperand(i: 0).getReg(), MRI,
3473 P: m_GAShr(L: m_GShl(L: m_Reg(R&: Src), R: m_ICstOrSplat(Cst&: ShlCst)),
3474 R: m_ICstOrSplat(Cst&: AshrCst))))
3475 return false;
3476 if (ShlCst != AshrCst)
3477 return false;
3478 if (!isLegalOrBeforeLegalizer(
3479 Query: {TargetOpcode::G_SEXT_INREG, {MRI.getType(Reg: Src)}}))
3480 return false;
3481 MatchInfo = std::make_tuple(args&: Src, args&: ShlCst);
3482 return true;
3483}
3484
3485void CombinerHelper::applyAshShlToSextInreg(
3486 MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {
3487 assert(MI.getOpcode() == TargetOpcode::G_ASHR);
3488 Register Src;
3489 int64_t ShiftAmt;
3490 std::tie(args&: Src, args&: ShiftAmt) = MatchInfo;
3491 unsigned Size = MRI.getType(Reg: Src).getScalarSizeInBits();
3492 Builder.buildSExtInReg(Res: MI.getOperand(i: 0).getReg(), Op: Src, ImmOp: Size - ShiftAmt);
3493 MI.eraseFromParent();
3494}
3495
3496/// and(and(x, C1), C2) -> C1&C2 ? and(x, C1&C2) : 0
3497bool CombinerHelper::matchOverlappingAnd(
3498 MachineInstr &MI,
3499 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
3500 assert(MI.getOpcode() == TargetOpcode::G_AND);
3501
3502 Register Dst = MI.getOperand(i: 0).getReg();
3503 LLT Ty = MRI.getType(Reg: Dst);
3504
3505 Register R;
3506 int64_t C1;
3507 int64_t C2;
3508 if (!mi_match(
3509 R: Dst, MRI,
3510 P: m_GAnd(L: m_GAnd(L: m_Reg(R), R: m_ICst(Cst&: C1)), R: m_ICst(Cst&: C2))))
3511 return false;
3512
3513 MatchInfo = [=](MachineIRBuilder &B) {
3514 if (C1 & C2) {
3515 B.buildAnd(Dst, Src0: R, Src1: B.buildConstant(Res: Ty, Val: C1 & C2));
3516 return;
3517 }
3518 auto Zero = B.buildConstant(Res: Ty, Val: 0);
3519 replaceRegWith(MRI, FromReg: Dst, ToReg: Zero->getOperand(i: 0).getReg());
3520 };
3521 return true;
3522}
3523
3524bool CombinerHelper::matchRedundantAnd(MachineInstr &MI,
3525 Register &Replacement) const {
3526 // Given
3527 //
3528 // %y:_(sN) = G_SOMETHING
3529 // %x:_(sN) = G_SOMETHING
3530 // %res:_(sN) = G_AND %x, %y
3531 //
3532 // Eliminate the G_AND when it is known that x & y == x or x & y == y.
3533 //
3534 // Patterns like this can appear as a result of legalization. E.g.
3535 //
3536 // %cmp:_(s32) = G_ICMP intpred(pred), %x(s32), %y
3537 // %one:_(s32) = G_CONSTANT i32 1
3538 // %and:_(s32) = G_AND %cmp, %one
3539 //
3540 // In this case, G_ICMP only produces a single bit, so x & 1 == x.
3541 assert(MI.getOpcode() == TargetOpcode::G_AND);
3542 if (!VT)
3543 return false;
3544
3545 Register AndDst = MI.getOperand(i: 0).getReg();
3546 Register LHS = MI.getOperand(i: 1).getReg();
3547 Register RHS = MI.getOperand(i: 2).getReg();
3548
3549 // Check the RHS (maybe a constant) first, and if we have no KnownBits there,
3550 // we can't do anything. If we do, then it depends on whether we have
3551 // KnownBits on the LHS.
3552 KnownBits RHSBits = VT->getKnownBits(R: RHS);
3553 if (RHSBits.isUnknown())
3554 return false;
3555
3556 KnownBits LHSBits = VT->getKnownBits(R: LHS);
3557
3558 // Check that x & Mask == x.
3559 // x & 1 == x, always
3560 // x & 0 == x, only if x is also 0
3561 // Meaning Mask has no effect if every bit is either one in Mask or zero in x.
3562 //
3563 // Check if we can replace AndDst with the LHS of the G_AND
3564 if (canReplaceReg(DstReg: AndDst, SrcReg: LHS, MRI) &&
3565 (LHSBits.Zero | RHSBits.One).isAllOnes()) {
3566 Replacement = LHS;
3567 return true;
3568 }
3569
3570 // Check if we can replace AndDst with the RHS of the G_AND
3571 if (canReplaceReg(DstReg: AndDst, SrcReg: RHS, MRI) &&
3572 (LHSBits.One | RHSBits.Zero).isAllOnes()) {
3573 Replacement = RHS;
3574 return true;
3575 }
3576
3577 return false;
3578}
3579
3580bool CombinerHelper::matchRedundantOr(MachineInstr &MI,
3581 Register &Replacement) const {
3582 // Given
3583 //
3584 // %y:_(sN) = G_SOMETHING
3585 // %x:_(sN) = G_SOMETHING
3586 // %res:_(sN) = G_OR %x, %y
3587 //
3588 // Eliminate the G_OR when it is known that x | y == x or x | y == y.
3589 assert(MI.getOpcode() == TargetOpcode::G_OR);
3590 if (!VT)
3591 return false;
3592
3593 Register OrDst = MI.getOperand(i: 0).getReg();
3594 Register LHS = MI.getOperand(i: 1).getReg();
3595 Register RHS = MI.getOperand(i: 2).getReg();
3596
3597 KnownBits LHSBits = VT->getKnownBits(R: LHS);
3598 KnownBits RHSBits = VT->getKnownBits(R: RHS);
3599
3600 // Check that x | Mask == x.
3601 // x | 0 == x, always
3602 // x | 1 == x, only if x is also 1
3603 // Meaning Mask has no effect if every bit is either zero in Mask or one in x.
3604 //
3605 // Check if we can replace OrDst with the LHS of the G_OR
3606 if (canReplaceReg(DstReg: OrDst, SrcReg: LHS, MRI) &&
3607 (LHSBits.One | RHSBits.Zero).isAllOnes()) {
3608 Replacement = LHS;
3609 return true;
3610 }
3611
3612 // Check if we can replace OrDst with the RHS of the G_OR
3613 if (canReplaceReg(DstReg: OrDst, SrcReg: RHS, MRI) &&
3614 (LHSBits.Zero | RHSBits.One).isAllOnes()) {
3615 Replacement = RHS;
3616 return true;
3617 }
3618
3619 return false;
3620}
3621
3622bool CombinerHelper::matchRedundantSExtInReg(MachineInstr &MI) const {
3623 // If the input is already sign extended, just drop the extension.
3624 Register Src = MI.getOperand(i: 1).getReg();
3625 unsigned ExtBits = MI.getOperand(i: 2).getImm();
3626 unsigned TypeSize = MRI.getType(Reg: Src).getScalarSizeInBits();
3627 return VT->computeNumSignBits(R: Src) >= (TypeSize - ExtBits + 1);
3628}
3629
3630static bool isConstValidTrue(const TargetLowering &TLI, unsigned ScalarSizeBits,
3631 int64_t Cst, bool IsVector, bool IsFP) {
3632 // For i1, Cst will always be -1 regardless of boolean contents.
3633 return (ScalarSizeBits == 1 && Cst == -1) ||
3634 isConstTrueVal(TLI, Val: Cst, IsVector, IsFP);
3635}
3636
3637// This pattern aims to match the following shape to avoid extra mov
3638// instructions
3639// G_BUILD_VECTOR(
3640// G_UNMERGE_VALUES(src, 0)
3641// G_UNMERGE_VALUES(src, 1)
3642// G_IMPLICIT_DEF
3643// G_IMPLICIT_DEF
3644// )
3645// ->
3646// G_CONCAT_VECTORS(
3647// src,
3648// undef
3649// )
3650bool CombinerHelper::matchCombineBuildUnmerge(MachineInstr &MI,
3651 MachineRegisterInfo &MRI,
3652 Register &UnmergeSrc) const {
3653 auto &BV = cast<GBuildVector>(Val&: MI);
3654
3655 unsigned BuildUseCount = BV.getNumSources();
3656 if (BuildUseCount % 2 != 0)
3657 return false;
3658
3659 unsigned NumUnmerge = BuildUseCount / 2;
3660
3661 auto *Unmerge = getOpcodeDef<GUnmerge>(Reg: BV.getSourceReg(I: 0), MRI);
3662
3663 // Check the first operand is an unmerge and has the correct number of
3664 // operands
3665 if (!Unmerge || Unmerge->getNumDefs() != NumUnmerge)
3666 return false;
3667
3668 UnmergeSrc = Unmerge->getSourceReg();
3669
3670 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3671 LLT UnmergeSrcTy = MRI.getType(Reg: UnmergeSrc);
3672
3673 if (!UnmergeSrcTy.isVector())
3674 return false;
3675
3676 // Ensure we only generate legal instructions post-legalizer
3677 if (!IsPreLegalize &&
3678 !isLegal(Query: {TargetOpcode::G_CONCAT_VECTORS, {DstTy, UnmergeSrcTy}}))
3679 return false;
3680
3681 // Check that all of the operands before the midpoint come from the same
3682 // unmerge and are in the same order as they are used in the build_vector
3683 for (unsigned I = 0; I < NumUnmerge; ++I) {
3684 auto MaybeUnmergeReg = BV.getSourceReg(I);
3685 auto *LoopUnmerge = getOpcodeDef<GUnmerge>(Reg: MaybeUnmergeReg, MRI);
3686
3687 if (!LoopUnmerge || LoopUnmerge != Unmerge)
3688 return false;
3689
3690 if (LoopUnmerge->getOperand(i: I).getReg() != MaybeUnmergeReg)
3691 return false;
3692 }
3693
3694 // Check that all of the unmerged values are used
3695 if (Unmerge->getNumDefs() != NumUnmerge)
3696 return false;
3697
3698 // Check that all of the operands after the mid point are undefs.
3699 for (unsigned I = NumUnmerge; I < BuildUseCount; ++I) {
3700 auto *Undef = getDefIgnoringCopies(Reg: BV.getSourceReg(I), MRI);
3701
3702 if (Undef->getOpcode() != TargetOpcode::G_IMPLICIT_DEF)
3703 return false;
3704 }
3705
3706 return true;
3707}
3708
3709void CombinerHelper::applyCombineBuildUnmerge(MachineInstr &MI,
3710 MachineRegisterInfo &MRI,
3711 MachineIRBuilder &B,
3712 Register &UnmergeSrc) const {
3713 assert(UnmergeSrc && "Expected there to be one matching G_UNMERGE_VALUES");
3714 B.setInstrAndDebugLoc(MI);
3715
3716 Register UndefVec = B.buildUndef(Res: MRI.getType(Reg: UnmergeSrc)).getReg(Idx: 0);
3717 B.buildConcatVectors(Res: MI.getOperand(i: 0), Ops: {UnmergeSrc, UndefVec});
3718
3719 MI.eraseFromParent();
3720}
3721
3722// This combine tries to reduce the number of scalarised G_TRUNC instructions by
3723// using vector truncates instead
3724//
3725// EXAMPLE:
3726// %a(i32), %b(i32) = G_UNMERGE_VALUES %src(<2 x i32>)
3727// %T_a(i16) = G_TRUNC %a(i32)
3728// %T_b(i16) = G_TRUNC %b(i32)
3729// %Undef(i16) = G_IMPLICIT_DEF(i16)
3730// %dst(v4i16) = G_BUILD_VECTORS %T_a(i16), %T_b(i16), %Undef(i16), %Undef(i16)
3731//
3732// ===>
3733// %Undef(<2 x i32>) = G_IMPLICIT_DEF(<2 x i32>)
3734// %Mid(<4 x s32>) = G_CONCAT_VECTORS %src(<2 x i32>), %Undef(<2 x i32>)
3735// %dst(<4 x s16>) = G_TRUNC %Mid(<4 x s32>)
3736//
3737// Only matches sources made up of G_TRUNCs followed by G_IMPLICIT_DEFs
3738bool CombinerHelper::matchUseVectorTruncate(MachineInstr &MI,
3739 Register &MatchInfo) const {
3740 auto BuildMI = cast<GBuildVector>(Val: &MI);
3741 unsigned NumOperands = BuildMI->getNumSources();
3742 LLT DstTy = MRI.getType(Reg: BuildMI->getReg(Idx: 0));
3743
3744 // Check the G_BUILD_VECTOR sources
3745 unsigned I;
3746 MachineInstr *UnmergeMI = nullptr;
3747
3748 // Check all source TRUNCs come from the same UNMERGE instruction
3749 // and that the element order matches (BUILD_VECTOR position I
3750 // corresponds to UNMERGE result I)
3751 for (I = 0; I < NumOperands; ++I) {
3752 auto SrcMI = MRI.getVRegDef(Reg: BuildMI->getSourceReg(I));
3753 auto SrcMIOpc = SrcMI->getOpcode();
3754
3755 // Check if the G_TRUNC instructions all come from the same MI
3756 if (SrcMIOpc == TargetOpcode::G_TRUNC) {
3757 Register TruncSrcReg = SrcMI->getOperand(i: 1).getReg();
3758 if (!UnmergeMI) {
3759 UnmergeMI = MRI.getVRegDef(Reg: TruncSrcReg);
3760 if (UnmergeMI->getOpcode() != TargetOpcode::G_UNMERGE_VALUES)
3761 return false;
3762 } else {
3763 auto UnmergeSrcMI = MRI.getVRegDef(Reg: TruncSrcReg);
3764 if (UnmergeMI != UnmergeSrcMI)
3765 return false;
3766 }
3767 // Verify element ordering: BUILD_VECTOR position I must use
3768 // UNMERGE result I, otherwise the fold would lose element reordering
3769 if (UnmergeMI->getOperand(i: I).getReg() != TruncSrcReg)
3770 return false;
3771 } else {
3772 break;
3773 }
3774 }
3775 if (I < 2)
3776 return false;
3777
3778 // Check the remaining source elements are only G_IMPLICIT_DEF
3779 for (; I < NumOperands; ++I) {
3780 auto SrcMI = MRI.getVRegDef(Reg: BuildMI->getSourceReg(I));
3781 auto SrcMIOpc = SrcMI->getOpcode();
3782
3783 if (SrcMIOpc != TargetOpcode::G_IMPLICIT_DEF)
3784 return false;
3785 }
3786
3787 // Check the size of unmerge source
3788 MatchInfo = cast<GUnmerge>(Val: UnmergeMI)->getSourceReg();
3789 LLT UnmergeSrcTy = MRI.getType(Reg: MatchInfo);
3790 if (!DstTy.getElementCount().isKnownMultipleOf(RHS: UnmergeSrcTy.getNumElements()))
3791 return false;
3792
3793 // Check the unmerge source and destination element types match
3794 LLT UnmergeSrcEltTy = UnmergeSrcTy.getElementType();
3795 Register UnmergeDstReg = UnmergeMI->getOperand(i: 0).getReg();
3796 LLT UnmergeDstEltTy = MRI.getType(Reg: UnmergeDstReg);
3797 if (UnmergeSrcEltTy != UnmergeDstEltTy)
3798 return false;
3799
3800 // Only generate legal instructions post-legalizer
3801 if (!IsPreLegalize) {
3802 LLT MidTy = DstTy.changeElementType(NewEltTy: UnmergeSrcTy.getScalarType());
3803
3804 if (DstTy.getElementCount() != UnmergeSrcTy.getElementCount() &&
3805 !isLegal(Query: {TargetOpcode::G_CONCAT_VECTORS, {MidTy, UnmergeSrcTy}}))
3806 return false;
3807
3808 if (!isLegal(Query: {TargetOpcode::G_TRUNC, {DstTy, MidTy}}))
3809 return false;
3810 }
3811
3812 return true;
3813}
3814
3815void CombinerHelper::applyUseVectorTruncate(MachineInstr &MI,
3816 Register &MatchInfo) const {
3817 Register MidReg;
3818 auto BuildMI = cast<GBuildVector>(Val: &MI);
3819 Register DstReg = BuildMI->getReg(Idx: 0);
3820 LLT DstTy = MRI.getType(Reg: DstReg);
3821 LLT UnmergeSrcTy = MRI.getType(Reg: MatchInfo);
3822 unsigned DstTyNumElt = DstTy.getNumElements();
3823 unsigned UnmergeSrcTyNumElt = UnmergeSrcTy.getNumElements();
3824
3825 // No need to pad vector if only G_TRUNC is needed
3826 if (DstTyNumElt / UnmergeSrcTyNumElt == 1) {
3827 MidReg = MatchInfo;
3828 } else {
3829 Register UndefReg = Builder.buildUndef(Res: UnmergeSrcTy).getReg(Idx: 0);
3830 SmallVector<Register> ConcatRegs = {MatchInfo};
3831 for (unsigned I = 1; I < DstTyNumElt / UnmergeSrcTyNumElt; ++I)
3832 ConcatRegs.push_back(Elt: UndefReg);
3833
3834 auto MidTy = DstTy.changeElementType(NewEltTy: UnmergeSrcTy.getScalarType());
3835 MidReg = Builder.buildConcatVectors(Res: MidTy, Ops: ConcatRegs).getReg(Idx: 0);
3836 }
3837
3838 Builder.buildTrunc(Res: DstReg, Op: MidReg);
3839 MI.eraseFromParent();
3840}
3841
3842bool CombinerHelper::matchNotCmp(
3843 MachineInstr &MI, SmallVectorImpl<Register> &RegsToNegate) const {
3844 assert(MI.getOpcode() == TargetOpcode::G_XOR);
3845 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3846 const auto &TLI = *Builder.getMF().getSubtarget().getTargetLowering();
3847 Register XorSrc;
3848 Register CstReg;
3849 // We match xor(src, true) here.
3850 if (!mi_match(R: MI.getOperand(i: 0).getReg(), MRI,
3851 P: m_GXor(L: m_Reg(R&: XorSrc), R: m_Reg(R&: CstReg))))
3852 return false;
3853
3854 if (!MRI.hasOneNonDBGUse(RegNo: XorSrc))
3855 return false;
3856
3857 // Check that XorSrc is the root of a tree of comparisons combined with ANDs
3858 // and ORs. The suffix of RegsToNegate starting from index I is used a work
3859 // list of tree nodes to visit.
3860 RegsToNegate.push_back(Elt: XorSrc);
3861 // Remember whether the comparisons are all integer or all floating point.
3862 bool IsInt = false;
3863 bool IsFP = false;
3864 for (unsigned I = 0; I < RegsToNegate.size(); ++I) {
3865 Register Reg = RegsToNegate[I];
3866 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
3867 return false;
3868 MachineInstr *Def = MRI.getVRegDef(Reg);
3869 switch (Def->getOpcode()) {
3870 default:
3871 // Don't match if the tree contains anything other than ANDs, ORs and
3872 // comparisons.
3873 return false;
3874 case TargetOpcode::G_ICMP:
3875 if (IsFP)
3876 return false;
3877 IsInt = true;
3878 // When we apply the combine we will invert the predicate.
3879 break;
3880 case TargetOpcode::G_FCMP:
3881 if (IsInt)
3882 return false;
3883 IsFP = true;
3884 // When we apply the combine we will invert the predicate.
3885 break;
3886 case TargetOpcode::G_AND:
3887 case TargetOpcode::G_OR:
3888 // Implement De Morgan's laws:
3889 // ~(x & y) -> ~x | ~y
3890 // ~(x | y) -> ~x & ~y
3891 // When we apply the combine we will change the opcode and recursively
3892 // negate the operands.
3893 RegsToNegate.push_back(Elt: Def->getOperand(i: 1).getReg());
3894 RegsToNegate.push_back(Elt: Def->getOperand(i: 2).getReg());
3895 break;
3896 }
3897 }
3898
3899 // Now we know whether the comparisons are integer or floating point, check
3900 // the constant in the xor.
3901 int64_t Cst;
3902 if (Ty.isVector()) {
3903 MachineInstr *CstDef = MRI.getVRegDef(Reg: CstReg);
3904 auto MaybeCst = getIConstantSplatSExtVal(MI: *CstDef, MRI);
3905 if (!MaybeCst)
3906 return false;
3907 if (!isConstValidTrue(TLI, ScalarSizeBits: Ty.getScalarSizeInBits(), Cst: *MaybeCst, IsVector: true, IsFP))
3908 return false;
3909 } else {
3910 if (!mi_match(R: CstReg, MRI, P: m_ICst(Cst)))
3911 return false;
3912 if (!isConstValidTrue(TLI, ScalarSizeBits: Ty.getSizeInBits(), Cst, IsVector: false, IsFP))
3913 return false;
3914 }
3915
3916 return true;
3917}
3918
3919void CombinerHelper::applyNotCmp(
3920 MachineInstr &MI, SmallVectorImpl<Register> &RegsToNegate) const {
3921 for (Register Reg : RegsToNegate) {
3922 MachineInstr *Def = MRI.getVRegDef(Reg);
3923 Observer.changingInstr(MI&: *Def);
3924 // For each comparison, invert the opcode. For each AND and OR, change the
3925 // opcode.
3926 switch (Def->getOpcode()) {
3927 default:
3928 llvm_unreachable("Unexpected opcode");
3929 case TargetOpcode::G_ICMP:
3930 case TargetOpcode::G_FCMP: {
3931 MachineOperand &PredOp = Def->getOperand(i: 1);
3932 CmpInst::Predicate NewP = CmpInst::getInversePredicate(
3933 pred: (CmpInst::Predicate)PredOp.getPredicate());
3934 PredOp.setPredicate(NewP);
3935 break;
3936 }
3937 case TargetOpcode::G_AND:
3938 Def->setDesc(Builder.getTII().get(Opcode: TargetOpcode::G_OR));
3939 break;
3940 case TargetOpcode::G_OR:
3941 Def->setDesc(Builder.getTII().get(Opcode: TargetOpcode::G_AND));
3942 break;
3943 }
3944 Observer.changedInstr(MI&: *Def);
3945 }
3946
3947 replaceRegWith(MRI, FromReg: MI.getOperand(i: 0).getReg(), ToReg: MI.getOperand(i: 1).getReg());
3948 MI.eraseFromParent();
3949}
3950
3951bool CombinerHelper::matchXorOfAndWithSameReg(
3952 MachineInstr &MI, std::pair<Register, Register> &MatchInfo) const {
3953 // Match (xor (and x, y), y) (or any of its commuted cases)
3954 assert(MI.getOpcode() == TargetOpcode::G_XOR);
3955 Register &X = MatchInfo.first;
3956 Register &Y = MatchInfo.second;
3957 Register AndReg = MI.getOperand(i: 1).getReg();
3958 Register SharedReg = MI.getOperand(i: 2).getReg();
3959
3960 // Find a G_AND on either side of the G_XOR.
3961 // Look for one of
3962 //
3963 // (xor (and x, y), SharedReg)
3964 // (xor SharedReg, (and x, y))
3965 if (!mi_match(R: AndReg, MRI, P: m_GAnd(L: m_Reg(R&: X), R: m_Reg(R&: Y)))) {
3966 std::swap(a&: AndReg, b&: SharedReg);
3967 if (!mi_match(R: AndReg, MRI, P: m_GAnd(L: m_Reg(R&: X), R: m_Reg(R&: Y))))
3968 return false;
3969 }
3970
3971 // Only do this if we'll eliminate the G_AND.
3972 if (!MRI.hasOneNonDBGUse(RegNo: AndReg))
3973 return false;
3974
3975 // We can combine if SharedReg is the same as either the LHS or RHS of the
3976 // G_AND.
3977 if (Y != SharedReg)
3978 std::swap(a&: X, b&: Y);
3979 return Y == SharedReg;
3980}
3981
3982void CombinerHelper::applyXorOfAndWithSameReg(
3983 MachineInstr &MI, std::pair<Register, Register> &MatchInfo) const {
3984 // Fold (xor (and x, y), y) -> (and (not x), y)
3985 Register X, Y;
3986 std::tie(args&: X, args&: Y) = MatchInfo;
3987 auto Not = Builder.buildNot(Dst: MRI.getType(Reg: X), Src0: X);
3988 Observer.changingInstr(MI);
3989 MI.setDesc(Builder.getTII().get(Opcode: TargetOpcode::G_AND));
3990 MI.getOperand(i: 1).setReg(Not->getOperand(i: 0).getReg());
3991 MI.getOperand(i: 2).setReg(Y);
3992 Observer.changedInstr(MI);
3993}
3994
3995bool CombinerHelper::matchPtrAddZero(MachineInstr &MI) const {
3996 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
3997 Register DstReg = PtrAdd.getReg(Idx: 0);
3998 LLT Ty = MRI.getType(Reg: DstReg);
3999 const DataLayout &DL = Builder.getMF().getDataLayout();
4000
4001 if (DL.isNonIntegralAddressSpace(AddrSpace: Ty.getScalarType().getAddressSpace()))
4002 return false;
4003
4004 if (Ty.isPointer()) {
4005 auto ConstVal = getIConstantVRegVal(VReg: PtrAdd.getBaseReg(), MRI);
4006 return ConstVal && *ConstVal == 0;
4007 }
4008
4009 assert(Ty.isVector() && "Expecting a vector type");
4010 const MachineInstr *VecMI = MRI.getVRegDef(Reg: PtrAdd.getBaseReg());
4011 return isBuildVectorAllZeros(MI: *VecMI, MRI);
4012}
4013
4014void CombinerHelper::applyPtrAddZero(MachineInstr &MI) const {
4015 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
4016 Builder.buildIntToPtr(Dst: PtrAdd.getReg(Idx: 0), Src: PtrAdd.getOffsetReg());
4017 PtrAdd.eraseFromParent();
4018}
4019
4020/// The second source operand is known to be a power of 2.
4021void CombinerHelper::applySimplifyURemByPow2(MachineInstr &MI) const {
4022 Register DstReg = MI.getOperand(i: 0).getReg();
4023 Register Src0 = MI.getOperand(i: 1).getReg();
4024 Register Pow2Src1 = MI.getOperand(i: 2).getReg();
4025 LLT Ty = MRI.getType(Reg: DstReg);
4026
4027 // Fold (urem x, pow2) -> (and x, pow2-1)
4028 auto NegOne = Builder.buildConstant(Res: Ty, Val: -1);
4029 auto Add = Builder.buildAdd(Dst: Ty, Src0: Pow2Src1, Src1: NegOne);
4030 Builder.buildAnd(Dst: DstReg, Src0, Src1: Add);
4031 MI.eraseFromParent();
4032}
4033
4034bool CombinerHelper::matchFoldBinOpIntoSelect(MachineInstr &MI,
4035 unsigned &SelectOpNo) const {
4036 Register LHS = MI.getOperand(i: 1).getReg();
4037 Register RHS = MI.getOperand(i: 2).getReg();
4038
4039 Register OtherOperandReg = RHS;
4040 SelectOpNo = 1;
4041 MachineInstr *Select = MRI.getVRegDef(Reg: LHS);
4042
4043 // Don't do this unless the old select is going away. We want to eliminate the
4044 // binary operator, not replace a binop with a select.
4045 if (Select->getOpcode() != TargetOpcode::G_SELECT ||
4046 !MRI.hasOneNonDBGUse(RegNo: LHS)) {
4047 OtherOperandReg = LHS;
4048 SelectOpNo = 2;
4049 Select = MRI.getVRegDef(Reg: RHS);
4050 if (Select->getOpcode() != TargetOpcode::G_SELECT ||
4051 !MRI.hasOneNonDBGUse(RegNo: RHS))
4052 return false;
4053 }
4054
4055 MachineInstr *SelectLHS = MRI.getVRegDef(Reg: Select->getOperand(i: 2).getReg());
4056 MachineInstr *SelectRHS = MRI.getVRegDef(Reg: Select->getOperand(i: 3).getReg());
4057
4058 if (!isConstantOrConstantVector(MI: *SelectLHS, MRI,
4059 /*AllowFP*/ true,
4060 /*AllowOpaqueConstants*/ false))
4061 return false;
4062 if (!isConstantOrConstantVector(MI: *SelectRHS, MRI,
4063 /*AllowFP*/ true,
4064 /*AllowOpaqueConstants*/ false))
4065 return false;
4066
4067 unsigned BinOpcode = MI.getOpcode();
4068
4069 // We know that one of the operands is a select of constants. Now verify that
4070 // the other binary operator operand is either a constant, or we can handle a
4071 // variable.
4072 bool CanFoldNonConst =
4073 (BinOpcode == TargetOpcode::G_AND || BinOpcode == TargetOpcode::G_OR) &&
4074 (isNullOrNullSplat(MI: *SelectLHS, MRI) ||
4075 isAllOnesOrAllOnesSplat(MI: *SelectLHS, MRI)) &&
4076 (isNullOrNullSplat(MI: *SelectRHS, MRI) ||
4077 isAllOnesOrAllOnesSplat(MI: *SelectRHS, MRI));
4078 if (CanFoldNonConst)
4079 return true;
4080
4081 return isConstantOrConstantVector(MI: *MRI.getVRegDef(Reg: OtherOperandReg), MRI,
4082 /*AllowFP*/ true,
4083 /*AllowOpaqueConstants*/ false);
4084}
4085
4086/// \p SelectOperand is the operand in binary operator \p MI that is the select
4087/// to fold.
4088void CombinerHelper::applyFoldBinOpIntoSelect(
4089 MachineInstr &MI, const unsigned &SelectOperand) const {
4090 Register Dst = MI.getOperand(i: 0).getReg();
4091 Register LHS = MI.getOperand(i: 1).getReg();
4092 Register RHS = MI.getOperand(i: 2).getReg();
4093 MachineInstr *Select = MRI.getVRegDef(Reg: MI.getOperand(i: SelectOperand).getReg());
4094
4095 Register SelectCond = Select->getOperand(i: 1).getReg();
4096 Register SelectTrue = Select->getOperand(i: 2).getReg();
4097 Register SelectFalse = Select->getOperand(i: 3).getReg();
4098
4099 LLT Ty = MRI.getType(Reg: Dst);
4100 unsigned BinOpcode = MI.getOpcode();
4101
4102 Register FoldTrue, FoldFalse;
4103
4104 // We have a select-of-constants followed by a binary operator with a
4105 // constant. Eliminate the binop by pulling the constant math into the select.
4106 // Example: add (select Cond, CT, CF), CBO --> select Cond, CT + CBO, CF + CBO
4107 if (SelectOperand == 1) {
4108 // TODO: SelectionDAG verifies this actually constant folds before
4109 // committing to the combine.
4110
4111 FoldTrue = Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {SelectTrue, RHS}).getReg(Idx: 0);
4112 FoldFalse =
4113 Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {SelectFalse, RHS}).getReg(Idx: 0);
4114 } else {
4115 FoldTrue = Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {LHS, SelectTrue}).getReg(Idx: 0);
4116 FoldFalse =
4117 Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {LHS, SelectFalse}).getReg(Idx: 0);
4118 }
4119
4120 Builder.buildSelect(Res: Dst, Tst: SelectCond, Op0: FoldTrue, Op1: FoldFalse, Flags: MI.getFlags());
4121 MI.eraseFromParent();
4122}
4123
4124std::optional<SmallVector<Register, 8>>
4125CombinerHelper::findCandidatesForLoadOrCombine(const MachineInstr *Root) const {
4126 assert(Root->getOpcode() == TargetOpcode::G_OR && "Expected G_OR only!");
4127 // We want to detect if Root is part of a tree which represents a bunch
4128 // of loads being merged into a larger load. We'll try to recognize patterns
4129 // like, for example:
4130 //
4131 // Reg Reg
4132 // \ /
4133 // OR_1 Reg
4134 // \ /
4135 // OR_2
4136 // \ Reg
4137 // .. /
4138 // Root
4139 //
4140 // Reg Reg Reg Reg
4141 // \ / \ /
4142 // OR_1 OR_2
4143 // \ /
4144 // \ /
4145 // ...
4146 // Root
4147 //
4148 // Each "Reg" may have been produced by a load + some arithmetic. This
4149 // function will save each of them.
4150 SmallVector<Register, 8> RegsToVisit;
4151 SmallVector<const MachineInstr *, 7> Ors = {Root};
4152
4153 // In the "worst" case, we're dealing with a load for each byte. So, there
4154 // are at most #bytes - 1 ORs.
4155 const unsigned MaxIter =
4156 MRI.getType(Reg: Root->getOperand(i: 0).getReg()).getSizeInBytes() - 1;
4157 for (unsigned Iter = 0; Iter < MaxIter; ++Iter) {
4158 if (Ors.empty())
4159 break;
4160 const MachineInstr *Curr = Ors.pop_back_val();
4161 Register OrLHS = Curr->getOperand(i: 1).getReg();
4162 Register OrRHS = Curr->getOperand(i: 2).getReg();
4163
4164 // In the combine, we want to elimate the entire tree.
4165 if (!MRI.hasOneNonDBGUse(RegNo: OrLHS) || !MRI.hasOneNonDBGUse(RegNo: OrRHS))
4166 return std::nullopt;
4167
4168 // If it's a G_OR, save it and continue to walk. If it's not, then it's
4169 // something that may be a load + arithmetic.
4170 if (const MachineInstr *Or = getOpcodeDef(Opcode: TargetOpcode::G_OR, Reg: OrLHS, MRI))
4171 Ors.push_back(Elt: Or);
4172 else
4173 RegsToVisit.push_back(Elt: OrLHS);
4174 if (const MachineInstr *Or = getOpcodeDef(Opcode: TargetOpcode::G_OR, Reg: OrRHS, MRI))
4175 Ors.push_back(Elt: Or);
4176 else
4177 RegsToVisit.push_back(Elt: OrRHS);
4178 }
4179
4180 // We're going to try and merge each register into a wider power-of-2 type,
4181 // so we ought to have an even number of registers.
4182 if (RegsToVisit.empty() || RegsToVisit.size() % 2 != 0)
4183 return std::nullopt;
4184 return RegsToVisit;
4185}
4186
4187/// Helper function for findLoadOffsetsForLoadOrCombine.
4188///
4189/// Check if \p Reg is the result of loading a \p MemSizeInBits wide value,
4190/// and then moving that value into a specific byte offset.
4191///
4192/// e.g. x[i] << 24
4193///
4194/// \returns The load instruction and the byte offset it is moved into.
4195static std::optional<std::pair<GZExtLoad *, int64_t>>
4196matchLoadAndBytePosition(Register Reg, unsigned MemSizeInBits,
4197 const MachineRegisterInfo &MRI) {
4198 assert(MRI.hasOneNonDBGUse(Reg) &&
4199 "Expected Reg to only have one non-debug use?");
4200 Register MaybeLoad;
4201 int64_t Shift;
4202 if (!mi_match(R: Reg, MRI,
4203 P: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: MaybeLoad), R: m_ICst(Cst&: Shift))))) {
4204 Shift = 0;
4205 MaybeLoad = Reg;
4206 }
4207
4208 if (Shift % MemSizeInBits != 0)
4209 return std::nullopt;
4210
4211 // TODO: Handle other types of loads.
4212 auto *Load = getOpcodeDef<GZExtLoad>(Reg: MaybeLoad, MRI);
4213 if (!Load)
4214 return std::nullopt;
4215
4216 if (!Load->isUnordered() || Load->getMemSizeInBits() != MemSizeInBits)
4217 return std::nullopt;
4218
4219 return std::make_pair(x&: Load, y: Shift / MemSizeInBits);
4220}
4221
4222std::optional<std::tuple<GZExtLoad *, int64_t, GZExtLoad *>>
4223CombinerHelper::findLoadOffsetsForLoadOrCombine(
4224 SmallDenseMap<int64_t, int64_t, 8> &MemOffset2Idx,
4225 const SmallVector<Register, 8> &RegsToVisit,
4226 const unsigned MemSizeInBits) const {
4227
4228 // Each load found for the pattern. There should be one for each RegsToVisit.
4229 SmallSetVector<const MachineInstr *, 8> Loads;
4230
4231 // The lowest index used in any load. (The lowest "i" for each x[i].)
4232 int64_t LowestIdx = INT64_MAX;
4233
4234 // The load which uses the lowest index.
4235 GZExtLoad *LowestIdxLoad = nullptr;
4236
4237 // Keeps track of the load indices we see. We shouldn't see any indices twice.
4238 SmallSet<int64_t, 8> SeenIdx;
4239
4240 // Ensure each load is in the same MBB.
4241 // TODO: Support multiple MachineBasicBlocks.
4242 MachineBasicBlock *MBB = nullptr;
4243 const MachineMemOperand *MMO = nullptr;
4244
4245 // Earliest instruction-order load in the pattern.
4246 GZExtLoad *EarliestLoad = nullptr;
4247
4248 // Latest instruction-order load in the pattern.
4249 GZExtLoad *LatestLoad = nullptr;
4250
4251 // Base pointer which every load should share.
4252 Register BasePtr;
4253
4254 // We want to find a load for each register. Each load should have some
4255 // appropriate bit twiddling arithmetic. During this loop, we will also keep
4256 // track of the load which uses the lowest index. Later, we will check if we
4257 // can use its pointer in the final, combined load.
4258 for (auto Reg : RegsToVisit) {
4259 // Find the load, and find the position that it will end up in (e.g. a
4260 // shifted) value.
4261 auto LoadAndPos = matchLoadAndBytePosition(Reg, MemSizeInBits, MRI);
4262 if (!LoadAndPos)
4263 return std::nullopt;
4264 GZExtLoad *Load;
4265 int64_t DstPos;
4266 std::tie(args&: Load, args&: DstPos) = *LoadAndPos;
4267
4268 // TODO: Handle multiple MachineBasicBlocks. Currently not handled because
4269 // it is difficult to check for stores/calls/etc between loads.
4270 MachineBasicBlock *LoadMBB = Load->getParent();
4271 if (!MBB)
4272 MBB = LoadMBB;
4273 if (LoadMBB != MBB)
4274 return std::nullopt;
4275
4276 // Make sure that the MachineMemOperands of every seen load are compatible.
4277 auto &LoadMMO = Load->getMMO();
4278 if (!MMO)
4279 MMO = &LoadMMO;
4280 if (MMO->getAddrSpace() != LoadMMO.getAddrSpace())
4281 return std::nullopt;
4282
4283 // Find out what the base pointer and index for the load is.
4284 Register LoadPtr;
4285 int64_t Idx;
4286 if (!mi_match(R: Load->getOperand(i: 1).getReg(), MRI,
4287 P: m_GPtrAdd(L: m_Reg(R&: LoadPtr), R: m_ICst(Cst&: Idx)))) {
4288 LoadPtr = Load->getOperand(i: 1).getReg();
4289 Idx = 0;
4290 }
4291
4292 // Don't combine things like a[i], a[i] -> a bigger load.
4293 if (!SeenIdx.insert(V: Idx).second)
4294 return std::nullopt;
4295
4296 // Every load must share the same base pointer; don't combine things like:
4297 //
4298 // a[i], b[i + 1] -> a bigger load.
4299 if (!BasePtr.isValid())
4300 BasePtr = LoadPtr;
4301 if (BasePtr != LoadPtr)
4302 return std::nullopt;
4303
4304 if (Idx < LowestIdx) {
4305 LowestIdx = Idx;
4306 LowestIdxLoad = Load;
4307 }
4308
4309 // Keep track of the byte offset that this load ends up at. If we have seen
4310 // the byte offset, then stop here. We do not want to combine:
4311 //
4312 // a[i] << 16, a[i + k] << 16 -> a bigger load.
4313 if (!MemOffset2Idx.try_emplace(Key: DstPos, Args&: Idx).second)
4314 return std::nullopt;
4315 Loads.insert(X: Load);
4316
4317 // Keep track of the position of the earliest/latest loads in the pattern.
4318 // We will check that there are no load fold barriers between them later
4319 // on.
4320 //
4321 // FIXME: Is there a better way to check for load fold barriers?
4322 if (!EarliestLoad || dominates(DefMI: *Load, UseMI: *EarliestLoad))
4323 EarliestLoad = Load;
4324 if (!LatestLoad || dominates(DefMI: *LatestLoad, UseMI: *Load))
4325 LatestLoad = Load;
4326 }
4327
4328 // We found a load for each register. Let's check if each load satisfies the
4329 // pattern.
4330 assert(Loads.size() == RegsToVisit.size() &&
4331 "Expected to find a load for each register?");
4332 assert(EarliestLoad != LatestLoad && EarliestLoad &&
4333 LatestLoad && "Expected at least two loads?");
4334
4335 // Check if there are any stores, calls, etc. between any of the loads. If
4336 // there are, then we can't safely perform the combine.
4337 //
4338 // MaxIter is chosen based off the (worst case) number of iterations it
4339 // typically takes to succeed in the LLVM test suite plus some padding.
4340 //
4341 // FIXME: Is there a better way to check for load fold barriers?
4342 const unsigned MaxIter = 20;
4343 unsigned Iter = 0;
4344 for (const auto &MI : instructionsWithoutDebug(It: EarliestLoad->getIterator(),
4345 End: LatestLoad->getIterator())) {
4346 if (Loads.count(key: &MI))
4347 continue;
4348 if (MI.isLoadFoldBarrier())
4349 return std::nullopt;
4350 if (Iter++ == MaxIter)
4351 return std::nullopt;
4352 }
4353
4354 return std::make_tuple(args&: LowestIdxLoad, args&: LowestIdx, args&: LatestLoad);
4355}
4356
4357bool CombinerHelper::matchLoadOrCombine(
4358 MachineInstr &MI,
4359 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4360 assert(MI.getOpcode() == TargetOpcode::G_OR);
4361 MachineFunction &MF = *MI.getMF();
4362 // Assuming a little-endian target, transform:
4363 // s8 *a = ...
4364 // s32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
4365 // =>
4366 // s32 val = *((i32)a)
4367 //
4368 // s8 *a = ...
4369 // s32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
4370 // =>
4371 // s32 val = BSWAP(*((s32)a))
4372 Register Dst = MI.getOperand(i: 0).getReg();
4373 LLT Ty = MRI.getType(Reg: Dst);
4374 if (Ty.isVector())
4375 return false;
4376
4377 // We need to combine at least two loads into this type. Since the smallest
4378 // possible load is into a byte, we need at least a 16-bit wide type.
4379 const unsigned WideMemSizeInBits = Ty.getSizeInBits();
4380 if (WideMemSizeInBits < 16 || WideMemSizeInBits % 8 != 0)
4381 return false;
4382
4383 // Match a collection of non-OR instructions in the pattern.
4384 auto RegsToVisit = findCandidatesForLoadOrCombine(Root: &MI);
4385 if (!RegsToVisit)
4386 return false;
4387
4388 // We have a collection of non-OR instructions. Figure out how wide each of
4389 // the small loads should be based off of the number of potential loads we
4390 // found.
4391 const unsigned NarrowMemSizeInBits = WideMemSizeInBits / RegsToVisit->size();
4392 if (NarrowMemSizeInBits % 8 != 0)
4393 return false;
4394
4395 // Check if each register feeding into each OR is a load from the same
4396 // base pointer + some arithmetic.
4397 //
4398 // e.g. a[0], a[1] << 8, a[2] << 16, etc.
4399 //
4400 // Also verify that each of these ends up putting a[i] into the same memory
4401 // offset as a load into a wide type would.
4402 SmallDenseMap<int64_t, int64_t, 8> MemOffset2Idx;
4403 GZExtLoad *LowestIdxLoad, *LatestLoad;
4404 int64_t LowestIdx;
4405 auto MaybeLoadInfo = findLoadOffsetsForLoadOrCombine(
4406 MemOffset2Idx, RegsToVisit: *RegsToVisit, MemSizeInBits: NarrowMemSizeInBits);
4407 if (!MaybeLoadInfo)
4408 return false;
4409 std::tie(args&: LowestIdxLoad, args&: LowestIdx, args&: LatestLoad) = *MaybeLoadInfo;
4410
4411 // We have a bunch of loads being OR'd together. Using the addresses + offsets
4412 // we found before, check if this corresponds to a big or little endian byte
4413 // pattern. If it does, then we can represent it using a load + possibly a
4414 // BSWAP.
4415 bool IsBigEndianTarget = MF.getDataLayout().isBigEndian();
4416 std::optional<bool> IsBigEndian = isBigEndian(MemOffset2Idx, LowestIdx);
4417 if (!IsBigEndian)
4418 return false;
4419 bool NeedsBSwap = IsBigEndianTarget != *IsBigEndian;
4420 if (NeedsBSwap && !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_BSWAP, {Ty}}))
4421 return false;
4422
4423 // Make sure that the load from the lowest index produces offset 0 in the
4424 // final value.
4425 //
4426 // This ensures that we won't combine something like this:
4427 //
4428 // load x[i] -> byte 2
4429 // load x[i+1] -> byte 0 ---> wide_load x[i]
4430 // load x[i+2] -> byte 1
4431 const unsigned NumLoadsInTy = WideMemSizeInBits / NarrowMemSizeInBits;
4432 const unsigned ZeroByteOffset =
4433 *IsBigEndian
4434 ? bigEndianByteAt(ByteWidth: NumLoadsInTy, I: 0)
4435 : littleEndianByteAt(ByteWidth: NumLoadsInTy, I: 0);
4436 auto ZeroOffsetIdx = MemOffset2Idx.find(Val: ZeroByteOffset);
4437 if (ZeroOffsetIdx == MemOffset2Idx.end() ||
4438 ZeroOffsetIdx->second != LowestIdx)
4439 return false;
4440
4441 // We wil reuse the pointer from the load which ends up at byte offset 0. It
4442 // may not use index 0.
4443 Register Ptr = LowestIdxLoad->getPointerReg();
4444 const MachineMemOperand &MMO = LowestIdxLoad->getMMO();
4445 LegalityQuery::MemDesc MMDesc(MMO);
4446 MMDesc.MemoryTy = Ty;
4447 if (!isLegalOrBeforeLegalizer(
4448 Query: {TargetOpcode::G_LOAD, {Ty, MRI.getType(Reg: Ptr)}, {MMDesc}}))
4449 return false;
4450 auto PtrInfo = MMO.getPointerInfo();
4451 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Size: WideMemSizeInBits / 8);
4452
4453 // Load must be allowed and fast on the target.
4454 LLVMContext &C = MF.getFunction().getContext();
4455 auto &DL = MF.getDataLayout();
4456 unsigned Fast = 0;
4457 if (!getTargetLowering().allowsMemoryAccess(Context&: C, DL, Ty, MMO: *NewMMO, Fast: &Fast) ||
4458 !Fast)
4459 return false;
4460
4461 MatchInfo = [=](MachineIRBuilder &MIB) {
4462 MIB.setInstrAndDebugLoc(*LatestLoad);
4463 Register LoadDst = NeedsBSwap ? MRI.cloneVirtualRegister(VReg: Dst) : Dst;
4464 MIB.buildLoad(Res: LoadDst, Addr: Ptr, MMO&: *NewMMO);
4465 if (NeedsBSwap)
4466 MIB.buildBSwap(Dst, Src0: LoadDst);
4467 };
4468 return true;
4469}
4470
4471bool CombinerHelper::matchExtendThroughPhis(MachineInstr &MI,
4472 MachineInstr *&ExtMI) const {
4473 auto &PHI = cast<GPhi>(Val&: MI);
4474 Register DstReg = PHI.getReg(Idx: 0);
4475
4476 // TODO: Extending a vector may be expensive, don't do this until heuristics
4477 // are better.
4478 if (MRI.getType(Reg: DstReg).isVector())
4479 return false;
4480
4481 // Try to match a phi, whose only use is an extend.
4482 if (!MRI.hasOneNonDBGUse(RegNo: DstReg))
4483 return false;
4484 ExtMI = &*MRI.use_instr_nodbg_begin(RegNo: DstReg);
4485 switch (ExtMI->getOpcode()) {
4486 case TargetOpcode::G_ANYEXT:
4487 return true; // G_ANYEXT is usually free.
4488 case TargetOpcode::G_ZEXT:
4489 case TargetOpcode::G_SEXT:
4490 break;
4491 default:
4492 return false;
4493 }
4494
4495 // If the target is likely to fold this extend away, don't propagate.
4496 if (Builder.getTII().isExtendLikelyToBeFolded(ExtMI&: *ExtMI, MRI))
4497 return false;
4498
4499 // We don't want to propagate the extends unless there's a good chance that
4500 // they'll be optimized in some way.
4501 // Collect the unique incoming values.
4502 SmallPtrSet<MachineInstr *, 4> InSrcs;
4503 for (unsigned I = 0; I < PHI.getNumIncomingValues(); ++I) {
4504 auto *DefMI = getDefIgnoringCopies(Reg: PHI.getIncomingValue(I), MRI);
4505 switch (DefMI->getOpcode()) {
4506 case TargetOpcode::G_LOAD:
4507 case TargetOpcode::G_TRUNC:
4508 case TargetOpcode::G_SEXT:
4509 case TargetOpcode::G_ZEXT:
4510 case TargetOpcode::G_ANYEXT:
4511 case TargetOpcode::G_CONSTANT:
4512 InSrcs.insert(Ptr: DefMI);
4513 // Don't try to propagate if there are too many places to create new
4514 // extends, chances are it'll increase code size.
4515 if (InSrcs.size() > 2)
4516 return false;
4517 break;
4518 default:
4519 return false;
4520 }
4521 }
4522 return true;
4523}
4524
4525void CombinerHelper::applyExtendThroughPhis(MachineInstr &MI,
4526 MachineInstr *&ExtMI) const {
4527 auto &PHI = cast<GPhi>(Val&: MI);
4528 Register DstReg = ExtMI->getOperand(i: 0).getReg();
4529 LLT ExtTy = MRI.getType(Reg: DstReg);
4530
4531 // Propagate the extension into the block of each incoming reg's block.
4532 // Use a SetVector here because PHIs can have duplicate edges, and we want
4533 // deterministic iteration order.
4534 SmallSetVector<MachineInstr *, 8> SrcMIs;
4535 SmallDenseMap<MachineInstr *, MachineInstr *, 8> OldToNewSrcMap;
4536 for (unsigned I = 0; I < PHI.getNumIncomingValues(); ++I) {
4537 auto SrcReg = PHI.getIncomingValue(I);
4538 auto *SrcMI = MRI.getVRegDef(Reg: SrcReg);
4539 if (!SrcMIs.insert(X: SrcMI))
4540 continue;
4541
4542 // Build an extend after each src inst.
4543 auto *MBB = SrcMI->getParent();
4544 MachineBasicBlock::iterator InsertPt = ++SrcMI->getIterator();
4545 if (InsertPt != MBB->end() && InsertPt->isPHI())
4546 InsertPt = MBB->getFirstNonPHI();
4547
4548 Builder.setInsertPt(MBB&: *SrcMI->getParent(), II: InsertPt);
4549 Builder.setDebugLoc(MI.getDebugLoc());
4550 auto NewExt = Builder.buildExtOrTrunc(ExtOpc: ExtMI->getOpcode(), Res: ExtTy, Op: SrcReg);
4551 OldToNewSrcMap[SrcMI] = NewExt;
4552 }
4553
4554 // Create a new phi with the extended inputs.
4555 Builder.setInstrAndDebugLoc(MI);
4556 auto NewPhi = Builder.buildInstrNoInsert(Opcode: TargetOpcode::G_PHI);
4557 NewPhi.addDef(RegNo: DstReg);
4558 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
4559 if (!MO.isReg()) {
4560 NewPhi.addMBB(MBB: MO.getMBB());
4561 continue;
4562 }
4563 auto *NewSrc = OldToNewSrcMap[MRI.getVRegDef(Reg: MO.getReg())];
4564 NewPhi.addUse(RegNo: NewSrc->getOperand(i: 0).getReg());
4565 }
4566 Builder.insertInstr(MIB: NewPhi);
4567 ExtMI->eraseFromParent();
4568}
4569
4570bool CombinerHelper::matchExtractVecEltBuildVec(MachineInstr &MI,
4571 Register &Reg) const {
4572 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
4573 // If we have a constant index, look for a G_BUILD_VECTOR source
4574 // and find the source register that the index maps to.
4575 Register SrcVec = MI.getOperand(i: 1).getReg();
4576 LLT SrcTy = MRI.getType(Reg: SrcVec);
4577 if (SrcTy.isScalableVector())
4578 return false;
4579
4580 auto Cst = getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
4581 if (!Cst || Cst->Value.getZExtValue() >= SrcTy.getNumElements())
4582 return false;
4583
4584 unsigned VecIdx = Cst->Value.getZExtValue();
4585
4586 // Check if we have a build_vector or build_vector_trunc with an optional
4587 // trunc in front.
4588 MachineInstr *SrcVecMI = MRI.getVRegDef(Reg: SrcVec);
4589 if (SrcVecMI->getOpcode() == TargetOpcode::G_TRUNC) {
4590 SrcVecMI = MRI.getVRegDef(Reg: SrcVecMI->getOperand(i: 1).getReg());
4591 }
4592
4593 if (SrcVecMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR &&
4594 SrcVecMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR_TRUNC)
4595 return false;
4596
4597 EVT Ty(getMVTForLLT(Ty: SrcTy));
4598 if (!MRI.hasOneNonDBGUse(RegNo: SrcVec) &&
4599 !getTargetLowering().aggressivelyPreferBuildVectorSources(VecVT: Ty))
4600 return false;
4601
4602 Reg = SrcVecMI->getOperand(i: VecIdx + 1).getReg();
4603 return true;
4604}
4605
4606void CombinerHelper::applyExtractVecEltBuildVec(MachineInstr &MI,
4607 Register &Reg) const {
4608 // Check the type of the register, since it may have come from a
4609 // G_BUILD_VECTOR_TRUNC.
4610 LLT ScalarTy = MRI.getType(Reg);
4611 Register DstReg = MI.getOperand(i: 0).getReg();
4612 LLT DstTy = MRI.getType(Reg: DstReg);
4613
4614 if (ScalarTy != DstTy) {
4615 assert(ScalarTy.getSizeInBits() > DstTy.getSizeInBits());
4616 Builder.buildTrunc(Res: DstReg, Op: Reg);
4617 MI.eraseFromParent();
4618 return;
4619 }
4620 replaceSingleDefInstWithReg(MI, Replacement: Reg);
4621}
4622
4623bool CombinerHelper::matchExtractAllEltsFromBuildVector(
4624 MachineInstr &MI,
4625 SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs) const {
4626 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4627 // This combine tries to find build_vector's which have every source element
4628 // extracted using G_EXTRACT_VECTOR_ELT. This can happen when transforms like
4629 // the masked load scalarization is run late in the pipeline. There's already
4630 // a combine for a similar pattern starting from the extract, but that
4631 // doesn't attempt to do it if there are multiple uses of the build_vector,
4632 // which in this case is true. Starting the combine from the build_vector
4633 // feels more natural than trying to find sibling nodes of extracts.
4634 // E.g.
4635 // %vec(<4 x s32>) = G_BUILD_VECTOR %s1(s32), %s2, %s3, %s4
4636 // %ext1 = G_EXTRACT_VECTOR_ELT %vec, 0
4637 // %ext2 = G_EXTRACT_VECTOR_ELT %vec, 1
4638 // %ext3 = G_EXTRACT_VECTOR_ELT %vec, 2
4639 // %ext4 = G_EXTRACT_VECTOR_ELT %vec, 3
4640 // ==>
4641 // replace ext{1,2,3,4} with %s{1,2,3,4}
4642
4643 Register DstReg = MI.getOperand(i: 0).getReg();
4644 LLT DstTy = MRI.getType(Reg: DstReg);
4645 unsigned NumElts = DstTy.getNumElements();
4646
4647 SmallBitVector ExtractedElts(NumElts);
4648 for (MachineInstr &II : MRI.use_nodbg_instructions(Reg: DstReg)) {
4649 if (II.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)
4650 return false;
4651 auto Cst = getIConstantVRegVal(VReg: II.getOperand(i: 2).getReg(), MRI);
4652 if (!Cst)
4653 return false;
4654 unsigned Idx = Cst->getZExtValue();
4655 if (Idx >= NumElts)
4656 return false; // Out of range.
4657 ExtractedElts.set(Idx);
4658 SrcDstPairs.emplace_back(
4659 Args: std::make_pair(x: MI.getOperand(i: Idx + 1).getReg(), y: &II));
4660 }
4661 // Match if every element was extracted.
4662 return ExtractedElts.all();
4663}
4664
4665void CombinerHelper::applyExtractAllEltsFromBuildVector(
4666 MachineInstr &MI,
4667 SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs) const {
4668 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4669 for (auto &Pair : SrcDstPairs) {
4670 auto *ExtMI = Pair.second;
4671 replaceRegWith(MRI, FromReg: ExtMI->getOperand(i: 0).getReg(), ToReg: Pair.first);
4672 ExtMI->eraseFromParent();
4673 }
4674 MI.eraseFromParent();
4675}
4676
4677void CombinerHelper::applyBuildFn(
4678 MachineInstr &MI,
4679 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4680 applyBuildFnNoErase(MI, MatchInfo);
4681 MI.eraseFromParent();
4682}
4683
4684void CombinerHelper::applyBuildFnNoErase(
4685 MachineInstr &MI,
4686 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4687 MatchInfo(Builder);
4688}
4689
4690bool CombinerHelper::matchOrShiftToFunnelShift(MachineInstr &MI,
4691 bool AllowScalarConstants,
4692 BuildFnTy &MatchInfo) const {
4693 assert(MI.getOpcode() == TargetOpcode::G_OR);
4694
4695 Register Dst = MI.getOperand(i: 0).getReg();
4696 LLT Ty = MRI.getType(Reg: Dst);
4697 unsigned BitWidth = Ty.getScalarSizeInBits();
4698
4699 Register ShlSrc, ShlAmt, LShrSrc, LShrAmt, Amt;
4700 unsigned FshOpc = 0;
4701
4702 // Match (or (shl ...), (lshr ...)).
4703 if (!mi_match(R: Dst, MRI,
4704 // m_GOr() handles the commuted version as well.
4705 P: m_GOr(L: m_GShl(L: m_Reg(R&: ShlSrc), R: m_Reg(R&: ShlAmt)),
4706 R: m_GLShr(L: m_Reg(R&: LShrSrc), R: m_Reg(R&: LShrAmt)))))
4707 return false;
4708
4709 // Given constants C0 and C1 such that C0 + C1 is bit-width:
4710 // (or (shl x, C0), (lshr y, C1)) -> (fshl x, y, C0) or (fshr x, y, C1)
4711 int64_t CstShlAmt = 0, CstLShrAmt;
4712 if (mi_match(R: ShlAmt, MRI, P: m_ICstOrSplat(Cst&: CstShlAmt)) &&
4713 mi_match(R: LShrAmt, MRI, P: m_ICstOrSplat(Cst&: CstLShrAmt)) &&
4714 CstShlAmt + CstLShrAmt == BitWidth) {
4715 FshOpc = TargetOpcode::G_FSHR;
4716 Amt = LShrAmt;
4717 } else if (mi_match(R: LShrAmt, MRI,
4718 P: m_GSub(L: m_SpecificICstOrSplat(RequestedValue: BitWidth), R: m_Reg(R&: Amt))) &&
4719 ShlAmt == Amt) {
4720 // (or (shl x, amt), (lshr y, (sub bw, amt))) -> (fshl x, y, amt)
4721 FshOpc = TargetOpcode::G_FSHL;
4722 } else if (mi_match(R: ShlAmt, MRI,
4723 P: m_GSub(L: m_SpecificICstOrSplat(RequestedValue: BitWidth), R: m_Reg(R&: Amt))) &&
4724 LShrAmt == Amt) {
4725 // (or (shl x, (sub bw, amt)), (lshr y, amt)) -> (fshr x, y, amt)
4726 FshOpc = TargetOpcode::G_FSHR;
4727 } else {
4728 return false;
4729 }
4730
4731 LLT AmtTy = MRI.getType(Reg: Amt);
4732 if (!isLegalOrBeforeLegalizer(Query: {FshOpc, {Ty, AmtTy}}) &&
4733 (!AllowScalarConstants || CstShlAmt == 0 || !Ty.isScalar()))
4734 return false;
4735
4736 MatchInfo = [=](MachineIRBuilder &B) {
4737 B.buildInstr(Opc: FshOpc, DstOps: {Dst}, SrcOps: {ShlSrc, LShrSrc, Amt});
4738 };
4739 return true;
4740}
4741
4742/// Match an FSHL or FSHR that can be combined to a ROTR or ROTL rotate.
4743bool CombinerHelper::matchFunnelShiftToRotate(MachineInstr &MI) const {
4744 unsigned Opc = MI.getOpcode();
4745 assert(Opc == TargetOpcode::G_FSHL || Opc == TargetOpcode::G_FSHR);
4746 Register X = MI.getOperand(i: 1).getReg();
4747 Register Y = MI.getOperand(i: 2).getReg();
4748 if (X != Y)
4749 return false;
4750 unsigned RotateOpc =
4751 Opc == TargetOpcode::G_FSHL ? TargetOpcode::G_ROTL : TargetOpcode::G_ROTR;
4752 return isLegalOrBeforeLegalizer(Query: {RotateOpc, {MRI.getType(Reg: X), MRI.getType(Reg: Y)}});
4753}
4754
4755void CombinerHelper::applyFunnelShiftToRotate(MachineInstr &MI) const {
4756 unsigned Opc = MI.getOpcode();
4757 assert(Opc == TargetOpcode::G_FSHL || Opc == TargetOpcode::G_FSHR);
4758 bool IsFSHL = Opc == TargetOpcode::G_FSHL;
4759 Observer.changingInstr(MI);
4760 MI.setDesc(Builder.getTII().get(Opcode: IsFSHL ? TargetOpcode::G_ROTL
4761 : TargetOpcode::G_ROTR));
4762 MI.removeOperand(OpNo: 2);
4763 Observer.changedInstr(MI);
4764}
4765
4766// Fold (rot x, c) -> (rot x, c % BitSize)
4767bool CombinerHelper::matchRotateOutOfRange(MachineInstr &MI) const {
4768 assert(MI.getOpcode() == TargetOpcode::G_ROTL ||
4769 MI.getOpcode() == TargetOpcode::G_ROTR);
4770 unsigned Bitsize =
4771 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getScalarSizeInBits();
4772 Register AmtReg = MI.getOperand(i: 2).getReg();
4773 bool OutOfRange = false;
4774 auto MatchOutOfRange = [Bitsize, &OutOfRange](const Constant *C) {
4775 if (auto *CI = dyn_cast<ConstantInt>(Val: C))
4776 OutOfRange |= CI->getValue().uge(RHS: Bitsize);
4777 return true;
4778 };
4779 return matchUnaryPredicate(MRI, Reg: AmtReg, Match: MatchOutOfRange) && OutOfRange;
4780}
4781
4782void CombinerHelper::applyRotateOutOfRange(MachineInstr &MI) const {
4783 assert(MI.getOpcode() == TargetOpcode::G_ROTL ||
4784 MI.getOpcode() == TargetOpcode::G_ROTR);
4785 unsigned Bitsize =
4786 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getScalarSizeInBits();
4787 Register Amt = MI.getOperand(i: 2).getReg();
4788 LLT AmtTy = MRI.getType(Reg: Amt);
4789 auto Bits = Builder.buildConstant(Res: AmtTy, Val: Bitsize);
4790 Amt = Builder.buildURem(Dst: AmtTy, Src0: MI.getOperand(i: 2).getReg(), Src1: Bits).getReg(Idx: 0);
4791 Observer.changingInstr(MI);
4792 MI.getOperand(i: 2).setReg(Amt);
4793 Observer.changedInstr(MI);
4794}
4795
4796bool CombinerHelper::matchICmpToTrueFalseKnownBits(MachineInstr &MI,
4797 int64_t &MatchInfo) const {
4798 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
4799 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
4800
4801 // We want to avoid calling KnownBits on the LHS if possible, as this combine
4802 // has no filter and runs on every G_ICMP instruction. We can avoid calling
4803 // KnownBits on the LHS in two cases:
4804 //
4805 // - The RHS is unknown: Constants are always on RHS. If the RHS is unknown
4806 // we cannot do any transforms so we can safely bail out early.
4807 // - The RHS is zero: we don't need to know the LHS to do unsigned <0 and
4808 // >=0.
4809 auto KnownRHS = VT->getKnownBits(R: MI.getOperand(i: 3).getReg());
4810 if (KnownRHS.isUnknown())
4811 return false;
4812
4813 std::optional<bool> KnownVal;
4814 if (KnownRHS.isZero()) {
4815 // ? uge 0 -> always true
4816 // ? ult 0 -> always false
4817 if (Pred == CmpInst::ICMP_UGE)
4818 KnownVal = true;
4819 else if (Pred == CmpInst::ICMP_ULT)
4820 KnownVal = false;
4821 }
4822
4823 if (!KnownVal) {
4824 auto KnownLHS = VT->getKnownBits(R: MI.getOperand(i: 2).getReg());
4825 KnownVal = ICmpInst::compare(LHS: KnownLHS, RHS: KnownRHS, Pred);
4826 }
4827
4828 if (!KnownVal)
4829 return false;
4830 MatchInfo =
4831 *KnownVal
4832 ? getICmpTrueVal(TLI: getTargetLowering(),
4833 /*IsVector = */
4834 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector(),
4835 /* IsFP = */ false)
4836 : 0;
4837 return true;
4838}
4839
4840bool CombinerHelper::matchICmpToLHSKnownBits(
4841 MachineInstr &MI,
4842 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4843 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
4844 // Given:
4845 //
4846 // %x = G_WHATEVER (... x is known to be 0 or 1 ...)
4847 // %cmp = G_ICMP ne %x, 0
4848 //
4849 // Or:
4850 //
4851 // %x = G_WHATEVER (... x is known to be 0 or 1 ...)
4852 // %cmp = G_ICMP eq %x, 1
4853 //
4854 // We can replace %cmp with %x assuming true is 1 on the target.
4855 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
4856 if (!CmpInst::isEquality(pred: Pred))
4857 return false;
4858 Register Dst = MI.getOperand(i: 0).getReg();
4859 LLT DstTy = MRI.getType(Reg: Dst);
4860 if (getICmpTrueVal(TLI: getTargetLowering(), IsVector: DstTy.isVector(),
4861 /* IsFP = */ false) != 1)
4862 return false;
4863 int64_t OneOrZero = Pred == CmpInst::ICMP_EQ;
4864 if (!mi_match(R: MI.getOperand(i: 3).getReg(), MRI, P: m_SpecificICst(RequestedValue: OneOrZero)))
4865 return false;
4866 Register LHS = MI.getOperand(i: 2).getReg();
4867 auto KnownLHS = VT->getKnownBits(R: LHS);
4868 if (KnownLHS.getMinValue() != 0 || KnownLHS.getMaxValue() != 1)
4869 return false;
4870 // Make sure replacing Dst with the LHS is a legal operation.
4871 LLT LHSTy = MRI.getType(Reg: LHS);
4872 unsigned LHSSize = LHSTy.getSizeInBits();
4873 unsigned DstSize = DstTy.getSizeInBits();
4874 unsigned Op = TargetOpcode::COPY;
4875 if (DstSize != LHSSize)
4876 Op = DstSize < LHSSize ? TargetOpcode::G_TRUNC : TargetOpcode::G_ZEXT;
4877 if (!isLegalOrBeforeLegalizer(Query: {Op, {DstTy, LHSTy}}))
4878 return false;
4879 MatchInfo = [=](MachineIRBuilder &B) { B.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {LHS}); };
4880 return true;
4881}
4882
4883// Replace (and (or x, c1), c2) with (and x, c2) iff c1 & c2 == 0
4884bool CombinerHelper::matchAndOrDisjointMask(
4885 MachineInstr &MI,
4886 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4887 assert(MI.getOpcode() == TargetOpcode::G_AND);
4888
4889 // Ignore vector types to simplify matching the two constants.
4890 // TODO: do this for vectors and scalars via a demanded bits analysis.
4891 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4892 if (Ty.isVector())
4893 return false;
4894
4895 Register Src;
4896 Register AndMaskReg;
4897 int64_t AndMaskBits;
4898 int64_t OrMaskBits;
4899 if (!mi_match(MI, MRI,
4900 P: m_GAnd(L: m_GOr(L: m_Reg(R&: Src), R: m_ICst(Cst&: OrMaskBits)),
4901 R: m_all_of(preds: m_ICst(Cst&: AndMaskBits), preds: m_Reg(R&: AndMaskReg)))))
4902 return false;
4903
4904 // Check if OrMask could turn on any bits in Src.
4905 if (AndMaskBits & OrMaskBits)
4906 return false;
4907
4908 MatchInfo = [=, &MI](MachineIRBuilder &B) {
4909 Observer.changingInstr(MI);
4910 // Canonicalize the result to have the constant on the RHS.
4911 if (MI.getOperand(i: 1).getReg() == AndMaskReg)
4912 MI.getOperand(i: 2).setReg(AndMaskReg);
4913 MI.getOperand(i: 1).setReg(Src);
4914 Observer.changedInstr(MI);
4915 };
4916 return true;
4917}
4918
4919/// Form a G_SBFX from a G_SEXT_INREG fed by a right shift.
4920bool CombinerHelper::matchBitfieldExtractFromSExtInReg(
4921 MachineInstr &MI,
4922 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4923 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
4924 Register Dst = MI.getOperand(i: 0).getReg();
4925 Register Src = MI.getOperand(i: 1).getReg();
4926 LLT Ty = MRI.getType(Reg: Src);
4927 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
4928 if (!LI || !LI->isLegalOrCustom(Query: {TargetOpcode::G_SBFX, {Ty, ExtractTy}}))
4929 return false;
4930 int64_t Width = MI.getOperand(i: 2).getImm();
4931 Register ShiftSrc;
4932 int64_t ShiftImm;
4933 if (!mi_match(
4934 R: Src, MRI,
4935 P: m_OneNonDBGUse(SP: m_any_of(preds: m_GAShr(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: ShiftImm)),
4936 preds: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: ShiftImm))))))
4937 return false;
4938 if (ShiftImm < 0 || ShiftImm + Width > Ty.getScalarSizeInBits())
4939 return false;
4940
4941 MatchInfo = [=](MachineIRBuilder &B) {
4942 auto Cst1 = B.buildConstant(Res: ExtractTy, Val: ShiftImm);
4943 auto Cst2 = B.buildConstant(Res: ExtractTy, Val: Width);
4944 B.buildSbfx(Dst, Src: ShiftSrc, LSB: Cst1, Width: Cst2);
4945 };
4946 return true;
4947}
4948
4949/// Form a G_UBFX from "(a srl b) & mask", where b and mask are constants.
4950bool CombinerHelper::matchBitfieldExtractFromAnd(MachineInstr &MI,
4951 BuildFnTy &MatchInfo) const {
4952 GAnd *And = cast<GAnd>(Val: &MI);
4953 Register Dst = And->getReg(Idx: 0);
4954 LLT Ty = MRI.getType(Reg: Dst);
4955 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
4956 // Note that isLegalOrBeforeLegalizer is stricter and does not take custom
4957 // into account.
4958 if (LI && !LI->isLegalOrCustom(Query: {TargetOpcode::G_UBFX, {Ty, ExtractTy}}))
4959 return false;
4960
4961 int64_t AndImm, LSBImm;
4962 Register ShiftSrc;
4963 const unsigned Size = Ty.getScalarSizeInBits();
4964 if (!mi_match(R: And->getReg(Idx: 0), MRI,
4965 P: m_GAnd(L: m_OneNonDBGUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: LSBImm))),
4966 R: m_ICst(Cst&: AndImm))))
4967 return false;
4968
4969 // The mask is a mask of the low bits iff imm & (imm+1) == 0.
4970 auto MaybeMask = static_cast<uint64_t>(AndImm);
4971 if (MaybeMask & (MaybeMask + 1))
4972 return false;
4973
4974 // LSB must fit within the register.
4975 if (static_cast<uint64_t>(LSBImm) >= Size)
4976 return false;
4977
4978 uint64_t Width = APInt(Size, AndImm).countr_one();
4979 MatchInfo = [=](MachineIRBuilder &B) {
4980 auto WidthCst = B.buildConstant(Res: ExtractTy, Val: Width);
4981 auto LSBCst = B.buildConstant(Res: ExtractTy, Val: LSBImm);
4982 B.buildInstr(Opc: TargetOpcode::G_UBFX, DstOps: {Dst}, SrcOps: {ShiftSrc, LSBCst, WidthCst});
4983 };
4984 return true;
4985}
4986
4987bool CombinerHelper::matchBitfieldExtractFromShr(
4988 MachineInstr &MI,
4989 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4990 const unsigned Opcode = MI.getOpcode();
4991 assert(Opcode == TargetOpcode::G_ASHR || Opcode == TargetOpcode::G_LSHR);
4992
4993 const Register Dst = MI.getOperand(i: 0).getReg();
4994
4995 const unsigned ExtrOpcode = Opcode == TargetOpcode::G_ASHR
4996 ? TargetOpcode::G_SBFX
4997 : TargetOpcode::G_UBFX;
4998
4999 // Check if the type we would use for the extract is legal
5000 LLT Ty = MRI.getType(Reg: Dst);
5001 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5002 if (!LI || !LI->isLegalOrCustom(Query: {ExtrOpcode, {Ty, ExtractTy}}))
5003 return false;
5004
5005 Register ShlSrc;
5006 int64_t ShrAmt;
5007 int64_t ShlAmt;
5008 const unsigned Size = Ty.getScalarSizeInBits();
5009
5010 // Try to match shr (shl x, c1), c2
5011 if (!mi_match(R: Dst, MRI,
5012 P: m_BinOp(Opcode,
5013 L: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: ShlSrc), R: m_ICst(Cst&: ShlAmt))),
5014 R: m_ICst(Cst&: ShrAmt))))
5015 return false;
5016
5017 // Make sure that the shift sizes can fit a bitfield extract
5018 if (ShlAmt < 0 || ShlAmt > ShrAmt || ShrAmt >= Size)
5019 return false;
5020
5021 // Skip this combine if the G_SEXT_INREG combine could handle it
5022 if (Opcode == TargetOpcode::G_ASHR && ShlAmt == ShrAmt)
5023 return false;
5024
5025 // Calculate start position and width of the extract
5026 const int64_t Pos = ShrAmt - ShlAmt;
5027 const int64_t Width = Size - ShrAmt;
5028
5029 MatchInfo = [=](MachineIRBuilder &B) {
5030 auto WidthCst = B.buildConstant(Res: ExtractTy, Val: Width);
5031 auto PosCst = B.buildConstant(Res: ExtractTy, Val: Pos);
5032 B.buildInstr(Opc: ExtrOpcode, DstOps: {Dst}, SrcOps: {ShlSrc, PosCst, WidthCst});
5033 };
5034 return true;
5035}
5036
5037bool CombinerHelper::matchBitfieldExtractFromShrAnd(
5038 MachineInstr &MI,
5039 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
5040 const unsigned Opcode = MI.getOpcode();
5041 assert(Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_ASHR);
5042
5043 const Register Dst = MI.getOperand(i: 0).getReg();
5044 LLT Ty = MRI.getType(Reg: Dst);
5045 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5046 if (LI && !LI->isLegalOrCustom(Query: {TargetOpcode::G_UBFX, {Ty, ExtractTy}}))
5047 return false;
5048
5049 // Try to match shr (and x, c1), c2
5050 Register AndSrc;
5051 int64_t ShrAmt;
5052 int64_t SMask;
5053 if (!mi_match(R: Dst, MRI,
5054 P: m_BinOp(Opcode,
5055 L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: AndSrc), R: m_ICst(Cst&: SMask))),
5056 R: m_ICst(Cst&: ShrAmt))))
5057 return false;
5058
5059 const unsigned Size = Ty.getScalarSizeInBits();
5060 if (ShrAmt < 0 || ShrAmt >= Size)
5061 return false;
5062
5063 // If the shift subsumes the mask, emit the 0 directly.
5064 if (0 == (SMask >> ShrAmt)) {
5065 MatchInfo = [=](MachineIRBuilder &B) {
5066 B.buildConstant(Res: Dst, Val: 0);
5067 };
5068 return true;
5069 }
5070
5071 // Check that ubfx can do the extraction, with no holes in the mask.
5072 uint64_t UMask = SMask;
5073 UMask |= maskTrailingOnes<uint64_t>(N: ShrAmt);
5074 UMask &= maskTrailingOnes<uint64_t>(N: Size);
5075 if (!isMask_64(Value: UMask))
5076 return false;
5077
5078 // Calculate start position and width of the extract.
5079 const int64_t Pos = ShrAmt;
5080 const int64_t Width = llvm::countr_one(Value: UMask) - ShrAmt;
5081
5082 // It's preferable to keep the shift, rather than form G_SBFX.
5083 // TODO: remove the G_AND via demanded bits analysis.
5084 if (Opcode == TargetOpcode::G_ASHR && Width + ShrAmt == Size)
5085 return false;
5086
5087 MatchInfo = [=](MachineIRBuilder &B) {
5088 auto WidthCst = B.buildConstant(Res: ExtractTy, Val: Width);
5089 auto PosCst = B.buildConstant(Res: ExtractTy, Val: Pos);
5090 B.buildInstr(Opc: TargetOpcode::G_UBFX, DstOps: {Dst}, SrcOps: {AndSrc, PosCst, WidthCst});
5091 };
5092 return true;
5093}
5094
5095bool CombinerHelper::reassociationCanBreakAddressingModePattern(
5096 MachineInstr &MI) const {
5097 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
5098
5099 Register Src1Reg = PtrAdd.getBaseReg();
5100 auto *Src1Def = getOpcodeDef<GPtrAdd>(Reg: Src1Reg, MRI);
5101 if (!Src1Def)
5102 return false;
5103
5104 Register Src2Reg = PtrAdd.getOffsetReg();
5105
5106 if (MRI.hasOneNonDBGUse(RegNo: Src1Reg))
5107 return false;
5108
5109 auto C1 = getIConstantVRegVal(VReg: Src1Def->getOffsetReg(), MRI);
5110 if (!C1)
5111 return false;
5112 auto C2 = getIConstantVRegVal(VReg: Src2Reg, MRI);
5113 if (!C2)
5114 return false;
5115
5116 const APInt &C1APIntVal = *C1;
5117 const APInt &C2APIntVal = *C2;
5118 const int64_t CombinedValue = (C1APIntVal + C2APIntVal).getSExtValue();
5119
5120 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: PtrAdd.getReg(Idx: 0))) {
5121 // This combine may end up running before ptrtoint/inttoptr combines
5122 // manage to eliminate redundant conversions, so try to look through them.
5123 MachineInstr *ConvUseMI = &UseMI;
5124 unsigned ConvUseOpc = ConvUseMI->getOpcode();
5125 while (ConvUseOpc == TargetOpcode::G_INTTOPTR ||
5126 ConvUseOpc == TargetOpcode::G_PTRTOINT) {
5127 Register DefReg = ConvUseMI->getOperand(i: 0).getReg();
5128 if (!MRI.hasOneNonDBGUse(RegNo: DefReg))
5129 break;
5130 ConvUseMI = &*MRI.use_instr_nodbg_begin(RegNo: DefReg);
5131 ConvUseOpc = ConvUseMI->getOpcode();
5132 }
5133 auto *LdStMI = dyn_cast<GLoadStore>(Val: ConvUseMI);
5134 if (!LdStMI)
5135 continue;
5136 // Is x[offset2] already not a legal addressing mode? If so then
5137 // reassociating the constants breaks nothing (we test offset2 because
5138 // that's the one we hope to fold into the load or store).
5139 TargetLoweringBase::AddrMode AM;
5140 AM.HasBaseReg = true;
5141 AM.BaseOffs = C2APIntVal.getSExtValue();
5142 unsigned AS = MRI.getType(Reg: LdStMI->getPointerReg()).getAddressSpace();
5143 Type *AccessTy = getTypeForLLT(Ty: LdStMI->getMMO().getMemoryType(),
5144 C&: PtrAdd.getMF()->getFunction().getContext());
5145 const auto &TLI = *PtrAdd.getMF()->getSubtarget().getTargetLowering();
5146 if (!TLI.isLegalAddressingMode(DL: PtrAdd.getMF()->getDataLayout(), AM,
5147 Ty: AccessTy, AddrSpace: AS))
5148 continue;
5149
5150 // Would x[offset1+offset2] still be a legal addressing mode?
5151 AM.BaseOffs = CombinedValue;
5152 if (!TLI.isLegalAddressingMode(DL: PtrAdd.getMF()->getDataLayout(), AM,
5153 Ty: AccessTy, AddrSpace: AS))
5154 return true;
5155 }
5156
5157 return false;
5158}
5159
5160bool CombinerHelper::matchReassocConstantInnerRHS(GPtrAdd &MI,
5161 MachineInstr *RHS,
5162 BuildFnTy &MatchInfo) const {
5163 // G_PTR_ADD(BASE, G_ADD(X, C)) -> G_PTR_ADD(G_PTR_ADD(BASE, X), C)
5164 Register Src1Reg = MI.getOperand(i: 1).getReg();
5165 if (RHS->getOpcode() != TargetOpcode::G_ADD)
5166 return false;
5167 auto C2 = getIConstantVRegVal(VReg: RHS->getOperand(i: 2).getReg(), MRI);
5168 if (!C2)
5169 return false;
5170
5171 // If both additions are nuw, the reassociated additions are also nuw.
5172 // If the original G_PTR_ADD is additionally nusw, X and C are both not
5173 // negative, so BASE+X is between BASE and BASE+(X+C). The new G_PTR_ADDs are
5174 // therefore also nusw.
5175 // If the original G_PTR_ADD is additionally inbounds (which implies nusw),
5176 // the new G_PTR_ADDs are then also inbounds.
5177 unsigned PtrAddFlags = MI.getFlags();
5178 unsigned AddFlags = RHS->getFlags();
5179 bool IsNoUWrap = PtrAddFlags & AddFlags & MachineInstr::MIFlag::NoUWrap;
5180 bool IsNoUSWrap = IsNoUWrap && (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap);
5181 bool IsInBounds = IsNoUWrap && (PtrAddFlags & MachineInstr::MIFlag::InBounds);
5182 unsigned Flags = 0;
5183 if (IsNoUWrap)
5184 Flags |= MachineInstr::MIFlag::NoUWrap;
5185 if (IsNoUSWrap)
5186 Flags |= MachineInstr::MIFlag::NoUSWrap;
5187 if (IsInBounds)
5188 Flags |= MachineInstr::MIFlag::InBounds;
5189
5190 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5191 LLT PtrTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5192
5193 auto NewBase =
5194 Builder.buildPtrAdd(Res: PtrTy, Op0: Src1Reg, Op1: RHS->getOperand(i: 1).getReg(), Flags);
5195 Observer.changingInstr(MI);
5196 MI.getOperand(i: 1).setReg(NewBase.getReg(Idx: 0));
5197 MI.getOperand(i: 2).setReg(RHS->getOperand(i: 2).getReg());
5198 MI.setFlags(Flags);
5199 Observer.changedInstr(MI);
5200 };
5201 return !reassociationCanBreakAddressingModePattern(MI);
5202}
5203
5204bool CombinerHelper::matchReassocConstantInnerLHS(GPtrAdd &MI,
5205 MachineInstr *LHS,
5206 MachineInstr *RHS,
5207 BuildFnTy &MatchInfo) const {
5208 // G_PTR_ADD (G_PTR_ADD X, C), Y) -> (G_PTR_ADD (G_PTR_ADD(X, Y), C)
5209 // if and only if (G_PTR_ADD X, C) has one use.
5210 Register LHSBase;
5211 std::optional<ValueAndVReg> LHSCstOff;
5212 if (!mi_match(R: MI.getBaseReg(), MRI,
5213 P: m_OneNonDBGUse(SP: m_GPtrAdd(L: m_Reg(R&: LHSBase), R: m_GCst(ValReg&: LHSCstOff)))))
5214 return false;
5215
5216 auto *LHSPtrAdd = cast<GPtrAdd>(Val: LHS);
5217
5218 // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
5219 // nuw and inbounds (which implies nusw), the offsets are both non-negative,
5220 // so the new G_PTR_ADDs are also inbounds.
5221 unsigned PtrAddFlags = MI.getFlags();
5222 unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();
5223 bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
5224 bool IsNoUSWrap = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &
5225 MachineInstr::MIFlag::NoUSWrap);
5226 bool IsInBounds = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &
5227 MachineInstr::MIFlag::InBounds);
5228 unsigned Flags = 0;
5229 if (IsNoUWrap)
5230 Flags |= MachineInstr::MIFlag::NoUWrap;
5231 if (IsNoUSWrap)
5232 Flags |= MachineInstr::MIFlag::NoUSWrap;
5233 if (IsInBounds)
5234 Flags |= MachineInstr::MIFlag::InBounds;
5235
5236 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5237 // When we change LHSPtrAdd's offset register we might cause it to use a reg
5238 // before its def. Sink the instruction so the outer PTR_ADD to ensure this
5239 // doesn't happen.
5240 LHSPtrAdd->moveBefore(MovePos: &MI);
5241 Register RHSReg = MI.getOffsetReg();
5242 // set VReg will cause type mismatch if it comes from extend/trunc
5243 auto NewCst = B.buildConstant(Res: MRI.getType(Reg: RHSReg), Val: LHSCstOff->Value);
5244 Observer.changingInstr(MI);
5245 MI.getOperand(i: 2).setReg(NewCst.getReg(Idx: 0));
5246 MI.setFlags(Flags);
5247 Observer.changedInstr(MI);
5248 Observer.changingInstr(MI&: *LHSPtrAdd);
5249 LHSPtrAdd->getOperand(i: 2).setReg(RHSReg);
5250 LHSPtrAdd->setFlags(Flags);
5251 Observer.changedInstr(MI&: *LHSPtrAdd);
5252 };
5253 return !reassociationCanBreakAddressingModePattern(MI);
5254}
5255
5256bool CombinerHelper::matchReassocFoldConstantsInSubTree(
5257 GPtrAdd &MI, MachineInstr *LHS, MachineInstr *RHS,
5258 BuildFnTy &MatchInfo) const {
5259 // G_PTR_ADD(G_PTR_ADD(BASE, C1), C2) -> G_PTR_ADD(BASE, C1+C2)
5260 auto *LHSPtrAdd = dyn_cast<GPtrAdd>(Val: LHS);
5261 if (!LHSPtrAdd)
5262 return false;
5263
5264 Register Src2Reg = MI.getOperand(i: 2).getReg();
5265 Register LHSSrc1 = LHSPtrAdd->getBaseReg();
5266 Register LHSSrc2 = LHSPtrAdd->getOffsetReg();
5267 auto C1 = getIConstantVRegVal(VReg: LHSSrc2, MRI);
5268 if (!C1)
5269 return false;
5270 auto C2 = getIConstantVRegVal(VReg: Src2Reg, MRI);
5271 if (!C2)
5272 return false;
5273
5274 // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
5275 // inbounds, reaching the same result in one G_PTR_ADD is also inbounds.
5276 // The nusw constraints are satisfied because imm1+imm2 cannot exceed the
5277 // largest signed integer that fits into the index type, which is the maximum
5278 // size of allocated objects according to the IR Language Reference.
5279 unsigned PtrAddFlags = MI.getFlags();
5280 unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();
5281 bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
5282 bool IsInBounds =
5283 PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::InBounds;
5284 unsigned Flags = 0;
5285 if (IsNoUWrap)
5286 Flags |= MachineInstr::MIFlag::NoUWrap;
5287 if (IsInBounds) {
5288 Flags |= MachineInstr::MIFlag::InBounds;
5289 Flags |= MachineInstr::MIFlag::NoUSWrap;
5290 }
5291
5292 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5293 auto NewCst = B.buildConstant(Res: MRI.getType(Reg: Src2Reg), Val: *C1 + *C2);
5294 Observer.changingInstr(MI);
5295 MI.getOperand(i: 1).setReg(LHSSrc1);
5296 MI.getOperand(i: 2).setReg(NewCst.getReg(Idx: 0));
5297 MI.setFlags(Flags);
5298 Observer.changedInstr(MI);
5299 };
5300 return !reassociationCanBreakAddressingModePattern(MI);
5301}
5302
5303bool CombinerHelper::matchReassocPtrAdd(MachineInstr &MI,
5304 BuildFnTy &MatchInfo) const {
5305 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
5306 // We're trying to match a few pointer computation patterns here for
5307 // re-association opportunities.
5308 // 1) Isolating a constant operand to be on the RHS, e.g.:
5309 // G_PTR_ADD(BASE, G_ADD(X, C)) -> G_PTR_ADD(G_PTR_ADD(BASE, X), C)
5310 //
5311 // 2) Folding two constants in each sub-tree as long as such folding
5312 // doesn't break a legal addressing mode.
5313 // G_PTR_ADD(G_PTR_ADD(BASE, C1), C2) -> G_PTR_ADD(BASE, C1+C2)
5314 //
5315 // 3) Move a constant from the LHS of an inner op to the RHS of the outer.
5316 // G_PTR_ADD (G_PTR_ADD X, C), Y) -> G_PTR_ADD (G_PTR_ADD(X, Y), C)
5317 // iif (G_PTR_ADD X, C) has one use.
5318 MachineInstr *LHS = MRI.getVRegDef(Reg: PtrAdd.getBaseReg());
5319 MachineInstr *RHS = MRI.getVRegDef(Reg: PtrAdd.getOffsetReg());
5320
5321 // Try to match example 2.
5322 if (matchReassocFoldConstantsInSubTree(MI&: PtrAdd, LHS, RHS, MatchInfo))
5323 return true;
5324
5325 // Try to match example 3.
5326 if (matchReassocConstantInnerLHS(MI&: PtrAdd, LHS, RHS, MatchInfo))
5327 return true;
5328
5329 // Try to match example 1.
5330 if (matchReassocConstantInnerRHS(MI&: PtrAdd, RHS, MatchInfo))
5331 return true;
5332
5333 return false;
5334}
5335bool CombinerHelper::tryReassocBinOp(unsigned Opc, Register DstReg,
5336 Register OpLHS, Register OpRHS,
5337 BuildFnTy &MatchInfo) const {
5338 LLT OpRHSTy = MRI.getType(Reg: OpRHS);
5339 MachineInstr *OpLHSDef = MRI.getVRegDef(Reg: OpLHS);
5340
5341 if (OpLHSDef->getOpcode() != Opc)
5342 return false;
5343
5344 Register OpLHSLHS = OpLHSDef->getOperand(i: 1).getReg();
5345 Register OpLHSRHS = OpLHSDef->getOperand(i: 2).getReg();
5346
5347 // If the inner op is (X op C), pull the constant out so it can be folded with
5348 // other constants in the expression tree. Folding is not guaranteed so we
5349 // might have (C1 op C2). In that case do not pull a constant out because it
5350 // won't help and can lead to infinite loops.
5351 if (isConstantOrConstantSplatVector(Def: OpLHSRHS, MRI) &&
5352 !isConstantOrConstantSplatVector(Def: OpLHSLHS, MRI)) {
5353 if (isConstantOrConstantSplatVector(Def: OpRHS, MRI)) {
5354 // (Opc (Opc X, C1), C2) -> (Opc X, (Opc C1, C2))
5355 MatchInfo = [=](MachineIRBuilder &B) {
5356 auto NewCst = B.buildInstr(Opc, DstOps: {OpRHSTy}, SrcOps: {OpLHSRHS, OpRHS});
5357 B.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {OpLHSLHS, NewCst});
5358 };
5359 return true;
5360 }
5361 if (getTargetLowering().isReassocProfitable(MRI, N0: OpLHS, N1: OpRHS)) {
5362 // Reassociate: (op (op x, c1), y) -> (op (op x, y), c1)
5363 // iff (op x, c1) has one use
5364 MatchInfo = [=](MachineIRBuilder &B) {
5365 auto NewLHSLHS = B.buildInstr(Opc, DstOps: {OpRHSTy}, SrcOps: {OpLHSLHS, OpRHS});
5366 B.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {NewLHSLHS, OpLHSRHS});
5367 };
5368 return true;
5369 }
5370 }
5371
5372 return false;
5373}
5374
5375bool CombinerHelper::matchReassocCommBinOp(MachineInstr &MI,
5376 BuildFnTy &MatchInfo) const {
5377 // We don't check if the reassociation will break a legal addressing mode
5378 // here since pointer arithmetic is handled by G_PTR_ADD.
5379 unsigned Opc = MI.getOpcode();
5380 Register DstReg = MI.getOperand(i: 0).getReg();
5381 Register LHSReg = MI.getOperand(i: 1).getReg();
5382 Register RHSReg = MI.getOperand(i: 2).getReg();
5383
5384 if (tryReassocBinOp(Opc, DstReg, OpLHS: LHSReg, OpRHS: RHSReg, MatchInfo))
5385 return true;
5386 if (tryReassocBinOp(Opc, DstReg, OpLHS: RHSReg, OpRHS: LHSReg, MatchInfo))
5387 return true;
5388 return false;
5389}
5390
5391bool CombinerHelper::matchConstantFoldCastOp(MachineInstr &MI,
5392 APInt &MatchInfo) const {
5393 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5394 Register SrcOp = MI.getOperand(i: 1).getReg();
5395
5396 if (auto MaybeCst = ConstantFoldCastOp(Opcode: MI.getOpcode(), DstTy, Op0: SrcOp, MRI)) {
5397 MatchInfo = *MaybeCst;
5398 return true;
5399 }
5400
5401 return false;
5402}
5403
5404bool CombinerHelper::matchConstantFoldUnaryIntOp(MachineInstr &MI,
5405 BuildFnTy &MatchInfo) const {
5406 Register Dst = MI.getOperand(i: 0).getReg();
5407 auto Csts = ConstantFoldUnaryIntOp(Opcode: MI.getOpcode(), DstTy: MRI.getType(Reg: Dst),
5408 Src: MI.getOperand(i: 1).getReg(), MRI);
5409 if (Csts.empty())
5410 return false;
5411
5412 MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
5413 if (Csts.size() == 1)
5414 B.buildConstant(Res: Dst, Val: Csts[0]);
5415 else
5416 B.buildBuildVectorConstant(Res: Dst, Ops: Csts);
5417 };
5418 return true;
5419}
5420
5421bool CombinerHelper::matchConstantFoldBinOp(MachineInstr &MI,
5422 APInt &MatchInfo) const {
5423 Register Op1 = MI.getOperand(i: 1).getReg();
5424 Register Op2 = MI.getOperand(i: 2).getReg();
5425 auto MaybeCst = ConstantFoldBinOp(Opcode: MI.getOpcode(), Op1, Op2, MRI);
5426 if (!MaybeCst)
5427 return false;
5428 MatchInfo = *MaybeCst;
5429 return true;
5430}
5431
5432bool CombinerHelper::matchConstantFoldFPBinOp(MachineInstr &MI,
5433 ConstantFP *&MatchInfo) const {
5434 Register Op1 = MI.getOperand(i: 1).getReg();
5435 Register Op2 = MI.getOperand(i: 2).getReg();
5436 auto MaybeCst = ConstantFoldFPBinOp(Opcode: MI.getOpcode(), Op1, Op2, MRI);
5437 if (!MaybeCst)
5438 return false;
5439 MatchInfo =
5440 ConstantFP::get(Context&: MI.getMF()->getFunction().getContext(), V: *MaybeCst);
5441 return true;
5442}
5443
5444bool CombinerHelper::matchConstantFoldFMA(MachineInstr &MI,
5445 ConstantFP *&MatchInfo) const {
5446 assert(MI.getOpcode() == TargetOpcode::G_FMA ||
5447 MI.getOpcode() == TargetOpcode::G_FMAD);
5448 auto [_, Op1, Op2, Op3] = MI.getFirst4Regs();
5449
5450 const ConstantFP *Op3Cst = getConstantFPVRegVal(VReg: Op3, MRI);
5451 if (!Op3Cst)
5452 return false;
5453
5454 const ConstantFP *Op2Cst = getConstantFPVRegVal(VReg: Op2, MRI);
5455 if (!Op2Cst)
5456 return false;
5457
5458 const ConstantFP *Op1Cst = getConstantFPVRegVal(VReg: Op1, MRI);
5459 if (!Op1Cst)
5460 return false;
5461
5462 APFloat Op1F = Op1Cst->getValueAPF();
5463 Op1F.fusedMultiplyAdd(Multiplicand: Op2Cst->getValueAPF(), Addend: Op3Cst->getValueAPF(),
5464 RM: APFloat::rmNearestTiesToEven);
5465 MatchInfo = ConstantFP::get(Context&: MI.getMF()->getFunction().getContext(), V: Op1F);
5466 return true;
5467}
5468
5469bool CombinerHelper::matchNarrowBinopFeedingAnd(
5470 MachineInstr &MI,
5471 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
5472 // Look for a binop feeding into an AND with a mask:
5473 //
5474 // %add = G_ADD %lhs, %rhs
5475 // %and = G_AND %add, 000...11111111
5476 //
5477 // Check if it's possible to perform the binop at a narrower width and zext
5478 // back to the original width like so:
5479 //
5480 // %narrow_lhs = G_TRUNC %lhs
5481 // %narrow_rhs = G_TRUNC %rhs
5482 // %narrow_add = G_ADD %narrow_lhs, %narrow_rhs
5483 // %new_add = G_ZEXT %narrow_add
5484 // %and = G_AND %new_add, 000...11111111
5485 //
5486 // This can allow later combines to eliminate the G_AND if it turns out
5487 // that the mask is irrelevant.
5488 assert(MI.getOpcode() == TargetOpcode::G_AND);
5489 Register Dst = MI.getOperand(i: 0).getReg();
5490 Register AndLHS = MI.getOperand(i: 1).getReg();
5491 Register AndRHS = MI.getOperand(i: 2).getReg();
5492 LLT WideTy = MRI.getType(Reg: Dst);
5493
5494 // If the potential binop has more than one use, then it's possible that one
5495 // of those uses will need its full width.
5496 if (!WideTy.isScalar() || !MRI.hasOneNonDBGUse(RegNo: AndLHS))
5497 return false;
5498
5499 // Check if the LHS feeding the AND is impacted by the high bits that we're
5500 // masking out.
5501 //
5502 // e.g. for 64-bit x, y:
5503 //
5504 // add_64(x, y) & 65535 == zext(add_16(trunc(x), trunc(y))) & 65535
5505 MachineInstr *LHSInst = getDefIgnoringCopies(Reg: AndLHS, MRI);
5506 if (!LHSInst)
5507 return false;
5508 unsigned LHSOpc = LHSInst->getOpcode();
5509 switch (LHSOpc) {
5510 default:
5511 return false;
5512 case TargetOpcode::G_ADD:
5513 case TargetOpcode::G_SUB:
5514 case TargetOpcode::G_MUL:
5515 case TargetOpcode::G_AND:
5516 case TargetOpcode::G_OR:
5517 case TargetOpcode::G_XOR:
5518 break;
5519 }
5520
5521 // Find the mask on the RHS.
5522 auto Cst = getIConstantVRegValWithLookThrough(VReg: AndRHS, MRI);
5523 if (!Cst)
5524 return false;
5525 auto Mask = Cst->Value;
5526 if (!Mask.isMask())
5527 return false;
5528
5529 // No point in combining if there's nothing to truncate.
5530 unsigned NarrowWidth = Mask.countr_one();
5531 if (NarrowWidth == WideTy.getSizeInBits())
5532 return false;
5533 LLT NarrowTy = LLT::integer(SizeInBits: NarrowWidth);
5534
5535 // Check if adding the zext + truncates could be harmful.
5536 auto &MF = *MI.getMF();
5537 const auto &TLI = getTargetLowering();
5538 LLVMContext &Ctx = MF.getFunction().getContext();
5539 if (!TLI.isTruncateFree(FromTy: WideTy, ToTy: NarrowTy, Ctx) ||
5540 !TLI.isZExtFree(FromTy: NarrowTy, ToTy: WideTy, Ctx))
5541 return false;
5542 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_TRUNC, {NarrowTy, WideTy}}) ||
5543 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ZEXT, {WideTy, NarrowTy}}))
5544 return false;
5545 Register BinOpLHS = LHSInst->getOperand(i: 1).getReg();
5546 Register BinOpRHS = LHSInst->getOperand(i: 2).getReg();
5547 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5548 auto NarrowLHS = Builder.buildTrunc(Res: NarrowTy, Op: BinOpLHS);
5549 auto NarrowRHS = Builder.buildTrunc(Res: NarrowTy, Op: BinOpRHS);
5550 auto NarrowBinOp =
5551 Builder.buildInstr(Opc: LHSOpc, DstOps: {NarrowTy}, SrcOps: {NarrowLHS, NarrowRHS});
5552 auto Ext = Builder.buildZExt(Res: WideTy, Op: NarrowBinOp);
5553 Observer.changingInstr(MI);
5554 MI.getOperand(i: 1).setReg(Ext.getReg(Idx: 0));
5555 Observer.changedInstr(MI);
5556 };
5557 return true;
5558}
5559
5560bool CombinerHelper::matchMulOBy2(MachineInstr &MI,
5561 BuildFnTy &MatchInfo) const {
5562 unsigned Opc = MI.getOpcode();
5563 assert(Opc == TargetOpcode::G_UMULO || Opc == TargetOpcode::G_SMULO);
5564
5565 if (!mi_match(R: MI.getOperand(i: 3).getReg(), MRI, P: m_SpecificICstOrSplat(RequestedValue: 2)))
5566 return false;
5567
5568 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5569 Observer.changingInstr(MI);
5570 unsigned NewOpc = Opc == TargetOpcode::G_UMULO ? TargetOpcode::G_UADDO
5571 : TargetOpcode::G_SADDO;
5572 MI.setDesc(Builder.getTII().get(Opcode: NewOpc));
5573 MI.getOperand(i: 3).setReg(MI.getOperand(i: 2).getReg());
5574 Observer.changedInstr(MI);
5575 };
5576 return true;
5577}
5578
5579bool CombinerHelper::matchMulOBy0(MachineInstr &MI,
5580 BuildFnTy &MatchInfo) const {
5581 // (G_*MULO x, 0) -> 0 + no carry out
5582 assert(MI.getOpcode() == TargetOpcode::G_UMULO ||
5583 MI.getOpcode() == TargetOpcode::G_SMULO);
5584 if (!mi_match(R: MI.getOperand(i: 3).getReg(), MRI, P: m_SpecificICstOrSplat(RequestedValue: 0)))
5585 return false;
5586 Register Dst = MI.getOperand(i: 0).getReg();
5587 Register Carry = MI.getOperand(i: 1).getReg();
5588 if (!isConstantLegalOrBeforeLegalizer(Ty: MRI.getType(Reg: Dst)) ||
5589 !isConstantLegalOrBeforeLegalizer(Ty: MRI.getType(Reg: Carry)))
5590 return false;
5591 MatchInfo = [=](MachineIRBuilder &B) {
5592 B.buildConstant(Res: Dst, Val: 0);
5593 B.buildConstant(Res: Carry, Val: 0);
5594 };
5595 return true;
5596}
5597
5598bool CombinerHelper::matchAddEToAddO(MachineInstr &MI,
5599 BuildFnTy &MatchInfo) const {
5600 // (G_*ADDE x, y, 0) -> (G_*ADDO x, y)
5601 // (G_*SUBE x, y, 0) -> (G_*SUBO x, y)
5602 assert(MI.getOpcode() == TargetOpcode::G_UADDE ||
5603 MI.getOpcode() == TargetOpcode::G_SADDE ||
5604 MI.getOpcode() == TargetOpcode::G_USUBE ||
5605 MI.getOpcode() == TargetOpcode::G_SSUBE);
5606 if (!mi_match(R: MI.getOperand(i: 4).getReg(), MRI, P: m_SpecificICstOrSplat(RequestedValue: 0)))
5607 return false;
5608 MatchInfo = [&](MachineIRBuilder &B) {
5609 unsigned NewOpcode;
5610 switch (MI.getOpcode()) {
5611 case TargetOpcode::G_UADDE:
5612 NewOpcode = TargetOpcode::G_UADDO;
5613 break;
5614 case TargetOpcode::G_SADDE:
5615 NewOpcode = TargetOpcode::G_SADDO;
5616 break;
5617 case TargetOpcode::G_USUBE:
5618 NewOpcode = TargetOpcode::G_USUBO;
5619 break;
5620 case TargetOpcode::G_SSUBE:
5621 NewOpcode = TargetOpcode::G_SSUBO;
5622 break;
5623 }
5624 Observer.changingInstr(MI);
5625 MI.setDesc(B.getTII().get(Opcode: NewOpcode));
5626 MI.removeOperand(OpNo: 4);
5627 Observer.changedInstr(MI);
5628 };
5629 return true;
5630}
5631
5632bool CombinerHelper::matchSubAddSameReg(MachineInstr &MI,
5633 BuildFnTy &MatchInfo) const {
5634 assert(MI.getOpcode() == TargetOpcode::G_SUB);
5635 Register Dst = MI.getOperand(i: 0).getReg();
5636 // (x + y) - z -> x (if y == z)
5637 // (x + y) - z -> y (if x == z)
5638 Register X, Y, Z;
5639 if (mi_match(R: Dst, MRI, P: m_GSub(L: m_GAdd(L: m_Reg(R&: X), R: m_Reg(R&: Y)), R: m_Reg(R&: Z)))) {
5640 Register ReplaceReg;
5641 int64_t CstX, CstY;
5642 if (Y == Z || (mi_match(R: Y, MRI, P: m_ICstOrSplat(Cst&: CstY)) &&
5643 mi_match(R: Z, MRI, P: m_SpecificICstOrSplat(RequestedValue: CstY))))
5644 ReplaceReg = X;
5645 else if (X == Z || (mi_match(R: X, MRI, P: m_ICstOrSplat(Cst&: CstX)) &&
5646 mi_match(R: Z, MRI, P: m_SpecificICstOrSplat(RequestedValue: CstX))))
5647 ReplaceReg = Y;
5648 if (ReplaceReg) {
5649 MatchInfo = [=](MachineIRBuilder &B) { B.buildCopy(Res: Dst, Op: ReplaceReg); };
5650 return true;
5651 }
5652 }
5653
5654 // x - (y + z) -> 0 - y (if x == z)
5655 // x - (y + z) -> 0 - z (if x == y)
5656 if (mi_match(R: Dst, MRI, P: m_GSub(L: m_Reg(R&: X), R: m_GAdd(L: m_Reg(R&: Y), R: m_Reg(R&: Z))))) {
5657 Register ReplaceReg;
5658 int64_t CstX;
5659 if (X == Z || (mi_match(R: X, MRI, P: m_ICstOrSplat(Cst&: CstX)) &&
5660 mi_match(R: Z, MRI, P: m_SpecificICstOrSplat(RequestedValue: CstX))))
5661 ReplaceReg = Y;
5662 else if (X == Y || (mi_match(R: X, MRI, P: m_ICstOrSplat(Cst&: CstX)) &&
5663 mi_match(R: Y, MRI, P: m_SpecificICstOrSplat(RequestedValue: CstX))))
5664 ReplaceReg = Z;
5665 if (ReplaceReg) {
5666 MatchInfo = [=](MachineIRBuilder &B) {
5667 auto Zero = B.buildConstant(Res: MRI.getType(Reg: Dst), Val: 0);
5668 B.buildSub(Dst, Src0: Zero, Src1: ReplaceReg);
5669 };
5670 return true;
5671 }
5672 }
5673 return false;
5674}
5675
5676MachineInstr *CombinerHelper::buildUDivOrURemUsingMul(MachineInstr &MI) const {
5677 unsigned Opcode = MI.getOpcode();
5678 assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
5679 auto &UDivorRem = cast<GenericMachineInstr>(Val&: MI);
5680 Register Dst = UDivorRem.getReg(Idx: 0);
5681 Register LHS = UDivorRem.getReg(Idx: 1);
5682 Register RHS = UDivorRem.getReg(Idx: 2);
5683 LLT Ty = MRI.getType(Reg: Dst);
5684 LLT ScalarTy = Ty.getScalarType();
5685 const unsigned EltBits = ScalarTy.getScalarSizeInBits();
5686 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5687 LLT ScalarShiftAmtTy = ShiftAmtTy.getScalarType();
5688
5689 auto &MIB = Builder;
5690
5691 bool UseSRL = false;
5692 SmallVector<Register, 16> Shifts, Factors;
5693 auto *RHSDefInstr = cast<GenericMachineInstr>(Val: getDefIgnoringCopies(Reg: RHS, MRI));
5694 bool IsSplat = getIConstantSplatVal(MI: *RHSDefInstr, MRI).has_value();
5695
5696 auto BuildExactUDIVPattern = [&](const Constant *C) {
5697 // Don't recompute inverses for each splat element.
5698 if (IsSplat && !Factors.empty()) {
5699 Shifts.push_back(Elt: Shifts[0]);
5700 Factors.push_back(Elt: Factors[0]);
5701 return true;
5702 }
5703
5704 auto *CI = cast<ConstantInt>(Val: C);
5705 APInt Divisor = CI->getValue();
5706 unsigned Shift = Divisor.countr_zero();
5707 if (Shift) {
5708 Divisor.lshrInPlace(ShiftAmt: Shift);
5709 UseSRL = true;
5710 }
5711
5712 // Calculate the multiplicative inverse modulo BW.
5713 APInt Factor = Divisor.multiplicativeInverse();
5714 Shifts.push_back(Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: Shift).getReg(Idx: 0));
5715 Factors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Factor).getReg(Idx: 0));
5716 return true;
5717 };
5718
5719 if (MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
5720 // Collect all magic values from the build vector.
5721 if (!matchUnaryPredicate(MRI, Reg: RHS, Match: BuildExactUDIVPattern))
5722 llvm_unreachable("Expected unary predicate match to succeed");
5723
5724 Register Shift, Factor;
5725 if (Ty.isVector()) {
5726 Shift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: Shifts).getReg(Idx: 0);
5727 Factor = MIB.buildBuildVector(Res: Ty, Ops: Factors).getReg(Idx: 0);
5728 } else {
5729 Shift = Shifts[0];
5730 Factor = Factors[0];
5731 }
5732
5733 Register Res = LHS;
5734
5735 if (UseSRL)
5736 Res = MIB.buildLShr(Dst: Ty, Src0: Res, Src1: Shift, Flags: MachineInstr::IsExact).getReg(Idx: 0);
5737
5738 return MIB.buildMul(Dst: Ty, Src0: Res, Src1: Factor);
5739 }
5740
5741 unsigned KnownLeadingZeros =
5742 VT ? VT->getKnownBits(R: LHS).countMinLeadingZeros() : 0;
5743
5744 bool UseNPQ = false;
5745 SmallVector<Register, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
5746 auto BuildUDIVPattern = [&](const Constant *C) {
5747 auto *CI = cast<ConstantInt>(Val: C);
5748 const APInt &Divisor = CI->getValue();
5749
5750 bool SelNPQ = false;
5751 APInt Magic(Divisor.getBitWidth(), 0);
5752 unsigned PreShift = 0, PostShift = 0;
5753
5754 // Magic algorithm doesn't work for division by 1. We need to emit a select
5755 // at the end.
5756 // TODO: Use undef values for divisor of 1.
5757 if (!Divisor.isOne()) {
5758
5759 // UnsignedDivisionByConstantInfo doesn't work correctly if leading zeros
5760 // in the dividend exceeds the leading zeros for the divisor.
5761 UnsignedDivisionByConstantInfo magics =
5762 UnsignedDivisionByConstantInfo::get(
5763 D: Divisor, LeadingZeros: std::min(a: KnownLeadingZeros, b: Divisor.countl_zero()));
5764
5765 Magic = std::move(magics.Magic);
5766
5767 assert(magics.PreShift < Divisor.getBitWidth() &&
5768 "We shouldn't generate an undefined shift!");
5769 assert(magics.PostShift < Divisor.getBitWidth() &&
5770 "We shouldn't generate an undefined shift!");
5771 assert((!magics.IsAdd || magics.PreShift == 0) && "Unexpected pre-shift");
5772 PreShift = magics.PreShift;
5773 PostShift = magics.PostShift;
5774 SelNPQ = magics.IsAdd;
5775 }
5776
5777 PreShifts.push_back(
5778 Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: PreShift).getReg(Idx: 0));
5779 MagicFactors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Magic).getReg(Idx: 0));
5780 NPQFactors.push_back(
5781 Elt: MIB.buildConstant(Res: ScalarTy,
5782 Val: SelNPQ ? APInt::getOneBitSet(numBits: EltBits, BitNo: EltBits - 1)
5783 : APInt::getZero(numBits: EltBits))
5784 .getReg(Idx: 0));
5785 PostShifts.push_back(
5786 Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: PostShift).getReg(Idx: 0));
5787 UseNPQ |= SelNPQ;
5788 return true;
5789 };
5790
5791 // Collect the shifts/magic values from each element.
5792 bool Matched = matchUnaryPredicate(MRI, Reg: RHS, Match: BuildUDIVPattern);
5793 (void)Matched;
5794 assert(Matched && "Expected unary predicate match to succeed");
5795
5796 Register PreShift, PostShift, MagicFactor, NPQFactor;
5797 auto *RHSDef = getOpcodeDef<GBuildVector>(Reg: RHS, MRI);
5798 if (RHSDef) {
5799 PreShift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: PreShifts).getReg(Idx: 0);
5800 MagicFactor = MIB.buildBuildVector(Res: Ty, Ops: MagicFactors).getReg(Idx: 0);
5801 NPQFactor = MIB.buildBuildVector(Res: Ty, Ops: NPQFactors).getReg(Idx: 0);
5802 PostShift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: PostShifts).getReg(Idx: 0);
5803 } else {
5804 assert(MRI.getType(RHS).isScalar() &&
5805 "Non-build_vector operation should have been a scalar");
5806 PreShift = PreShifts[0];
5807 MagicFactor = MagicFactors[0];
5808 PostShift = PostShifts[0];
5809 }
5810
5811 Register Q = LHS;
5812 Q = MIB.buildLShr(Dst: Ty, Src0: Q, Src1: PreShift).getReg(Idx: 0);
5813
5814 // Multiply the numerator (operand 0) by the magic value.
5815 Q = MIB.buildUMulH(Dst: Ty, Src0: Q, Src1: MagicFactor).getReg(Idx: 0);
5816
5817 if (UseNPQ) {
5818 Register NPQ = MIB.buildSub(Dst: Ty, Src0: LHS, Src1: Q).getReg(Idx: 0);
5819
5820 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
5821 // G_UMULH to act as a SRL-by-1 for NPQ, else multiply by zero.
5822 if (Ty.isVector())
5823 NPQ = MIB.buildUMulH(Dst: Ty, Src0: NPQ, Src1: NPQFactor).getReg(Idx: 0);
5824 else
5825 NPQ = MIB.buildLShr(Dst: Ty, Src0: NPQ, Src1: MIB.buildConstant(Res: ShiftAmtTy, Val: 1)).getReg(Idx: 0);
5826
5827 Q = MIB.buildAdd(Dst: Ty, Src0: NPQ, Src1: Q).getReg(Idx: 0);
5828 }
5829
5830 Q = MIB.buildLShr(Dst: Ty, Src0: Q, Src1: PostShift).getReg(Idx: 0);
5831 auto One = MIB.buildConstant(Res: Ty, Val: 1);
5832 auto IsOne = MIB.buildICmp(
5833 Pred: CmpInst::Predicate::ICMP_EQ,
5834 Res: Ty.isScalar() ? LLT::integer(SizeInBits: 1) : Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1)),
5835 Op0: RHS, Op1: One);
5836 auto ret = MIB.buildSelect(Res: Ty, Tst: IsOne, Op0: LHS, Op1: Q);
5837
5838 if (Opcode == TargetOpcode::G_UREM) {
5839 auto Prod = MIB.buildMul(Dst: Ty, Src0: ret, Src1: RHS);
5840 return MIB.buildSub(Dst: Ty, Src0: LHS, Src1: Prod);
5841 }
5842 return ret;
5843}
5844
5845bool CombinerHelper::matchUDivOrURemByConst(MachineInstr &MI) const {
5846 unsigned Opcode = MI.getOpcode();
5847 assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
5848 Register Dst = MI.getOperand(i: 0).getReg();
5849 Register RHS = MI.getOperand(i: 2).getReg();
5850 LLT DstTy = MRI.getType(Reg: Dst);
5851
5852 auto &MF = *MI.getMF();
5853 AttributeList Attr = MF.getFunction().getAttributes();
5854 const auto &TLI = getTargetLowering();
5855 LLVMContext &Ctx = MF.getFunction().getContext();
5856 if (DstTy.getScalarSizeInBits() == 1 ||
5857 TLI.isIntDivCheap(VT: getApproximateEVTForLLT(Ty: DstTy, Ctx), Attr))
5858 return false;
5859
5860 // Don't do this for minsize because the instruction sequence is usually
5861 // larger.
5862 if (MF.getFunction().hasMinSize())
5863 return false;
5864
5865 if (Opcode == TargetOpcode::G_UDIV &&
5866 MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
5867 return matchUnaryPredicate(
5868 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5869 }
5870
5871 auto *RHSDef = MRI.getVRegDef(Reg: RHS);
5872 if (!isConstantOrConstantVector(MI&: *RHSDef, MRI))
5873 return false;
5874
5875 // Don't do this if the types are not going to be legal.
5876 if (LI) {
5877 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_MUL, {DstTy, DstTy}}))
5878 return false;
5879 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_UMULH, {DstTy}}))
5880 return false;
5881 if (!isLegalOrBeforeLegalizer(
5882 Query: {TargetOpcode::G_ICMP,
5883 {DstTy.isVector() ? DstTy.changeElementSize(NewEltSize: 1) : LLT::scalar(SizeInBits: 1),
5884 DstTy}}))
5885 return false;
5886 if (Opcode == TargetOpcode::G_UREM &&
5887 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SUB, {DstTy, DstTy}}))
5888 return false;
5889 }
5890
5891 return matchUnaryPredicate(
5892 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5893}
5894
5895void CombinerHelper::applyUDivOrURemByConst(MachineInstr &MI) const {
5896 auto *NewMI = buildUDivOrURemUsingMul(MI);
5897 replaceSingleDefInstWithReg(MI, Replacement: NewMI->getOperand(i: 0).getReg());
5898}
5899
5900bool CombinerHelper::matchSDivOrSRemByConst(MachineInstr &MI) const {
5901 unsigned Opcode = MI.getOpcode();
5902 assert(Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM);
5903 Register Dst = MI.getOperand(i: 0).getReg();
5904 Register RHS = MI.getOperand(i: 2).getReg();
5905 LLT DstTy = MRI.getType(Reg: Dst);
5906 auto SizeInBits = DstTy.getScalarSizeInBits();
5907 LLT WideTy = DstTy.changeElementSize(NewEltSize: SizeInBits * 2);
5908
5909 auto &MF = *MI.getMF();
5910 AttributeList Attr = MF.getFunction().getAttributes();
5911 const auto &TLI = getTargetLowering();
5912 LLVMContext &Ctx = MF.getFunction().getContext();
5913 if (DstTy.getScalarSizeInBits() < 3 ||
5914 TLI.isIntDivCheap(VT: getApproximateEVTForLLT(Ty: DstTy, Ctx), Attr))
5915 return false;
5916
5917 // Don't do this for minsize because the instruction sequence is usually
5918 // larger.
5919 if (MF.getFunction().hasMinSize())
5920 return false;
5921
5922 // If the sdiv has an 'exact' flag we can use a simpler lowering.
5923 if (Opcode == TargetOpcode::G_SDIV &&
5924 MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
5925 return matchUnaryPredicate(
5926 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5927 }
5928
5929 auto *RHSDef = MRI.getVRegDef(Reg: RHS);
5930 if (!isConstantOrConstantVector(MI&: *RHSDef, MRI))
5931 return false;
5932
5933 // Don't do this if the types are not going to be legal.
5934 if (LI) {
5935 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_MUL, {DstTy, DstTy}}))
5936 return false;
5937 if (!isLegal(Query: {TargetOpcode::G_SMULH, {DstTy}}) &&
5938 !isLegalOrHasWidenScalar(Query: {TargetOpcode::G_MUL, {WideTy, WideTy}}))
5939 return false;
5940 if (Opcode == TargetOpcode::G_SREM &&
5941 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SUB, {DstTy, DstTy}}))
5942 return false;
5943 }
5944
5945 return matchUnaryPredicate(
5946 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5947}
5948
5949void CombinerHelper::applySDivOrSRemByConst(MachineInstr &MI) const {
5950 auto *NewMI = buildSDivOrSRemUsingMul(MI);
5951 replaceSingleDefInstWithReg(MI, Replacement: NewMI->getOperand(i: 0).getReg());
5952}
5953
5954MachineInstr *CombinerHelper::buildSDivOrSRemUsingMul(MachineInstr &MI) const {
5955 unsigned Opcode = MI.getOpcode();
5956 assert(MI.getOpcode() == TargetOpcode::G_SDIV ||
5957 Opcode == TargetOpcode::G_SREM);
5958 auto &SDivorRem = cast<GenericMachineInstr>(Val&: MI);
5959 Register Dst = SDivorRem.getReg(Idx: 0);
5960 Register LHS = SDivorRem.getReg(Idx: 1);
5961 Register RHS = SDivorRem.getReg(Idx: 2);
5962 LLT Ty = MRI.getType(Reg: Dst);
5963 LLT ScalarTy = Ty.getScalarType();
5964 const unsigned EltBits = ScalarTy.getScalarSizeInBits();
5965 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5966 LLT ScalarShiftAmtTy = ShiftAmtTy.getScalarType();
5967 auto &MIB = Builder;
5968
5969 bool UseSRA = false;
5970 SmallVector<Register, 16> ExactShifts, ExactFactors;
5971
5972 auto *RHSDefInstr = cast<GenericMachineInstr>(Val: getDefIgnoringCopies(Reg: RHS, MRI));
5973 bool IsSplat = getIConstantSplatVal(MI: *RHSDefInstr, MRI).has_value();
5974
5975 auto BuildExactSDIVPattern = [&](const Constant *C) {
5976 // Don't recompute inverses for each splat element.
5977 if (IsSplat && !ExactFactors.empty()) {
5978 ExactShifts.push_back(Elt: ExactShifts[0]);
5979 ExactFactors.push_back(Elt: ExactFactors[0]);
5980 return true;
5981 }
5982
5983 auto *CI = cast<ConstantInt>(Val: C);
5984 APInt Divisor = CI->getValue();
5985 unsigned Shift = Divisor.countr_zero();
5986 if (Shift) {
5987 Divisor.ashrInPlace(ShiftAmt: Shift);
5988 UseSRA = true;
5989 }
5990
5991 // Calculate the multiplicative inverse modulo BW.
5992 // 2^W requires W + 1 bits, so we have to extend and then truncate.
5993 APInt Factor = Divisor.multiplicativeInverse();
5994 ExactShifts.push_back(Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: Shift).getReg(Idx: 0));
5995 ExactFactors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Factor).getReg(Idx: 0));
5996 return true;
5997 };
5998
5999 if (MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
6000 // Collect all magic values from the build vector.
6001 bool Matched = matchUnaryPredicate(MRI, Reg: RHS, Match: BuildExactSDIVPattern);
6002 (void)Matched;
6003 assert(Matched && "Expected unary predicate match to succeed");
6004
6005 Register Shift, Factor;
6006 if (Ty.isVector()) {
6007 Shift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: ExactShifts).getReg(Idx: 0);
6008 Factor = MIB.buildBuildVector(Res: Ty, Ops: ExactFactors).getReg(Idx: 0);
6009 } else {
6010 Shift = ExactShifts[0];
6011 Factor = ExactFactors[0];
6012 }
6013
6014 Register Res = LHS;
6015
6016 if (UseSRA)
6017 Res = MIB.buildAShr(Dst: Ty, Src0: Res, Src1: Shift, Flags: MachineInstr::IsExact).getReg(Idx: 0);
6018
6019 return MIB.buildMul(Dst: Ty, Src0: Res, Src1: Factor);
6020 }
6021
6022 SmallVector<Register, 16> MagicFactors, Factors, Shifts, ShiftMasks;
6023
6024 auto BuildSDIVPattern = [&](const Constant *C) {
6025 auto *CI = cast<ConstantInt>(Val: C);
6026 const APInt &Divisor = CI->getValue();
6027
6028 SignedDivisionByConstantInfo Magics =
6029 SignedDivisionByConstantInfo::get(D: Divisor);
6030 int NumeratorFactor = 0;
6031 int ShiftMask = -1;
6032
6033 if (Divisor.isOne() || Divisor.isAllOnes()) {
6034 // If d is +1/-1, we just multiply the numerator by +1/-1.
6035 NumeratorFactor = Divisor.getSExtValue();
6036 Magics.Magic = 0;
6037 Magics.ShiftAmount = 0;
6038 ShiftMask = 0;
6039 } else if (Divisor.isStrictlyPositive() && Magics.Magic.isNegative()) {
6040 // If d > 0 and m < 0, add the numerator.
6041 NumeratorFactor = 1;
6042 } else if (Divisor.isNegative() && Magics.Magic.isStrictlyPositive()) {
6043 // If d < 0 and m > 0, subtract the numerator.
6044 NumeratorFactor = -1;
6045 }
6046
6047 MagicFactors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Magics.Magic).getReg(Idx: 0));
6048 Factors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: NumeratorFactor).getReg(Idx: 0));
6049 Shifts.push_back(
6050 Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: Magics.ShiftAmount).getReg(Idx: 0));
6051 ShiftMasks.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: ShiftMask).getReg(Idx: 0));
6052
6053 return true;
6054 };
6055
6056 // Collect the shifts/magic values from each element.
6057 bool Matched = matchUnaryPredicate(MRI, Reg: RHS, Match: BuildSDIVPattern);
6058 (void)Matched;
6059 assert(Matched && "Expected unary predicate match to succeed");
6060
6061 Register MagicFactor, Factor, Shift, ShiftMask;
6062 auto *RHSDef = getOpcodeDef<GBuildVector>(Reg: RHS, MRI);
6063 if (RHSDef) {
6064 MagicFactor = MIB.buildBuildVector(Res: Ty, Ops: MagicFactors).getReg(Idx: 0);
6065 Factor = MIB.buildBuildVector(Res: Ty, Ops: Factors).getReg(Idx: 0);
6066 Shift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: Shifts).getReg(Idx: 0);
6067 ShiftMask = MIB.buildBuildVector(Res: Ty, Ops: ShiftMasks).getReg(Idx: 0);
6068 } else {
6069 assert(MRI.getType(RHS).isScalar() &&
6070 "Non-build_vector operation should have been a scalar");
6071 MagicFactor = MagicFactors[0];
6072 Factor = Factors[0];
6073 Shift = Shifts[0];
6074 ShiftMask = ShiftMasks[0];
6075 }
6076
6077 Register Q = LHS;
6078 Q = MIB.buildSMulH(Dst: Ty, Src0: LHS, Src1: MagicFactor).getReg(Idx: 0);
6079
6080 // (Optionally) Add/subtract the numerator using Factor.
6081 Factor = MIB.buildMul(Dst: Ty, Src0: LHS, Src1: Factor).getReg(Idx: 0);
6082 Q = MIB.buildAdd(Dst: Ty, Src0: Q, Src1: Factor).getReg(Idx: 0);
6083
6084 // Shift right algebraic by shift value.
6085 Q = MIB.buildAShr(Dst: Ty, Src0: Q, Src1: Shift).getReg(Idx: 0);
6086
6087 // Extract the sign bit, mask it and add it to the quotient.
6088 auto SignShift = MIB.buildConstant(Res: ShiftAmtTy, Val: EltBits - 1);
6089 auto T = MIB.buildLShr(Dst: Ty, Src0: Q, Src1: SignShift);
6090 T = MIB.buildAnd(Dst: Ty, Src0: T, Src1: ShiftMask);
6091 auto ret = MIB.buildAdd(Dst: Ty, Src0: Q, Src1: T);
6092
6093 if (Opcode == TargetOpcode::G_SREM) {
6094 auto Prod = MIB.buildMul(Dst: Ty, Src0: ret, Src1: RHS);
6095 return MIB.buildSub(Dst: Ty, Src0: LHS, Src1: Prod);
6096 }
6097 return ret;
6098}
6099
6100bool CombinerHelper::matchDivByPow2(MachineInstr &MI, bool IsSigned) const {
6101 assert((MI.getOpcode() == TargetOpcode::G_SDIV ||
6102 MI.getOpcode() == TargetOpcode::G_UDIV) &&
6103 "Expected SDIV or UDIV");
6104 auto &Div = cast<GenericMachineInstr>(Val&: MI);
6105 Register RHS = Div.getReg(Idx: 2);
6106 auto MatchPow2 = [&](const Constant *C) {
6107 auto *CI = dyn_cast<ConstantInt>(Val: C);
6108 return CI && (CI->getValue().isPowerOf2() ||
6109 (IsSigned && CI->getValue().isNegatedPowerOf2()));
6110 };
6111 return matchUnaryPredicate(MRI, Reg: RHS, Match: MatchPow2, /*AllowUndefs=*/false);
6112}
6113
6114void CombinerHelper::applySDivByPow2(MachineInstr &MI) const {
6115 assert(MI.getOpcode() == TargetOpcode::G_SDIV && "Expected SDIV");
6116 auto &SDiv = cast<GenericMachineInstr>(Val&: MI);
6117 Register Dst = SDiv.getReg(Idx: 0);
6118 Register LHS = SDiv.getReg(Idx: 1);
6119 Register RHS = SDiv.getReg(Idx: 2);
6120 LLT Ty = MRI.getType(Reg: Dst);
6121 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
6122 LLT CCVT = Ty.isVector() ? LLT::vector(EC: Ty.getElementCount(), ScalarTy: LLT::integer(SizeInBits: 1))
6123 : LLT::integer(SizeInBits: 1);
6124
6125 // Effectively we want to lower G_SDIV %lhs, %rhs, where %rhs is a power of 2,
6126 // to the following version:
6127 //
6128 // %c1 = G_CTTZ %rhs
6129 // %inexact = G_SUB $bitwidth, %c1
6130 // %sign = %G_ASHR %lhs, $(bitwidth - 1)
6131 // %lshr = G_LSHR %sign, %inexact
6132 // %add = G_ADD %lhs, %lshr
6133 // %ashr = G_ASHR %add, %c1
6134 // %ashr = G_SELECT, %isoneorallones, %lhs, %ashr
6135 // %zero = G_CONSTANT $0
6136 // %neg = G_NEG %ashr
6137 // %isneg = G_ICMP SLT %rhs, %zero
6138 // %res = G_SELECT %isneg, %neg, %ashr
6139
6140 unsigned BitWidth = Ty.getScalarSizeInBits();
6141 auto Zero = Builder.buildConstant(Res: Ty, Val: 0);
6142
6143 auto Bits = Builder.buildConstant(Res: ShiftAmtTy, Val: BitWidth);
6144 auto C1 = Builder.buildCTTZ(Dst: ShiftAmtTy, Src0: RHS);
6145 auto Inexact = Builder.buildSub(Dst: ShiftAmtTy, Src0: Bits, Src1: C1);
6146 // Splat the sign bit into the register
6147 auto Sign = Builder.buildAShr(
6148 Dst: Ty, Src0: LHS, Src1: Builder.buildConstant(Res: ShiftAmtTy, Val: BitWidth - 1));
6149
6150 // Add (LHS < 0) ? abs2 - 1 : 0;
6151 auto LSrl = Builder.buildLShr(Dst: Ty, Src0: Sign, Src1: Inexact);
6152 auto Add = Builder.buildAdd(Dst: Ty, Src0: LHS, Src1: LSrl);
6153 auto AShr = Builder.buildAShr(Dst: Ty, Src0: Add, Src1: C1);
6154
6155 // Special case: (sdiv X, 1) -> X
6156 // Special Case: (sdiv X, -1) -> 0-X
6157 auto One = Builder.buildConstant(Res: Ty, Val: 1);
6158 auto MinusOne = Builder.buildConstant(Res: Ty, Val: -1);
6159 auto IsOne = Builder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: CCVT, Op0: RHS, Op1: One);
6160 auto IsMinusOne =
6161 Builder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: CCVT, Op0: RHS, Op1: MinusOne);
6162 auto IsOneOrMinusOne = Builder.buildOr(Dst: CCVT, Src0: IsOne, Src1: IsMinusOne);
6163 AShr = Builder.buildSelect(Res: Ty, Tst: IsOneOrMinusOne, Op0: LHS, Op1: AShr);
6164
6165 // If divided by a positive value, we're done. Otherwise, the result must be
6166 // negated.
6167 auto Neg = Builder.buildNeg(Dst: Ty, Src0: AShr);
6168 auto IsNeg = Builder.buildICmp(Pred: CmpInst::Predicate::ICMP_SLT, Res: CCVT, Op0: RHS, Op1: Zero);
6169 Builder.buildSelect(Res: MI.getOperand(i: 0).getReg(), Tst: IsNeg, Op0: Neg, Op1: AShr);
6170 MI.eraseFromParent();
6171}
6172
6173void CombinerHelper::applyUDivByPow2(MachineInstr &MI) const {
6174 assert(MI.getOpcode() == TargetOpcode::G_UDIV && "Expected UDIV");
6175 auto &UDiv = cast<GenericMachineInstr>(Val&: MI);
6176 Register Dst = UDiv.getReg(Idx: 0);
6177 Register LHS = UDiv.getReg(Idx: 1);
6178 Register RHS = UDiv.getReg(Idx: 2);
6179 LLT Ty = MRI.getType(Reg: Dst);
6180 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
6181
6182 auto C1 = Builder.buildCTTZ(Dst: ShiftAmtTy, Src0: RHS);
6183 Builder.buildLShr(Dst: MI.getOperand(i: 0).getReg(), Src0: LHS, Src1: C1);
6184 MI.eraseFromParent();
6185}
6186
6187void CombinerHelper::applySimplifySRemByPow2(MachineInstr &MI) const {
6188 assert(MI.getOpcode() == TargetOpcode::G_SREM && "Expected SREM");
6189 auto &SRem = cast<GBinOp>(Val&: MI);
6190 Register Dst = SRem.getReg(Idx: 0);
6191 Register LHS = SRem.getLHSReg();
6192 Register RHS = SRem.getRHSReg();
6193 LLT Ty = MRI.getType(Reg: Dst);
6194 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
6195
6196 // Effectively we want to lower G_SREM %lhs, %rhs, where %rhs is +/- a power
6197 // of 2, to the following branch-free bias-and-mask version:
6198 //
6199 // %abs = G_ABS %rhs
6200 // %mask = G_SUB %abs, 1
6201 // %sign = G_ASHR %lhs, $(bitwidth - 1)
6202 // %bias = G_AND %sign, %mask
6203 // %biased = G_ADD %lhs, %bias
6204 // %masked = G_AND %biased, %mask
6205 // %res = G_SUB %masked, %bias
6206 //
6207 // The bias adds (|%rhs| - 1) for negative %lhs, correcting rounding towards
6208 // zero (instead of towards -inf that a plain mask would give). Constant
6209 // divisors collapse %mask to a single G_CONSTANT via the CSEMIRBuilder folds
6210 // for G_ABS and G_SUB.
6211
6212 unsigned BitWidth = Ty.getScalarSizeInBits();
6213 auto AbsRHS = Builder.buildAbs(Dst: Ty, Src: RHS);
6214 auto Mask = Builder.buildSub(Dst: Ty, Src0: AbsRHS, Src1: Builder.buildConstant(Res: Ty, Val: 1));
6215 auto BWMinusOne = Builder.buildConstant(Res: ShiftAmtTy, Val: BitWidth - 1);
6216 auto Sign = Builder.buildAShr(Dst: Ty, Src0: LHS, Src1: BWMinusOne);
6217 auto Bias = Builder.buildAnd(Dst: Ty, Src0: Sign, Src1: Mask);
6218 auto Biased = Builder.buildAdd(Dst: Ty, Src0: LHS, Src1: Bias);
6219 auto Masked = Builder.buildAnd(Dst: Ty, Src0: Biased, Src1: Mask);
6220 Builder.buildSub(Dst, Src0: Masked, Src1: Bias);
6221 MI.eraseFromParent();
6222}
6223
6224bool CombinerHelper::matchUMulHToLShr(MachineInstr &MI) const {
6225 assert(MI.getOpcode() == TargetOpcode::G_UMULH);
6226 Register RHS = MI.getOperand(i: 2).getReg();
6227 Register Dst = MI.getOperand(i: 0).getReg();
6228 LLT Ty = MRI.getType(Reg: Dst);
6229 LLT RHSTy = MRI.getType(Reg: RHS);
6230 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
6231 auto MatchPow2ExceptOne = [&](const Constant *C) {
6232 if (auto *CI = dyn_cast<ConstantInt>(Val: C))
6233 return CI->getValue().isPowerOf2() && !CI->getValue().isOne();
6234 return false;
6235 };
6236 if (!matchUnaryPredicate(MRI, Reg: RHS, Match: MatchPow2ExceptOne, AllowUndefs: false))
6237 return false;
6238 // We need to check both G_LSHR and G_CTLZ because the combine uses G_CTLZ to
6239 // get log base 2, and it is not always legal for on a target.
6240 return isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_LSHR, {Ty, ShiftAmtTy}}) &&
6241 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_CTLZ, {RHSTy, RHSTy}});
6242}
6243
6244void CombinerHelper::applyUMulHToLShr(MachineInstr &MI) const {
6245 Register LHS = MI.getOperand(i: 1).getReg();
6246 Register RHS = MI.getOperand(i: 2).getReg();
6247 Register Dst = MI.getOperand(i: 0).getReg();
6248 LLT Ty = MRI.getType(Reg: Dst);
6249 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
6250 unsigned NumEltBits = Ty.getScalarSizeInBits();
6251
6252 auto LogBase2 = buildLogBase2(V: RHS, MIB&: Builder);
6253 auto ShiftAmt =
6254 Builder.buildSub(Dst: Ty, Src0: Builder.buildConstant(Res: Ty, Val: NumEltBits), Src1: LogBase2);
6255 auto Trunc = Builder.buildZExtOrTrunc(Res: ShiftAmtTy, Op: ShiftAmt);
6256 Builder.buildLShr(Dst, Src0: LHS, Src1: Trunc);
6257 MI.eraseFromParent();
6258}
6259
6260bool CombinerHelper::matchTruncSSatS(MachineInstr &MI,
6261 Register &MatchInfo) const {
6262 Register Dst = MI.getOperand(i: 0).getReg();
6263 Register Src = MI.getOperand(i: 1).getReg();
6264 LLT DstTy = MRI.getType(Reg: Dst);
6265 LLT SrcTy = MRI.getType(Reg: Src);
6266 unsigned NumDstBits = DstTy.getScalarSizeInBits();
6267 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6268 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
6269
6270 if (!LI || !isLegalOrHasFewerElements(
6271 Query: {TargetOpcode::G_TRUNC_SSAT_S, {DstTy, SrcTy}}))
6272 return false;
6273
6274 APInt SignedMax = APInt::getSignedMaxValue(numBits: NumDstBits).sext(width: NumSrcBits);
6275 APInt SignedMin = APInt::getSignedMinValue(numBits: NumDstBits).sext(width: NumSrcBits);
6276 return mi_match(R: Src, MRI,
6277 P: m_GSMin(L: m_GSMax(L: m_Reg(R&: MatchInfo),
6278 R: m_SpecificICstOrSplat(RequestedValue: SignedMin)),
6279 R: m_SpecificICstOrSplat(RequestedValue: SignedMax))) ||
6280 mi_match(R: Src, MRI,
6281 P: m_GSMax(L: m_GSMin(L: m_Reg(R&: MatchInfo),
6282 R: m_SpecificICstOrSplat(RequestedValue: SignedMax)),
6283 R: m_SpecificICstOrSplat(RequestedValue: SignedMin)));
6284}
6285
6286void CombinerHelper::applyTruncSSatS(MachineInstr &MI,
6287 Register &MatchInfo) const {
6288 Register Dst = MI.getOperand(i: 0).getReg();
6289 Builder.buildTruncSSatS(Res: Dst, Op: MatchInfo);
6290 MI.eraseFromParent();
6291}
6292
6293bool CombinerHelper::matchTruncSSatU(MachineInstr &MI,
6294 Register &MatchInfo) const {
6295 Register Dst = MI.getOperand(i: 0).getReg();
6296 Register Src = MI.getOperand(i: 1).getReg();
6297 LLT DstTy = MRI.getType(Reg: Dst);
6298 LLT SrcTy = MRI.getType(Reg: Src);
6299 unsigned NumDstBits = DstTy.getScalarSizeInBits();
6300 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6301 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
6302
6303 if (!LI || !isLegalOrHasFewerElements(
6304 Query: {TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))
6305 return false;
6306 APInt UnsignedMax = APInt::getMaxValue(numBits: NumDstBits).zext(width: NumSrcBits);
6307 return mi_match(R: Src, MRI,
6308 P: m_GSMin(L: m_GSMax(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: 0)),
6309 R: m_SpecificICstOrSplat(RequestedValue: UnsignedMax))) ||
6310 mi_match(R: Src, MRI,
6311 P: m_GSMax(L: m_GSMin(L: m_Reg(R&: MatchInfo),
6312 R: m_SpecificICstOrSplat(RequestedValue: UnsignedMax)),
6313 R: m_SpecificICstOrSplat(RequestedValue: 0))) ||
6314 mi_match(R: Src, MRI,
6315 P: m_GUMin(L: m_GSMax(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: 0)),
6316 R: m_SpecificICstOrSplat(RequestedValue: UnsignedMax)));
6317}
6318
6319void CombinerHelper::applyTruncSSatU(MachineInstr &MI,
6320 Register &MatchInfo) const {
6321 Register Dst = MI.getOperand(i: 0).getReg();
6322 Builder.buildTruncSSatU(Res: Dst, Op: MatchInfo);
6323 MI.eraseFromParent();
6324}
6325
6326bool CombinerHelper::matchTruncUSatU(MachineInstr &MI,
6327 MachineInstr &MinMI) const {
6328 Register Min = MinMI.getOperand(i: 2).getReg();
6329 Register Val = MinMI.getOperand(i: 1).getReg();
6330 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6331 LLT SrcTy = MRI.getType(Reg: Val);
6332 unsigned NumDstBits = DstTy.getScalarSizeInBits();
6333 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6334 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
6335
6336 if (!LI || !isLegalOrHasFewerElements(
6337 Query: {TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))
6338 return false;
6339 APInt UnsignedMax = APInt::getMaxValue(numBits: NumDstBits).zext(width: NumSrcBits);
6340 return mi_match(R: Min, MRI, P: m_SpecificICstOrSplat(RequestedValue: UnsignedMax)) &&
6341 !mi_match(R: Val, MRI, P: m_GSMax(L: m_Reg(), R: m_Reg()));
6342}
6343
6344bool CombinerHelper::matchTruncUSatUToFPTOUISat(MachineInstr &MI,
6345 MachineInstr &SrcMI) const {
6346 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6347 LLT SrcTy = MRI.getType(Reg: SrcMI.getOperand(i: 1).getReg());
6348
6349 return LI &&
6350 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FPTOUI_SAT, {DstTy, SrcTy}});
6351}
6352
6353bool CombinerHelper::matchRedundantNegOperands(MachineInstr &MI,
6354 BuildFnTy &MatchInfo) const {
6355 unsigned Opc = MI.getOpcode();
6356 assert(Opc == TargetOpcode::G_FADD || Opc == TargetOpcode::G_FSUB ||
6357 Opc == TargetOpcode::G_FMUL || Opc == TargetOpcode::G_FDIV ||
6358 Opc == TargetOpcode::G_FMAD || Opc == TargetOpcode::G_FMA);
6359
6360 Register Dst = MI.getOperand(i: 0).getReg();
6361 Register X = MI.getOperand(i: 1).getReg();
6362 Register Y = MI.getOperand(i: 2).getReg();
6363 LLT Type = MRI.getType(Reg: Dst);
6364
6365 // fold (fadd x, fneg(y)) -> (fsub x, y)
6366 // fold (fadd fneg(y), x) -> (fsub x, y)
6367 // G_ADD is commutative so both cases are checked by m_GFAdd
6368 if (mi_match(R: Dst, MRI, P: m_GFAdd(L: m_Reg(R&: X), R: m_GFNeg(Src: m_Reg(R&: Y)))) &&
6369 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FSUB, {Type}})) {
6370 Opc = TargetOpcode::G_FSUB;
6371 }
6372 /// fold (fsub x, fneg(y)) -> (fadd x, y)
6373 else if (mi_match(R: Dst, MRI, P: m_GFSub(L: m_Reg(R&: X), R: m_GFNeg(Src: m_Reg(R&: Y)))) &&
6374 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FADD, {Type}})) {
6375 Opc = TargetOpcode::G_FADD;
6376 }
6377 // fold (fmul fneg(x), fneg(y)) -> (fmul x, y)
6378 // fold (fdiv fneg(x), fneg(y)) -> (fdiv x, y)
6379 // fold (fmad fneg(x), fneg(y), z) -> (fmad x, y, z)
6380 // fold (fma fneg(x), fneg(y), z) -> (fma x, y, z)
6381 else if ((Opc == TargetOpcode::G_FMUL || Opc == TargetOpcode::G_FDIV ||
6382 Opc == TargetOpcode::G_FMAD || Opc == TargetOpcode::G_FMA) &&
6383 mi_match(R: X, MRI, P: m_GFNeg(Src: m_Reg(R&: X))) &&
6384 mi_match(R: Y, MRI, P: m_GFNeg(Src: m_Reg(R&: Y)))) {
6385 // no opcode change
6386 } else
6387 return false;
6388
6389 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6390 Observer.changingInstr(MI);
6391 MI.setDesc(B.getTII().get(Opcode: Opc));
6392 MI.getOperand(i: 1).setReg(X);
6393 MI.getOperand(i: 2).setReg(Y);
6394 Observer.changedInstr(MI);
6395 };
6396 return true;
6397}
6398
6399bool CombinerHelper::matchFsubToFneg(MachineInstr &MI,
6400 Register &MatchInfo) const {
6401 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6402
6403 Register LHS = MI.getOperand(i: 1).getReg();
6404 MatchInfo = MI.getOperand(i: 2).getReg();
6405 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6406
6407 const auto LHSCst = Ty.isVector()
6408 ? getFConstantSplat(VReg: LHS, MRI, /* allowUndef */ AllowUndef: true)
6409 : getFConstantVRegValWithLookThrough(VReg: LHS, MRI);
6410 if (!LHSCst)
6411 return false;
6412
6413 // -0.0 is always allowed
6414 if (LHSCst->Value.isNegZero())
6415 return true;
6416
6417 // +0.0 is only allowed if nsz is set.
6418 if (LHSCst->Value.isPosZero())
6419 return MI.getFlag(Flag: MachineInstr::FmNsz);
6420
6421 return false;
6422}
6423
6424void CombinerHelper::applyFsubToFneg(MachineInstr &MI,
6425 Register &MatchInfo) const {
6426 Register Dst = MI.getOperand(i: 0).getReg();
6427 Builder.buildFNeg(
6428 Dst, Src0: Builder.buildFCanonicalize(Dst: MRI.getType(Reg: Dst), Src0: MatchInfo).getReg(Idx: 0));
6429 eraseInst(MI);
6430}
6431
6432/// Checks if \p MI is TargetOpcode::G_FMUL and contractable either
6433/// due to global flags or MachineInstr flags.
6434static bool isContractableFMul(MachineInstr &MI, bool AllowFusionGlobally) {
6435 if (MI.getOpcode() != TargetOpcode::G_FMUL)
6436 return false;
6437 return AllowFusionGlobally || MI.getFlag(Flag: MachineInstr::MIFlag::FmContract);
6438}
6439
6440static bool hasMoreUses(const MachineInstr &MI0, const MachineInstr &MI1,
6441 const MachineRegisterInfo &MRI) {
6442 return std::distance(first: MRI.use_instr_nodbg_begin(RegNo: MI0.getOperand(i: 0).getReg()),
6443 last: MRI.use_instr_nodbg_end()) >
6444 std::distance(first: MRI.use_instr_nodbg_begin(RegNo: MI1.getOperand(i: 0).getReg()),
6445 last: MRI.use_instr_nodbg_end());
6446}
6447
6448bool CombinerHelper::canCombineFMadOrFMA(MachineInstr &MI,
6449 bool &AllowFusionGlobally,
6450 bool &HasFMAD, bool &Aggressive,
6451 bool CanReassociate) const {
6452
6453 auto *MF = MI.getMF();
6454 const auto &TLI = *MF->getSubtarget().getTargetLowering();
6455 const TargetOptions &Options = MF->getTarget().Options;
6456 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6457
6458 if (CanReassociate && !MI.getFlag(Flag: MachineInstr::MIFlag::FmReassoc))
6459 return false;
6460
6461 // Floating-point multiply-add with intermediate rounding.
6462 HasFMAD = (!isPreLegalize() && TLI.isFMADLegal(MI, Ty: DstType));
6463 // Floating-point multiply-add without intermediate rounding.
6464 bool HasFMA = TLI.isFMAFasterThanFMulAndFAdd(MF: *MF, DstType) &&
6465 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FMA, {DstType}});
6466 // No valid opcode, do not combine.
6467 if (!HasFMAD && !HasFMA)
6468 return false;
6469
6470 AllowFusionGlobally = Options.AllowFPOpFusion == FPOpFusion::Fast || HasFMAD;
6471 // If the addition is not contractable, do not combine.
6472 if (!AllowFusionGlobally && !MI.getFlag(Flag: MachineInstr::MIFlag::FmContract))
6473 return false;
6474
6475 Aggressive = TLI.enableAggressiveFMAFusion(Ty: DstType);
6476 return true;
6477}
6478
6479bool CombinerHelper::matchCombineFAddFMulToFMadOrFMA(
6480 MachineInstr &MI,
6481 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6482 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6483
6484 bool AllowFusionGlobally, HasFMAD, Aggressive;
6485 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6486 return false;
6487
6488 Register Op1 = MI.getOperand(i: 1).getReg();
6489 Register Op2 = MI.getOperand(i: 2).getReg();
6490 DefinitionAndSourceRegister LHS = {.MI: MRI.getVRegDef(Reg: Op1), .Reg: Op1};
6491 DefinitionAndSourceRegister RHS = {.MI: MRI.getVRegDef(Reg: Op2), .Reg: Op2};
6492 unsigned PreferredFusedOpcode =
6493 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6494
6495 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
6496 // prefer to fold the multiply with fewer uses.
6497 if (Aggressive && isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6498 isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally)) {
6499 if (hasMoreUses(MI0: *LHS.MI, MI1: *RHS.MI, MRI))
6500 std::swap(a&: LHS, b&: RHS);
6501 }
6502
6503 // fold (fadd (fmul x, y), z) -> (fma x, y, z)
6504 if (isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6505 (Aggressive || MRI.hasOneNonDBGUse(RegNo: LHS.Reg))) {
6506 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6507 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6508 SrcOps: {LHS.MI->getOperand(i: 1).getReg(),
6509 LHS.MI->getOperand(i: 2).getReg(), RHS.Reg});
6510 };
6511 return true;
6512 }
6513
6514 // fold (fadd x, (fmul y, z)) -> (fma y, z, x)
6515 if (isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally) &&
6516 (Aggressive || MRI.hasOneNonDBGUse(RegNo: RHS.Reg))) {
6517 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6518 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6519 SrcOps: {RHS.MI->getOperand(i: 1).getReg(),
6520 RHS.MI->getOperand(i: 2).getReg(), LHS.Reg});
6521 };
6522 return true;
6523 }
6524
6525 return false;
6526}
6527
6528bool CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMA(
6529 MachineInstr &MI,
6530 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6531 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6532
6533 bool AllowFusionGlobally, HasFMAD, Aggressive;
6534 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6535 return false;
6536
6537 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
6538 Register Op1 = MI.getOperand(i: 1).getReg();
6539 Register Op2 = MI.getOperand(i: 2).getReg();
6540 DefinitionAndSourceRegister LHS = {.MI: MRI.getVRegDef(Reg: Op1), .Reg: Op1};
6541 DefinitionAndSourceRegister RHS = {.MI: MRI.getVRegDef(Reg: Op2), .Reg: Op2};
6542 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6543
6544 unsigned PreferredFusedOpcode =
6545 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6546
6547 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
6548 // prefer to fold the multiply with fewer uses.
6549 if (Aggressive && isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6550 isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally)) {
6551 if (hasMoreUses(MI0: *LHS.MI, MI1: *RHS.MI, MRI))
6552 std::swap(a&: LHS, b&: RHS);
6553 }
6554
6555 // fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
6556 MachineInstr *FpExtSrc;
6557 if (mi_match(R: LHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FpExtSrc))) &&
6558 isContractableFMul(MI&: *FpExtSrc, AllowFusionGlobally) &&
6559 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6560 SrcTy: MRI.getType(Reg: FpExtSrc->getOperand(i: 1).getReg()))) {
6561 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6562 auto FpExtX = B.buildFPExt(Res: DstType, Op: FpExtSrc->getOperand(i: 1).getReg());
6563 auto FpExtY = B.buildFPExt(Res: DstType, Op: FpExtSrc->getOperand(i: 2).getReg());
6564 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6565 SrcOps: {FpExtX.getReg(Idx: 0), FpExtY.getReg(Idx: 0), RHS.Reg});
6566 };
6567 return true;
6568 }
6569
6570 // fold (fadd z, (fpext (fmul x, y))) -> (fma (fpext x), (fpext y), z)
6571 // Note: Commutes FADD operands.
6572 if (mi_match(R: RHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FpExtSrc))) &&
6573 isContractableFMul(MI&: *FpExtSrc, AllowFusionGlobally) &&
6574 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6575 SrcTy: MRI.getType(Reg: FpExtSrc->getOperand(i: 1).getReg()))) {
6576 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6577 auto FpExtX = B.buildFPExt(Res: DstType, Op: FpExtSrc->getOperand(i: 1).getReg());
6578 auto FpExtY = B.buildFPExt(Res: DstType, Op: FpExtSrc->getOperand(i: 2).getReg());
6579 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6580 SrcOps: {FpExtX.getReg(Idx: 0), FpExtY.getReg(Idx: 0), LHS.Reg});
6581 };
6582 return true;
6583 }
6584
6585 return false;
6586}
6587
6588bool CombinerHelper::matchCombineFAddFMAFMulToFMadOrFMA(
6589 MachineInstr &MI,
6590 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6591 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6592
6593 bool AllowFusionGlobally, HasFMAD, Aggressive;
6594 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive, CanReassociate: true))
6595 return false;
6596
6597 Register Op1 = MI.getOperand(i: 1).getReg();
6598 Register Op2 = MI.getOperand(i: 2).getReg();
6599 DefinitionAndSourceRegister LHS = {.MI: MRI.getVRegDef(Reg: Op1), .Reg: Op1};
6600 DefinitionAndSourceRegister RHS = {.MI: MRI.getVRegDef(Reg: Op2), .Reg: Op2};
6601 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6602
6603 unsigned PreferredFusedOpcode =
6604 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6605
6606 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
6607 // prefer to fold the multiply with fewer uses.
6608 if (Aggressive && isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6609 isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally)) {
6610 if (hasMoreUses(MI0: *LHS.MI, MI1: *RHS.MI, MRI))
6611 std::swap(a&: LHS, b&: RHS);
6612 }
6613
6614 MachineInstr *FMA = nullptr;
6615 Register Z;
6616 // fold (fadd (fma x, y, (fmul u, v)), z) -> (fma x, y, (fma u, v, z))
6617 if (LHS.MI->getOpcode() == PreferredFusedOpcode &&
6618 (MRI.getVRegDef(Reg: LHS.MI->getOperand(i: 3).getReg())->getOpcode() ==
6619 TargetOpcode::G_FMUL) &&
6620 MRI.hasOneNonDBGUse(RegNo: LHS.MI->getOperand(i: 0).getReg()) &&
6621 MRI.hasOneNonDBGUse(RegNo: LHS.MI->getOperand(i: 3).getReg())) {
6622 FMA = LHS.MI;
6623 Z = RHS.Reg;
6624 }
6625 // fold (fadd z, (fma x, y, (fmul u, v))) -> (fma x, y, (fma u, v, z))
6626 else if (RHS.MI->getOpcode() == PreferredFusedOpcode &&
6627 (MRI.getVRegDef(Reg: RHS.MI->getOperand(i: 3).getReg())->getOpcode() ==
6628 TargetOpcode::G_FMUL) &&
6629 MRI.hasOneNonDBGUse(RegNo: RHS.MI->getOperand(i: 0).getReg()) &&
6630 MRI.hasOneNonDBGUse(RegNo: RHS.MI->getOperand(i: 3).getReg())) {
6631 Z = LHS.Reg;
6632 FMA = RHS.MI;
6633 }
6634
6635 if (FMA) {
6636 MachineInstr *FMulMI = MRI.getVRegDef(Reg: FMA->getOperand(i: 3).getReg());
6637 Register X = FMA->getOperand(i: 1).getReg();
6638 Register Y = FMA->getOperand(i: 2).getReg();
6639 Register U = FMulMI->getOperand(i: 1).getReg();
6640 Register V = FMulMI->getOperand(i: 2).getReg();
6641
6642 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6643 Register InnerFMA = MRI.createGenericVirtualRegister(Ty: DstTy);
6644 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {InnerFMA}, SrcOps: {U, V, Z});
6645 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6646 SrcOps: {X, Y, InnerFMA});
6647 };
6648 return true;
6649 }
6650
6651 return false;
6652}
6653
6654bool CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMAAggressive(
6655 MachineInstr &MI,
6656 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6657 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6658
6659 bool AllowFusionGlobally, HasFMAD, Aggressive;
6660 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6661 return false;
6662
6663 if (!Aggressive)
6664 return false;
6665
6666 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
6667 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6668 Register Op1 = MI.getOperand(i: 1).getReg();
6669 Register Op2 = MI.getOperand(i: 2).getReg();
6670 DefinitionAndSourceRegister LHS = {.MI: MRI.getVRegDef(Reg: Op1), .Reg: Op1};
6671 DefinitionAndSourceRegister RHS = {.MI: MRI.getVRegDef(Reg: Op2), .Reg: Op2};
6672
6673 unsigned PreferredFusedOpcode =
6674 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6675
6676 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
6677 // prefer to fold the multiply with fewer uses.
6678 if (Aggressive && isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6679 isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally)) {
6680 if (hasMoreUses(MI0: *LHS.MI, MI1: *RHS.MI, MRI))
6681 std::swap(a&: LHS, b&: RHS);
6682 }
6683
6684 // Builds: (fma x, y, (fma (fpext u), (fpext v), z))
6685 auto buildMatchInfo = [=, &MI](Register U, Register V, Register Z, Register X,
6686 Register Y, MachineIRBuilder &B) {
6687 Register FpExtU = B.buildFPExt(Res: DstType, Op: U).getReg(Idx: 0);
6688 Register FpExtV = B.buildFPExt(Res: DstType, Op: V).getReg(Idx: 0);
6689 Register InnerFMA =
6690 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {DstType}, SrcOps: {FpExtU, FpExtV, Z})
6691 .getReg(Idx: 0);
6692 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6693 SrcOps: {X, Y, InnerFMA});
6694 };
6695
6696 MachineInstr *FMulMI, *FMAMI;
6697 // fold (fadd (fma x, y, (fpext (fmul u, v))), z)
6698 // -> (fma x, y, (fma (fpext u), (fpext v), z))
6699 if (LHS.MI->getOpcode() == PreferredFusedOpcode &&
6700 mi_match(R: LHS.MI->getOperand(i: 3).getReg(), MRI,
6701 P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6702 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6703 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6704 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
6705 MatchInfo = [=](MachineIRBuilder &B) {
6706 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6707 FMulMI->getOperand(i: 2).getReg(), RHS.Reg,
6708 LHS.MI->getOperand(i: 1).getReg(),
6709 LHS.MI->getOperand(i: 2).getReg(), B);
6710 };
6711 return true;
6712 }
6713
6714 // fold (fadd (fpext (fma x, y, (fmul u, v))), z)
6715 // -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z))
6716 // FIXME: This turns two single-precision and one double-precision
6717 // operation into two double-precision operations, which might not be
6718 // interesting for all targets, especially GPUs.
6719 if (mi_match(R: LHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMAMI))) &&
6720 FMAMI->getOpcode() == PreferredFusedOpcode) {
6721 MachineInstr *FMulMI = MRI.getVRegDef(Reg: FMAMI->getOperand(i: 3).getReg());
6722 if (isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6723 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6724 SrcTy: MRI.getType(Reg: FMAMI->getOperand(i: 0).getReg()))) {
6725 MatchInfo = [=](MachineIRBuilder &B) {
6726 Register X = FMAMI->getOperand(i: 1).getReg();
6727 Register Y = FMAMI->getOperand(i: 2).getReg();
6728 X = B.buildFPExt(Res: DstType, Op: X).getReg(Idx: 0);
6729 Y = B.buildFPExt(Res: DstType, Op: Y).getReg(Idx: 0);
6730 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6731 FMulMI->getOperand(i: 2).getReg(), RHS.Reg, X, Y, B);
6732 };
6733
6734 return true;
6735 }
6736 }
6737
6738 // fold (fadd z, (fma x, y, (fpext (fmul u, v)))
6739 // -> (fma x, y, (fma (fpext u), (fpext v), z))
6740 if (RHS.MI->getOpcode() == PreferredFusedOpcode &&
6741 mi_match(R: RHS.MI->getOperand(i: 3).getReg(), MRI,
6742 P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6743 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6744 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6745 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
6746 MatchInfo = [=](MachineIRBuilder &B) {
6747 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6748 FMulMI->getOperand(i: 2).getReg(), LHS.Reg,
6749 RHS.MI->getOperand(i: 1).getReg(),
6750 RHS.MI->getOperand(i: 2).getReg(), B);
6751 };
6752 return true;
6753 }
6754
6755 // fold (fadd z, (fpext (fma x, y, (fmul u, v)))
6756 // -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z))
6757 // FIXME: This turns two single-precision and one double-precision
6758 // operation into two double-precision operations, which might not be
6759 // interesting for all targets, especially GPUs.
6760 if (mi_match(R: RHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMAMI))) &&
6761 FMAMI->getOpcode() == PreferredFusedOpcode) {
6762 MachineInstr *FMulMI = MRI.getVRegDef(Reg: FMAMI->getOperand(i: 3).getReg());
6763 if (isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6764 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6765 SrcTy: MRI.getType(Reg: FMAMI->getOperand(i: 0).getReg()))) {
6766 MatchInfo = [=](MachineIRBuilder &B) {
6767 Register X = FMAMI->getOperand(i: 1).getReg();
6768 Register Y = FMAMI->getOperand(i: 2).getReg();
6769 X = B.buildFPExt(Res: DstType, Op: X).getReg(Idx: 0);
6770 Y = B.buildFPExt(Res: DstType, Op: Y).getReg(Idx: 0);
6771 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6772 FMulMI->getOperand(i: 2).getReg(), LHS.Reg, X, Y, B);
6773 };
6774 return true;
6775 }
6776 }
6777
6778 return false;
6779}
6780
6781bool CombinerHelper::matchCombineFSubFMulToFMadOrFMA(
6782 MachineInstr &MI,
6783 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6784 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6785
6786 bool AllowFusionGlobally, HasFMAD, Aggressive;
6787 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6788 return false;
6789
6790 Register Op1 = MI.getOperand(i: 1).getReg();
6791 Register Op2 = MI.getOperand(i: 2).getReg();
6792 DefinitionAndSourceRegister LHS = {.MI: MRI.getVRegDef(Reg: Op1), .Reg: Op1};
6793 DefinitionAndSourceRegister RHS = {.MI: MRI.getVRegDef(Reg: Op2), .Reg: Op2};
6794 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6795
6796 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
6797 // prefer to fold the multiply with fewer uses.
6798 int FirstMulHasFewerUses = true;
6799 if (isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6800 isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally) &&
6801 hasMoreUses(MI0: *LHS.MI, MI1: *RHS.MI, MRI))
6802 FirstMulHasFewerUses = false;
6803
6804 unsigned PreferredFusedOpcode =
6805 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6806
6807 // fold (fsub (fmul x, y), z) -> (fma x, y, -z)
6808 if (FirstMulHasFewerUses &&
6809 (isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6810 (Aggressive || MRI.hasOneNonDBGUse(RegNo: LHS.Reg)))) {
6811 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6812 Register NegZ = B.buildFNeg(Dst: DstTy, Src0: RHS.Reg).getReg(Idx: 0);
6813 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6814 SrcOps: {LHS.MI->getOperand(i: 1).getReg(),
6815 LHS.MI->getOperand(i: 2).getReg(), NegZ});
6816 };
6817 return true;
6818 }
6819 // fold (fsub x, (fmul y, z)) -> (fma -y, z, x)
6820 else if ((isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally) &&
6821 (Aggressive || MRI.hasOneNonDBGUse(RegNo: RHS.Reg)))) {
6822 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6823 Register NegY =
6824 B.buildFNeg(Dst: DstTy, Src0: RHS.MI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6825 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6826 SrcOps: {NegY, RHS.MI->getOperand(i: 2).getReg(), LHS.Reg});
6827 };
6828 return true;
6829 }
6830
6831 return false;
6832}
6833
6834bool CombinerHelper::matchCombineFSubFNegFMulToFMadOrFMA(
6835 MachineInstr &MI,
6836 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6837 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6838
6839 bool AllowFusionGlobally, HasFMAD, Aggressive;
6840 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6841 return false;
6842
6843 Register LHSReg = MI.getOperand(i: 1).getReg();
6844 Register RHSReg = MI.getOperand(i: 2).getReg();
6845 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6846
6847 unsigned PreferredFusedOpcode =
6848 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6849
6850 MachineInstr *FMulMI;
6851 // fold (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
6852 if (mi_match(R: LHSReg, MRI, P: m_GFNeg(Src: m_MInstr(MI&: FMulMI))) &&
6853 (Aggressive || (MRI.hasOneNonDBGUse(RegNo: LHSReg) &&
6854 MRI.hasOneNonDBGUse(RegNo: FMulMI->getOperand(i: 0).getReg()))) &&
6855 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally)) {
6856 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6857 Register NegX =
6858 B.buildFNeg(Dst: DstTy, Src0: FMulMI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6859 Register NegZ = B.buildFNeg(Dst: DstTy, Src0: RHSReg).getReg(Idx: 0);
6860 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6861 SrcOps: {NegX, FMulMI->getOperand(i: 2).getReg(), NegZ});
6862 };
6863 return true;
6864 }
6865
6866 // fold (fsub x, (fneg (fmul, y, z))) -> (fma y, z, x)
6867 if (mi_match(R: RHSReg, MRI, P: m_GFNeg(Src: m_MInstr(MI&: FMulMI))) &&
6868 (Aggressive || (MRI.hasOneNonDBGUse(RegNo: RHSReg) &&
6869 MRI.hasOneNonDBGUse(RegNo: FMulMI->getOperand(i: 0).getReg()))) &&
6870 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally)) {
6871 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6872 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6873 SrcOps: {FMulMI->getOperand(i: 1).getReg(),
6874 FMulMI->getOperand(i: 2).getReg(), LHSReg});
6875 };
6876 return true;
6877 }
6878
6879 return false;
6880}
6881
6882bool CombinerHelper::matchCombineFSubFpExtFMulToFMadOrFMA(
6883 MachineInstr &MI,
6884 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6885 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6886
6887 bool AllowFusionGlobally, HasFMAD, Aggressive;
6888 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6889 return false;
6890
6891 Register LHSReg = MI.getOperand(i: 1).getReg();
6892 Register RHSReg = MI.getOperand(i: 2).getReg();
6893 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6894
6895 unsigned PreferredFusedOpcode =
6896 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6897
6898 MachineInstr *FMulMI;
6899 // fold (fsub (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), (fneg z))
6900 if (mi_match(R: LHSReg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6901 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6902 (Aggressive || MRI.hasOneNonDBGUse(RegNo: LHSReg))) {
6903 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6904 Register FpExtX =
6905 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6906 Register FpExtY =
6907 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 2).getReg()).getReg(Idx: 0);
6908 Register NegZ = B.buildFNeg(Dst: DstTy, Src0: RHSReg).getReg(Idx: 0);
6909 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6910 SrcOps: {FpExtX, FpExtY, NegZ});
6911 };
6912 return true;
6913 }
6914
6915 // fold (fsub x, (fpext (fmul y, z))) -> (fma (fneg (fpext y)), (fpext z), x)
6916 if (mi_match(R: RHSReg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6917 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6918 (Aggressive || MRI.hasOneNonDBGUse(RegNo: RHSReg))) {
6919 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6920 Register FpExtY =
6921 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6922 Register NegY = B.buildFNeg(Dst: DstTy, Src0: FpExtY).getReg(Idx: 0);
6923 Register FpExtZ =
6924 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 2).getReg()).getReg(Idx: 0);
6925 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6926 SrcOps: {NegY, FpExtZ, LHSReg});
6927 };
6928 return true;
6929 }
6930
6931 return false;
6932}
6933
6934bool CombinerHelper::matchCombineFSubFpExtFNegFMulToFMadOrFMA(
6935 MachineInstr &MI,
6936 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6937 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6938
6939 bool AllowFusionGlobally, HasFMAD, Aggressive;
6940 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6941 return false;
6942
6943 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
6944 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6945 Register LHSReg = MI.getOperand(i: 1).getReg();
6946 Register RHSReg = MI.getOperand(i: 2).getReg();
6947
6948 unsigned PreferredFusedOpcode =
6949 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6950
6951 auto buildMatchInfo = [=](Register Dst, Register X, Register Y, Register Z,
6952 MachineIRBuilder &B) {
6953 Register FpExtX = B.buildFPExt(Res: DstTy, Op: X).getReg(Idx: 0);
6954 Register FpExtY = B.buildFPExt(Res: DstTy, Op: Y).getReg(Idx: 0);
6955 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {Dst}, SrcOps: {FpExtX, FpExtY, Z});
6956 };
6957
6958 MachineInstr *FMulMI;
6959 // fold (fsub (fpext (fneg (fmul x, y))), z) ->
6960 // (fneg (fma (fpext x), (fpext y), z))
6961 // fold (fsub (fneg (fpext (fmul x, y))), z) ->
6962 // (fneg (fma (fpext x), (fpext y), z))
6963 if ((mi_match(R: LHSReg, MRI, P: m_GFPExt(Src: m_GFNeg(Src: m_MInstr(MI&: FMulMI)))) ||
6964 mi_match(R: LHSReg, MRI, P: m_GFNeg(Src: m_GFPExt(Src: m_MInstr(MI&: FMulMI))))) &&
6965 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6966 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstTy,
6967 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
6968 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6969 Register FMAReg = MRI.createGenericVirtualRegister(Ty: DstTy);
6970 buildMatchInfo(FMAReg, FMulMI->getOperand(i: 1).getReg(),
6971 FMulMI->getOperand(i: 2).getReg(), RHSReg, B);
6972 B.buildFNeg(Dst: MI.getOperand(i: 0).getReg(), Src0: FMAReg);
6973 };
6974 return true;
6975 }
6976
6977 // fold (fsub x, (fpext (fneg (fmul y, z)))) -> (fma (fpext y), (fpext z), x)
6978 // fold (fsub x, (fneg (fpext (fmul y, z)))) -> (fma (fpext y), (fpext z), x)
6979 if ((mi_match(R: RHSReg, MRI, P: m_GFPExt(Src: m_GFNeg(Src: m_MInstr(MI&: FMulMI)))) ||
6980 mi_match(R: RHSReg, MRI, P: m_GFNeg(Src: m_GFPExt(Src: m_MInstr(MI&: FMulMI))))) &&
6981 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6982 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstTy,
6983 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
6984 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6985 buildMatchInfo(MI.getOperand(i: 0).getReg(), FMulMI->getOperand(i: 1).getReg(),
6986 FMulMI->getOperand(i: 2).getReg(), LHSReg, B);
6987 };
6988 return true;
6989 }
6990
6991 return false;
6992}
6993
6994bool CombinerHelper::matchCombineFMinMaxNaN(MachineInstr &MI,
6995 unsigned &IdxToPropagate) const {
6996 bool PropagateNaN;
6997 switch (MI.getOpcode()) {
6998 default:
6999 return false;
7000 case TargetOpcode::G_FMINNUM:
7001 case TargetOpcode::G_FMAXNUM:
7002 PropagateNaN = false;
7003 break;
7004 case TargetOpcode::G_FMINIMUM:
7005 case TargetOpcode::G_FMAXIMUM:
7006 PropagateNaN = true;
7007 break;
7008 }
7009
7010 auto MatchNaN = [&](unsigned Idx) {
7011 Register MaybeNaNReg = MI.getOperand(i: Idx).getReg();
7012 const ConstantFP *MaybeCst = getConstantFPVRegVal(VReg: MaybeNaNReg, MRI);
7013 if (!MaybeCst || !MaybeCst->getValueAPF().isNaN())
7014 return false;
7015 IdxToPropagate = PropagateNaN ? Idx : (Idx == 1 ? 2 : 1);
7016 return true;
7017 };
7018
7019 return MatchNaN(1) || MatchNaN(2);
7020}
7021
7022// Combine multiple FDIVs with the same divisor into multiple FMULs by the
7023// reciprocal.
7024// E.g., (a / Y; b / Y;) -> (recip = 1.0 / Y; a * recip; b * recip)
7025bool CombinerHelper::matchRepeatedFPDivisor(
7026 MachineInstr &MI, SmallVector<MachineInstr *> &MatchInfo) const {
7027 assert(MI.getOpcode() == TargetOpcode::G_FDIV);
7028
7029 Register X = MI.getOperand(i: 1).getReg();
7030 Register Y = MI.getOperand(i: 2).getReg();
7031
7032 if (!MI.getFlag(Flag: MachineInstr::MIFlag::FmArcp))
7033 return false;
7034
7035 auto IsOne = [this](Register X) {
7036 auto N0CFP = isConstantOrConstantSplatVectorFP(Def: X, MRI);
7037 return N0CFP && (N0CFP->isOne() || N0CFP->isMinusOne());
7038 };
7039
7040 // Skip if current node is a reciprocal/fneg-reciprocal.
7041 if (IsOne(X))
7042 return false;
7043
7044 // Exit early if the target does not want this transform or if there can't
7045 // possibly be enough uses of the divisor to make the transform worthwhile.
7046 unsigned MinUses = getTargetLowering().combineRepeatedFPDivisors();
7047 if (!MinUses)
7048 return false;
7049
7050 // Find all FDIV users of the same divisor. For the moment we limit all
7051 // instructions to a single BB and use the first Instr in MatchInfo as the
7052 // dominating position.
7053 MatchInfo.push_back(Elt: &MI);
7054 for (auto &U : MRI.use_nodbg_instructions(Reg: Y)) {
7055 if (&U == &MI || U.getParent() != MI.getParent())
7056 continue;
7057 if (U.getOpcode() == TargetOpcode::G_FDIV &&
7058 U.getOperand(i: 2).getReg() == Y && U.getOperand(i: 1).getReg() != Y &&
7059 !IsOne(U.getOperand(i: 1).getReg())) {
7060 // This division is eligible for optimization only if global unsafe math
7061 // is enabled or if this division allows reciprocal formation.
7062 if (U.getFlag(Flag: MachineInstr::MIFlag::FmArcp)) {
7063 MatchInfo.push_back(Elt: &U);
7064 if (dominates(DefMI: U, UseMI: *MatchInfo[0]))
7065 std::swap(a&: MatchInfo[0], b&: MatchInfo.back());
7066 }
7067 }
7068 }
7069
7070 // Now that we have the actual number of divisor uses, make sure it meets
7071 // the minimum threshold specified by the target.
7072 return MatchInfo.size() >= MinUses;
7073}
7074
7075void CombinerHelper::applyRepeatedFPDivisor(
7076 SmallVector<MachineInstr *> &MatchInfo) const {
7077 // Generate the new div at the position of the first instruction, that we have
7078 // ensured will dominate all other instructions.
7079 Builder.setInsertPt(MBB&: *MatchInfo[0]->getParent(), II: MatchInfo[0]);
7080 LLT Ty = MRI.getType(Reg: MatchInfo[0]->getOperand(i: 0).getReg());
7081 auto Div = Builder.buildFDiv(Dst: Ty, Src0: Builder.buildFConstant(Res: Ty, Val: 1.0),
7082 Src1: MatchInfo[0]->getOperand(i: 2).getReg(),
7083 Flags: MatchInfo[0]->getFlags());
7084
7085 // Replace all found div's with fmul instructions.
7086 for (MachineInstr *MI : MatchInfo) {
7087 Builder.setInsertPt(MBB&: *MI->getParent(), II: MI);
7088 Builder.buildFMul(Dst: MI->getOperand(i: 0).getReg(), Src0: MI->getOperand(i: 1).getReg(),
7089 Src1: Div->getOperand(i: 0).getReg(), Flags: MI->getFlags());
7090 MI->eraseFromParent();
7091 }
7092}
7093
7094bool CombinerHelper::matchAddSubSameReg(MachineInstr &MI, Register &Src) const {
7095 assert(MI.getOpcode() == TargetOpcode::G_ADD && "Expected a G_ADD");
7096 Register LHS = MI.getOperand(i: 1).getReg();
7097 Register RHS = MI.getOperand(i: 2).getReg();
7098
7099 // Helper lambda to check for opportunities for
7100 // A + (B - A) -> B
7101 // (B - A) + A -> B
7102 auto CheckFold = [&](Register MaybeSub, Register MaybeSameReg) {
7103 Register Reg;
7104 return mi_match(R: MaybeSub, MRI, P: m_GSub(L: m_Reg(R&: Src), R: m_Reg(R&: Reg))) &&
7105 Reg == MaybeSameReg;
7106 };
7107 return CheckFold(LHS, RHS) || CheckFold(RHS, LHS);
7108}
7109
7110bool CombinerHelper::matchBuildVectorIdentityFold(MachineInstr &MI,
7111 Register &MatchInfo) const {
7112 // This combine folds the following patterns:
7113 //
7114 // G_BUILD_VECTOR_TRUNC (G_BITCAST(x), G_LSHR(G_BITCAST(x), k))
7115 // G_BUILD_VECTOR(G_TRUNC(G_BITCAST(x)), G_TRUNC(G_LSHR(G_BITCAST(x), k)))
7116 // into
7117 // x
7118 // if
7119 // k == sizeof(VecEltTy)/2
7120 // type(x) == type(dst)
7121 //
7122 // G_BUILD_VECTOR(G_TRUNC(G_BITCAST(x)), undef)
7123 // into
7124 // x
7125 // if
7126 // type(x) == type(dst)
7127
7128 LLT DstVecTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7129 LLT DstEltTy = DstVecTy.getElementType();
7130
7131 Register Lo, Hi;
7132
7133 if (mi_match(
7134 MI, MRI,
7135 P: m_GBuildVector(L: m_GTrunc(Src: m_GBitcast(Src: m_Reg(R&: Lo))), R: m_GImplicitDef()))) {
7136 MatchInfo = Lo;
7137 return MRI.getType(Reg: MatchInfo) == DstVecTy;
7138 }
7139
7140 std::optional<ValueAndVReg> ShiftAmount;
7141 const auto LoPattern = m_GBitcast(Src: m_Reg(R&: Lo));
7142 const auto HiPattern = m_GLShr(L: m_GBitcast(Src: m_Reg(R&: Hi)), R: m_GCst(ValReg&: ShiftAmount));
7143 if (mi_match(
7144 MI, MRI,
7145 P: m_any_of(preds: m_GBuildVectorTrunc(L: LoPattern, R: HiPattern),
7146 preds: m_GBuildVector(L: m_GTrunc(Src: LoPattern), R: m_GTrunc(Src: HiPattern))))) {
7147 if (Lo == Hi && ShiftAmount->Value == DstEltTy.getSizeInBits()) {
7148 MatchInfo = Lo;
7149 return MRI.getType(Reg: MatchInfo) == DstVecTy;
7150 }
7151 }
7152
7153 return false;
7154}
7155
7156bool CombinerHelper::matchTruncBuildVectorFold(MachineInstr &MI,
7157 Register &MatchInfo) const {
7158 // Replace (G_TRUNC (G_BITCAST (G_BUILD_VECTOR x, y)) with just x
7159 // if type(x) == type(G_TRUNC)
7160 if (!mi_match(R: MI.getOperand(i: 1).getReg(), MRI,
7161 P: m_GBitcast(Src: m_GBuildVector(L: m_Reg(R&: MatchInfo), R: m_Reg()))))
7162 return false;
7163
7164 return MRI.getType(Reg: MatchInfo) == MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7165}
7166
7167bool CombinerHelper::matchTruncLshrBuildVectorFold(MachineInstr &MI,
7168 Register &MatchInfo) const {
7169 // Replace (G_TRUNC (G_LSHR (G_BITCAST (G_BUILD_VECTOR x, y)), K)) with
7170 // y if K == size of vector element type
7171 std::optional<ValueAndVReg> ShiftAmt;
7172 if (!mi_match(R: MI.getOperand(i: 1).getReg(), MRI,
7173 P: m_GLShr(L: m_GBitcast(Src: m_GBuildVector(L: m_Reg(), R: m_Reg(R&: MatchInfo))),
7174 R: m_GCst(ValReg&: ShiftAmt))))
7175 return false;
7176
7177 LLT MatchTy = MRI.getType(Reg: MatchInfo);
7178 return ShiftAmt->Value.getZExtValue() == MatchTy.getSizeInBits() &&
7179 MatchTy == MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7180}
7181
7182unsigned CombinerHelper::getFPMinMaxOpcForSelect(
7183 CmpInst::Predicate Pred, LLT DstTy,
7184 SelectPatternNaNBehaviour VsNaNRetVal) const {
7185 assert(VsNaNRetVal != SelectPatternNaNBehaviour::NOT_APPLICABLE &&
7186 "Expected a NaN behaviour?");
7187 // Choose an opcode based off of legality or the behaviour when one of the
7188 // LHS/RHS may be NaN.
7189 switch (Pred) {
7190 default:
7191 return 0;
7192 case CmpInst::FCMP_UGT:
7193 case CmpInst::FCMP_UGE:
7194 case CmpInst::FCMP_OGT:
7195 case CmpInst::FCMP_OGE:
7196 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
7197 return TargetOpcode::G_FMAXNUM;
7198 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
7199 return TargetOpcode::G_FMAXIMUM;
7200 if (isLegal(Query: {TargetOpcode::G_FMAXNUM, {DstTy}}))
7201 return TargetOpcode::G_FMAXNUM;
7202 if (isLegal(Query: {TargetOpcode::G_FMAXIMUM, {DstTy}}))
7203 return TargetOpcode::G_FMAXIMUM;
7204 return 0;
7205 case CmpInst::FCMP_ULT:
7206 case CmpInst::FCMP_ULE:
7207 case CmpInst::FCMP_OLT:
7208 case CmpInst::FCMP_OLE:
7209 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
7210 return TargetOpcode::G_FMINNUM;
7211 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
7212 return TargetOpcode::G_FMINIMUM;
7213 if (isLegal(Query: {TargetOpcode::G_FMINNUM, {DstTy}}))
7214 return TargetOpcode::G_FMINNUM;
7215 if (!isLegal(Query: {TargetOpcode::G_FMINIMUM, {DstTy}}))
7216 return 0;
7217 return TargetOpcode::G_FMINIMUM;
7218 }
7219}
7220
7221CombinerHelper::SelectPatternNaNBehaviour
7222CombinerHelper::computeRetValAgainstNaN(Register LHS, Register RHS,
7223 bool IsOrderedComparison) const {
7224 bool LHSSafe = VT->isKnownNeverNaN(Val: LHS);
7225 bool RHSSafe = VT->isKnownNeverNaN(Val: RHS);
7226 // Completely unsafe.
7227 if (!LHSSafe && !RHSSafe)
7228 return SelectPatternNaNBehaviour::NOT_APPLICABLE;
7229 if (LHSSafe && RHSSafe)
7230 return SelectPatternNaNBehaviour::RETURNS_ANY;
7231 // An ordered comparison will return false when given a NaN, so it
7232 // returns the RHS.
7233 if (IsOrderedComparison)
7234 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_NAN
7235 : SelectPatternNaNBehaviour::RETURNS_OTHER;
7236 // An unordered comparison will return true when given a NaN, so it
7237 // returns the LHS.
7238 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_OTHER
7239 : SelectPatternNaNBehaviour::RETURNS_NAN;
7240}
7241
7242bool CombinerHelper::matchFPSelectToMinMax(Register Dst, Register Cond,
7243 Register TrueVal, Register FalseVal,
7244 BuildFnTy &MatchInfo) const {
7245 // Match: select (fcmp cond x, y) x, y
7246 // select (fcmp cond x, y) y, x
7247 // And turn it into fminnum/fmaxnum or fmin/fmax based off of the condition.
7248 LLT DstTy = MRI.getType(Reg: Dst);
7249 // Bail out early on pointers, since we'll never want to fold to a min/max.
7250 if (DstTy.isPointer())
7251 return false;
7252 // Match a floating point compare with a less-than/greater-than predicate.
7253 // TODO: Allow multiple users of the compare if they are all selects.
7254 CmpInst::Predicate Pred;
7255 Register CmpLHS, CmpRHS;
7256 if (!mi_match(R: Cond, MRI,
7257 P: m_OneNonDBGUse(
7258 SP: m_GFCmp(P: m_Pred(P&: Pred), L: m_Reg(R&: CmpLHS), R: m_Reg(R&: CmpRHS)))) ||
7259 CmpInst::isEquality(pred: Pred))
7260 return false;
7261 SelectPatternNaNBehaviour ResWithKnownNaNInfo =
7262 computeRetValAgainstNaN(LHS: CmpLHS, RHS: CmpRHS, IsOrderedComparison: CmpInst::isOrdered(predicate: Pred));
7263 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::NOT_APPLICABLE)
7264 return false;
7265 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
7266 std::swap(a&: CmpLHS, b&: CmpRHS);
7267 Pred = CmpInst::getSwappedPredicate(pred: Pred);
7268 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_NAN)
7269 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_OTHER;
7270 else if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_OTHER)
7271 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_NAN;
7272 }
7273 if (TrueVal != CmpLHS || FalseVal != CmpRHS)
7274 return false;
7275 // Decide what type of max/min this should be based off of the predicate.
7276 unsigned Opc = getFPMinMaxOpcForSelect(Pred, DstTy, VsNaNRetVal: ResWithKnownNaNInfo);
7277 if (!Opc || !isLegal(Query: {Opc, {DstTy}}))
7278 return false;
7279 // Comparisons between signed zero and zero may have different results...
7280 // unless we have fmaximum/fminimum. In that case, we know -0 < 0.
7281 if (Opc != TargetOpcode::G_FMAXIMUM && Opc != TargetOpcode::G_FMINIMUM) {
7282 // We don't know if a comparison between two 0s will give us a consistent
7283 // result. Be conservative and only proceed if at least one side is
7284 // non-zero.
7285 auto KnownNonZeroSide = getFConstantVRegValWithLookThrough(VReg: CmpLHS, MRI);
7286 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero()) {
7287 KnownNonZeroSide = getFConstantVRegValWithLookThrough(VReg: CmpRHS, MRI);
7288 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero())
7289 return false;
7290 }
7291 }
7292 MatchInfo = [=](MachineIRBuilder &B) {
7293 B.buildInstr(Opc, DstOps: {Dst}, SrcOps: {CmpLHS, CmpRHS});
7294 };
7295 return true;
7296}
7297
7298bool CombinerHelper::matchSimplifySelectToMinMax(MachineInstr &MI,
7299 BuildFnTy &MatchInfo) const {
7300 // TODO: Handle integer cases.
7301 assert(MI.getOpcode() == TargetOpcode::G_SELECT);
7302 // Condition may be fed by a truncated compare.
7303 Register Cond = MI.getOperand(i: 1).getReg();
7304 Register MaybeTrunc;
7305 if (mi_match(R: Cond, MRI, P: m_OneNonDBGUse(SP: m_GTrunc(Src: m_Reg(R&: MaybeTrunc)))))
7306 Cond = MaybeTrunc;
7307 Register Dst = MI.getOperand(i: 0).getReg();
7308 Register TrueVal = MI.getOperand(i: 2).getReg();
7309 Register FalseVal = MI.getOperand(i: 3).getReg();
7310 return matchFPSelectToMinMax(Dst, Cond, TrueVal, FalseVal, MatchInfo);
7311}
7312
7313bool CombinerHelper::matchRedundantBinOpInEquality(MachineInstr &MI,
7314 BuildFnTy &MatchInfo) const {
7315 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
7316 // (X + Y) == X --> Y == 0
7317 // (X + Y) != X --> Y != 0
7318 // (X - Y) == X --> Y == 0
7319 // (X - Y) != X --> Y != 0
7320 // (X ^ Y) == X --> Y == 0
7321 // (X ^ Y) != X --> Y != 0
7322 Register Dst = MI.getOperand(i: 0).getReg();
7323 CmpInst::Predicate Pred;
7324 Register X, Y, OpLHS, OpRHS;
7325 bool MatchedSub = mi_match(
7326 R: Dst, MRI,
7327 P: m_c_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: X), R: m_GSub(L: m_Reg(R&: OpLHS), R: m_Reg(R&: Y))));
7328 if (MatchedSub && X != OpLHS)
7329 return false;
7330 if (!MatchedSub) {
7331 if (!mi_match(R: Dst, MRI,
7332 P: m_c_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: X),
7333 R: m_any_of(preds: m_GAdd(L: m_Reg(R&: OpLHS), R: m_Reg(R&: OpRHS)),
7334 preds: m_GXor(L: m_Reg(R&: OpLHS), R: m_Reg(R&: OpRHS))))))
7335 return false;
7336 Y = X == OpLHS ? OpRHS : X == OpRHS ? OpLHS : Register();
7337 }
7338 MatchInfo = [=](MachineIRBuilder &B) {
7339 auto Zero = B.buildConstant(Res: MRI.getType(Reg: Y), Val: 0);
7340 B.buildICmp(Pred, Res: Dst, Op0: Y, Op1: Zero);
7341 };
7342 return CmpInst::isEquality(pred: Pred) && Y.isValid();
7343}
7344
7345/// Return the minimum useless shift amount that results in complete loss of the
7346/// source value. Return std::nullopt when it cannot determine a value.
7347static std::optional<unsigned>
7348getMinUselessShift(KnownBits ValueKB, unsigned Opcode,
7349 std::optional<int64_t> &Result) {
7350 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR ||
7351 Opcode == TargetOpcode::G_ASHR) &&
7352 "Expect G_SHL, G_LSHR or G_ASHR.");
7353 auto SignificantBits = 0;
7354 switch (Opcode) {
7355 case TargetOpcode::G_SHL:
7356 SignificantBits = ValueKB.countMinTrailingZeros();
7357 Result = 0;
7358 break;
7359 case TargetOpcode::G_LSHR:
7360 Result = 0;
7361 SignificantBits = ValueKB.countMinLeadingZeros();
7362 break;
7363 case TargetOpcode::G_ASHR:
7364 if (ValueKB.isNonNegative()) {
7365 SignificantBits = ValueKB.countMinLeadingZeros();
7366 Result = 0;
7367 } else if (ValueKB.isNegative()) {
7368 SignificantBits = ValueKB.countMinLeadingOnes();
7369 Result = -1;
7370 } else {
7371 // Cannot determine shift result.
7372 Result = std::nullopt;
7373 }
7374 break;
7375 default:
7376 break;
7377 }
7378 return ValueKB.getBitWidth() - SignificantBits;
7379}
7380
7381bool CombinerHelper::matchShiftsTooBig(
7382 MachineInstr &MI, std::optional<int64_t> &MatchInfo) const {
7383 Register ShiftVal = MI.getOperand(i: 1).getReg();
7384 Register ShiftReg = MI.getOperand(i: 2).getReg();
7385 LLT ResTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7386 auto IsShiftTooBig = [&](const Constant *C) {
7387 auto *CI = dyn_cast<ConstantInt>(Val: C);
7388 if (!CI)
7389 return false;
7390 if (CI->uge(Num: ResTy.getScalarSizeInBits())) {
7391 MatchInfo = std::nullopt;
7392 return true;
7393 }
7394 auto OptMaxUsefulShift = getMinUselessShift(ValueKB: VT->getKnownBits(R: ShiftVal),
7395 Opcode: MI.getOpcode(), Result&: MatchInfo);
7396 return OptMaxUsefulShift && CI->uge(Num: *OptMaxUsefulShift);
7397 };
7398 return matchUnaryPredicate(MRI, Reg: ShiftReg, Match: IsShiftTooBig);
7399}
7400
7401bool CombinerHelper::matchCommuteConstantToRHS(MachineInstr &MI) const {
7402 unsigned LHSOpndIdx = 1;
7403 unsigned RHSOpndIdx = 2;
7404 switch (MI.getOpcode()) {
7405 case TargetOpcode::G_UADDO:
7406 case TargetOpcode::G_SADDO:
7407 case TargetOpcode::G_UMULO:
7408 case TargetOpcode::G_SMULO:
7409 LHSOpndIdx = 2;
7410 RHSOpndIdx = 3;
7411 break;
7412 default:
7413 break;
7414 }
7415 Register LHS = MI.getOperand(i: LHSOpndIdx).getReg();
7416 Register RHS = MI.getOperand(i: RHSOpndIdx).getReg();
7417 if (!getIConstantVRegVal(VReg: LHS, MRI)) {
7418 // Skip commuting if LHS is not a constant. But, LHS may be a
7419 // G_CONSTANT_FOLD_BARRIER. If so we commute as long as we don't already
7420 // have a constant on the RHS.
7421 if (MRI.getVRegDef(Reg: LHS)->getOpcode() !=
7422 TargetOpcode::G_CONSTANT_FOLD_BARRIER)
7423 return false;
7424 }
7425 // Commute as long as RHS is not a constant or G_CONSTANT_FOLD_BARRIER.
7426 return MRI.getVRegDef(Reg: RHS)->getOpcode() !=
7427 TargetOpcode::G_CONSTANT_FOLD_BARRIER &&
7428 !getIConstantVRegVal(VReg: RHS, MRI);
7429}
7430
7431bool CombinerHelper::matchCommuteFPConstantToRHS(MachineInstr &MI) const {
7432 Register LHS = MI.getOperand(i: 1).getReg();
7433 Register RHS = MI.getOperand(i: 2).getReg();
7434 std::optional<FPValueAndVReg> ValAndVReg;
7435 if (!mi_match(R: LHS, MRI, P: m_GFCstOrSplat(FPValReg&: ValAndVReg)))
7436 return false;
7437 return !mi_match(R: RHS, MRI, P: m_GFCstOrSplat(FPValReg&: ValAndVReg));
7438}
7439
7440void CombinerHelper::applyCommuteBinOpOperands(MachineInstr &MI) const {
7441 Observer.changingInstr(MI);
7442 unsigned LHSOpndIdx = 1;
7443 unsigned RHSOpndIdx = 2;
7444 switch (MI.getOpcode()) {
7445 case TargetOpcode::G_UADDO:
7446 case TargetOpcode::G_SADDO:
7447 case TargetOpcode::G_UMULO:
7448 case TargetOpcode::G_SMULO:
7449 LHSOpndIdx = 2;
7450 RHSOpndIdx = 3;
7451 break;
7452 default:
7453 break;
7454 }
7455 Register LHSReg = MI.getOperand(i: LHSOpndIdx).getReg();
7456 Register RHSReg = MI.getOperand(i: RHSOpndIdx).getReg();
7457 MI.getOperand(i: LHSOpndIdx).setReg(RHSReg);
7458 MI.getOperand(i: RHSOpndIdx).setReg(LHSReg);
7459 Observer.changedInstr(MI);
7460}
7461
7462bool CombinerHelper::isOneOrOneSplat(Register Src, bool AllowUndefs) const {
7463 LLT SrcTy = MRI.getType(Reg: Src);
7464 if (SrcTy.isFixedVector())
7465 return isConstantSplatVector(Src, SplatValue: 1, AllowUndefs);
7466 if (SrcTy.isScalar()) {
7467 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Reg: Src, MRI) != nullptr)
7468 return true;
7469 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7470 return IConstant && IConstant->Value == 1;
7471 }
7472 return false; // scalable vector
7473}
7474
7475bool CombinerHelper::isZeroOrZeroSplat(Register Src, bool AllowUndefs) const {
7476 LLT SrcTy = MRI.getType(Reg: Src);
7477 if (SrcTy.isFixedVector())
7478 return isConstantSplatVector(Src, SplatValue: 0, AllowUndefs);
7479 if (SrcTy.isScalar()) {
7480 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Reg: Src, MRI) != nullptr)
7481 return true;
7482 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7483 return IConstant && IConstant->Value == 0;
7484 }
7485 return false; // scalable vector
7486}
7487
7488// Ignores COPYs during conformance checks.
7489// FIXME scalable vectors.
7490bool CombinerHelper::isConstantSplatVector(Register Src, int64_t SplatValue,
7491 bool AllowUndefs) const {
7492 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
7493 if (!BuildVector)
7494 return false;
7495 unsigned NumSources = BuildVector->getNumSources();
7496
7497 for (unsigned I = 0; I < NumSources; ++I) {
7498 GImplicitDef *ImplicitDef =
7499 getOpcodeDef<GImplicitDef>(Reg: BuildVector->getSourceReg(I), MRI);
7500 if (ImplicitDef && AllowUndefs)
7501 continue;
7502 if (ImplicitDef && !AllowUndefs)
7503 return false;
7504 std::optional<ValueAndVReg> IConstant =
7505 getIConstantVRegValWithLookThrough(VReg: BuildVector->getSourceReg(I), MRI);
7506 if (IConstant && IConstant->Value == SplatValue)
7507 continue;
7508 return false;
7509 }
7510 return true;
7511}
7512
7513// Ignores COPYs during lookups.
7514// FIXME scalable vectors
7515std::optional<APInt>
7516CombinerHelper::getConstantOrConstantSplatVector(Register Src) const {
7517 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7518 if (IConstant)
7519 return IConstant->Value;
7520
7521 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
7522 if (!BuildVector)
7523 return std::nullopt;
7524 unsigned NumSources = BuildVector->getNumSources();
7525
7526 std::optional<APInt> Value = std::nullopt;
7527 for (unsigned I = 0; I < NumSources; ++I) {
7528 std::optional<ValueAndVReg> IConstant =
7529 getIConstantVRegValWithLookThrough(VReg: BuildVector->getSourceReg(I), MRI);
7530 if (!IConstant)
7531 return std::nullopt;
7532 if (!Value)
7533 Value = IConstant->Value;
7534 else if (*Value != IConstant->Value)
7535 return std::nullopt;
7536 }
7537 return Value;
7538}
7539
7540// FIXME G_SPLAT_VECTOR
7541bool CombinerHelper::isConstantOrConstantVectorI(Register Src) const {
7542 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7543 if (IConstant)
7544 return true;
7545
7546 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
7547 if (!BuildVector)
7548 return false;
7549
7550 unsigned NumSources = BuildVector->getNumSources();
7551 for (unsigned I = 0; I < NumSources; ++I) {
7552 std::optional<ValueAndVReg> IConstant =
7553 getIConstantVRegValWithLookThrough(VReg: BuildVector->getSourceReg(I), MRI);
7554 if (!IConstant)
7555 return false;
7556 }
7557 return true;
7558}
7559
7560// TODO: use knownbits to determine zeros
7561bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select,
7562 BuildFnTy &MatchInfo) const {
7563 uint32_t Flags = Select->getFlags();
7564 Register Dest = Select->getReg(Idx: 0);
7565 Register Cond = Select->getCondReg();
7566 Register True = Select->getTrueReg();
7567 Register False = Select->getFalseReg();
7568 LLT CondTy = MRI.getType(Reg: Select->getCondReg());
7569 LLT TrueTy = MRI.getType(Reg: Select->getTrueReg());
7570
7571 // We only do this combine for scalar boolean conditions.
7572 if (CondTy != LLT::scalar(SizeInBits: 1))
7573 return false;
7574
7575 if (TrueTy.isPointer())
7576 return false;
7577
7578 // Both are scalars.
7579 std::optional<ValueAndVReg> TrueOpt =
7580 getIConstantVRegValWithLookThrough(VReg: True, MRI);
7581 std::optional<ValueAndVReg> FalseOpt =
7582 getIConstantVRegValWithLookThrough(VReg: False, MRI);
7583
7584 if (!TrueOpt || !FalseOpt)
7585 return false;
7586
7587 APInt TrueValue = TrueOpt->Value;
7588 APInt FalseValue = FalseOpt->Value;
7589
7590 // select Cond, 1, 0 --> zext (Cond)
7591 if (TrueValue.isOne() && FalseValue.isZero()) {
7592 MatchInfo = [=](MachineIRBuilder &B) {
7593 B.setInstrAndDebugLoc(*Select);
7594 B.buildZExtOrTrunc(Res: Dest, Op: Cond);
7595 };
7596 return true;
7597 }
7598
7599 // select Cond, -1, 0 --> sext (Cond)
7600 if (TrueValue.isAllOnes() && FalseValue.isZero()) {
7601 MatchInfo = [=](MachineIRBuilder &B) {
7602 B.setInstrAndDebugLoc(*Select);
7603 B.buildSExtOrTrunc(Res: Dest, Op: Cond);
7604 };
7605 return true;
7606 }
7607
7608 // select Cond, 0, 1 --> zext (!Cond)
7609 if (TrueValue.isZero() && FalseValue.isOne()) {
7610 MatchInfo = [=](MachineIRBuilder &B) {
7611 B.setInstrAndDebugLoc(*Select);
7612 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7613 B.buildNot(Dst: Inner, Src0: Cond);
7614 B.buildZExtOrTrunc(Res: Dest, Op: Inner);
7615 };
7616 return true;
7617 }
7618
7619 // select Cond, 0, -1 --> sext (!Cond)
7620 if (TrueValue.isZero() && FalseValue.isAllOnes()) {
7621 MatchInfo = [=](MachineIRBuilder &B) {
7622 B.setInstrAndDebugLoc(*Select);
7623 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7624 B.buildNot(Dst: Inner, Src0: Cond);
7625 B.buildSExtOrTrunc(Res: Dest, Op: Inner);
7626 };
7627 return true;
7628 }
7629
7630 // select Cond, C1, C1-1 --> add (zext Cond), C1-1
7631 if (TrueValue - 1 == FalseValue) {
7632 MatchInfo = [=](MachineIRBuilder &B) {
7633 B.setInstrAndDebugLoc(*Select);
7634 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7635 B.buildZExtOrTrunc(Res: Inner, Op: Cond);
7636 B.buildAdd(Dst: Dest, Src0: Inner, Src1: False);
7637 };
7638 return true;
7639 }
7640
7641 // select Cond, C1, C1+1 --> add (sext Cond), C1+1
7642 if (TrueValue + 1 == FalseValue) {
7643 MatchInfo = [=](MachineIRBuilder &B) {
7644 B.setInstrAndDebugLoc(*Select);
7645 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7646 B.buildSExtOrTrunc(Res: Inner, Op: Cond);
7647 B.buildAdd(Dst: Dest, Src0: Inner, Src1: False);
7648 };
7649 return true;
7650 }
7651
7652 // select Cond, Pow2, 0 --> (zext Cond) << log2(Pow2)
7653 if (TrueValue.isPowerOf2() && FalseValue.isZero()) {
7654 MatchInfo = [=](MachineIRBuilder &B) {
7655 B.setInstrAndDebugLoc(*Select);
7656 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7657 B.buildZExtOrTrunc(Res: Inner, Op: Cond);
7658 // The shift amount must be scalar.
7659 LLT ShiftTy = TrueTy.isVector() ? TrueTy.getElementType() : TrueTy;
7660 auto ShAmtC = B.buildConstant(Res: ShiftTy, Val: TrueValue.exactLogBase2());
7661 B.buildShl(Dst: Dest, Src0: Inner, Src1: ShAmtC, Flags);
7662 };
7663 return true;
7664 }
7665
7666 // select Cond, 0, Pow2 --> (zext (!Cond)) << log2(Pow2)
7667 if (FalseValue.isPowerOf2() && TrueValue.isZero()) {
7668 MatchInfo = [=](MachineIRBuilder &B) {
7669 B.setInstrAndDebugLoc(*Select);
7670 Register Not = MRI.createGenericVirtualRegister(Ty: CondTy);
7671 B.buildNot(Dst: Not, Src0: Cond);
7672 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7673 B.buildZExtOrTrunc(Res: Inner, Op: Not);
7674 // The shift amount must be scalar.
7675 LLT ShiftTy = TrueTy.isVector() ? TrueTy.getElementType() : TrueTy;
7676 auto ShAmtC = B.buildConstant(Res: ShiftTy, Val: FalseValue.exactLogBase2());
7677 B.buildShl(Dst: Dest, Src0: Inner, Src1: ShAmtC, Flags);
7678 };
7679 return true;
7680 }
7681
7682 // select Cond, -1, C --> or (sext Cond), C
7683 if (TrueValue.isAllOnes()) {
7684 MatchInfo = [=](MachineIRBuilder &B) {
7685 B.setInstrAndDebugLoc(*Select);
7686 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7687 B.buildSExtOrTrunc(Res: Inner, Op: Cond);
7688 B.buildOr(Dst: Dest, Src0: Inner, Src1: False, Flags);
7689 };
7690 return true;
7691 }
7692
7693 // select Cond, C, -1 --> or (sext (not Cond)), C
7694 if (FalseValue.isAllOnes()) {
7695 MatchInfo = [=](MachineIRBuilder &B) {
7696 B.setInstrAndDebugLoc(*Select);
7697 Register Not = MRI.createGenericVirtualRegister(Ty: CondTy);
7698 B.buildNot(Dst: Not, Src0: Cond);
7699 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7700 B.buildSExtOrTrunc(Res: Inner, Op: Not);
7701 B.buildOr(Dst: Dest, Src0: Inner, Src1: True, Flags);
7702 };
7703 return true;
7704 }
7705
7706 return false;
7707}
7708
7709// TODO: use knownbits to determine zeros
7710bool CombinerHelper::tryFoldBoolSelectToLogic(GSelect *Select,
7711 BuildFnTy &MatchInfo) const {
7712 uint32_t Flags = Select->getFlags();
7713 Register DstReg = Select->getReg(Idx: 0);
7714 Register Cond = Select->getCondReg();
7715 Register True = Select->getTrueReg();
7716 Register False = Select->getFalseReg();
7717 LLT CondTy = MRI.getType(Reg: Select->getCondReg());
7718 LLT TrueTy = MRI.getType(Reg: Select->getTrueReg());
7719
7720 // Boolean or fixed vector of booleans.
7721 if (CondTy.isScalableVector() ||
7722 (CondTy.isFixedVector() &&
7723 CondTy.getElementType().getScalarSizeInBits() != 1) ||
7724 CondTy.getScalarSizeInBits() != 1)
7725 return false;
7726
7727 if (CondTy != TrueTy)
7728 return false;
7729
7730 // select Cond, Cond, F --> or Cond, F
7731 // select Cond, 1, F --> or Cond, F
7732 if ((Cond == True) || isOneOrOneSplat(Src: True, /* AllowUndefs */ true)) {
7733 MatchInfo = [=](MachineIRBuilder &B) {
7734 B.setInstrAndDebugLoc(*Select);
7735 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7736 B.buildZExtOrTrunc(Res: Ext, Op: Cond);
7737 auto FreezeFalse = B.buildFreeze(Dst: TrueTy, Src: False);
7738 B.buildOr(Dst: DstReg, Src0: Ext, Src1: FreezeFalse, Flags);
7739 };
7740 return true;
7741 }
7742
7743 // select Cond, T, Cond --> and Cond, T
7744 // select Cond, T, 0 --> and Cond, T
7745 if ((Cond == False) || isZeroOrZeroSplat(Src: False, /* AllowUndefs */ true)) {
7746 MatchInfo = [=](MachineIRBuilder &B) {
7747 B.setInstrAndDebugLoc(*Select);
7748 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7749 B.buildZExtOrTrunc(Res: Ext, Op: Cond);
7750 auto FreezeTrue = B.buildFreeze(Dst: TrueTy, Src: True);
7751 B.buildAnd(Dst: DstReg, Src0: Ext, Src1: FreezeTrue);
7752 };
7753 return true;
7754 }
7755
7756 // select Cond, T, 1 --> or (not Cond), T
7757 if (isOneOrOneSplat(Src: False, /* AllowUndefs */ true)) {
7758 MatchInfo = [=](MachineIRBuilder &B) {
7759 B.setInstrAndDebugLoc(*Select);
7760 // First the not.
7761 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7762 B.buildNot(Dst: Inner, Src0: Cond);
7763 // Then an ext to match the destination register.
7764 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7765 B.buildZExtOrTrunc(Res: Ext, Op: Inner);
7766 auto FreezeTrue = B.buildFreeze(Dst: TrueTy, Src: True);
7767 B.buildOr(Dst: DstReg, Src0: Ext, Src1: FreezeTrue, Flags);
7768 };
7769 return true;
7770 }
7771
7772 // select Cond, 0, F --> and (not Cond), F
7773 if (isZeroOrZeroSplat(Src: True, /* AllowUndefs */ true)) {
7774 MatchInfo = [=](MachineIRBuilder &B) {
7775 B.setInstrAndDebugLoc(*Select);
7776 // First the not.
7777 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7778 B.buildNot(Dst: Inner, Src0: Cond);
7779 // Then an ext to match the destination register.
7780 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7781 B.buildZExtOrTrunc(Res: Ext, Op: Inner);
7782 auto FreezeFalse = B.buildFreeze(Dst: TrueTy, Src: False);
7783 B.buildAnd(Dst: DstReg, Src0: Ext, Src1: FreezeFalse);
7784 };
7785 return true;
7786 }
7787
7788 return false;
7789}
7790
7791bool CombinerHelper::matchSelectIMinMax(const MachineOperand &MO,
7792 BuildFnTy &MatchInfo) const {
7793 GSelect *Select = cast<GSelect>(Val: MRI.getVRegDef(Reg: MO.getReg()));
7794 GICmp *Cmp = cast<GICmp>(Val: MRI.getVRegDef(Reg: Select->getCondReg()));
7795
7796 Register DstReg = Select->getReg(Idx: 0);
7797 Register True = Select->getTrueReg();
7798 Register False = Select->getFalseReg();
7799 LLT DstTy = MRI.getType(Reg: DstReg);
7800
7801 if (DstTy.isPointerOrPointerVector())
7802 return false;
7803
7804 // We want to fold the icmp and replace the select.
7805 if (!MRI.hasOneNonDBGUse(RegNo: Cmp->getReg(Idx: 0)))
7806 return false;
7807
7808 CmpInst::Predicate Pred = Cmp->getCond();
7809 // We need a larger or smaller predicate for
7810 // canonicalization.
7811 if (CmpInst::isEquality(pred: Pred))
7812 return false;
7813
7814 Register CmpLHS = Cmp->getLHSReg();
7815 Register CmpRHS = Cmp->getRHSReg();
7816
7817 // We can swap CmpLHS and CmpRHS for higher hitrate.
7818 if (True == CmpRHS && False == CmpLHS) {
7819 std::swap(a&: CmpLHS, b&: CmpRHS);
7820 Pred = CmpInst::getSwappedPredicate(pred: Pred);
7821 }
7822
7823 // (icmp X, Y) ? X : Y -> integer minmax.
7824 // see matchSelectPattern in ValueTracking.
7825 // Legality between G_SELECT and integer minmax can differ.
7826 if (True != CmpLHS || False != CmpRHS)
7827 return false;
7828
7829 switch (Pred) {
7830 case ICmpInst::ICMP_UGT:
7831 case ICmpInst::ICMP_UGE: {
7832 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_UMAX, DstTy}))
7833 return false;
7834 MatchInfo = [=](MachineIRBuilder &B) { B.buildUMax(Dst: DstReg, Src0: True, Src1: False); };
7835 return true;
7836 }
7837 case ICmpInst::ICMP_SGT:
7838 case ICmpInst::ICMP_SGE: {
7839 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SMAX, DstTy}))
7840 return false;
7841 MatchInfo = [=](MachineIRBuilder &B) { B.buildSMax(Dst: DstReg, Src0: True, Src1: False); };
7842 return true;
7843 }
7844 case ICmpInst::ICMP_ULT:
7845 case ICmpInst::ICMP_ULE: {
7846 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_UMIN, DstTy}))
7847 return false;
7848 MatchInfo = [=](MachineIRBuilder &B) { B.buildUMin(Dst: DstReg, Src0: True, Src1: False); };
7849 return true;
7850 }
7851 case ICmpInst::ICMP_SLT:
7852 case ICmpInst::ICMP_SLE: {
7853 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SMIN, DstTy}))
7854 return false;
7855 MatchInfo = [=](MachineIRBuilder &B) { B.buildSMin(Dst: DstReg, Src0: True, Src1: False); };
7856 return true;
7857 }
7858 default:
7859 return false;
7860 }
7861}
7862
7863// (neg (min/max x, (neg x))) --> (max/min x, (neg x))
7864bool CombinerHelper::matchSimplifyNegMinMax(MachineInstr &MI,
7865 BuildFnTy &MatchInfo) const {
7866 assert(MI.getOpcode() == TargetOpcode::G_SUB);
7867 Register DestReg = MI.getOperand(i: 0).getReg();
7868 LLT DestTy = MRI.getType(Reg: DestReg);
7869
7870 Register X;
7871 Register Sub0;
7872 auto NegPattern = m_all_of(preds: m_Neg(Src: m_DeferredReg(R&: X)), preds: m_Reg(R&: Sub0));
7873 if (mi_match(R: DestReg, MRI,
7874 P: m_Neg(Src: m_OneUse(SP: m_any_of(preds: m_GSMin(L: m_Reg(R&: X), R: NegPattern),
7875 preds: m_GSMax(L: m_Reg(R&: X), R: NegPattern),
7876 preds: m_GUMin(L: m_Reg(R&: X), R: NegPattern),
7877 preds: m_GUMax(L: m_Reg(R&: X), R: NegPattern)))))) {
7878 MachineInstr *MinMaxMI = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
7879 unsigned NewOpc = getInverseGMinMaxOpcode(MinMaxOpc: MinMaxMI->getOpcode());
7880 if (isLegal(Query: {NewOpc, {DestTy}})) {
7881 MatchInfo = [=](MachineIRBuilder &B) {
7882 B.buildInstr(Opc: NewOpc, DstOps: {DestReg}, SrcOps: {X, Sub0});
7883 };
7884 return true;
7885 }
7886 }
7887
7888 return false;
7889}
7890
7891bool CombinerHelper::matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo) const {
7892 GSelect *Select = cast<GSelect>(Val: &MI);
7893
7894 if (tryFoldSelectOfConstants(Select, MatchInfo))
7895 return true;
7896
7897 if (tryFoldBoolSelectToLogic(Select, MatchInfo))
7898 return true;
7899
7900 return false;
7901}
7902
7903/// Fold (icmp Pred1 V1, C1) && (icmp Pred2 V2, C2)
7904/// or (icmp Pred1 V1, C1) || (icmp Pred2 V2, C2)
7905/// into a single comparison using range-based reasoning.
7906/// see InstCombinerImpl::foldAndOrOfICmpsUsingRanges.
7907bool CombinerHelper::tryFoldAndOrOrICmpsUsingRanges(
7908 GLogicalBinOp *Logic, BuildFnTy &MatchInfo) const {
7909 assert(Logic->getOpcode() != TargetOpcode::G_XOR && "unexpected xor");
7910 bool IsAnd = Logic->getOpcode() == TargetOpcode::G_AND;
7911 Register DstReg = Logic->getReg(Idx: 0);
7912 Register LHS = Logic->getLHSReg();
7913 Register RHS = Logic->getRHSReg();
7914 unsigned Flags = Logic->getFlags();
7915
7916 // We need an G_ICMP on the LHS register.
7917 GICmp *Cmp1 = getOpcodeDef<GICmp>(Reg: LHS, MRI);
7918 if (!Cmp1)
7919 return false;
7920
7921 // We need an G_ICMP on the RHS register.
7922 GICmp *Cmp2 = getOpcodeDef<GICmp>(Reg: RHS, MRI);
7923 if (!Cmp2)
7924 return false;
7925
7926 // We want to fold the icmps.
7927 if (!MRI.hasOneNonDBGUse(RegNo: Cmp1->getReg(Idx: 0)) ||
7928 !MRI.hasOneNonDBGUse(RegNo: Cmp2->getReg(Idx: 0)))
7929 return false;
7930
7931 APInt C1;
7932 APInt C2;
7933 std::optional<ValueAndVReg> MaybeC1 =
7934 getIConstantVRegValWithLookThrough(VReg: Cmp1->getRHSReg(), MRI);
7935 if (!MaybeC1)
7936 return false;
7937 C1 = MaybeC1->Value;
7938
7939 std::optional<ValueAndVReg> MaybeC2 =
7940 getIConstantVRegValWithLookThrough(VReg: Cmp2->getRHSReg(), MRI);
7941 if (!MaybeC2)
7942 return false;
7943 C2 = MaybeC2->Value;
7944
7945 Register R1 = Cmp1->getLHSReg();
7946 Register R2 = Cmp2->getLHSReg();
7947 CmpInst::Predicate Pred1 = Cmp1->getCond();
7948 CmpInst::Predicate Pred2 = Cmp2->getCond();
7949 LLT CmpTy = MRI.getType(Reg: Cmp1->getReg(Idx: 0));
7950 LLT CmpOperandTy = MRI.getType(Reg: R1);
7951
7952 if (CmpOperandTy.isPointer())
7953 return false;
7954
7955 // We build ands, adds, and constants of type CmpOperandTy.
7956 // They must be legal to build.
7957 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_AND, CmpOperandTy}) ||
7958 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ADD, CmpOperandTy}) ||
7959 !isConstantLegalOrBeforeLegalizer(Ty: CmpOperandTy))
7960 return false;
7961
7962 // Look through add of a constant offset on R1, R2, or both operands. This
7963 // allows us to interpret the R + C' < C'' range idiom into a proper range.
7964 std::optional<APInt> Offset1;
7965 std::optional<APInt> Offset2;
7966 if (R1 != R2) {
7967 if (GAdd *Add = getOpcodeDef<GAdd>(Reg: R1, MRI)) {
7968 std::optional<ValueAndVReg> MaybeOffset1 =
7969 getIConstantVRegValWithLookThrough(VReg: Add->getRHSReg(), MRI);
7970 if (MaybeOffset1) {
7971 R1 = Add->getLHSReg();
7972 Offset1 = MaybeOffset1->Value;
7973 }
7974 }
7975 if (GAdd *Add = getOpcodeDef<GAdd>(Reg: R2, MRI)) {
7976 std::optional<ValueAndVReg> MaybeOffset2 =
7977 getIConstantVRegValWithLookThrough(VReg: Add->getRHSReg(), MRI);
7978 if (MaybeOffset2) {
7979 R2 = Add->getLHSReg();
7980 Offset2 = MaybeOffset2->Value;
7981 }
7982 }
7983 }
7984
7985 if (R1 != R2)
7986 return false;
7987
7988 // We calculate the icmp ranges including maybe offsets.
7989 ConstantRange CR1 = ConstantRange::makeExactICmpRegion(
7990 Pred: IsAnd ? ICmpInst::getInversePredicate(pred: Pred1) : Pred1, Other: C1);
7991 if (Offset1)
7992 CR1 = CR1.subtract(CI: *Offset1);
7993
7994 ConstantRange CR2 = ConstantRange::makeExactICmpRegion(
7995 Pred: IsAnd ? ICmpInst::getInversePredicate(pred: Pred2) : Pred2, Other: C2);
7996 if (Offset2)
7997 CR2 = CR2.subtract(CI: *Offset2);
7998
7999 bool CreateMask = false;
8000 APInt LowerDiff;
8001 std::optional<ConstantRange> CR = CR1.exactUnionWith(CR: CR2);
8002 if (!CR) {
8003 // We need non-wrapping ranges.
8004 if (CR1.isWrappedSet() || CR2.isWrappedSet())
8005 return false;
8006
8007 // Check whether we have equal-size ranges that only differ by one bit.
8008 // In that case we can apply a mask to map one range onto the other.
8009 LowerDiff = CR1.getLower() ^ CR2.getLower();
8010 APInt UpperDiff = (CR1.getUpper() - 1) ^ (CR2.getUpper() - 1);
8011 APInt CR1Size = CR1.getUpper() - CR1.getLower();
8012 if (!LowerDiff.isPowerOf2() || LowerDiff != UpperDiff ||
8013 CR1Size != CR2.getUpper() - CR2.getLower())
8014 return false;
8015
8016 CR = CR1.getLower().ult(RHS: CR2.getLower()) ? CR1 : CR2;
8017 CreateMask = true;
8018 }
8019
8020 if (IsAnd)
8021 CR = CR->inverse();
8022
8023 CmpInst::Predicate NewPred;
8024 APInt NewC, Offset;
8025 CR->getEquivalentICmp(Pred&: NewPred, RHS&: NewC, Offset);
8026
8027 // We take the result type of one of the original icmps, CmpTy, for
8028 // the to be build icmp. The operand type, CmpOperandTy, is used for
8029 // the other instructions and constants to be build. The types of
8030 // the parameters and output are the same for add and and. CmpTy
8031 // and the type of DstReg might differ. That is why we zext or trunc
8032 // the icmp into the destination register.
8033
8034 MatchInfo = [=](MachineIRBuilder &B) {
8035 if (CreateMask && Offset != 0) {
8036 auto TildeLowerDiff = B.buildConstant(Res: CmpOperandTy, Val: ~LowerDiff);
8037 auto And = B.buildAnd(Dst: CmpOperandTy, Src0: R1, Src1: TildeLowerDiff); // the mask.
8038 auto OffsetC = B.buildConstant(Res: CmpOperandTy, Val: Offset);
8039 auto Add = B.buildAdd(Dst: CmpOperandTy, Src0: And, Src1: OffsetC, Flags);
8040 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
8041 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: Add, Op1: NewCon);
8042 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
8043 } else if (CreateMask && Offset == 0) {
8044 auto TildeLowerDiff = B.buildConstant(Res: CmpOperandTy, Val: ~LowerDiff);
8045 auto And = B.buildAnd(Dst: CmpOperandTy, Src0: R1, Src1: TildeLowerDiff); // the mask.
8046 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
8047 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: And, Op1: NewCon);
8048 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
8049 } else if (!CreateMask && Offset != 0) {
8050 auto OffsetC = B.buildConstant(Res: CmpOperandTy, Val: Offset);
8051 auto Add = B.buildAdd(Dst: CmpOperandTy, Src0: R1, Src1: OffsetC, Flags);
8052 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
8053 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: Add, Op1: NewCon);
8054 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
8055 } else if (!CreateMask && Offset == 0) {
8056 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
8057 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: R1, Op1: NewCon);
8058 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
8059 } else {
8060 llvm_unreachable("unexpected configuration of CreateMask and Offset");
8061 }
8062 };
8063 return true;
8064}
8065
8066bool CombinerHelper::tryFoldLogicOfFCmps(GLogicalBinOp *Logic,
8067 BuildFnTy &MatchInfo) const {
8068 assert(Logic->getOpcode() != TargetOpcode::G_XOR && "unexpecte xor");
8069 Register DestReg = Logic->getReg(Idx: 0);
8070 Register LHS = Logic->getLHSReg();
8071 Register RHS = Logic->getRHSReg();
8072 bool IsAnd = Logic->getOpcode() == TargetOpcode::G_AND;
8073
8074 // We need a compare on the LHS register.
8075 GFCmp *Cmp1 = getOpcodeDef<GFCmp>(Reg: LHS, MRI);
8076 if (!Cmp1)
8077 return false;
8078
8079 // We need a compare on the RHS register.
8080 GFCmp *Cmp2 = getOpcodeDef<GFCmp>(Reg: RHS, MRI);
8081 if (!Cmp2)
8082 return false;
8083
8084 LLT CmpTy = MRI.getType(Reg: Cmp1->getReg(Idx: 0));
8085 LLT CmpOperandTy = MRI.getType(Reg: Cmp1->getLHSReg());
8086
8087 // We build one fcmp, want to fold the fcmps, replace the logic op,
8088 // and the fcmps must have the same shape.
8089 if (!isLegalOrBeforeLegalizer(
8090 Query: {TargetOpcode::G_FCMP, {CmpTy, CmpOperandTy}}) ||
8091 !MRI.hasOneNonDBGUse(RegNo: Logic->getReg(Idx: 0)) ||
8092 !MRI.hasOneNonDBGUse(RegNo: Cmp1->getReg(Idx: 0)) ||
8093 !MRI.hasOneNonDBGUse(RegNo: Cmp2->getReg(Idx: 0)) ||
8094 MRI.getType(Reg: Cmp1->getLHSReg()) != MRI.getType(Reg: Cmp2->getLHSReg()))
8095 return false;
8096
8097 CmpInst::Predicate PredL = Cmp1->getCond();
8098 CmpInst::Predicate PredR = Cmp2->getCond();
8099 Register LHS0 = Cmp1->getLHSReg();
8100 Register LHS1 = Cmp1->getRHSReg();
8101 Register RHS0 = Cmp2->getLHSReg();
8102 Register RHS1 = Cmp2->getRHSReg();
8103
8104 if (LHS0 == RHS1 && LHS1 == RHS0) {
8105 // Swap RHS operands to match LHS.
8106 PredR = CmpInst::getSwappedPredicate(pred: PredR);
8107 std::swap(a&: RHS0, b&: RHS1);
8108 }
8109
8110 if (LHS0 == RHS0 && LHS1 == RHS1) {
8111 // We determine the new predicate.
8112 unsigned CmpCodeL = getFCmpCode(CC: PredL);
8113 unsigned CmpCodeR = getFCmpCode(CC: PredR);
8114 unsigned NewPred = IsAnd ? CmpCodeL & CmpCodeR : CmpCodeL | CmpCodeR;
8115 unsigned Flags = Cmp1->getFlags() | Cmp2->getFlags();
8116 MatchInfo = [=](MachineIRBuilder &B) {
8117 // The fcmp predicates fill the lower part of the enum.
8118 FCmpInst::Predicate Pred = static_cast<FCmpInst::Predicate>(NewPred);
8119 if (Pred == FCmpInst::FCMP_FALSE &&
8120 isConstantLegalOrBeforeLegalizer(Ty: CmpTy)) {
8121 auto False = B.buildConstant(Res: CmpTy, Val: 0);
8122 B.buildZExtOrTrunc(Res: DestReg, Op: False);
8123 } else if (Pred == FCmpInst::FCMP_TRUE &&
8124 isConstantLegalOrBeforeLegalizer(Ty: CmpTy)) {
8125 auto True =
8126 B.buildConstant(Res: CmpTy, Val: getICmpTrueVal(TLI: getTargetLowering(),
8127 IsVector: CmpTy.isVector() /*isVector*/,
8128 IsFP: true /*isFP*/));
8129 B.buildZExtOrTrunc(Res: DestReg, Op: True);
8130 } else { // We take the predicate without predicate optimizations.
8131 auto Cmp = B.buildFCmp(Pred, Res: CmpTy, Op0: LHS0, Op1: LHS1, Flags);
8132 B.buildZExtOrTrunc(Res: DestReg, Op: Cmp);
8133 }
8134 };
8135 return true;
8136 }
8137
8138 return false;
8139}
8140
8141bool CombinerHelper::matchAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const {
8142 GAnd *And = cast<GAnd>(Val: &MI);
8143
8144 if (tryFoldAndOrOrICmpsUsingRanges(Logic: And, MatchInfo))
8145 return true;
8146
8147 if (tryFoldLogicOfFCmps(Logic: And, MatchInfo))
8148 return true;
8149
8150 return false;
8151}
8152
8153bool CombinerHelper::matchOr(MachineInstr &MI, BuildFnTy &MatchInfo) const {
8154 GOr *Or = cast<GOr>(Val: &MI);
8155
8156 if (tryFoldAndOrOrICmpsUsingRanges(Logic: Or, MatchInfo))
8157 return true;
8158
8159 if (tryFoldLogicOfFCmps(Logic: Or, MatchInfo))
8160 return true;
8161
8162 return false;
8163}
8164
8165bool CombinerHelper::matchAddOverflow(MachineInstr &MI,
8166 BuildFnTy &MatchInfo) const {
8167 GAddCarryOut *Add = cast<GAddCarryOut>(Val: &MI);
8168
8169 // Addo has no flags
8170 Register Dst = Add->getReg(Idx: 0);
8171 Register Carry = Add->getReg(Idx: 1);
8172 Register LHS = Add->getLHSReg();
8173 Register RHS = Add->getRHSReg();
8174 bool IsSigned = Add->isSigned();
8175 LLT DstTy = MRI.getType(Reg: Dst);
8176 LLT CarryTy = MRI.getType(Reg: Carry);
8177
8178 // Fold addo, if the carry is dead -> add, undef.
8179 if (MRI.use_nodbg_empty(RegNo: Carry) &&
8180 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ADD, {DstTy}})) {
8181 MatchInfo = [=](MachineIRBuilder &B) {
8182 B.buildAdd(Dst, Src0: LHS, Src1: RHS);
8183 B.buildUndef(Res: Carry);
8184 };
8185 return true;
8186 }
8187
8188 // Canonicalize constant to RHS.
8189 if (isConstantOrConstantVectorI(Src: LHS) && !isConstantOrConstantVectorI(Src: RHS)) {
8190 if (IsSigned) {
8191 MatchInfo = [=](MachineIRBuilder &B) {
8192 B.buildSAddo(Res: Dst, CarryOut: Carry, Op0: RHS, Op1: LHS);
8193 };
8194 return true;
8195 }
8196 // !IsSigned
8197 MatchInfo = [=](MachineIRBuilder &B) {
8198 B.buildUAddo(Res: Dst, CarryOut: Carry, Op0: RHS, Op1: LHS);
8199 };
8200 return true;
8201 }
8202
8203 std::optional<APInt> MaybeLHS = getConstantOrConstantSplatVector(Src: LHS);
8204 std::optional<APInt> MaybeRHS = getConstantOrConstantSplatVector(Src: RHS);
8205
8206 // Fold addo(c1, c2) -> c3, carry.
8207 if (MaybeLHS && MaybeRHS && isConstantLegalOrBeforeLegalizer(Ty: DstTy) &&
8208 isConstantLegalOrBeforeLegalizer(Ty: CarryTy)) {
8209 bool Overflow;
8210 APInt Result = IsSigned ? MaybeLHS->sadd_ov(RHS: *MaybeRHS, Overflow)
8211 : MaybeLHS->uadd_ov(RHS: *MaybeRHS, Overflow);
8212 MatchInfo = [=](MachineIRBuilder &B) {
8213 B.buildConstant(Res: Dst, Val: Result);
8214 B.buildConstant(Res: Carry, Val: Overflow);
8215 };
8216 return true;
8217 }
8218
8219 // Fold (addo x, 0) -> x, no carry
8220 if (MaybeRHS && *MaybeRHS == 0 && isConstantLegalOrBeforeLegalizer(Ty: CarryTy)) {
8221 MatchInfo = [=](MachineIRBuilder &B) {
8222 B.buildCopy(Res: Dst, Op: LHS);
8223 B.buildConstant(Res: Carry, Val: 0);
8224 };
8225 return true;
8226 }
8227
8228 // Given 2 constant operands whose sum does not overflow:
8229 // uaddo (X +nuw C0), C1 -> uaddo X, C0 + C1
8230 // saddo (X +nsw C0), C1 -> saddo X, C0 + C1
8231 GAdd *AddLHS = getOpcodeDef<GAdd>(Reg: LHS, MRI);
8232 if (MaybeRHS && AddLHS && MRI.hasOneNonDBGUse(RegNo: Add->getReg(Idx: 0)) &&
8233 ((IsSigned && AddLHS->getFlag(Flag: MachineInstr::MIFlag::NoSWrap)) ||
8234 (!IsSigned && AddLHS->getFlag(Flag: MachineInstr::MIFlag::NoUWrap)))) {
8235 std::optional<APInt> MaybeAddRHS =
8236 getConstantOrConstantSplatVector(Src: AddLHS->getRHSReg());
8237 if (MaybeAddRHS) {
8238 bool Overflow;
8239 APInt NewC = IsSigned ? MaybeAddRHS->sadd_ov(RHS: *MaybeRHS, Overflow)
8240 : MaybeAddRHS->uadd_ov(RHS: *MaybeRHS, Overflow);
8241 if (!Overflow && isConstantLegalOrBeforeLegalizer(Ty: DstTy)) {
8242 if (IsSigned) {
8243 MatchInfo = [=](MachineIRBuilder &B) {
8244 auto ConstRHS = B.buildConstant(Res: DstTy, Val: NewC);
8245 B.buildSAddo(Res: Dst, CarryOut: Carry, Op0: AddLHS->getLHSReg(), Op1: ConstRHS);
8246 };
8247 return true;
8248 }
8249 // !IsSigned
8250 MatchInfo = [=](MachineIRBuilder &B) {
8251 auto ConstRHS = B.buildConstant(Res: DstTy, Val: NewC);
8252 B.buildUAddo(Res: Dst, CarryOut: Carry, Op0: AddLHS->getLHSReg(), Op1: ConstRHS);
8253 };
8254 return true;
8255 }
8256 }
8257 };
8258
8259 // We try to combine addo to non-overflowing add.
8260 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ADD, {DstTy}}) ||
8261 !isConstantLegalOrBeforeLegalizer(Ty: CarryTy))
8262 return false;
8263
8264 // We try to combine uaddo to non-overflowing add.
8265 if (!IsSigned) {
8266 ConstantRange CRLHS =
8267 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: LHS), /*IsSigned=*/false);
8268 ConstantRange CRRHS =
8269 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: RHS), /*IsSigned=*/false);
8270
8271 switch (CRLHS.unsignedAddMayOverflow(Other: CRRHS)) {
8272 case ConstantRange::OverflowResult::MayOverflow:
8273 return false;
8274 case ConstantRange::OverflowResult::NeverOverflows: {
8275 MatchInfo = [=](MachineIRBuilder &B) {
8276 B.buildAdd(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoUWrap);
8277 B.buildConstant(Res: Carry, Val: 0);
8278 };
8279 return true;
8280 }
8281 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
8282 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
8283 MatchInfo = [=](MachineIRBuilder &B) {
8284 B.buildAdd(Dst, Src0: LHS, Src1: RHS);
8285 B.buildConstant(Res: Carry, Val: 1);
8286 };
8287 return true;
8288 }
8289 }
8290 return false;
8291 }
8292
8293 // We try to combine saddo to non-overflowing add.
8294
8295 // If LHS and RHS each have at least two sign bits, then there is no signed
8296 // overflow.
8297 if (VT->computeNumSignBits(R: RHS) > 1 && VT->computeNumSignBits(R: LHS) > 1) {
8298 MatchInfo = [=](MachineIRBuilder &B) {
8299 B.buildAdd(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoSWrap);
8300 B.buildConstant(Res: Carry, Val: 0);
8301 };
8302 return true;
8303 }
8304
8305 ConstantRange CRLHS =
8306 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: LHS), /*IsSigned=*/true);
8307 ConstantRange CRRHS =
8308 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: RHS), /*IsSigned=*/true);
8309
8310 switch (CRLHS.signedAddMayOverflow(Other: CRRHS)) {
8311 case ConstantRange::OverflowResult::MayOverflow:
8312 return false;
8313 case ConstantRange::OverflowResult::NeverOverflows: {
8314 MatchInfo = [=](MachineIRBuilder &B) {
8315 B.buildAdd(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoSWrap);
8316 B.buildConstant(Res: Carry, Val: 0);
8317 };
8318 return true;
8319 }
8320 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
8321 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
8322 MatchInfo = [=](MachineIRBuilder &B) {
8323 B.buildAdd(Dst, Src0: LHS, Src1: RHS);
8324 B.buildConstant(Res: Carry, Val: 1);
8325 };
8326 return true;
8327 }
8328 }
8329
8330 return false;
8331}
8332
8333void CombinerHelper::applyBuildFnMO(const MachineOperand &MO,
8334 BuildFnTy &MatchInfo) const {
8335 MachineInstr *Root = getDefIgnoringCopies(Reg: MO.getReg(), MRI);
8336 MatchInfo(Builder);
8337 Root->eraseFromParent();
8338}
8339
8340bool CombinerHelper::matchFPowIExpansion(MachineInstr &MI,
8341 int64_t Exponent) const {
8342 bool OptForSize = MI.getMF()->getFunction().hasOptSize();
8343 return getTargetLowering().isBeneficialToExpandPowI(Exponent, OptForSize);
8344}
8345
8346void CombinerHelper::applyExpandFPowI(MachineInstr &MI,
8347 int64_t Exponent) const {
8348 auto [Dst, Base] = MI.getFirst2Regs();
8349 LLT Ty = MRI.getType(Reg: Dst);
8350 int64_t ExpVal = Exponent;
8351
8352 if (ExpVal == 0) {
8353 Builder.buildFConstant(Res: Dst, Val: 1.0);
8354 MI.removeFromParent();
8355 return;
8356 }
8357
8358 if (ExpVal < 0)
8359 ExpVal = -ExpVal;
8360
8361 // We use the simple binary decomposition method from SelectionDAG ExpandPowI
8362 // to generate the multiply sequence. There are more optimal ways to do this
8363 // (for example, powi(x,15) generates one more multiply than it should), but
8364 // this has the benefit of being both really simple and much better than a
8365 // libcall.
8366 std::optional<SrcOp> Res;
8367 SrcOp CurSquare = Base;
8368 while (ExpVal > 0) {
8369 if (ExpVal & 1) {
8370 if (!Res)
8371 Res = CurSquare;
8372 else
8373 Res = Builder.buildFMul(Dst: Ty, Src0: *Res, Src1: CurSquare);
8374 }
8375
8376 CurSquare = Builder.buildFMul(Dst: Ty, Src0: CurSquare, Src1: CurSquare);
8377 ExpVal >>= 1;
8378 }
8379
8380 // If the original exponent was negative, invert the result, producing
8381 // 1/(x*x*x).
8382 if (Exponent < 0)
8383 Res = Builder.buildFDiv(Dst: Ty, Src0: Builder.buildFConstant(Res: Ty, Val: 1.0), Src1: *Res,
8384 Flags: MI.getFlags());
8385
8386 Builder.buildCopy(Res: Dst, Op: *Res);
8387 MI.eraseFromParent();
8388}
8389
8390bool CombinerHelper::matchFoldAPlusC1MinusC2(const MachineInstr &MI,
8391 BuildFnTy &MatchInfo) const {
8392 // fold (A+C1)-C2 -> A+(C1-C2)
8393 const GSub *Sub = cast<GSub>(Val: &MI);
8394 GAdd *Add = cast<GAdd>(Val: MRI.getVRegDef(Reg: Sub->getLHSReg()));
8395
8396 if (!MRI.hasOneNonDBGUse(RegNo: Add->getReg(Idx: 0)))
8397 return false;
8398
8399 APInt C2 = getIConstantFromReg(VReg: Sub->getRHSReg(), MRI);
8400 APInt C1 = getIConstantFromReg(VReg: Add->getRHSReg(), MRI);
8401
8402 Register Dst = Sub->getReg(Idx: 0);
8403 LLT DstTy = MRI.getType(Reg: Dst);
8404
8405 MatchInfo = [=](MachineIRBuilder &B) {
8406 auto Const = B.buildConstant(Res: DstTy, Val: C1 - C2);
8407 B.buildAdd(Dst, Src0: Add->getLHSReg(), Src1: Const);
8408 };
8409
8410 return true;
8411}
8412
8413bool CombinerHelper::matchFoldC2MinusAPlusC1(const MachineInstr &MI,
8414 BuildFnTy &MatchInfo) const {
8415 // fold C2-(A+C1) -> (C2-C1)-A
8416 const GSub *Sub = cast<GSub>(Val: &MI);
8417 GAdd *Add = cast<GAdd>(Val: MRI.getVRegDef(Reg: Sub->getRHSReg()));
8418
8419 if (!MRI.hasOneNonDBGUse(RegNo: Add->getReg(Idx: 0)))
8420 return false;
8421
8422 APInt C2 = getIConstantFromReg(VReg: Sub->getLHSReg(), MRI);
8423 APInt C1 = getIConstantFromReg(VReg: Add->getRHSReg(), MRI);
8424
8425 Register Dst = Sub->getReg(Idx: 0);
8426 LLT DstTy = MRI.getType(Reg: Dst);
8427
8428 MatchInfo = [=](MachineIRBuilder &B) {
8429 auto Const = B.buildConstant(Res: DstTy, Val: C2 - C1);
8430 B.buildSub(Dst, Src0: Const, Src1: Add->getLHSReg());
8431 };
8432
8433 return true;
8434}
8435
8436bool CombinerHelper::matchFoldAMinusC1MinusC2(const MachineInstr &MI,
8437 BuildFnTy &MatchInfo) const {
8438 // fold (A-C1)-C2 -> A-(C1+C2)
8439 const GSub *Sub1 = cast<GSub>(Val: &MI);
8440 GSub *Sub2 = cast<GSub>(Val: MRI.getVRegDef(Reg: Sub1->getLHSReg()));
8441
8442 if (!MRI.hasOneNonDBGUse(RegNo: Sub2->getReg(Idx: 0)))
8443 return false;
8444
8445 APInt C2 = getIConstantFromReg(VReg: Sub1->getRHSReg(), MRI);
8446 APInt C1 = getIConstantFromReg(VReg: Sub2->getRHSReg(), MRI);
8447
8448 Register Dst = Sub1->getReg(Idx: 0);
8449 LLT DstTy = MRI.getType(Reg: Dst);
8450
8451 MatchInfo = [=](MachineIRBuilder &B) {
8452 auto Const = B.buildConstant(Res: DstTy, Val: C1 + C2);
8453 B.buildSub(Dst, Src0: Sub2->getLHSReg(), Src1: Const);
8454 };
8455
8456 return true;
8457}
8458
8459bool CombinerHelper::matchFoldC1Minus2MinusC2(const MachineInstr &MI,
8460 BuildFnTy &MatchInfo) const {
8461 // fold (C1-A)-C2 -> (C1-C2)-A
8462 const GSub *Sub1 = cast<GSub>(Val: &MI);
8463 GSub *Sub2 = cast<GSub>(Val: MRI.getVRegDef(Reg: Sub1->getLHSReg()));
8464
8465 if (!MRI.hasOneNonDBGUse(RegNo: Sub2->getReg(Idx: 0)))
8466 return false;
8467
8468 APInt C2 = getIConstantFromReg(VReg: Sub1->getRHSReg(), MRI);
8469 APInt C1 = getIConstantFromReg(VReg: Sub2->getLHSReg(), MRI);
8470
8471 Register Dst = Sub1->getReg(Idx: 0);
8472 LLT DstTy = MRI.getType(Reg: Dst);
8473
8474 MatchInfo = [=](MachineIRBuilder &B) {
8475 auto Const = B.buildConstant(Res: DstTy, Val: C1 - C2);
8476 B.buildSub(Dst, Src0: Const, Src1: Sub2->getRHSReg());
8477 };
8478
8479 return true;
8480}
8481
8482bool CombinerHelper::matchFoldAMinusC1PlusC2(const MachineInstr &MI,
8483 BuildFnTy &MatchInfo) const {
8484 // fold ((A-C1)+C2) -> (A+(C2-C1))
8485 const GAdd *Add = cast<GAdd>(Val: &MI);
8486 GSub *Sub = cast<GSub>(Val: MRI.getVRegDef(Reg: Add->getLHSReg()));
8487
8488 if (!MRI.hasOneNonDBGUse(RegNo: Sub->getReg(Idx: 0)))
8489 return false;
8490
8491 APInt C2 = getIConstantFromReg(VReg: Add->getRHSReg(), MRI);
8492 APInt C1 = getIConstantFromReg(VReg: Sub->getRHSReg(), MRI);
8493
8494 Register Dst = Add->getReg(Idx: 0);
8495 LLT DstTy = MRI.getType(Reg: Dst);
8496
8497 MatchInfo = [=](MachineIRBuilder &B) {
8498 auto Const = B.buildConstant(Res: DstTy, Val: C2 - C1);
8499 B.buildAdd(Dst, Src0: Sub->getLHSReg(), Src1: Const);
8500 };
8501
8502 return true;
8503}
8504
8505bool CombinerHelper::matchUnmergeValuesAnyExtBuildVector(
8506 const MachineInstr &MI, BuildFnTy &MatchInfo) const {
8507 const GUnmerge *Unmerge = cast<GUnmerge>(Val: &MI);
8508
8509 if (!MRI.hasOneNonDBGUse(RegNo: Unmerge->getSourceReg()))
8510 return false;
8511
8512 const MachineInstr *Source = MRI.getVRegDef(Reg: Unmerge->getSourceReg());
8513
8514 LLT DstTy = MRI.getType(Reg: Unmerge->getReg(Idx: 0));
8515
8516 // $bv:_(<8 x s8>) = G_BUILD_VECTOR ....
8517 // $any:_(<8 x s16>) = G_ANYEXT $bv
8518 // $uv:_(<4 x s16>), $uv1:_(<4 x s16>) = G_UNMERGE_VALUES $any
8519 //
8520 // ->
8521 //
8522 // $any:_(s16) = G_ANYEXT $bv[0]
8523 // $any1:_(s16) = G_ANYEXT $bv[1]
8524 // $any2:_(s16) = G_ANYEXT $bv[2]
8525 // $any3:_(s16) = G_ANYEXT $bv[3]
8526 // $any4:_(s16) = G_ANYEXT $bv[4]
8527 // $any5:_(s16) = G_ANYEXT $bv[5]
8528 // $any6:_(s16) = G_ANYEXT $bv[6]
8529 // $any7:_(s16) = G_ANYEXT $bv[7]
8530 // $uv:_(<4 x s16>) = G_BUILD_VECTOR $any, $any1, $any2, $any3
8531 // $uv1:_(<4 x s16>) = G_BUILD_VECTOR $any4, $any5, $any6, $any7
8532
8533 // We want to unmerge into vectors.
8534 if (!DstTy.isFixedVector())
8535 return false;
8536
8537 const GAnyExt *Any = dyn_cast<GAnyExt>(Val: Source);
8538 if (!Any)
8539 return false;
8540
8541 const MachineInstr *NextSource = MRI.getVRegDef(Reg: Any->getSrcReg());
8542
8543 if (const GBuildVector *BV = dyn_cast<GBuildVector>(Val: NextSource)) {
8544 // G_UNMERGE_VALUES G_ANYEXT G_BUILD_VECTOR
8545
8546 if (!MRI.hasOneNonDBGUse(RegNo: BV->getReg(Idx: 0)))
8547 return false;
8548
8549 // FIXME: check element types?
8550 if (BV->getNumSources() % Unmerge->getNumDefs() != 0)
8551 return false;
8552
8553 LLT BigBvTy = MRI.getType(Reg: BV->getReg(Idx: 0));
8554 LLT SmallBvTy = DstTy;
8555 LLT SmallBvElemenTy = SmallBvTy.getElementType();
8556
8557 if (!isLegalOrBeforeLegalizer(
8558 Query: {TargetOpcode::G_BUILD_VECTOR, {SmallBvTy, SmallBvElemenTy}}))
8559 return false;
8560
8561 // We check the legality of scalar anyext.
8562 if (!isLegalOrBeforeLegalizer(
8563 Query: {TargetOpcode::G_ANYEXT,
8564 {SmallBvElemenTy, BigBvTy.getElementType()}}))
8565 return false;
8566
8567 MatchInfo = [=](MachineIRBuilder &B) {
8568 // Build into each G_UNMERGE_VALUES def
8569 // a small build vector with anyext from the source build vector.
8570 for (unsigned I = 0; I < Unmerge->getNumDefs(); ++I) {
8571 SmallVector<Register> Ops;
8572 for (unsigned J = 0; J < SmallBvTy.getNumElements(); ++J) {
8573 Register SourceArray =
8574 BV->getSourceReg(I: I * SmallBvTy.getNumElements() + J);
8575 auto AnyExt = B.buildAnyExt(Res: SmallBvElemenTy, Op: SourceArray);
8576 Ops.push_back(Elt: AnyExt.getReg(Idx: 0));
8577 }
8578 B.buildBuildVector(Res: Unmerge->getOperand(i: I).getReg(), Ops);
8579 };
8580 };
8581 return true;
8582 };
8583
8584 return false;
8585}
8586
8587bool CombinerHelper::matchShuffleUndefRHS(MachineInstr &MI,
8588 BuildFnTy &MatchInfo) const {
8589
8590 bool Changed = false;
8591 auto &Shuffle = cast<GShuffleVector>(Val&: MI);
8592 ArrayRef<int> OrigMask = Shuffle.getMask();
8593 SmallVector<int, 16> NewMask;
8594 const LLT SrcTy = MRI.getType(Reg: Shuffle.getSrc1Reg());
8595 const unsigned NumSrcElems = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
8596 const unsigned NumDstElts = OrigMask.size();
8597 for (unsigned i = 0; i != NumDstElts; ++i) {
8598 int Idx = OrigMask[i];
8599 if (Idx >= (int)NumSrcElems) {
8600 Idx = -1;
8601 Changed = true;
8602 }
8603 NewMask.push_back(Elt: Idx);
8604 }
8605
8606 if (!Changed)
8607 return false;
8608
8609 MatchInfo = [&, NewMask = std::move(NewMask)](MachineIRBuilder &B) {
8610 B.buildShuffleVector(Res: MI.getOperand(i: 0), Src1: MI.getOperand(i: 1), Src2: MI.getOperand(i: 2),
8611 Mask: std::move(NewMask));
8612 };
8613
8614 return true;
8615}
8616
8617static void commuteMask(MutableArrayRef<int> Mask, const unsigned NumElems) {
8618 const unsigned MaskSize = Mask.size();
8619 for (unsigned I = 0; I < MaskSize; ++I) {
8620 int Idx = Mask[I];
8621 if (Idx < 0)
8622 continue;
8623
8624 if (Idx < (int)NumElems)
8625 Mask[I] = Idx + NumElems;
8626 else
8627 Mask[I] = Idx - NumElems;
8628 }
8629}
8630
8631bool CombinerHelper::matchShuffleDisjointMask(MachineInstr &MI,
8632 BuildFnTy &MatchInfo) const {
8633
8634 auto &Shuffle = cast<GShuffleVector>(Val&: MI);
8635 // If any of the two inputs is already undef, don't check the mask again to
8636 // prevent infinite loop
8637 if (getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: Shuffle.getSrc1Reg(), MRI))
8638 return false;
8639
8640 if (getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: Shuffle.getSrc2Reg(), MRI))
8641 return false;
8642
8643 const LLT DstTy = MRI.getType(Reg: Shuffle.getReg(Idx: 0));
8644 const LLT Src1Ty = MRI.getType(Reg: Shuffle.getSrc1Reg());
8645 if (!isLegalOrBeforeLegalizer(
8646 Query: {TargetOpcode::G_SHUFFLE_VECTOR, {DstTy, Src1Ty}}))
8647 return false;
8648
8649 ArrayRef<int> Mask = Shuffle.getMask();
8650 const unsigned NumSrcElems = Src1Ty.getNumElements();
8651
8652 bool TouchesSrc1 = false;
8653 bool TouchesSrc2 = false;
8654 const unsigned NumElems = Mask.size();
8655 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
8656 if (Mask[Idx] < 0)
8657 continue;
8658
8659 if (Mask[Idx] < (int)NumSrcElems)
8660 TouchesSrc1 = true;
8661 else
8662 TouchesSrc2 = true;
8663 }
8664
8665 if (TouchesSrc1 == TouchesSrc2)
8666 return false;
8667
8668 Register NewSrc1 = Shuffle.getSrc1Reg();
8669 SmallVector<int, 16> NewMask(Mask);
8670 if (TouchesSrc2) {
8671 NewSrc1 = Shuffle.getSrc2Reg();
8672 commuteMask(Mask: NewMask, NumElems: NumSrcElems);
8673 }
8674
8675 MatchInfo = [=, &Shuffle](MachineIRBuilder &B) {
8676 auto Undef = B.buildUndef(Res: Src1Ty);
8677 B.buildShuffleVector(Res: Shuffle.getReg(Idx: 0), Src1: NewSrc1, Src2: Undef, Mask: NewMask);
8678 };
8679
8680 return true;
8681}
8682
8683bool CombinerHelper::matchSuboCarryOut(const MachineInstr &MI,
8684 BuildFnTy &MatchInfo) const {
8685 const GSubCarryOut *Subo = cast<GSubCarryOut>(Val: &MI);
8686
8687 Register Dst = Subo->getReg(Idx: 0);
8688 Register LHS = Subo->getLHSReg();
8689 Register RHS = Subo->getRHSReg();
8690 Register Carry = Subo->getCarryOutReg();
8691 LLT DstTy = MRI.getType(Reg: Dst);
8692 LLT CarryTy = MRI.getType(Reg: Carry);
8693
8694 // Check legality before known bits.
8695 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SUB, {DstTy}}) ||
8696 !isConstantLegalOrBeforeLegalizer(Ty: CarryTy))
8697 return false;
8698
8699 ConstantRange KBLHS =
8700 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: LHS),
8701 /* IsSigned=*/Subo->isSigned());
8702 ConstantRange KBRHS =
8703 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: RHS),
8704 /* IsSigned=*/Subo->isSigned());
8705
8706 if (Subo->isSigned()) {
8707 // G_SSUBO
8708 switch (KBLHS.signedSubMayOverflow(Other: KBRHS)) {
8709 case ConstantRange::OverflowResult::MayOverflow:
8710 return false;
8711 case ConstantRange::OverflowResult::NeverOverflows: {
8712 MatchInfo = [=](MachineIRBuilder &B) {
8713 B.buildSub(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoSWrap);
8714 B.buildConstant(Res: Carry, Val: 0);
8715 };
8716 return true;
8717 }
8718 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
8719 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
8720 MatchInfo = [=](MachineIRBuilder &B) {
8721 B.buildSub(Dst, Src0: LHS, Src1: RHS);
8722 B.buildConstant(Res: Carry, Val: getICmpTrueVal(TLI: getTargetLowering(),
8723 /*isVector=*/IsVector: CarryTy.isVector(),
8724 /*isFP=*/IsFP: false));
8725 };
8726 return true;
8727 }
8728 }
8729 return false;
8730 }
8731
8732 // G_USUBO
8733 switch (KBLHS.unsignedSubMayOverflow(Other: KBRHS)) {
8734 case ConstantRange::OverflowResult::MayOverflow:
8735 return false;
8736 case ConstantRange::OverflowResult::NeverOverflows: {
8737 MatchInfo = [=](MachineIRBuilder &B) {
8738 B.buildSub(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoUWrap);
8739 B.buildConstant(Res: Carry, Val: 0);
8740 };
8741 return true;
8742 }
8743 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
8744 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
8745 MatchInfo = [=](MachineIRBuilder &B) {
8746 B.buildSub(Dst, Src0: LHS, Src1: RHS);
8747 B.buildConstant(Res: Carry, Val: getICmpTrueVal(TLI: getTargetLowering(),
8748 /*isVector=*/IsVector: CarryTy.isVector(),
8749 /*isFP=*/IsFP: false));
8750 };
8751 return true;
8752 }
8753 }
8754
8755 return false;
8756}
8757
8758// Fold (ctlz (xor x, (sra x, bitwidth-1))) -> (add (ctls x), 1).
8759// Fold (ctlz (or (shl (xor x, (sra x, bitwidth-1)), 1), 1) -> (ctls x)
8760bool CombinerHelper::matchCtls(MachineInstr &CtlzMI,
8761 BuildFnTy &MatchInfo) const {
8762 assert((CtlzMI.getOpcode() == TargetOpcode::G_CTLZ ||
8763 CtlzMI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_POISON) &&
8764 "Expected G_CTLZ variant");
8765
8766 const Register Dst = CtlzMI.getOperand(i: 0).getReg();
8767 Register Src = CtlzMI.getOperand(i: 1).getReg();
8768
8769 LLT Ty = MRI.getType(Reg: Dst);
8770 LLT SrcTy = MRI.getType(Reg: Src);
8771
8772 if (!(Ty.isValid() && Ty.isScalar()))
8773 return false;
8774
8775 if (!LI)
8776 return false;
8777
8778 SmallVector<LLT, 2> QueryTypes = {Ty, SrcTy};
8779 LegalityQuery Query(TargetOpcode::G_CTLS, QueryTypes);
8780
8781 switch (LI->getAction(Query).Action) {
8782 default:
8783 return false;
8784 case LegalizeActions::Legal:
8785 case LegalizeActions::Custom:
8786 case LegalizeActions::WidenScalar:
8787 break;
8788 }
8789
8790 // Src = or(shl(V, 1), 1) -> Src=V; NeedAdd = False
8791 Register V;
8792 bool NeedAdd = true;
8793 if (mi_match(R: Src, MRI,
8794 P: m_OneUse(SP: m_GOr(L: m_OneUse(SP: m_GShl(L: m_Reg(R&: V), R: m_SpecificICst(RequestedValue: 1))),
8795 R: m_SpecificICst(RequestedValue: 1))))) {
8796 NeedAdd = false;
8797 Src = V;
8798 }
8799
8800 unsigned BitWidth = Ty.getScalarSizeInBits();
8801
8802 Register X;
8803 if (!mi_match(R: Src, MRI,
8804 P: m_OneUse(SP: m_GXor(L: m_Reg(R&: X), R: m_OneUse(SP: m_GAShr(
8805 L: m_DeferredReg(R&: X),
8806 R: m_SpecificICst(RequestedValue: BitWidth - 1)))))))
8807 return false;
8808
8809 MatchInfo = [=](MachineIRBuilder &B) {
8810 if (!NeedAdd) {
8811 B.buildCTLS(Dst, Src0: X);
8812 return;
8813 }
8814
8815 auto Ctls = B.buildCTLS(Dst: Ty, Src0: X);
8816 auto One = B.buildConstant(Res: Ty, Val: 1);
8817
8818 B.buildAdd(Dst, Src0: Ctls, Src1: One);
8819 };
8820
8821 return true;
8822}
8823
8824// Fold shr ( add ( ext X, ext Y ), 1 ) -> avgfloor ( x, y )
8825// Fold shr ( add ( ext X, ext Y, 1 ), 1 ) -> avgceil ( x, y )
8826bool CombinerHelper::matchAVG(MachineInstr &MI, MachineRegisterInfo &MRI,
8827 Register X, Register Y,
8828 unsigned TargetOpc) const {
8829 assert((MI.getOpcode() == TargetOpcode::G_LSHR ||
8830 MI.getOpcode() == TargetOpcode::G_ASHR) &&
8831 "Expected G_LSHR/G_ASHR");
8832
8833 LLT XTy = MRI.getType(Reg: X);
8834 return XTy == MRI.getType(Reg: Y) && isLegal(Query: {TargetOpc, {XTy}});
8835}
8836
8837static unsigned getCountZeroPoisonOpcode(const MachineInstr &MI) {
8838 assert((MI.getOpcode() == TargetOpcode::G_CTLZ ||
8839 MI.getOpcode() == TargetOpcode::G_CTTZ) &&
8840 "Expected count-zero opcode");
8841 switch (MI.getOpcode()) {
8842 case TargetOpcode::G_CTLZ:
8843 return TargetOpcode::G_CTLZ_ZERO_POISON;
8844 case TargetOpcode::G_CTTZ:
8845 return TargetOpcode::G_CTTZ_ZERO_POISON;
8846 default:
8847 llvm_unreachable("Unexpected count-zero opcode");
8848 }
8849}
8850
8851bool CombinerHelper::matchCountZeroToZeroPoison(MachineInstr &MI) const {
8852 if (!VT)
8853 return false;
8854
8855 unsigned ZPOpc = getCountZeroPoisonOpcode(MI);
8856 Register Src = MI.getOperand(i: 1).getReg();
8857 if (!VT->isKnownNeverZero(R: Src))
8858 return false;
8859
8860 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
8861 LLT SrcTy = MRI.getType(Reg: Src);
8862 return isLegalOrBeforeLegalizer(Query: {ZPOpc, {DstTy, SrcTy}});
8863}
8864
8865void CombinerHelper::applyCountZeroToZeroPoison(MachineInstr &MI) const {
8866 replaceOpcodeWith(FromMI&: MI, ToOpcode: getCountZeroPoisonOpcode(MI));
8867}
8868