1//===-- lib/CodeGen/GlobalISel/GICombinerHelper.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
9#include "llvm/ADT/APFloat.h"
10#include "llvm/ADT/STLExtras.h"
11#include "llvm/ADT/SetVector.h"
12#include "llvm/ADT/SmallBitVector.h"
13#include "llvm/Analysis/CmpInstAnalysis.h"
14#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
15#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
16#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
17#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
18#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
19#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
20#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
21#include "llvm/CodeGen/GlobalISel/Utils.h"
22#include "llvm/CodeGen/LowLevelTypeUtils.h"
23#include "llvm/CodeGen/MachineBasicBlock.h"
24#include "llvm/CodeGen/MachineDominators.h"
25#include "llvm/CodeGen/MachineInstr.h"
26#include "llvm/CodeGen/MachineMemOperand.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/Register.h"
29#include "llvm/CodeGen/RegisterBankInfo.h"
30#include "llvm/CodeGen/TargetInstrInfo.h"
31#include "llvm/CodeGen/TargetLowering.h"
32#include "llvm/CodeGen/TargetOpcodes.h"
33#include "llvm/IR/ConstantRange.h"
34#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/InstrTypes.h"
36#include "llvm/Support/Casting.h"
37#include "llvm/Support/DivisionByConstantInfo.h"
38#include "llvm/Support/ErrorHandling.h"
39#include "llvm/Support/MathExtras.h"
40#include "llvm/Target/TargetMachine.h"
41#include <cmath>
42#include <optional>
43#include <tuple>
44
45#define DEBUG_TYPE "gi-combiner"
46
47using namespace llvm;
48using namespace MIPatternMatch;
49
50// Option to allow testing of the combiner while no targets know about indexed
51// addressing.
52static cl::opt<bool>
53 ForceLegalIndexing("force-legal-indexing", cl::Hidden, cl::init(Val: false),
54 cl::desc("Force all indexed operations to be "
55 "legal for the GlobalISel combiner"));
56
57CombinerHelper::CombinerHelper(GISelChangeObserver &Observer,
58 MachineIRBuilder &B, bool IsPreLegalize,
59 GISelValueTracking *VT,
60 MachineDominatorTree *MDT,
61 const LegalizerInfo *LI)
62 : Builder(B), MRI(Builder.getMF().getRegInfo()), Observer(Observer), VT(VT),
63 MDT(MDT), IsPreLegalize(IsPreLegalize), LI(LI),
64 TII(Builder.getMF().getSubtarget().getInstrInfo()),
65 RBI(Builder.getMF().getSubtarget().getRegBankInfo()),
66 TRI(Builder.getMF().getSubtarget().getRegisterInfo()) {
67 (void)this->VT;
68}
69
70const TargetLowering &CombinerHelper::getTargetLowering() const {
71 return *Builder.getMF().getSubtarget().getTargetLowering();
72}
73
74const MachineFunction &CombinerHelper::getMachineFunction() const {
75 return Builder.getMF();
76}
77
78const DataLayout &CombinerHelper::getDataLayout() const {
79 return getMachineFunction().getDataLayout();
80}
81
82LLVMContext &CombinerHelper::getContext() const { return Builder.getContext(); }
83
84/// \returns The little endian in-memory byte position of byte \p I in a
85/// \p ByteWidth bytes wide type.
86///
87/// E.g. Given a 4-byte type x, x[0] -> byte 0
88static unsigned littleEndianByteAt(const unsigned ByteWidth, const unsigned I) {
89 assert(I < ByteWidth && "I must be in [0, ByteWidth)");
90 return I;
91}
92
93/// Determines the LogBase2 value for a non-null input value using the
94/// transform: LogBase2(V) = (EltBits - 1) - ctlz(V).
95static Register buildLogBase2(Register V, MachineIRBuilder &MIB) {
96 auto &MRI = *MIB.getMRI();
97 LLT Ty = MRI.getType(Reg: V);
98 auto Ctlz = MIB.buildCTLZ(Dst: Ty, Src0: V);
99 auto Base = MIB.buildConstant(Res: Ty, Val: Ty.getScalarSizeInBits() - 1);
100 return MIB.buildSub(Dst: Ty, Src0: Base, Src1: Ctlz).getReg(Idx: 0);
101}
102
103/// \returns The big endian in-memory byte position of byte \p I in a
104/// \p ByteWidth bytes wide type.
105///
106/// E.g. Given a 4-byte type x, x[0] -> byte 3
107static unsigned bigEndianByteAt(const unsigned ByteWidth, const unsigned I) {
108 assert(I < ByteWidth && "I must be in [0, ByteWidth)");
109 return ByteWidth - I - 1;
110}
111
112/// Given a map from byte offsets in memory to indices in a load/store,
113/// determine if that map corresponds to a little or big endian byte pattern.
114///
115/// \param MemOffset2Idx maps memory offsets to address offsets.
116/// \param LowestIdx is the lowest index in \p MemOffset2Idx.
117///
118/// \returns true if the map corresponds to a big endian byte pattern, false if
119/// it corresponds to a little endian byte pattern, and std::nullopt otherwise.
120///
121/// E.g. given a 32-bit type x, and x[AddrOffset], the in-memory byte patterns
122/// are as follows:
123///
124/// AddrOffset Little endian Big endian
125/// 0 0 3
126/// 1 1 2
127/// 2 2 1
128/// 3 3 0
129static std::optional<bool>
130isBigEndian(const SmallDenseMap<int64_t, int64_t, 8> &MemOffset2Idx,
131 int64_t LowestIdx) {
132 // Need at least two byte positions to decide on endianness.
133 unsigned Width = MemOffset2Idx.size();
134 if (Width < 2)
135 return std::nullopt;
136 bool BigEndian = true, LittleEndian = true;
137 for (unsigned MemOffset = 0; MemOffset < Width; ++ MemOffset) {
138 auto MemOffsetAndIdx = MemOffset2Idx.find(Val: MemOffset);
139 if (MemOffsetAndIdx == MemOffset2Idx.end())
140 return std::nullopt;
141 const int64_t Idx = MemOffsetAndIdx->second - LowestIdx;
142 assert(Idx >= 0 && "Expected non-negative byte offset?");
143 LittleEndian &= Idx == littleEndianByteAt(ByteWidth: Width, I: MemOffset);
144 BigEndian &= Idx == bigEndianByteAt(ByteWidth: Width, I: MemOffset);
145 if (!BigEndian && !LittleEndian)
146 return std::nullopt;
147 }
148
149 assert((BigEndian != LittleEndian) &&
150 "Pattern cannot be both big and little endian!");
151 return BigEndian;
152}
153
154bool CombinerHelper::isPreLegalize() const { return IsPreLegalize; }
155
156bool CombinerHelper::isLegal(const LegalityQuery &Query) const {
157 assert(LI && "Must have LegalizerInfo to query isLegal!");
158 return LI->getAction(Query).Action == LegalizeActions::Legal;
159}
160
161bool CombinerHelper::isLegalOrBeforeLegalizer(
162 const LegalityQuery &Query) const {
163 return isPreLegalize() || isLegal(Query);
164}
165
166bool CombinerHelper::isLegalOrHasWidenScalar(const LegalityQuery &Query) const {
167 return isLegal(Query) ||
168 LI->getAction(Query).Action == LegalizeActions::WidenScalar;
169}
170
171bool CombinerHelper::isLegalOrHasFewerElements(
172 const LegalityQuery &Query) const {
173 LegalizeAction Action = LI->getAction(Query).Action;
174 return Action == LegalizeActions::Legal ||
175 Action == LegalizeActions::FewerElements;
176}
177
178bool CombinerHelper::isConstantLegalOrBeforeLegalizer(const LLT Ty) const {
179 if (!Ty.isVector())
180 return isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_CONSTANT, {Ty}});
181 // Vector constants are represented as a G_BUILD_VECTOR of scalar G_CONSTANTs.
182 if (isPreLegalize())
183 return true;
184 LLT EltTy = Ty.getElementType();
185 return isLegal(Query: {TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}}) &&
186 isLegal(Query: {TargetOpcode::G_CONSTANT, {EltTy}});
187}
188
189void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg,
190 Register ToReg) const {
191 Observer.changingAllUsesOfReg(MRI, Reg: FromReg);
192
193 if (MRI.constrainRegAttrs(Reg: ToReg, ConstrainingReg: FromReg))
194 MRI.replaceRegWith(FromReg, ToReg);
195 else
196 Builder.buildCopy(Res: FromReg, Op: ToReg);
197
198 Observer.finishedChangingAllUsesOfReg();
199}
200
201void CombinerHelper::replaceRegOpWith(MachineRegisterInfo &MRI,
202 MachineOperand &FromRegOp,
203 Register ToReg) const {
204 assert(FromRegOp.getParent() && "Expected an operand in an MI");
205 Observer.changingInstr(MI&: *FromRegOp.getParent());
206
207 FromRegOp.setReg(ToReg);
208
209 Observer.changedInstr(MI&: *FromRegOp.getParent());
210}
211
212void CombinerHelper::replaceOpcodeWith(MachineInstr &FromMI,
213 unsigned ToOpcode) const {
214 Observer.changingInstr(MI&: FromMI);
215
216 FromMI.setDesc(Builder.getTII().get(Opcode: ToOpcode));
217
218 Observer.changedInstr(MI&: FromMI);
219}
220
221const RegisterBank *CombinerHelper::getRegBank(Register Reg) const {
222 return RBI->getRegBank(Reg, MRI, TRI: *TRI);
223}
224
225void CombinerHelper::setRegBank(Register Reg,
226 const RegisterBank *RegBank) const {
227 if (RegBank)
228 MRI.setRegBank(Reg, RegBank: *RegBank);
229}
230
231bool CombinerHelper::tryCombineCopy(MachineInstr &MI) const {
232 if (matchCombineCopy(MI)) {
233 applyCombineCopy(MI);
234 return true;
235 }
236 return false;
237}
238bool CombinerHelper::matchCombineCopy(MachineInstr &MI) const {
239 if (MI.getOpcode() != TargetOpcode::COPY)
240 return false;
241 Register DstReg = MI.getOperand(i: 0).getReg();
242 Register SrcReg = MI.getOperand(i: 1).getReg();
243 return canReplaceReg(DstReg, SrcReg, MRI);
244}
245void CombinerHelper::applyCombineCopy(MachineInstr &MI) const {
246 Register DstReg = MI.getOperand(i: 0).getReg();
247 Register SrcReg = MI.getOperand(i: 1).getReg();
248 replaceRegWith(MRI, FromReg: DstReg, ToReg: SrcReg);
249 MI.eraseFromParent();
250}
251
252bool CombinerHelper::matchFreezeOfSingleMaybePoisonOperand(
253 MachineInstr &MI, BuildFnTy &MatchInfo) const {
254 assert(MI.getOpcode() == TargetOpcode::G_FREEZE && "Invalid instruction");
255
256 // Ported from InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating.
257 Register DstOp = MI.getOperand(i: 0).getReg();
258 Register OrigOp = MI.getOperand(i: 1).getReg();
259
260 if (!MRI.hasOneNonDBGUse(RegNo: OrigOp))
261 return false;
262
263 MachineInstr *OrigDef;
264 if (!mi_match(R: OrigOp, MRI, P: m_MInstr(MI&: OrigDef)))
265 return false;
266 // Even if only a single operand of the PHI is not guaranteed non-poison,
267 // moving freeze() backwards across a PHI can cause optimization issues for
268 // other users of that operand.
269 //
270 // Moving freeze() from one of the output registers of a G_UNMERGE_VALUES to
271 // the source register is unprofitable because it makes the freeze() more
272 // strict than is necessary (it would affect the whole register instead of
273 // just the subreg being frozen).
274 if (OrigDef->isPHI() || isa<GUnmerge>(Val: OrigDef))
275 return false;
276
277 if (canCreateUndefOrPoison(Reg: OrigOp, MRI,
278 /*ConsiderFlagsAndMetadata=*/false))
279 return false;
280
281 std::optional<MachineOperand> MaybePoisonOperand;
282 for (MachineOperand &Operand : OrigDef->uses()) {
283 if (!Operand.isReg())
284 return false;
285
286 if (isGuaranteedNotToBeUndefOrPoison(Reg: Operand.getReg(), MRI))
287 continue;
288
289 if (!MaybePoisonOperand)
290 MaybePoisonOperand = Operand;
291 else {
292 // We have more than one maybe-poison operand. Moving the freeze is
293 // unsafe.
294 return false;
295 }
296 }
297
298 // Eliminate freeze if all operands are guaranteed non-poison.
299 if (!MaybePoisonOperand) {
300 MatchInfo = [=](MachineIRBuilder &B) {
301 Observer.changingInstr(MI&: *OrigDef);
302 cast<GenericMachineInstr>(Val: OrigDef)->dropPoisonGeneratingFlags();
303 Observer.changedInstr(MI&: *OrigDef);
304 B.buildCopy(Res: DstOp, Op: OrigOp);
305 };
306 return true;
307 }
308
309 Register MaybePoisonOperandReg = MaybePoisonOperand->getReg();
310 LLT MaybePoisonOperandRegTy = MRI.getType(Reg: MaybePoisonOperandReg);
311
312 if (!isLegalOrBeforeLegalizer(
313 Query: {TargetOpcode::G_FREEZE, {MaybePoisonOperandRegTy}}))
314 return false;
315
316 MatchInfo = [=](MachineIRBuilder &B) mutable {
317 Observer.changingInstr(MI&: *OrigDef);
318 cast<GenericMachineInstr>(Val: OrigDef)->dropPoisonGeneratingFlags();
319 Observer.changedInstr(MI&: *OrigDef);
320 B.setInsertPt(MBB&: *OrigDef->getParent(), II: OrigDef->getIterator());
321 auto Freeze = B.buildFreeze(Dst: MaybePoisonOperandRegTy, Src: MaybePoisonOperandReg);
322 replaceRegOpWith(
323 MRI, FromRegOp&: *OrigDef->findRegisterUseOperand(Reg: MaybePoisonOperandReg, TRI),
324 ToReg: Freeze.getReg(Idx: 0));
325 replaceRegWith(MRI, FromReg: DstOp, ToReg: OrigOp);
326 };
327 return true;
328}
329
330bool CombinerHelper::matchCombineConcatVectors(
331 MachineInstr &MI, SmallVector<Register> &Ops) const {
332 assert(MI.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
333 "Invalid instruction");
334 bool IsUndef = true;
335 MachineInstr *Undef = nullptr;
336
337 // Walk over all the operands of concat vectors and check if they are
338 // build_vector themselves or undef.
339 // Then collect their operands in Ops.
340 for (const MachineOperand &MO : MI.uses()) {
341 Register Reg = MO.getReg();
342 MachineInstr *Def;
343 if (!mi_match(R: Reg, MRI, P: m_MInstr(MI&: Def)))
344 return false;
345 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
346 return false;
347 switch (Def->getOpcode()) {
348 case TargetOpcode::G_BUILD_VECTOR:
349 IsUndef = false;
350 // Remember the operands of the build_vector to fold
351 // them into the yet-to-build flattened concat vectors.
352 for (const MachineOperand &BuildVecMO : Def->uses())
353 Ops.push_back(Elt: BuildVecMO.getReg());
354 break;
355 case TargetOpcode::G_IMPLICIT_DEF: {
356 LLT OpType = MRI.getType(Reg);
357 // Keep one undef value for all the undef operands.
358 if (!Undef) {
359 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
360 Undef = Builder.buildUndef(Res: OpType.getScalarType());
361 }
362 assert(MRI.getType(Undef->getOperand(0).getReg()) ==
363 OpType.getScalarType() &&
364 "All undefs should have the same type");
365 // Break the undef vector in as many scalar elements as needed
366 // for the flattening.
367 for (unsigned EltIdx = 0, EltEnd = OpType.getNumElements();
368 EltIdx != EltEnd; ++EltIdx)
369 Ops.push_back(Elt: Undef->getOperand(i: 0).getReg());
370 break;
371 }
372 default:
373 return false;
374 }
375 }
376
377 // Check if the combine is illegal
378 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
379 if (!isLegalOrBeforeLegalizer(
380 Query: {TargetOpcode::G_BUILD_VECTOR, {DstTy, MRI.getType(Reg: Ops[0])}})) {
381 return false;
382 }
383
384 if (IsUndef)
385 Ops.clear();
386
387 return true;
388}
389void CombinerHelper::applyCombineConcatVectors(
390 MachineInstr &MI, SmallVector<Register> &Ops) const {
391 // We determined that the concat_vectors can be flatten.
392 // Generate the flattened build_vector.
393 Register DstReg = MI.getOperand(i: 0).getReg();
394 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
395 Register NewDstReg = MRI.cloneVirtualRegister(VReg: DstReg);
396
397 // Note: IsUndef is sort of redundant. We could have determine it by
398 // checking that at all Ops are undef. Alternatively, we could have
399 // generate a build_vector of undefs and rely on another combine to
400 // clean that up. For now, given we already gather this information
401 // in matchCombineConcatVectors, just save compile time and issue the
402 // right thing.
403 if (Ops.empty())
404 Builder.buildUndef(Res: NewDstReg);
405 else
406 Builder.buildBuildVector(Res: NewDstReg, Ops);
407 replaceRegWith(MRI, FromReg: DstReg, ToReg: NewDstReg);
408 MI.eraseFromParent();
409}
410
411bool CombinerHelper::matchCombineBuildVectorOfBitcast(
412 MachineInstr &MI, SmallVector<Register> &Ops) const {
413 auto &BV = cast<GBuildVector>(Val&: MI);
414
415 // Look at the first operand for a unmerge(bitcast) from a scalar type.
416 GUnmerge *Unmerge = getOpcodeDef<GUnmerge>(Reg: BV.getSourceReg(I: 0), MRI);
417 if (!Unmerge || Unmerge->getReg(Idx: 0) != BV.getSourceReg(I: 0))
418 return false;
419 Register BCSrc;
420 if (!mi_match(R: Unmerge->getSourceReg(), MRI, P: m_GBitcast(Src: m_Reg(R&: BCSrc))))
421 return false;
422 LLT InputTy = MRI.getType(Reg: BCSrc);
423 unsigned Factor = Unmerge->getNumDefs();
424 if (!InputTy.isScalar() || BV.getNumSources() % Factor != 0)
425 return false;
426
427 // Check if the build_vector is legal
428 LLT BVDstTy = LLT::fixed_vector(NumElements: BV.getNumSources() / Factor, ScalarTy: InputTy);
429 if (!isLegal(Query: {TargetOpcode::G_BUILD_VECTOR, {BVDstTy, InputTy}}))
430 return false;
431
432 // Check all other operands are bitcasts or undef.
433 for (unsigned Idx = 0; Idx < BV.getNumSources(); Idx += Factor) {
434 GUnmerge *Unmerge = getOpcodeDef<GUnmerge>(Reg: BV.getSourceReg(I: Idx), MRI);
435 if (!all_of(Range: iota_range<unsigned>(0, Factor, false), P: [&](unsigned J) {
436 if (mi_match(R: BV.getSourceReg(I: Idx + J), MRI, P: m_GImplicitDef()))
437 return true;
438 return Unmerge && BV.getSourceReg(I: Idx + J) == Unmerge->getReg(Idx: J);
439 }))
440 return false;
441 if (!Unmerge)
442 Ops.push_back(Elt: 0);
443 else {
444 Register BCSrc;
445 if (!mi_match(
446 R: Unmerge->getSourceReg(), MRI,
447 P: m_GBitcast(Src: m_all_of(preds: m_Reg(R&: BCSrc), preds: m_SpecificType(Ty: InputTy)))))
448 return false;
449 Ops.push_back(Elt: BCSrc);
450 }
451 }
452
453 return true;
454}
455
456void CombinerHelper::applyCombineBuildVectorOfBitcast(
457 MachineInstr &MI, SmallVector<Register> &Ops) const {
458 LLT SrcTy = MRI.getType(Reg: Ops[0]);
459 // Build undef if any operations require it.
460 Register Undef = 0;
461 for (Register &Op : Ops) {
462 if (!Op) {
463 if (!Undef)
464 Undef = Builder.buildUndef(Res: SrcTy).getReg(Idx: 0);
465 Op = Undef;
466 }
467 }
468
469 LLT BVDstTy = LLT::fixed_vector(NumElements: Ops.size(), ScalarTy: SrcTy);
470 auto BV = Builder.buildBuildVector(Res: BVDstTy, Ops);
471 Builder.buildBitcast(Dst: MI.getOperand(i: 0).getReg(), Src: BV);
472 MI.eraseFromParent();
473}
474
475void CombinerHelper::applyCombineShuffleToBuildVector(MachineInstr &MI) const {
476 auto &Shuffle = cast<GShuffleVector>(Val&: MI);
477
478 Register SrcVec1 = Shuffle.getSrc1Reg();
479 Register SrcVec2 = Shuffle.getSrc2Reg();
480 LLT EltTy = MRI.getType(Reg: SrcVec1).getElementType();
481 int Width = MRI.getType(Reg: SrcVec1).getNumElements();
482
483 auto Unmerge1 = Builder.buildUnmerge(Res: EltTy, Op: SrcVec1);
484 auto Unmerge2 = Builder.buildUnmerge(Res: EltTy, Op: SrcVec2);
485
486 SmallVector<Register> Extracts;
487 // Select only applicable elements from unmerged values.
488 for (int Val : Shuffle.getMask()) {
489 if (Val == -1)
490 Extracts.push_back(Elt: Builder.buildUndef(Res: EltTy).getReg(Idx: 0));
491 else if (Val < Width)
492 Extracts.push_back(Elt: Unmerge1.getReg(Idx: Val));
493 else
494 Extracts.push_back(Elt: Unmerge2.getReg(Idx: Val - Width));
495 }
496 assert(Extracts.size() > 0 && "Expected at least one element in the shuffle");
497 if (Extracts.size() == 1)
498 Builder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: Extracts[0]);
499 else
500 Builder.buildBuildVector(Res: MI.getOperand(i: 0).getReg(), Ops: Extracts);
501 MI.eraseFromParent();
502}
503
504bool CombinerHelper::matchCombineShuffleConcat(
505 MachineInstr &MI, SmallVector<Register> &Ops) const {
506 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
507 GConcatVectors *ConcatMI1, *ConcatMI2;
508 if (!mi_match(R: MI.getOperand(i: 1).getReg(), MRI, P: m_GConcatVectors(Inst&: ConcatMI1)) ||
509 !mi_match(R: MI.getOperand(i: 2).getReg(), MRI, P: m_GConcatVectors(Inst&: ConcatMI2)))
510 return false;
511
512 // Check that the sources of the Concat instructions have the same type
513 if (MRI.getType(Reg: ConcatMI1->getSourceReg(I: 0)) !=
514 MRI.getType(Reg: ConcatMI2->getSourceReg(I: 0)))
515 return false;
516
517 LLT ConcatSrcTy = MRI.getType(Reg: ConcatMI1->getReg(Idx: 1));
518 LLT ShuffleSrcTy1 = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
519 unsigned ConcatSrcNumElt = ConcatSrcTy.getNumElements();
520 for (unsigned i = 0; i < Mask.size(); i += ConcatSrcNumElt) {
521 // Check if the index takes a whole source register from G_CONCAT_VECTORS
522 // Assumes that all Sources of G_CONCAT_VECTORS are the same type
523 if (Mask[i] == -1) {
524 for (unsigned j = 1; j < ConcatSrcNumElt; j++) {
525 if (i + j >= Mask.size())
526 return false;
527 if (Mask[i + j] != -1)
528 return false;
529 }
530 if (!isLegalOrBeforeLegalizer(
531 Query: {TargetOpcode::G_IMPLICIT_DEF, {ConcatSrcTy}}))
532 return false;
533 Ops.push_back(Elt: 0);
534 } else if (Mask[i] % ConcatSrcNumElt == 0) {
535 for (unsigned j = 1; j < ConcatSrcNumElt; j++) {
536 if (i + j >= Mask.size())
537 return false;
538 if (Mask[i + j] != Mask[i] + static_cast<int>(j))
539 return false;
540 }
541 // Retrieve the source register from its respective G_CONCAT_VECTORS
542 // instruction
543 if (Mask[i] < ShuffleSrcTy1.getNumElements()) {
544 Ops.push_back(Elt: ConcatMI1->getSourceReg(I: Mask[i] / ConcatSrcNumElt));
545 } else {
546 Ops.push_back(Elt: ConcatMI2->getSourceReg(I: Mask[i] / ConcatSrcNumElt -
547 ConcatMI1->getNumSources()));
548 }
549 } else {
550 return false;
551 }
552 }
553
554 if (!isLegalOrBeforeLegalizer(
555 Query: {TargetOpcode::G_CONCAT_VECTORS,
556 {MRI.getType(Reg: MI.getOperand(i: 0).getReg()), ConcatSrcTy}}))
557 return false;
558
559 return !Ops.empty();
560}
561
562void CombinerHelper::applyCombineShuffleConcat(
563 MachineInstr &MI, SmallVector<Register> &Ops) const {
564 LLT SrcTy;
565 for (Register &Reg : Ops) {
566 if (Reg != 0)
567 SrcTy = MRI.getType(Reg);
568 }
569 assert(SrcTy.isValid() && "Unexpected full undef vector in concat combine");
570
571 Register UndefReg = 0;
572
573 for (Register &Reg : Ops) {
574 if (Reg == 0) {
575 if (UndefReg == 0)
576 UndefReg = Builder.buildUndef(Res: SrcTy).getReg(Idx: 0);
577 Reg = UndefReg;
578 }
579 }
580
581 if (Ops.size() > 1)
582 Builder.buildConcatVectors(Res: MI.getOperand(i: 0).getReg(), Ops);
583 else
584 Builder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: Ops[0]);
585 MI.eraseFromParent();
586}
587
588bool CombinerHelper::matchCombineShuffleVector(
589 MachineInstr &MI, SmallVectorImpl<Register> &Ops) const {
590 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
591 "Invalid instruction kind");
592 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
593 Register Src1 = MI.getOperand(i: 1).getReg();
594 LLT SrcType = MRI.getType(Reg: Src1);
595
596 unsigned DstNumElts = DstType.getNumElements();
597 unsigned SrcNumElts = SrcType.getNumElements();
598
599 // If the resulting vector is smaller than the size of the source
600 // vectors being concatenated, we won't be able to replace the
601 // shuffle vector into a concat_vectors.
602 //
603 // Note: We may still be able to produce a concat_vectors fed by
604 // extract_vector_elt and so on. It is less clear that would
605 // be better though, so don't bother for now.
606 //
607 // If the destination is a scalar, the size of the sources doesn't
608 // matter. we will lower the shuffle to a plain copy. This will
609 // work only if the source and destination have the same size. But
610 // that's covered by the next condition.
611 //
612 // TODO: If the size between the source and destination don't match
613 // we could still emit an extract vector element in that case.
614 if (DstNumElts < 2 * SrcNumElts)
615 return false;
616
617 // Check that the shuffle mask can be broken evenly between the
618 // different sources.
619 if (DstNumElts % SrcNumElts != 0)
620 return false;
621
622 // Mask length is a multiple of the source vector length.
623 // Check if the shuffle is some kind of concatenation of the input
624 // vectors.
625 unsigned NumConcat = DstNumElts / SrcNumElts;
626 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
627 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
628 for (unsigned i = 0; i != DstNumElts; ++i) {
629 int Idx = Mask[i];
630 // Undef value.
631 if (Idx < 0)
632 continue;
633 // Ensure the indices in each SrcType sized piece are sequential and that
634 // the same source is used for the whole piece.
635 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
636 (ConcatSrcs[i / SrcNumElts] >= 0 &&
637 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts)))
638 return false;
639 // Remember which source this index came from.
640 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
641 }
642
643 // The shuffle is concatenating multiple vectors together.
644 // Collect the different operands for that.
645 Register UndefReg;
646 Register Src2 = MI.getOperand(i: 2).getReg();
647 for (auto Src : ConcatSrcs) {
648 if (Src < 0) {
649 if (!UndefReg) {
650 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
651 UndefReg = Builder.buildUndef(Res: SrcType).getReg(Idx: 0);
652 }
653 Ops.push_back(Elt: UndefReg);
654 } else if (Src == 0)
655 Ops.push_back(Elt: Src1);
656 else
657 Ops.push_back(Elt: Src2);
658 }
659 return true;
660}
661
662void CombinerHelper::applyCombineShuffleVector(MachineInstr &MI,
663 ArrayRef<Register> Ops) const {
664 Register DstReg = MI.getOperand(i: 0).getReg();
665 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI);
666 Register NewDstReg = MRI.cloneVirtualRegister(VReg: DstReg);
667
668 if (Ops.size() == 1)
669 Builder.buildCopy(Res: NewDstReg, Op: Ops[0]);
670 else
671 Builder.buildMergeLikeInstr(Res: NewDstReg, Ops);
672
673 replaceRegWith(MRI, FromReg: DstReg, ToReg: NewDstReg);
674 MI.eraseFromParent();
675}
676
677namespace {
678
679/// Select a preference between two uses. CurrentUse is the current preference
680/// while *ForCandidate is attributes of the candidate under consideration.
681PreferredTuple ChoosePreferredUse(MachineInstr &LoadMI,
682 PreferredTuple &CurrentUse,
683 const LLT TyForCandidate,
684 unsigned OpcodeForCandidate,
685 MachineInstr *MIForCandidate) {
686 if (!CurrentUse.Ty.isValid()) {
687 if (CurrentUse.ExtendOpcode == OpcodeForCandidate ||
688 CurrentUse.ExtendOpcode == TargetOpcode::G_ANYEXT)
689 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
690 return CurrentUse;
691 }
692
693 // We permit the extend to hoist through basic blocks but this is only
694 // sensible if the target has extending loads. If you end up lowering back
695 // into a load and extend during the legalizer then the end result is
696 // hoisting the extend up to the load.
697
698 // Prefer defined extensions to undefined extensions as these are more
699 // likely to reduce the number of instructions.
700 if (OpcodeForCandidate == TargetOpcode::G_ANYEXT &&
701 CurrentUse.ExtendOpcode != TargetOpcode::G_ANYEXT)
702 return CurrentUse;
703 else if (CurrentUse.ExtendOpcode == TargetOpcode::G_ANYEXT &&
704 OpcodeForCandidate != TargetOpcode::G_ANYEXT)
705 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
706
707 // Prefer sign extensions to zero extensions as sign-extensions tend to be
708 // more expensive. Don't do this if the load is already a zero-extend load
709 // though, otherwise we'll rewrite a zero-extend load into a sign-extend
710 // later.
711 if (!isa<GZExtLoad>(Val: LoadMI) && CurrentUse.Ty == TyForCandidate) {
712 if (CurrentUse.ExtendOpcode == TargetOpcode::G_SEXT &&
713 OpcodeForCandidate == TargetOpcode::G_ZEXT)
714 return CurrentUse;
715 else if (CurrentUse.ExtendOpcode == TargetOpcode::G_ZEXT &&
716 OpcodeForCandidate == TargetOpcode::G_SEXT)
717 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
718 }
719
720 // This is potentially target specific. We've chosen the largest type
721 // because G_TRUNC is usually free. One potential catch with this is that
722 // some targets have a reduced number of larger registers than smaller
723 // registers and this choice potentially increases the live-range for the
724 // larger value.
725 if (TyForCandidate.getSizeInBits() > CurrentUse.Ty.getSizeInBits()) {
726 return {.Ty: TyForCandidate, .ExtendOpcode: OpcodeForCandidate, .MI: MIForCandidate};
727 }
728 return CurrentUse;
729}
730
731/// Find a suitable place to insert some instructions and insert them. This
732/// function accounts for special cases like inserting before a PHI node.
733/// The current strategy for inserting before PHI's is to duplicate the
734/// instructions for each predecessor. However, while that's ok for G_TRUNC
735/// on most targets since it generally requires no code, other targets/cases may
736/// want to try harder to find a dominating block.
737static void InsertInsnsWithoutSideEffectsBeforeUse(
738 MachineIRBuilder &Builder, MachineInstr &DefMI, MachineOperand &UseMO,
739 std::function<void(MachineBasicBlock *, MachineBasicBlock::iterator,
740 MachineOperand &UseMO)>
741 Inserter) {
742 MachineInstr &UseMI = *UseMO.getParent();
743
744 MachineBasicBlock *InsertBB = UseMI.getParent();
745
746 // If the use is a PHI then we want the predecessor block instead.
747 if (UseMI.isPHI()) {
748 MachineOperand *PredBB = std::next(x: &UseMO);
749 InsertBB = PredBB->getMBB();
750 }
751
752 // If the block is the same block as the def then we want to insert just after
753 // the def instead of at the start of the block.
754 if (InsertBB == DefMI.getParent()) {
755 MachineBasicBlock::iterator InsertPt = &DefMI;
756 Inserter(InsertBB, std::next(x: InsertPt), UseMO);
757 return;
758 }
759
760 // Otherwise we want the start of the BB
761 Inserter(InsertBB, InsertBB->getFirstNonPHI(), UseMO);
762}
763} // end anonymous namespace
764
765bool CombinerHelper::tryCombineExtendingLoads(MachineInstr &MI) const {
766 PreferredTuple Preferred;
767 if (matchCombineExtendingLoads(MI, MatchInfo&: Preferred)) {
768 applyCombineExtendingLoads(MI, MatchInfo&: Preferred);
769 return true;
770 }
771 return false;
772}
773
774static unsigned getExtLoadOpcForExtend(unsigned ExtOpc) {
775 unsigned CandidateLoadOpc;
776 switch (ExtOpc) {
777 case TargetOpcode::G_ANYEXT:
778 CandidateLoadOpc = TargetOpcode::G_LOAD;
779 break;
780 case TargetOpcode::G_SEXT:
781 CandidateLoadOpc = TargetOpcode::G_SEXTLOAD;
782 break;
783 case TargetOpcode::G_ZEXT:
784 CandidateLoadOpc = TargetOpcode::G_ZEXTLOAD;
785 break;
786 default:
787 llvm_unreachable("Unexpected extend opc");
788 }
789 return CandidateLoadOpc;
790}
791
792bool CombinerHelper::matchCombineExtendingLoads(
793 MachineInstr &MI, PreferredTuple &Preferred) const {
794 // We match the loads and follow the uses to the extend instead of matching
795 // the extends and following the def to the load. This is because the load
796 // must remain in the same position for correctness (unless we also add code
797 // to find a safe place to sink it) whereas the extend is freely movable.
798 // It also prevents us from duplicating the load for the volatile case or just
799 // for performance.
800 GAnyLoad *LoadMI = dyn_cast<GAnyLoad>(Val: &MI);
801 if (!LoadMI)
802 return false;
803
804 Register LoadReg = LoadMI->getDstReg();
805
806 LLT LoadValueTy = MRI.getType(Reg: LoadReg);
807 if (!LoadValueTy.isScalar())
808 return false;
809
810 // Most architectures are going to legalize <s8 loads into at least a 1 byte
811 // load, and the MMOs can only describe memory accesses in multiples of bytes.
812 // If we try to perform extload combining on those, we can end up with
813 // %a(s8) = extload %ptr (load 1 byte from %ptr)
814 // ... which is an illegal extload instruction.
815 if (LoadValueTy.getSizeInBits() < 8)
816 return false;
817
818 // For non power-of-2 types, they will very likely be legalized into multiple
819 // loads. Don't bother trying to match them into extending loads.
820 if (!llvm::has_single_bit<uint32_t>(Value: LoadValueTy.getSizeInBits()))
821 return false;
822
823 // Find the preferred type aside from the any-extends (unless it's the only
824 // one) and non-extending ops. We'll emit an extending load to that type and
825 // and emit a variant of (extend (trunc X)) for the others according to the
826 // relative type sizes. At the same time, pick an extend to use based on the
827 // extend involved in the chosen type.
828 unsigned PreferredOpcode =
829 isa<GLoad>(Val: &MI)
830 ? TargetOpcode::G_ANYEXT
831 : isa<GSExtLoad>(Val: &MI) ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
832 Preferred = {.Ty: LLT(), .ExtendOpcode: PreferredOpcode, .MI: nullptr};
833 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: LoadReg)) {
834 if (UseMI.getOpcode() == TargetOpcode::G_SEXT ||
835 UseMI.getOpcode() == TargetOpcode::G_ZEXT ||
836 (UseMI.getOpcode() == TargetOpcode::G_ANYEXT)) {
837 const auto &MMO = LoadMI->getMMO();
838 // Don't do anything for atomics.
839 if (MMO.isAtomic())
840 continue;
841 // Check for legality.
842 if (!isPreLegalize()) {
843 LegalityQuery::MemDesc MMDesc(MMO);
844 unsigned CandidateLoadOpc = getExtLoadOpcForExtend(ExtOpc: UseMI.getOpcode());
845 LLT UseTy = MRI.getType(Reg: UseMI.getOperand(i: 0).getReg());
846 LLT SrcTy = MRI.getType(Reg: LoadMI->getPointerReg());
847 if (LI->getAction(Query: {CandidateLoadOpc, {UseTy, SrcTy}, {MMDesc}})
848 .Action != LegalizeActions::Legal)
849 continue;
850 }
851 Preferred = ChoosePreferredUse(LoadMI&: MI, CurrentUse&: Preferred,
852 TyForCandidate: MRI.getType(Reg: UseMI.getOperand(i: 0).getReg()),
853 OpcodeForCandidate: UseMI.getOpcode(), MIForCandidate: &UseMI);
854 }
855 }
856
857 // There were no extends
858 if (!Preferred.MI)
859 return false;
860 // It should be impossible to chose an extend without selecting a different
861 // type since by definition the result of an extend is larger.
862 assert(Preferred.Ty != LoadValueTy && "Extending to same type?");
863
864 LLVM_DEBUG(dbgs() << "Preferred use is: " << *Preferred.MI);
865 return true;
866}
867
868void CombinerHelper::applyCombineExtendingLoads(
869 MachineInstr &MI, PreferredTuple &Preferred) const {
870 // Rewrite the load to the chosen extending load.
871 Register ChosenDstReg = Preferred.MI->getOperand(i: 0).getReg();
872
873 // Inserter to insert a truncate back to the original type at a given point
874 // with some basic CSE to limit truncate duplication to one per BB.
875 DenseMap<MachineBasicBlock *, MachineInstr *> EmittedInsns;
876 auto InsertTruncAt = [&](MachineBasicBlock *InsertIntoBB,
877 MachineBasicBlock::iterator InsertBefore,
878 MachineOperand &UseMO) {
879 MachineInstr *PreviouslyEmitted = EmittedInsns.lookup(Val: InsertIntoBB);
880 if (PreviouslyEmitted) {
881 Observer.changingInstr(MI&: *UseMO.getParent());
882 UseMO.setReg(PreviouslyEmitted->getOperand(i: 0).getReg());
883 Observer.changedInstr(MI&: *UseMO.getParent());
884 return;
885 }
886
887 Builder.setInsertPt(MBB&: *InsertIntoBB, II: InsertBefore);
888 Register NewDstReg = MRI.cloneVirtualRegister(VReg: MI.getOperand(i: 0).getReg());
889 MachineInstr *NewMI = Builder.buildTrunc(Res: NewDstReg, Op: ChosenDstReg);
890 EmittedInsns[InsertIntoBB] = NewMI;
891 replaceRegOpWith(MRI, FromRegOp&: UseMO, ToReg: NewDstReg);
892 };
893
894 Observer.changingInstr(MI);
895 unsigned LoadOpc = getExtLoadOpcForExtend(ExtOpc: Preferred.ExtendOpcode);
896 MI.setDesc(Builder.getTII().get(Opcode: LoadOpc));
897
898 // Rewrite all the uses to fix up the types.
899 auto &LoadValue = MI.getOperand(i: 0);
900 SmallVector<MachineOperand *, 4> Uses(
901 llvm::make_pointer_range(Range: MRI.use_operands(Reg: LoadValue.getReg())));
902
903 for (auto *UseMO : Uses) {
904 MachineInstr *UseMI = UseMO->getParent();
905
906 // If the extend is compatible with the preferred extend then we should fix
907 // up the type and extend so that it uses the preferred use.
908 if (UseMI->getOpcode() == Preferred.ExtendOpcode ||
909 UseMI->getOpcode() == TargetOpcode::G_ANYEXT) {
910 Register UseDstReg = UseMI->getOperand(i: 0).getReg();
911 MachineOperand &UseSrcMO = UseMI->getOperand(i: 1);
912 const LLT UseDstTy = MRI.getType(Reg: UseDstReg);
913 if (UseDstReg != ChosenDstReg) {
914 if (Preferred.Ty == UseDstTy) {
915 // If the use has the same type as the preferred use, then merge
916 // the vregs and erase the extend. For example:
917 // %1:_(s8) = G_LOAD ...
918 // %2:_(s32) = G_SEXT %1(s8)
919 // %3:_(s32) = G_ANYEXT %1(s8)
920 // ... = ... %3(s32)
921 // rewrites to:
922 // %2:_(s32) = G_SEXTLOAD ...
923 // ... = ... %2(s32)
924 replaceRegWith(MRI, FromReg: UseDstReg, ToReg: ChosenDstReg);
925 Observer.erasingInstr(MI&: *UseMO->getParent());
926 UseMO->getParent()->eraseFromParent();
927 } else if (Preferred.Ty.getSizeInBits() < UseDstTy.getSizeInBits()) {
928 // If the preferred size is smaller, then keep the extend but extend
929 // from the result of the extending load. For example:
930 // %1:_(s8) = G_LOAD ...
931 // %2:_(s32) = G_SEXT %1(s8)
932 // %3:_(s64) = G_ANYEXT %1(s8)
933 // ... = ... %3(s64)
934 /// rewrites to:
935 // %2:_(s32) = G_SEXTLOAD ...
936 // %3:_(s64) = G_ANYEXT %2:_(s32)
937 // ... = ... %3(s64)
938 replaceRegOpWith(MRI, FromRegOp&: UseSrcMO, ToReg: ChosenDstReg);
939 } else {
940 // If the preferred size is large, then insert a truncate. For
941 // example:
942 // %1:_(s8) = G_LOAD ...
943 // %2:_(s64) = G_SEXT %1(s8)
944 // %3:_(s32) = G_ZEXT %1(s8)
945 // ... = ... %3(s32)
946 /// rewrites to:
947 // %2:_(s64) = G_SEXTLOAD ...
948 // %4:_(s8) = G_TRUNC %2:_(s32)
949 // %3:_(s64) = G_ZEXT %2:_(s8)
950 // ... = ... %3(s64)
951 InsertInsnsWithoutSideEffectsBeforeUse(Builder, DefMI&: MI, UseMO&: *UseMO,
952 Inserter: InsertTruncAt);
953 }
954 continue;
955 }
956 // The use is (one of) the uses of the preferred use we chose earlier.
957 // We're going to update the load to def this value later so just erase
958 // the old extend.
959 Observer.erasingInstr(MI&: *UseMO->getParent());
960 UseMO->getParent()->eraseFromParent();
961 continue;
962 }
963
964 // The use isn't an extend. Truncate back to the type we originally loaded.
965 // This is free on many targets.
966 InsertInsnsWithoutSideEffectsBeforeUse(Builder, DefMI&: MI, UseMO&: *UseMO, Inserter: InsertTruncAt);
967 }
968
969 MI.getOperand(i: 0).setReg(ChosenDstReg);
970 Observer.changedInstr(MI);
971}
972
973bool CombinerHelper::matchCombineLoadWithAndMask(MachineInstr &MI,
974 BuildFnTy &MatchInfo) const {
975 assert(MI.getOpcode() == TargetOpcode::G_AND);
976
977 // If we have the following code:
978 // %mask = G_CONSTANT 255
979 // %ld = G_LOAD %ptr, (load s16)
980 // %and = G_AND %ld, %mask
981 //
982 // Try to fold it into
983 // %ld = G_ZEXTLOAD %ptr, (load s8)
984
985 Register Dst = MI.getOperand(i: 0).getReg();
986 if (MRI.getType(Reg: Dst).isVector())
987 return false;
988
989 auto MaybeMask =
990 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
991 if (!MaybeMask)
992 return false;
993
994 APInt MaskVal = MaybeMask->Value;
995
996 if (!MaskVal.isMask())
997 return false;
998
999 Register SrcReg = MI.getOperand(i: 1).getReg();
1000 // Don't use getOpcodeDef() here since intermediate instructions may have
1001 // multiple users.
1002 GAnyLoad *LoadMI;
1003 Register PtrReg;
1004 const MachineMemOperand *MMO;
1005 if (!mi_match(R: SrcReg, MRI, P: m_GAnyLoad(Inst&: LoadMI, Ptr: m_Reg(R&: PtrReg), MMO: m_MMO(MMO))))
1006 return false;
1007
1008 Register LoadReg = LoadMI->getDstReg();
1009 LLT RegTy = MRI.getType(Reg: LoadReg);
1010 unsigned RegSize = RegTy.getSizeInBits();
1011 unsigned LoadSizeBits = MMO->getSizeInBits().getValue();
1012 unsigned MaskSizeBits = MaskVal.countr_one();
1013
1014 if ((isa<GSExtLoad>(Val: LoadMI) || MaskSizeBits < LoadSizeBits) &&
1015 !MRI.hasOneNonDBGUse(RegNo: LoadReg))
1016 return false;
1017
1018 // The mask may not be larger than the in-memory type, as it might cover sign
1019 // extended bits
1020 if (MaskSizeBits > LoadSizeBits)
1021 return false;
1022
1023 // If the mask covers the whole destination register, there's nothing to
1024 // extend
1025 if (MaskSizeBits >= RegSize)
1026 return false;
1027
1028 // Most targets cannot deal with loads of size < 8 and need to re-legalize to
1029 // at least byte loads. Avoid creating such loads here
1030 if (MaskSizeBits < 8 || !isPowerOf2_32(Value: MaskSizeBits))
1031 return false;
1032
1033 LegalityQuery::MemDesc MemDesc(*MMO);
1034
1035 // Don't modify the memory access size if this is atomic/volatile, but we can
1036 // still adjust the opcode to indicate the high bit behavior.
1037 if (!MMO->isAtomic() && !MMO->isVolatile())
1038 MemDesc.MemoryTy = LLT::scalar(SizeInBits: MaskSizeBits);
1039 else if (LoadSizeBits > MaskSizeBits || LoadSizeBits == RegSize)
1040 return false;
1041
1042 // TODO: Could check if it's legal with the reduced or original memory size.
1043 if (!isLegalOrBeforeLegalizer(
1044 Query: {TargetOpcode::G_ZEXTLOAD, {RegTy, MRI.getType(Reg: PtrReg)}, {MemDesc}}))
1045 return false;
1046
1047 MatchInfo = [=](MachineIRBuilder &B) {
1048 B.setInstrAndDebugLoc(*LoadMI);
1049 auto &MF = B.getMF();
1050 auto PtrInfo = MMO->getPointerInfo();
1051 auto *NewMMO = MF.getMachineMemOperand(MMO, PtrInfo, Ty: MemDesc.MemoryTy);
1052 B.buildLoadInstr(Opcode: TargetOpcode::G_ZEXTLOAD, Res: Dst, Addr: PtrReg, MMO&: *NewMMO);
1053 replaceRegWith(MRI, FromReg: LoadReg, ToReg: Dst);
1054 LoadMI->eraseFromParent();
1055 };
1056 return true;
1057}
1058
1059bool CombinerHelper::isPredecessor(const MachineInstr &DefMI,
1060 const MachineInstr &UseMI) const {
1061 assert(!DefMI.isDebugInstr() && !UseMI.isDebugInstr() &&
1062 "shouldn't consider debug uses");
1063 assert(DefMI.getParent() == UseMI.getParent());
1064 if (&DefMI == &UseMI)
1065 return true;
1066 const MachineBasicBlock &MBB = *DefMI.getParent();
1067 auto DefOrUse = find_if(Range: MBB, P: [&DefMI, &UseMI](const MachineInstr &MI) {
1068 return &MI == &DefMI || &MI == &UseMI;
1069 });
1070 if (DefOrUse == MBB.end())
1071 llvm_unreachable("Block must contain both DefMI and UseMI!");
1072 return &*DefOrUse == &DefMI;
1073}
1074
1075bool CombinerHelper::dominates(const MachineInstr &DefMI,
1076 const MachineInstr &UseMI) const {
1077 assert(!DefMI.isDebugInstr() && !UseMI.isDebugInstr() &&
1078 "shouldn't consider debug uses");
1079 if (MDT)
1080 return MDT->dominates(A: &DefMI, B: &UseMI);
1081 else if (DefMI.getParent() != UseMI.getParent())
1082 return false;
1083
1084 return isPredecessor(DefMI, UseMI);
1085}
1086
1087bool CombinerHelper::matchSextTruncSextLoad(MachineInstr &MI) const {
1088 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1089 Register SrcReg = MI.getOperand(i: 1).getReg();
1090 Register LoadUser = SrcReg;
1091
1092 if (MRI.getType(Reg: SrcReg).isVector())
1093 return false;
1094
1095 Register TruncSrc;
1096 if (mi_match(R: SrcReg, MRI, P: m_GTrunc(Src: m_Reg(R&: TruncSrc))))
1097 LoadUser = TruncSrc;
1098
1099 uint64_t SizeInBits = MI.getOperand(i: 2).getImm();
1100 // If the source is a G_SEXTLOAD from the same bit width, then we don't
1101 // need any extend at all, just a truncate.
1102 if (auto *LoadMI = getOpcodeDef<GSExtLoad>(Reg: LoadUser, MRI)) {
1103 // If truncating more than the original extended value, abort.
1104 auto LoadSizeBits = LoadMI->getMemSizeInBits();
1105 if (TruncSrc &&
1106 MRI.getType(Reg: TruncSrc).getSizeInBits() < LoadSizeBits.getValue())
1107 return false;
1108 if (LoadSizeBits == SizeInBits)
1109 return true;
1110 }
1111 return false;
1112}
1113
1114void CombinerHelper::applySextTruncSextLoad(MachineInstr &MI) const {
1115 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1116 Builder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: MI.getOperand(i: 1).getReg());
1117 MI.eraseFromParent();
1118}
1119
1120bool CombinerHelper::matchSextInRegOfLoad(
1121 MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) const {
1122 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1123
1124 Register DstReg = MI.getOperand(i: 0).getReg();
1125 LLT RegTy = MRI.getType(Reg: DstReg);
1126
1127 // Only supports scalars for now.
1128 if (RegTy.isVector())
1129 return false;
1130
1131 Register SrcReg = MI.getOperand(i: 1).getReg();
1132 Register PtrReg;
1133 const MachineMemOperand *MMO;
1134 if (!mi_match(R: SrcReg, MRI, P: m_GLoad(Ptr: m_Reg(R&: PtrReg), MMO: m_MMO(MMO))))
1135 return false;
1136
1137 uint64_t MemBits = MMO->getSizeInBits().getValue();
1138 uint64_t ExtFrom = MI.getOperand(i: 2).getImm();
1139
1140 if (MemBits > ExtFrom && !MRI.hasOneNonDBGUse(RegNo: SrcReg))
1141 return false;
1142
1143 // If the sign extend extends from a narrower width than the load's width,
1144 // then we can narrow the load width when we combine to a G_SEXTLOAD.
1145 // Avoid widening the load at all.
1146 unsigned NewSizeBits = std::min(a: ExtFrom, b: MemBits);
1147
1148 // Don't generate G_SEXTLOADs with a < 1 byte width.
1149 if (NewSizeBits < 8)
1150 return false;
1151 // Don't bother creating a non-power-2 sextload, it will likely be broken up
1152 // anyway for most targets.
1153 if (!isPowerOf2_32(Value: NewSizeBits))
1154 return false;
1155
1156 LegalityQuery::MemDesc MMDesc(*MMO);
1157
1158 // Don't modify the memory access size if this is atomic/volatile, but we can
1159 // still adjust the opcode to indicate the high bit behavior.
1160 if (!MMO->isAtomic() && !MMO->isVolatile())
1161 MMDesc.MemoryTy = LLT::scalar(SizeInBits: NewSizeBits);
1162 else if (MemBits > NewSizeBits || MemBits == RegTy.getSizeInBits())
1163 return false;
1164
1165 // TODO: Could check if it's legal with the reduced or original memory size.
1166 if (!isLegalOrBeforeLegalizer(
1167 Query: {TargetOpcode::G_SEXTLOAD, {RegTy, MRI.getType(Reg: PtrReg)}, {MMDesc}}))
1168 return false;
1169
1170 MatchInfo = std::make_tuple(args&: SrcReg, args&: NewSizeBits);
1171 return true;
1172}
1173
1174void CombinerHelper::applySextInRegOfLoad(
1175 MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) const {
1176 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1177 Register LoadReg;
1178 unsigned ScalarSizeBits;
1179 std::tie(args&: LoadReg, args&: ScalarSizeBits) = MatchInfo;
1180 GLoad *LoadDef = cast<GLoad>(Val: MRI.getVRegDef(Reg: LoadReg));
1181
1182 // If we have the following:
1183 // %ld = G_LOAD %ptr, (load 2)
1184 // %ext = G_SEXT_INREG %ld, 8
1185 // ==>
1186 // %ld = G_SEXTLOAD %ptr (load 1)
1187
1188 auto &MMO = LoadDef->getMMO();
1189 Builder.setInstrAndDebugLoc(*LoadDef);
1190 auto &MF = Builder.getMF();
1191 auto PtrInfo = MMO.getPointerInfo();
1192 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Size: ScalarSizeBits / 8);
1193 Builder.buildLoadInstr(Opcode: TargetOpcode::G_SEXTLOAD, Res: MI.getOperand(i: 0).getReg(),
1194 Addr: LoadDef->getPointerReg(), MMO&: *NewMMO);
1195 replaceRegWith(MRI, FromReg: LoadReg, ToReg: MI.getOperand(i: 0).getReg());
1196 MI.eraseFromParent();
1197
1198 // Not all loads can be deleted, so make sure the old one is removed.
1199 LoadDef->eraseFromParent();
1200}
1201
1202/// Return true if 'MI' is a load or a store that may be fold it's address
1203/// operand into the load / store addressing mode.
1204static bool canFoldInAddressingMode(GLoadStore *MI, const TargetLowering &TLI,
1205 MachineRegisterInfo &MRI) {
1206 TargetLowering::AddrMode AM;
1207 auto *MF = MI->getMF();
1208 auto *Addr = getOpcodeDef<GPtrAdd>(Reg: MI->getPointerReg(), MRI);
1209 if (!Addr)
1210 return false;
1211
1212 AM.HasBaseReg = true;
1213 if (auto CstOff = getIConstantVRegVal(VReg: Addr->getOffsetReg(), MRI))
1214 AM.BaseOffs = CstOff->getSExtValue(); // [reg +/- imm]
1215 else
1216 AM.Scale = 1; // [reg +/- reg]
1217
1218 return TLI.isLegalAddressingMode(
1219 DL: MF->getDataLayout(), AM,
1220 Ty: getTypeForLLT(Ty: MI->getMMO().getMemoryType(),
1221 C&: MF->getFunction().getContext()),
1222 AddrSpace: MI->getMMO().getAddrSpace());
1223}
1224
1225static unsigned getIndexedOpc(unsigned LdStOpc) {
1226 switch (LdStOpc) {
1227 case TargetOpcode::G_LOAD:
1228 return TargetOpcode::G_INDEXED_LOAD;
1229 case TargetOpcode::G_STORE:
1230 return TargetOpcode::G_INDEXED_STORE;
1231 case TargetOpcode::G_ZEXTLOAD:
1232 return TargetOpcode::G_INDEXED_ZEXTLOAD;
1233 case TargetOpcode::G_SEXTLOAD:
1234 return TargetOpcode::G_INDEXED_SEXTLOAD;
1235 default:
1236 llvm_unreachable("Unexpected opcode");
1237 }
1238}
1239
1240bool CombinerHelper::isIndexedLoadStoreLegal(GLoadStore &LdSt) const {
1241 // Check for legality.
1242 LLT PtrTy = MRI.getType(Reg: LdSt.getPointerReg());
1243 LLT Ty = MRI.getType(Reg: LdSt.getReg(Idx: 0));
1244 LLT MemTy = LdSt.getMMO().getMemoryType();
1245 SmallVector<LegalityQuery::MemDesc, 2> MemDescrs(
1246 {{MemTy, MemTy.getSizeInBits().getKnownMinValue(),
1247 AtomicOrdering::NotAtomic, AtomicOrdering::NotAtomic}});
1248 unsigned IndexedOpc = getIndexedOpc(LdStOpc: LdSt.getOpcode());
1249 SmallVector<LLT> OpTys;
1250 if (IndexedOpc == TargetOpcode::G_INDEXED_STORE)
1251 OpTys = {PtrTy, Ty, Ty};
1252 else
1253 OpTys = {Ty, PtrTy}; // For G_INDEXED_LOAD, G_INDEXED_[SZ]EXTLOAD
1254
1255 LegalityQuery Q(IndexedOpc, OpTys, MemDescrs);
1256 return isLegal(Query: Q);
1257}
1258
1259static cl::opt<unsigned> PostIndexUseThreshold(
1260 "post-index-use-threshold", cl::Hidden, cl::init(Val: 32),
1261 cl::desc("Number of uses of a base pointer to check before it is no longer "
1262 "considered for post-indexing."));
1263
1264bool CombinerHelper::findPostIndexCandidate(GLoadStore &LdSt, Register &Addr,
1265 Register &Base, Register &Offset,
1266 bool &RematOffset) const {
1267 // We're looking for the following pattern, for either load or store:
1268 // %baseptr:_(p0) = ...
1269 // G_STORE %val(s64), %baseptr(p0)
1270 // %offset:_(s64) = G_CONSTANT i64 -256
1271 // %new_addr:_(p0) = G_PTR_ADD %baseptr, %offset(s64)
1272 const auto &TLI = getTargetLowering();
1273
1274 Register Ptr = LdSt.getPointerReg();
1275 // If the store is the only use, don't bother.
1276 if (MRI.hasOneNonDBGUse(RegNo: Ptr))
1277 return false;
1278
1279 if (!isIndexedLoadStoreLegal(LdSt))
1280 return false;
1281
1282 if (getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Ptr, MRI))
1283 return false;
1284
1285 MachineInstr *StoredValDef = getDefIgnoringCopies(Reg: LdSt.getReg(Idx: 0), MRI);
1286 MachineInstr *PtrDef;
1287 if (!mi_match(R: Ptr, MRI, P: m_MInstr(MI&: PtrDef)))
1288 return false;
1289
1290 unsigned NumUsesChecked = 0;
1291 for (auto &Use : MRI.use_nodbg_instructions(Reg: Ptr)) {
1292 if (++NumUsesChecked > PostIndexUseThreshold)
1293 return false; // Try to avoid exploding compile time.
1294
1295 auto *PtrAdd = dyn_cast<GPtrAdd>(Val: &Use);
1296 // The use itself might be dead. This can happen during combines if DCE
1297 // hasn't had a chance to run yet. Don't allow it to form an indexed op.
1298 if (!PtrAdd || MRI.use_nodbg_empty(RegNo: PtrAdd->getReg(Idx: 0)))
1299 continue;
1300
1301 // Check the user of this isn't the store, otherwise we'd be generate a
1302 // indexed store defining its own use.
1303 if (StoredValDef == &Use)
1304 continue;
1305
1306 Offset = PtrAdd->getOffsetReg();
1307 if (!ForceLegalIndexing &&
1308 !TLI.isIndexingLegal(MI&: LdSt, Base: PtrAdd->getBaseReg(), Offset,
1309 /*IsPre*/ false, MRI))
1310 continue;
1311
1312 // Make sure the offset calculation is before the potentially indexed op.
1313 MachineInstr *OffsetDef;
1314 if (!mi_match(R: Offset, MRI, P: m_MInstr(MI&: OffsetDef)))
1315 continue;
1316 RematOffset = false;
1317 if (!dominates(DefMI: *OffsetDef, UseMI: LdSt)) {
1318 // If the offset however is just a G_CONSTANT, we can always just
1319 // rematerialize it where we need it.
1320 if (OffsetDef->getOpcode() != TargetOpcode::G_CONSTANT)
1321 continue;
1322 RematOffset = true;
1323 }
1324
1325 for (auto &BasePtrUse : MRI.use_nodbg_instructions(Reg: PtrAdd->getBaseReg())) {
1326 if (&BasePtrUse == PtrDef)
1327 continue;
1328
1329 // If the user is a later load/store that can be post-indexed, then don't
1330 // combine this one.
1331 auto *BasePtrLdSt = dyn_cast<GLoadStore>(Val: &BasePtrUse);
1332 if (BasePtrLdSt && BasePtrLdSt != &LdSt &&
1333 dominates(DefMI: LdSt, UseMI: *BasePtrLdSt) &&
1334 isIndexedLoadStoreLegal(LdSt&: *BasePtrLdSt))
1335 return false;
1336
1337 // Now we're looking for the key G_PTR_ADD instruction, which contains
1338 // the offset add that we want to fold.
1339 if (auto *BasePtrUseDef = dyn_cast<GPtrAdd>(Val: &BasePtrUse)) {
1340 Register PtrAddDefReg = BasePtrUseDef->getReg(Idx: 0);
1341 for (auto &BaseUseUse : MRI.use_nodbg_instructions(Reg: PtrAddDefReg)) {
1342 // If the use is in a different block, then we may produce worse code
1343 // due to the extra register pressure.
1344 if (BaseUseUse.getParent() != LdSt.getParent())
1345 return false;
1346
1347 if (auto *UseUseLdSt = dyn_cast<GLoadStore>(Val: &BaseUseUse))
1348 if (canFoldInAddressingMode(MI: UseUseLdSt, TLI, MRI))
1349 return false;
1350 }
1351 if (!dominates(DefMI: LdSt, UseMI: BasePtrUse))
1352 return false; // All use must be dominated by the load/store.
1353 }
1354 }
1355
1356 Addr = PtrAdd->getReg(Idx: 0);
1357 Base = PtrAdd->getBaseReg();
1358 return true;
1359 }
1360
1361 return false;
1362}
1363
1364bool CombinerHelper::findPreIndexCandidate(GLoadStore &LdSt, Register &Addr,
1365 Register &Base,
1366 Register &Offset) const {
1367 auto &MF = *LdSt.getParent()->getParent();
1368 const auto &TLI = *MF.getSubtarget().getTargetLowering();
1369
1370 Addr = LdSt.getPointerReg();
1371 if (!mi_match(R: Addr, MRI, P: m_GPtrAdd(L: m_Reg(R&: Base), R: m_Reg(R&: Offset))) ||
1372 MRI.hasOneNonDBGUse(RegNo: Addr))
1373 return false;
1374
1375 if (!ForceLegalIndexing &&
1376 !TLI.isIndexingLegal(MI&: LdSt, Base, Offset, /*IsPre*/ true, MRI))
1377 return false;
1378
1379 if (!isIndexedLoadStoreLegal(LdSt))
1380 return false;
1381
1382 MachineInstr *BaseDef = getDefIgnoringCopies(Reg: Base, MRI);
1383 if (BaseDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
1384 return false;
1385
1386 if (auto *St = dyn_cast<GStore>(Val: &LdSt)) {
1387 // Would require a copy.
1388 if (Base == St->getValueReg())
1389 return false;
1390
1391 // We're expecting one use of Addr in MI, but it could also be the
1392 // value stored, which isn't actually dominated by the instruction.
1393 if (St->getValueReg() == Addr)
1394 return false;
1395 }
1396
1397 // Avoid increasing cross-block register pressure.
1398 for (auto &AddrUse : MRI.use_nodbg_instructions(Reg: Addr))
1399 if (AddrUse.getParent() != LdSt.getParent())
1400 return false;
1401
1402 // FIXME: check whether all uses of the base pointer are constant PtrAdds.
1403 // That might allow us to end base's liveness here by adjusting the constant.
1404 bool RealUse = false;
1405 for (auto &AddrUse : MRI.use_nodbg_instructions(Reg: Addr)) {
1406 if (!dominates(DefMI: LdSt, UseMI: AddrUse))
1407 return false; // All use must be dominated by the load/store.
1408
1409 // If Ptr may be folded in addressing mode of other use, then it's
1410 // not profitable to do this transformation.
1411 if (auto *UseLdSt = dyn_cast<GLoadStore>(Val: &AddrUse)) {
1412 if (!canFoldInAddressingMode(MI: UseLdSt, TLI, MRI))
1413 RealUse = true;
1414 } else {
1415 RealUse = true;
1416 }
1417 }
1418 return RealUse;
1419}
1420
1421bool CombinerHelper::matchCombineExtractedVectorLoad(
1422 MachineInstr &MI, BuildFnTy &MatchInfo) const {
1423 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
1424
1425 // Check if there is a load that defines the vector being extracted from.
1426 auto *LoadMI = getOpcodeDef<GLoad>(Reg: MI.getOperand(i: 1).getReg(), MRI);
1427 if (!LoadMI)
1428 return false;
1429
1430 Register Vector = MI.getOperand(i: 1).getReg();
1431 LLT VecEltTy = MRI.getType(Reg: Vector).getElementType();
1432
1433 assert(MRI.getType(MI.getOperand(0).getReg()) == VecEltTy);
1434
1435 // Checking whether we should reduce the load width.
1436 if (!MRI.hasOneNonDBGUse(RegNo: Vector))
1437 return false;
1438
1439 // Check if the defining load is simple.
1440 if (!LoadMI->isSimple())
1441 return false;
1442
1443 // If the vector element type is not a multiple of a byte then we are unable
1444 // to correctly compute an address to load only the extracted element as a
1445 // scalar.
1446 if (!VecEltTy.isByteSized())
1447 return false;
1448
1449 // Check for load fold barriers between the extraction and the load.
1450 if (MI.getParent() != LoadMI->getParent())
1451 return false;
1452 const unsigned MaxIter = 20;
1453 unsigned Iter = 0;
1454 for (auto II = LoadMI->getIterator(), IE = MI.getIterator(); II != IE; ++II) {
1455 if (II->isLoadFoldBarrier())
1456 return false;
1457 if (Iter++ == MaxIter)
1458 return false;
1459 }
1460
1461 // Check if the new load that we are going to create is legal
1462 // if we are in the post-legalization phase.
1463 MachineMemOperand MMO = LoadMI->getMMO();
1464 Align Alignment = MMO.getAlign();
1465 MachinePointerInfo PtrInfo;
1466 uint64_t Offset;
1467
1468 // Finding the appropriate PtrInfo if offset is a known constant.
1469 // This is required to create the memory operand for the narrowed load.
1470 // This machine memory operand object helps us infer about legality
1471 // before we proceed to combine the instruction.
1472 if (auto CVal = getIConstantVRegVal(VReg: Vector, MRI)) {
1473 int Elt = CVal->getZExtValue();
1474 // FIXME: should be (ABI size)*Elt.
1475 Offset = VecEltTy.getSizeInBits() * Elt / 8;
1476 PtrInfo = MMO.getPointerInfo().getWithOffset(O: Offset);
1477 } else {
1478 // Discard the pointer info except the address space because the memory
1479 // operand can't represent this new access since the offset is variable.
1480 Offset = VecEltTy.getSizeInBits() / 8;
1481 PtrInfo = MachinePointerInfo(MMO.getPointerInfo().getAddrSpace());
1482 }
1483
1484 Alignment = commonAlignment(A: Alignment, Offset);
1485
1486 Register VecPtr = LoadMI->getPointerReg();
1487 LLT PtrTy = MRI.getType(Reg: VecPtr);
1488
1489 MachineFunction &MF = *MI.getMF();
1490 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Ty: VecEltTy);
1491
1492 LegalityQuery::MemDesc MMDesc(*NewMMO);
1493
1494 if (!isLegalOrBeforeLegalizer(
1495 Query: {TargetOpcode::G_LOAD, {VecEltTy, PtrTy}, {MMDesc}}))
1496 return false;
1497
1498 // Load must be allowed and fast on the target.
1499 LLVMContext &C = MF.getFunction().getContext();
1500 auto &DL = MF.getDataLayout();
1501 unsigned Fast = 0;
1502 if (!getTargetLowering().allowsMemoryAccess(Context&: C, DL, Ty: VecEltTy, MMO: *NewMMO,
1503 Fast: &Fast) ||
1504 !Fast)
1505 return false;
1506
1507 Register Result = MI.getOperand(i: 0).getReg();
1508 Register Index = MI.getOperand(i: 2).getReg();
1509
1510 MatchInfo = [=](MachineIRBuilder &B) {
1511 GISelObserverWrapper DummyObserver;
1512 LegalizerHelper Helper(B.getMF(), DummyObserver, B);
1513 //// Get pointer to the vector element.
1514 Register finalPtr = Helper.getVectorElementPointer(
1515 VecPtr: LoadMI->getPointerReg(), VecTy: MRI.getType(Reg: LoadMI->getOperand(i: 0).getReg()),
1516 Index);
1517 // New G_LOAD instruction.
1518 B.buildLoad(Res: Result, Addr: finalPtr, PtrInfo, Alignment);
1519 // Remove original GLOAD instruction.
1520 LoadMI->eraseFromParent();
1521 };
1522
1523 return true;
1524}
1525
1526bool CombinerHelper::matchCombineIndexedLoadStore(
1527 MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) const {
1528 auto &LdSt = cast<GLoadStore>(Val&: MI);
1529
1530 if (LdSt.isAtomic())
1531 return false;
1532
1533 MatchInfo.IsPre = findPreIndexCandidate(LdSt, Addr&: MatchInfo.Addr, Base&: MatchInfo.Base,
1534 Offset&: MatchInfo.Offset);
1535 if (!MatchInfo.IsPre &&
1536 !findPostIndexCandidate(LdSt, Addr&: MatchInfo.Addr, Base&: MatchInfo.Base,
1537 Offset&: MatchInfo.Offset, RematOffset&: MatchInfo.RematOffset))
1538 return false;
1539
1540 return true;
1541}
1542
1543void CombinerHelper::applyCombineIndexedLoadStore(
1544 MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) const {
1545 MachineInstr &AddrDef = *MRI.getVRegDef(Reg: MatchInfo.Addr);
1546 unsigned Opcode = MI.getOpcode();
1547 bool IsStore = Opcode == TargetOpcode::G_STORE;
1548 unsigned NewOpcode = getIndexedOpc(LdStOpc: Opcode);
1549
1550 // If the offset constant didn't happen to dominate the load/store, we can
1551 // just clone it as needed.
1552 if (MatchInfo.RematOffset) {
1553 auto *OldCst = MRI.getVRegDef(Reg: MatchInfo.Offset);
1554 auto NewCst = Builder.buildConstant(Res: MRI.getType(Reg: MatchInfo.Offset),
1555 Val: *OldCst->getOperand(i: 1).getCImm());
1556 MatchInfo.Offset = NewCst.getReg(Idx: 0);
1557 }
1558
1559 auto MIB = Builder.buildInstr(Opcode: NewOpcode);
1560 if (IsStore) {
1561 MIB.addDef(RegNo: MatchInfo.Addr);
1562 MIB.addUse(RegNo: MI.getOperand(i: 0).getReg());
1563 } else {
1564 MIB.addDef(RegNo: MI.getOperand(i: 0).getReg());
1565 MIB.addDef(RegNo: MatchInfo.Addr);
1566 }
1567
1568 MIB.addUse(RegNo: MatchInfo.Base);
1569 MIB.addUse(RegNo: MatchInfo.Offset);
1570 MIB.addImm(Val: MatchInfo.IsPre);
1571 MIB->cloneMemRefs(MF&: *MI.getMF(), MI);
1572 MI.eraseFromParent();
1573 AddrDef.eraseFromParent();
1574
1575 LLVM_DEBUG(dbgs() << " Combinined to indexed operation");
1576}
1577
1578bool CombinerHelper::matchCombineDivRem(MachineInstr &MI,
1579 MachineInstr *&OtherMI) const {
1580 unsigned Opcode = MI.getOpcode();
1581 bool IsDiv, IsSigned;
1582
1583 switch (Opcode) {
1584 default:
1585 llvm_unreachable("Unexpected opcode!");
1586 case TargetOpcode::G_SDIV:
1587 case TargetOpcode::G_UDIV: {
1588 IsDiv = true;
1589 IsSigned = Opcode == TargetOpcode::G_SDIV;
1590 break;
1591 }
1592 case TargetOpcode::G_SREM:
1593 case TargetOpcode::G_UREM: {
1594 IsDiv = false;
1595 IsSigned = Opcode == TargetOpcode::G_SREM;
1596 break;
1597 }
1598 }
1599
1600 Register Src1 = MI.getOperand(i: 1).getReg();
1601 unsigned DivOpcode, RemOpcode, DivremOpcode;
1602 if (IsSigned) {
1603 DivOpcode = TargetOpcode::G_SDIV;
1604 RemOpcode = TargetOpcode::G_SREM;
1605 DivremOpcode = TargetOpcode::G_SDIVREM;
1606 } else {
1607 DivOpcode = TargetOpcode::G_UDIV;
1608 RemOpcode = TargetOpcode::G_UREM;
1609 DivremOpcode = TargetOpcode::G_UDIVREM;
1610 }
1611
1612 if (!isLegalOrBeforeLegalizer(Query: {DivremOpcode, {MRI.getType(Reg: Src1)}}))
1613 return false;
1614
1615 // Combine:
1616 // %div:_ = G_[SU]DIV %src1:_, %src2:_
1617 // %rem:_ = G_[SU]REM %src1:_, %src2:_
1618 // into:
1619 // %div:_, %rem:_ = G_[SU]DIVREM %src1:_, %src2:_
1620
1621 // Combine:
1622 // %rem:_ = G_[SU]REM %src1:_, %src2:_
1623 // %div:_ = G_[SU]DIV %src1:_, %src2:_
1624 // into:
1625 // %div:_, %rem:_ = G_[SU]DIVREM %src1:_, %src2:_
1626
1627 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: Src1)) {
1628 if (MI.getParent() == UseMI.getParent() &&
1629 ((IsDiv && UseMI.getOpcode() == RemOpcode) ||
1630 (!IsDiv && UseMI.getOpcode() == DivOpcode)) &&
1631 matchEqualDefs(MOP1: MI.getOperand(i: 2), MOP2: UseMI.getOperand(i: 2)) &&
1632 matchEqualDefs(MOP1: MI.getOperand(i: 1), MOP2: UseMI.getOperand(i: 1))) {
1633 OtherMI = &UseMI;
1634 return true;
1635 }
1636 }
1637
1638 return false;
1639}
1640
1641void CombinerHelper::applyCombineDivRem(MachineInstr &MI,
1642 MachineInstr *&OtherMI) const {
1643 unsigned Opcode = MI.getOpcode();
1644 assert(OtherMI && "OtherMI shouldn't be empty.");
1645
1646 Register DestDivReg, DestRemReg;
1647 if (Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_UDIV) {
1648 DestDivReg = MI.getOperand(i: 0).getReg();
1649 DestRemReg = OtherMI->getOperand(i: 0).getReg();
1650 } else {
1651 DestDivReg = OtherMI->getOperand(i: 0).getReg();
1652 DestRemReg = MI.getOperand(i: 0).getReg();
1653 }
1654
1655 bool IsSigned =
1656 Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM;
1657
1658 // Check which instruction is first in the block so we don't break def-use
1659 // deps by "moving" the instruction incorrectly. Also keep track of which
1660 // instruction is first so we pick it's operands, avoiding use-before-def
1661 // bugs.
1662 MachineInstr *FirstInst = dominates(DefMI: MI, UseMI: *OtherMI) ? &MI : OtherMI;
1663 Builder.setInstrAndDebugLoc(*FirstInst);
1664
1665 Builder.buildInstr(Opc: IsSigned ? TargetOpcode::G_SDIVREM
1666 : TargetOpcode::G_UDIVREM,
1667 DstOps: {DestDivReg, DestRemReg},
1668 SrcOps: { FirstInst->getOperand(i: 1), FirstInst->getOperand(i: 2) });
1669 MI.eraseFromParent();
1670 OtherMI->eraseFromParent();
1671}
1672
1673bool CombinerHelper::matchOptBrCondByInvertingCond(
1674 MachineInstr &MI, MachineInstr *&BrCond) const {
1675 assert(MI.getOpcode() == TargetOpcode::G_BR);
1676
1677 // Try to match the following:
1678 // bb1:
1679 // G_BRCOND %c1, %bb2
1680 // G_BR %bb3
1681 // bb2:
1682 // ...
1683 // bb3:
1684
1685 // The above pattern does not have a fall through to the successor bb2, always
1686 // resulting in a branch no matter which path is taken. Here we try to find
1687 // and replace that pattern with conditional branch to bb3 and otherwise
1688 // fallthrough to bb2. This is generally better for branch predictors.
1689
1690 MachineBasicBlock *MBB = MI.getParent();
1691 MachineBasicBlock::iterator BrIt(MI);
1692 if (BrIt == MBB->begin())
1693 return false;
1694 assert(std::next(BrIt) == MBB->end() && "expected G_BR to be a terminator");
1695
1696 BrCond = &*std::prev(x: BrIt);
1697 if (BrCond->getOpcode() != TargetOpcode::G_BRCOND)
1698 return false;
1699
1700 // Check that the next block is the conditional branch target. Also make sure
1701 // that it isn't the same as the G_BR's target (otherwise, this will loop.)
1702 MachineBasicBlock *BrCondTarget = BrCond->getOperand(i: 1).getMBB();
1703 return BrCondTarget != MI.getOperand(i: 0).getMBB() &&
1704 MBB->isLayoutSuccessor(MBB: BrCondTarget);
1705}
1706
1707void CombinerHelper::applyOptBrCondByInvertingCond(
1708 MachineInstr &MI, MachineInstr *&BrCond) const {
1709 MachineBasicBlock *BrTarget = MI.getOperand(i: 0).getMBB();
1710 Builder.setInstrAndDebugLoc(*BrCond);
1711 LLT Ty = MRI.getType(Reg: BrCond->getOperand(i: 0).getReg());
1712 // FIXME: Does int/fp matter for this? If so, we might need to restrict
1713 // this to i1 only since we might not know for sure what kind of
1714 // compare generated the condition value.
1715 auto True = Builder.buildConstant(
1716 Res: Ty, Val: getICmpTrueVal(TLI: getTargetLowering(), IsVector: false, IsFP: false));
1717 auto Xor = Builder.buildXor(Dst: Ty, Src0: BrCond->getOperand(i: 0), Src1: True);
1718
1719 auto *FallthroughBB = BrCond->getOperand(i: 1).getMBB();
1720 Observer.changingInstr(MI);
1721 MI.getOperand(i: 0).setMBB(FallthroughBB);
1722 Observer.changedInstr(MI);
1723
1724 // Change the conditional branch to use the inverted condition and
1725 // new target block.
1726 Observer.changingInstr(MI&: *BrCond);
1727 BrCond->getOperand(i: 0).setReg(Xor.getReg(Idx: 0));
1728 BrCond->getOperand(i: 1).setMBB(BrTarget);
1729 Observer.changedInstr(MI&: *BrCond);
1730}
1731
1732bool CombinerHelper::matchCombineMemCpyFamily(
1733 MachineInstr &MI, MemCpyFamilyLoweringInfo &MatchInfo,
1734 unsigned MaxLen) const {
1735 auto &[Dst, Src, KnownLen, Alignment, DstAlignCanChange, MemOps] = MatchInfo;
1736 return canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, Alignment,
1737 DstAlignCanChange, MemOps);
1738}
1739
1740void CombinerHelper::applyCombineMemCpyFamily(
1741 MachineInstr &MI, MemCpyFamilyLoweringInfo &MatchInfo) const {
1742 auto &[Dst, Src, KnownLen, Alignment, DstAlignCanChange, MemOps] = MatchInfo;
1743 MachineIRBuilder HelperBuilder(MI);
1744 GISelObserverWrapper DummyObserver;
1745 LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);
1746 bool Changed = Helper.lowerMemCpyFamily(MI, Dst, Src, KnownLen, Alignment,
1747 DstAlignCanChange, MemOps) ==
1748 LegalizerHelper::LegalizeResult::Legalized;
1749 assert(Changed && "expected memcpy-family instruction to lower");
1750 (void)Changed;
1751}
1752
1753bool CombinerHelper::tryCombineMemCpyFamily(MachineInstr &MI,
1754 unsigned MaxLen) const {
1755 MachineIRBuilder HelperBuilder(MI);
1756 GISelObserverWrapper DummyObserver;
1757 LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);
1758 return Helper.lowerMemCpyFamily(MI, MaxLen) ==
1759 LegalizerHelper::LegalizeResult::Legalized;
1760}
1761
1762static APFloat constantFoldFpUnary(const MachineInstr &MI,
1763 const MachineRegisterInfo &MRI,
1764 const APFloat &Val) {
1765 APFloat Result(Val);
1766 switch (MI.getOpcode()) {
1767 default:
1768 llvm_unreachable("Unexpected opcode!");
1769 case TargetOpcode::G_FNEG: {
1770 Result.changeSign();
1771 return Result;
1772 }
1773 case TargetOpcode::G_FABS: {
1774 Result.clearSign();
1775 return Result;
1776 }
1777 case TargetOpcode::G_FCEIL:
1778 Result.roundToIntegral(RM: APFloat::rmTowardPositive);
1779 return Result;
1780 case TargetOpcode::G_FFLOOR:
1781 Result.roundToIntegral(RM: APFloat::rmTowardNegative);
1782 return Result;
1783 case TargetOpcode::G_INTRINSIC_TRUNC:
1784 Result.roundToIntegral(RM: APFloat::rmTowardZero);
1785 return Result;
1786 case TargetOpcode::G_INTRINSIC_ROUND:
1787 Result.roundToIntegral(RM: APFloat::rmNearestTiesToAway);
1788 return Result;
1789 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
1790 Result.roundToIntegral(RM: APFloat::rmNearestTiesToEven);
1791 return Result;
1792 case TargetOpcode::G_FRINT:
1793 case TargetOpcode::G_FNEARBYINT:
1794 // Use default rounding mode (round to nearest, ties to even)
1795 Result.roundToIntegral(RM: APFloat::rmNearestTiesToEven);
1796 return Result;
1797 case TargetOpcode::G_FPEXT:
1798 case TargetOpcode::G_FPTRUNC: {
1799 bool Unused;
1800 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1801 Result.convert(ToSemantics: getFltSemanticForLLT(Ty: DstTy), RM: APFloat::rmNearestTiesToEven,
1802 losesInfo: &Unused);
1803 return Result;
1804 }
1805 case TargetOpcode::G_FSQRT: {
1806 bool Unused;
1807 Result.convert(ToSemantics: APFloat::IEEEdouble(), RM: APFloat::rmNearestTiesToEven,
1808 losesInfo: &Unused);
1809 Result = APFloat(sqrt(x: Result.convertToDouble()));
1810 break;
1811 }
1812 case TargetOpcode::G_FLOG2: {
1813 bool Unused;
1814 Result.convert(ToSemantics: APFloat::IEEEdouble(), RM: APFloat::rmNearestTiesToEven,
1815 losesInfo: &Unused);
1816 Result = APFloat(log2(x: Result.convertToDouble()));
1817 break;
1818 }
1819 }
1820 // Convert `APFloat` to appropriate IEEE type depending on `DstTy`. Otherwise,
1821 // `buildFConstant` will assert on size mismatch. Only `G_FSQRT`, and
1822 // `G_FLOG2` reach here.
1823 bool Unused;
1824 Result.convert(ToSemantics: Val.getSemantics(), RM: APFloat::rmNearestTiesToEven, losesInfo: &Unused);
1825 return Result;
1826}
1827
1828void CombinerHelper::applyCombineConstantFoldFpUnary(
1829 MachineInstr &MI, const ConstantFP *Cst) const {
1830 APFloat Folded = constantFoldFpUnary(MI, MRI, Val: Cst->getValue());
1831 const ConstantFP *NewCst = ConstantFP::get(Context&: Builder.getContext(), V: Folded);
1832 Builder.buildFConstant(Res: MI.getOperand(i: 0), Val: *NewCst);
1833 MI.eraseFromParent();
1834}
1835
1836bool CombinerHelper::matchPtrAddImmedChain(MachineInstr &MI,
1837 PtrAddChain &MatchInfo) const {
1838 // We're trying to match the following pattern:
1839 // %t1 = G_PTR_ADD %base, G_CONSTANT imm1
1840 // %root = G_PTR_ADD %t1, G_CONSTANT imm2
1841 // -->
1842 // %root = G_PTR_ADD %base, G_CONSTANT (imm1 + imm2)
1843
1844 if (MI.getOpcode() != TargetOpcode::G_PTR_ADD)
1845 return false;
1846
1847 Register Add2 = MI.getOperand(i: 1).getReg();
1848 Register Imm1 = MI.getOperand(i: 2).getReg();
1849 auto MaybeImmVal = getIConstantVRegValWithLookThrough(VReg: Imm1, MRI);
1850 if (!MaybeImmVal)
1851 return false;
1852
1853 Register Base, Imm2;
1854 uint32_t LHSPtrAddFlags;
1855 if (!mi_match(R: Add2, MRI,
1856 P: m_GPtrAdd(L: m_Reg(R&: Base), R: m_Reg(R&: Imm2), Flags: m_MIFlags(Flags&: LHSPtrAddFlags))))
1857 return false;
1858
1859 auto MaybeImm2Val = getIConstantVRegValWithLookThrough(VReg: Imm2, MRI);
1860 if (!MaybeImm2Val)
1861 return false;
1862
1863 // Check if the new combined immediate forms an illegal addressing mode.
1864 // Do not combine if it was legal before but would get illegal.
1865 // To do so, we need to find a load/store user of the pointer to get
1866 // the access type.
1867 Type *AccessTy = nullptr;
1868 auto &MF = *MI.getMF();
1869 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg())) {
1870 if (auto *LdSt = dyn_cast<GLoadStore>(Val: &UseMI)) {
1871 AccessTy = getTypeForLLT(Ty: MRI.getType(Reg: LdSt->getReg(Idx: 0)),
1872 C&: MF.getFunction().getContext());
1873 break;
1874 }
1875 }
1876 TargetLoweringBase::AddrMode AMNew;
1877 APInt CombinedImm = MaybeImmVal->Value + MaybeImm2Val->Value;
1878 AMNew.BaseOffs = CombinedImm.getSExtValue();
1879 if (AccessTy) {
1880 AMNew.HasBaseReg = true;
1881 TargetLoweringBase::AddrMode AMOld;
1882 AMOld.BaseOffs = MaybeImmVal->Value.getSExtValue();
1883 AMOld.HasBaseReg = true;
1884 unsigned AS = MRI.getType(Reg: Add2).getAddressSpace();
1885 const auto &TLI = *MF.getSubtarget().getTargetLowering();
1886 if (TLI.isLegalAddressingMode(DL: MF.getDataLayout(), AM: AMOld, Ty: AccessTy, AddrSpace: AS) &&
1887 !TLI.isLegalAddressingMode(DL: MF.getDataLayout(), AM: AMNew, Ty: AccessTy, AddrSpace: AS))
1888 return false;
1889 }
1890
1891 // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
1892 // inbounds, reaching the same result in one G_PTR_ADD is also inbounds.
1893 // The nusw constraints are satisfied because imm1+imm2 cannot exceed the
1894 // largest signed integer that fits into the index type, which is the maximum
1895 // size of allocated objects according to the IR Language Reference.
1896 unsigned PtrAddFlags = MI.getFlags();
1897 bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
1898 bool IsInBounds =
1899 PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::InBounds;
1900 unsigned Flags = 0;
1901 if (IsNoUWrap)
1902 Flags |= MachineInstr::MIFlag::NoUWrap;
1903 if (IsInBounds) {
1904 Flags |= MachineInstr::MIFlag::InBounds;
1905 Flags |= MachineInstr::MIFlag::NoUSWrap;
1906 }
1907
1908 // Pass the combined immediate to the apply function.
1909 MatchInfo.Imm = AMNew.BaseOffs;
1910 MatchInfo.Base = Base;
1911 MatchInfo.Bank = getRegBank(Reg: Imm2);
1912 MatchInfo.Flags = Flags;
1913 return true;
1914}
1915
1916void CombinerHelper::applyPtrAddImmedChain(MachineInstr &MI,
1917 PtrAddChain &MatchInfo) const {
1918 assert(MI.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD");
1919 MachineIRBuilder MIB(MI);
1920 LLT OffsetTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
1921 auto NewOffset = MIB.buildConstant(Res: OffsetTy, Val: MatchInfo.Imm);
1922 setRegBank(Reg: NewOffset.getReg(Idx: 0), RegBank: MatchInfo.Bank);
1923 Observer.changingInstr(MI);
1924 MI.getOperand(i: 1).setReg(MatchInfo.Base);
1925 MI.getOperand(i: 2).setReg(NewOffset.getReg(Idx: 0));
1926 MI.setFlags(MatchInfo.Flags);
1927 Observer.changedInstr(MI);
1928}
1929
1930bool CombinerHelper::matchShiftImmedChain(MachineInstr &MI,
1931 RegisterImmPair &MatchInfo) const {
1932 // We're trying to match the following pattern with any of
1933 // G_SHL/G_ASHR/G_LSHR/G_SSHLSAT/G_USHLSAT shift instructions:
1934 // %t1 = SHIFT %base, G_CONSTANT imm1
1935 // %root = SHIFT %t1, G_CONSTANT imm2
1936 // -->
1937 // %root = SHIFT %base, G_CONSTANT (imm1 + imm2)
1938
1939 unsigned Opcode = MI.getOpcode();
1940 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1941 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1942 Opcode == TargetOpcode::G_USHLSAT) &&
1943 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1944
1945 Register Shl2 = MI.getOperand(i: 1).getReg();
1946 Register Imm1 = MI.getOperand(i: 2).getReg();
1947 auto MaybeImmVal = getIConstantVRegValWithLookThrough(VReg: Imm1, MRI);
1948 if (!MaybeImmVal)
1949 return false;
1950
1951 MachineInstr *Shl2Def;
1952 if (!mi_match(R: Shl2, MRI, P: m_MInstr(MI&: Shl2Def)) || Shl2Def->getOpcode() != Opcode)
1953 return false;
1954
1955 Register Base = Shl2Def->getOperand(i: 1).getReg();
1956 Register Imm2 = Shl2Def->getOperand(i: 2).getReg();
1957 auto MaybeImm2Val = getIConstantVRegValWithLookThrough(VReg: Imm2, MRI);
1958 if (!MaybeImm2Val)
1959 return false;
1960
1961 // Pass the combined immediate to the apply function.
1962 MatchInfo.Imm =
1963 (MaybeImmVal->Value.getZExtValue() + MaybeImm2Val->Value).getZExtValue();
1964 MatchInfo.Reg = Base;
1965
1966 // There is no simple replacement for a saturating unsigned left shift that
1967 // exceeds the scalar size.
1968 if (Opcode == TargetOpcode::G_USHLSAT &&
1969 MatchInfo.Imm >= MRI.getType(Reg: Shl2).getScalarSizeInBits())
1970 return false;
1971
1972 return true;
1973}
1974
1975void CombinerHelper::applyShiftImmedChain(MachineInstr &MI,
1976 RegisterImmPair &MatchInfo) const {
1977 unsigned Opcode = MI.getOpcode();
1978 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1979 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1980 Opcode == TargetOpcode::G_USHLSAT) &&
1981 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1982
1983 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1984 unsigned const ScalarSizeInBits = Ty.getScalarSizeInBits();
1985 auto Imm = MatchInfo.Imm;
1986
1987 if (Imm >= ScalarSizeInBits) {
1988 // Any logical shift that exceeds scalar size will produce zero.
1989 if (Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR) {
1990 Builder.buildConstant(Res: MI.getOperand(i: 0), Val: 0);
1991 MI.eraseFromParent();
1992 return;
1993 }
1994 // Arithmetic shift and saturating signed left shift have no effect beyond
1995 // scalar size.
1996 Imm = ScalarSizeInBits - 1;
1997 }
1998
1999 LLT ImmTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
2000 Register NewImm = Builder.buildConstant(Res: ImmTy, Val: Imm).getReg(Idx: 0);
2001 Observer.changingInstr(MI);
2002 MI.getOperand(i: 1).setReg(MatchInfo.Reg);
2003 MI.getOperand(i: 2).setReg(NewImm);
2004 Observer.changedInstr(MI);
2005}
2006
2007bool CombinerHelper::matchShiftOfShiftedLogic(
2008 MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo) const {
2009 // We're trying to match the following pattern with any of
2010 // G_SHL/G_ASHR/G_LSHR/G_USHLSAT/G_SSHLSAT shift instructions in combination
2011 // with any of G_AND/G_OR/G_XOR logic instructions.
2012 // %t1 = SHIFT %X, G_CONSTANT C0
2013 // %t2 = LOGIC %t1, %Y
2014 // %root = SHIFT %t2, G_CONSTANT C1
2015 // -->
2016 // %t3 = SHIFT %X, G_CONSTANT (C0+C1)
2017 // %t4 = SHIFT %Y, G_CONSTANT C1
2018 // %root = LOGIC %t3, %t4
2019 unsigned ShiftOpcode = MI.getOpcode();
2020 assert((ShiftOpcode == TargetOpcode::G_SHL ||
2021 ShiftOpcode == TargetOpcode::G_ASHR ||
2022 ShiftOpcode == TargetOpcode::G_LSHR ||
2023 ShiftOpcode == TargetOpcode::G_USHLSAT ||
2024 ShiftOpcode == TargetOpcode::G_SSHLSAT) &&
2025 "Expected G_SHL, G_ASHR, G_LSHR, G_USHLSAT and G_SSHLSAT");
2026
2027 // Match a one-use bitwise logic op.
2028 Register LogicDest = MI.getOperand(i: 1).getReg();
2029 if (!MRI.hasOneNonDBGUse(RegNo: LogicDest))
2030 return false;
2031
2032 MachineInstr *LogicMI;
2033 if (!mi_match(R: LogicDest, MRI, P: m_MInstr(MI&: LogicMI)))
2034 return false;
2035 unsigned LogicOpcode = LogicMI->getOpcode();
2036 if (LogicOpcode != TargetOpcode::G_AND && LogicOpcode != TargetOpcode::G_OR &&
2037 LogicOpcode != TargetOpcode::G_XOR)
2038 return false;
2039
2040 // Find a matching one-use shift by constant.
2041 const Register C1 = MI.getOperand(i: 2).getReg();
2042 auto MaybeImmVal = getIConstantVRegValWithLookThrough(VReg: C1, MRI);
2043 if (!MaybeImmVal || MaybeImmVal->Value == 0)
2044 return false;
2045
2046 const uint64_t C1Val = MaybeImmVal->Value.getZExtValue();
2047
2048 auto matchFirstShift = [&](const MachineInstr *MI, uint64_t &ShiftVal) {
2049 // Shift should match previous one and should be a one-use.
2050 if (MI->getOpcode() != ShiftOpcode ||
2051 !MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
2052 return false;
2053
2054 // Must be a constant.
2055 auto MaybeImmVal =
2056 getIConstantVRegValWithLookThrough(VReg: MI->getOperand(i: 2).getReg(), MRI);
2057 if (!MaybeImmVal)
2058 return false;
2059
2060 ShiftVal = MaybeImmVal->Value.getSExtValue();
2061 return true;
2062 };
2063
2064 // Logic ops are commutative, so check each operand for a match.
2065 Register LogicMIReg1 = LogicMI->getOperand(i: 1).getReg();
2066 MachineInstr *LogicMIOp1;
2067 Register LogicMIReg2 = LogicMI->getOperand(i: 2).getReg();
2068 MachineInstr *LogicMIOp2;
2069 if (!mi_match(R: LogicMIReg1, MRI, P: m_MInstr(MI&: LogicMIOp1)) ||
2070 !mi_match(R: LogicMIReg2, MRI, P: m_MInstr(MI&: LogicMIOp2)))
2071 return false;
2072 uint64_t C0Val;
2073
2074 if (matchFirstShift(LogicMIOp1, C0Val)) {
2075 MatchInfo.LogicNonShiftReg = LogicMIReg2;
2076 MatchInfo.Shift2 = LogicMIOp1;
2077 } else if (matchFirstShift(LogicMIOp2, C0Val)) {
2078 MatchInfo.LogicNonShiftReg = LogicMIReg1;
2079 MatchInfo.Shift2 = LogicMIOp2;
2080 } else
2081 return false;
2082
2083 MatchInfo.ValSum = C0Val + C1Val;
2084
2085 // The fold is not valid if the sum of the shift values exceeds bitwidth.
2086 if (MatchInfo.ValSum >= MRI.getType(Reg: LogicDest).getScalarSizeInBits())
2087 return false;
2088
2089 MatchInfo.Logic = LogicMI;
2090 return true;
2091}
2092
2093void CombinerHelper::applyShiftOfShiftedLogic(
2094 MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo) const {
2095 unsigned Opcode = MI.getOpcode();
2096 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
2097 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_USHLSAT ||
2098 Opcode == TargetOpcode::G_SSHLSAT) &&
2099 "Expected G_SHL, G_ASHR, G_LSHR, G_USHLSAT and G_SSHLSAT");
2100
2101 LLT ShlType = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
2102 LLT DestType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2103
2104 Register Const = Builder.buildConstant(Res: ShlType, Val: MatchInfo.ValSum).getReg(Idx: 0);
2105
2106 Register Shift1Base = MatchInfo.Shift2->getOperand(i: 1).getReg();
2107 Register Shift1 =
2108 Builder.buildInstr(Opc: Opcode, DstOps: {DestType}, SrcOps: {Shift1Base, Const}).getReg(Idx: 0);
2109
2110 // If LogicNonShiftReg is the same to Shift1Base, and shift1 const is the same
2111 // to MatchInfo.Shift2 const, CSEMIRBuilder will reuse the old shift1 when
2112 // build shift2. So, if we erase MatchInfo.Shift2 at the end, actually we
2113 // remove old shift1. And it will cause crash later. So erase it earlier to
2114 // avoid the crash.
2115 MatchInfo.Shift2->eraseFromParent();
2116
2117 Register Shift2Const = MI.getOperand(i: 2).getReg();
2118 Register Shift2 = Builder
2119 .buildInstr(Opc: Opcode, DstOps: {DestType},
2120 SrcOps: {MatchInfo.LogicNonShiftReg, Shift2Const})
2121 .getReg(Idx: 0);
2122
2123 Register Dest = MI.getOperand(i: 0).getReg();
2124 Builder.buildInstr(Opc: MatchInfo.Logic->getOpcode(), DstOps: {Dest}, SrcOps: {Shift1, Shift2});
2125
2126 // This was one use so it's safe to remove it.
2127 MatchInfo.Logic->eraseFromParent();
2128
2129 MI.eraseFromParent();
2130}
2131
2132bool CombinerHelper::matchCommuteShift(MachineInstr &MI,
2133 BuildFnTy &MatchInfo) const {
2134 assert(MI.getOpcode() == TargetOpcode::G_SHL && "Expected G_SHL");
2135 // Combine (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
2136 // Combine (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
2137 auto &Shl = cast<GenericMachineInstr>(Val&: MI);
2138 Register DstReg = Shl.getReg(Idx: 0);
2139 Register SrcReg = Shl.getReg(Idx: 1);
2140 Register ShiftReg = Shl.getReg(Idx: 2);
2141 Register X, C1;
2142
2143 if (!getTargetLowering().isDesirableToCommuteWithShift(MI, IsAfterLegal: !isPreLegalize()))
2144 return false;
2145
2146 MachineInstr *SrcDef;
2147 if (!mi_match(R: SrcReg, MRI,
2148 P: m_OneNonDBGUse(SP: m_any_of(preds: m_GAdd(L: m_Reg(R&: X), R: m_Reg(R&: C1)),
2149 preds: m_GOr(L: m_Reg(R&: X), R: m_Reg(R&: C1))))) ||
2150 !mi_match(R: SrcReg, MRI, P: m_MInstr(MI&: SrcDef)))
2151 return false;
2152
2153 APInt C1Val, C2Val;
2154 if (!mi_match(R: C1, MRI, P: m_ICstOrSplat(Cst&: C1Val)) ||
2155 !mi_match(R: ShiftReg, MRI, P: m_ICstOrSplat(Cst&: C2Val)))
2156 return false;
2157
2158 unsigned SrcOpc = SrcDef->getOpcode();
2159 LLT SrcTy = MRI.getType(Reg: SrcReg);
2160 MatchInfo = [=](MachineIRBuilder &B) {
2161 auto S1 = B.buildShl(Dst: SrcTy, Src0: X, Src1: ShiftReg);
2162 auto S2 = B.buildShl(Dst: SrcTy, Src0: C1, Src1: ShiftReg);
2163 B.buildInstr(Opc: SrcOpc, DstOps: {DstReg}, SrcOps: {S1, S2});
2164 };
2165 return true;
2166}
2167
2168bool CombinerHelper::matchLshrOfTruncOfLshr(MachineInstr &MI,
2169 LshrOfTruncOfLshr &MatchInfo,
2170 MachineInstr &ShiftMI) const {
2171 assert(MI.getOpcode() == TargetOpcode::G_LSHR && "Expected a G_LSHR");
2172
2173 Register N0 = MI.getOperand(i: 1).getReg();
2174 Register N1 = MI.getOperand(i: 2).getReg();
2175 unsigned OpSizeInBits = MRI.getType(Reg: N0).getScalarSizeInBits();
2176
2177 APInt N1C, N001C;
2178 if (!mi_match(R: N1, MRI, P: m_ICstOrSplat(Cst&: N1C)))
2179 return false;
2180 auto N001 = ShiftMI.getOperand(i: 2).getReg();
2181 if (!mi_match(R: N001, MRI, P: m_ICstOrSplat(Cst&: N001C)))
2182 return false;
2183
2184 if (N001C.getBitWidth() > N1C.getBitWidth())
2185 N1C = N1C.zext(width: N001C.getBitWidth());
2186 else
2187 N001C = N001C.zext(width: N1C.getBitWidth());
2188
2189 Register InnerShift = ShiftMI.getOperand(i: 0).getReg();
2190 LLT InnerShiftTy = MRI.getType(Reg: InnerShift);
2191 uint64_t InnerShiftSize = InnerShiftTy.getScalarSizeInBits();
2192 if ((N1C + N001C).ult(RHS: InnerShiftSize)) {
2193 MatchInfo.Src = ShiftMI.getOperand(i: 1).getReg();
2194 MatchInfo.ShiftAmt = N1C + N001C;
2195 MatchInfo.ShiftAmtTy = MRI.getType(Reg: N001);
2196 MatchInfo.InnerShiftTy = InnerShiftTy;
2197
2198 if ((N001C + OpSizeInBits) == InnerShiftSize)
2199 return true;
2200 if (MRI.hasOneUse(RegNo: N0) && MRI.hasOneUse(RegNo: InnerShift)) {
2201 MatchInfo.Mask = true;
2202 MatchInfo.MaskVal = APInt(N1C.getBitWidth(), OpSizeInBits) - N1C;
2203 return true;
2204 }
2205 }
2206 return false;
2207}
2208
2209void CombinerHelper::applyLshrOfTruncOfLshr(
2210 MachineInstr &MI, LshrOfTruncOfLshr &MatchInfo) const {
2211 assert(MI.getOpcode() == TargetOpcode::G_LSHR && "Expected a G_LSHR");
2212
2213 Register Dst = MI.getOperand(i: 0).getReg();
2214 auto ShiftAmt =
2215 Builder.buildConstant(Res: MatchInfo.ShiftAmtTy, Val: MatchInfo.ShiftAmt);
2216 auto Shift =
2217 Builder.buildLShr(Dst: MatchInfo.InnerShiftTy, Src0: MatchInfo.Src, Src1: ShiftAmt);
2218 if (MatchInfo.Mask == true) {
2219 APInt MaskVal =
2220 APInt::getLowBitsSet(numBits: MatchInfo.InnerShiftTy.getScalarSizeInBits(),
2221 loBitsSet: MatchInfo.MaskVal.getZExtValue());
2222 auto Mask = Builder.buildConstant(Res: MatchInfo.InnerShiftTy, Val: MaskVal);
2223 auto And = Builder.buildAnd(Dst: MatchInfo.InnerShiftTy, Src0: Shift, Src1: Mask);
2224 Builder.buildTrunc(Res: Dst, Op: And);
2225 } else
2226 Builder.buildTrunc(Res: Dst, Op: Shift);
2227 MI.eraseFromParent();
2228}
2229
2230bool CombinerHelper::matchCombineMulToShl(MachineInstr &MI,
2231 unsigned &ShiftVal) const {
2232 assert(MI.getOpcode() == TargetOpcode::G_MUL && "Expected a G_MUL");
2233 auto MaybeImmVal =
2234 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
2235 if (!MaybeImmVal)
2236 return false;
2237
2238 ShiftVal = MaybeImmVal->Value.exactLogBase2();
2239 return (static_cast<int32_t>(ShiftVal) != -1);
2240}
2241
2242void CombinerHelper::applyCombineMulToShl(MachineInstr &MI,
2243 unsigned &ShiftVal) const {
2244 assert(MI.getOpcode() == TargetOpcode::G_MUL && "Expected a G_MUL");
2245 MachineIRBuilder MIB(MI);
2246 LLT ShiftTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2247 auto ShiftCst = MIB.buildConstant(Res: ShiftTy, Val: ShiftVal);
2248 Observer.changingInstr(MI);
2249 MI.setDesc(MIB.getTII().get(Opcode: TargetOpcode::G_SHL));
2250 MI.getOperand(i: 2).setReg(ShiftCst.getReg(Idx: 0));
2251 if (ShiftVal == ShiftTy.getScalarSizeInBits() - 1)
2252 MI.clearFlag(Flag: MachineInstr::MIFlag::NoSWrap);
2253 Observer.changedInstr(MI);
2254}
2255
2256bool CombinerHelper::matchCombineSubToAdd(MachineInstr &MI,
2257 BuildFnTy &MatchInfo) const {
2258 GSub &Sub = cast<GSub>(Val&: MI);
2259
2260 LLT Ty = MRI.getType(Reg: Sub.getReg(Idx: 0));
2261
2262 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ADD, {Ty}}))
2263 return false;
2264
2265 if (!isConstantLegalOrBeforeLegalizer(Ty))
2266 return false;
2267
2268 APInt Imm = getIConstantFromReg(VReg: Sub.getRHSReg(), MRI);
2269
2270 MatchInfo = [=, &MI](MachineIRBuilder &B) {
2271 auto NegCst = B.buildConstant(Res: Ty, Val: -Imm);
2272 Observer.changingInstr(MI);
2273 MI.setDesc(B.getTII().get(Opcode: TargetOpcode::G_ADD));
2274 MI.getOperand(i: 2).setReg(NegCst.getReg(Idx: 0));
2275 MI.clearFlag(Flag: MachineInstr::MIFlag::NoUWrap);
2276 if (Imm.isMinSignedValue())
2277 MI.clearFlags(flags: MachineInstr::MIFlag::NoSWrap);
2278 Observer.changedInstr(MI);
2279 };
2280 return true;
2281}
2282
2283// shl ([sza]ext x), y => zext (shl x, y), if shift does not overflow source
2284bool CombinerHelper::matchCombineShlOfExtend(MachineInstr &MI,
2285 RegisterImmPair &MatchData) const {
2286 assert(MI.getOpcode() == TargetOpcode::G_SHL && VT);
2287 if (!getTargetLowering().isDesirableToPullExtFromShl(MI))
2288 return false;
2289
2290 Register LHS = MI.getOperand(i: 1).getReg();
2291
2292 Register ExtSrc;
2293 if (!mi_match(R: LHS, MRI, P: m_GAnyExt(Src: m_Reg(R&: ExtSrc))) &&
2294 !mi_match(R: LHS, MRI, P: m_GZExt(Src: m_Reg(R&: ExtSrc))) &&
2295 !mi_match(R: LHS, MRI, P: m_GSExt(Src: m_Reg(R&: ExtSrc))))
2296 return false;
2297
2298 Register RHS = MI.getOperand(i: 2).getReg();
2299 auto MaybeShiftAmtVal = isConstantOrConstantSplatVector(Def: RHS, MRI);
2300 if (!MaybeShiftAmtVal)
2301 return false;
2302
2303 if (LI) {
2304 LLT SrcTy = MRI.getType(Reg: ExtSrc);
2305
2306 // We only really care about the legality with the shifted value. We can
2307 // pick any type the constant shift amount, so ask the target what to
2308 // use. Otherwise we would have to guess and hope it is reported as legal.
2309 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: SrcTy);
2310 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SHL, {SrcTy, ShiftAmtTy}}))
2311 return false;
2312 }
2313
2314 int64_t ShiftAmt = MaybeShiftAmtVal->getSExtValue();
2315 MatchData.Reg = ExtSrc;
2316 MatchData.Imm = ShiftAmt;
2317
2318 unsigned MinLeadingZeros = VT->getKnownZeroes(R: ExtSrc).countl_one();
2319 unsigned SrcTySize = MRI.getType(Reg: ExtSrc).getScalarSizeInBits();
2320 return MinLeadingZeros >= ShiftAmt && ShiftAmt < SrcTySize;
2321}
2322
2323void CombinerHelper::applyCombineShlOfExtend(
2324 MachineInstr &MI, const RegisterImmPair &MatchData) const {
2325 Register ExtSrcReg = MatchData.Reg;
2326 int64_t ShiftAmtVal = MatchData.Imm;
2327
2328 LLT ExtSrcTy = MRI.getType(Reg: ExtSrcReg);
2329 auto ShiftAmt = Builder.buildConstant(Res: ExtSrcTy, Val: ShiftAmtVal);
2330 auto NarrowShift =
2331 Builder.buildShl(Dst: ExtSrcTy, Src0: ExtSrcReg, Src1: ShiftAmt, Flags: MI.getFlags());
2332 Builder.buildZExt(Res: MI.getOperand(i: 0), Op: NarrowShift);
2333 MI.eraseFromParent();
2334}
2335
2336bool CombinerHelper::matchCombineMergeUnmerge(MachineInstr &MI,
2337 Register &MatchInfo) const {
2338 GMerge &Merge = cast<GMerge>(Val&: MI);
2339 SmallVector<Register, 16> MergedValues;
2340 for (unsigned I = 0; I < Merge.getNumSources(); ++I)
2341 MergedValues.emplace_back(Args: Merge.getSourceReg(I));
2342
2343 auto *Unmerge = getOpcodeDef<GUnmerge>(Reg: MergedValues[0], MRI);
2344 if (!Unmerge || Unmerge->getNumDefs() != Merge.getNumSources())
2345 return false;
2346
2347 for (unsigned I = 0; I < MergedValues.size(); ++I)
2348 if (MergedValues[I] != Unmerge->getReg(Idx: I))
2349 return false;
2350
2351 MatchInfo = Unmerge->getSourceReg();
2352 return true;
2353}
2354
2355static Register peekThroughBitcast(Register Reg,
2356 const MachineRegisterInfo &MRI) {
2357 while (mi_match(R: Reg, MRI, P: m_GBitcast(Src: m_Reg(R&: Reg))))
2358 ;
2359
2360 return Reg;
2361}
2362
2363bool CombinerHelper::matchCombineUnmergeMergeToPlainValues(
2364 MachineInstr &MI, SmallVectorImpl<Register> &Operands) const {
2365 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2366 "Expected an unmerge");
2367 auto &Unmerge = cast<GUnmerge>(Val&: MI);
2368 Register SrcReg = peekThroughBitcast(Reg: Unmerge.getSourceReg(), MRI);
2369
2370 auto *SrcInstr = getOpcodeDef<GMergeLikeInstr>(Reg: SrcReg, MRI);
2371 if (!SrcInstr)
2372 return false;
2373
2374 // Check the source type of the merge.
2375 LLT SrcMergeTy = MRI.getType(Reg: SrcInstr->getSourceReg(I: 0));
2376 LLT Dst0Ty = MRI.getType(Reg: Unmerge.getReg(Idx: 0));
2377 bool SameSize = Dst0Ty.getSizeInBits() == SrcMergeTy.getSizeInBits();
2378 if (SrcMergeTy != Dst0Ty && !SameSize)
2379 return false;
2380 // They are the same now (modulo a bitcast).
2381 // We can collect all the src registers.
2382 for (unsigned Idx = 0; Idx < SrcInstr->getNumSources(); ++Idx)
2383 Operands.push_back(Elt: SrcInstr->getSourceReg(I: Idx));
2384 return true;
2385}
2386
2387void CombinerHelper::applyCombineUnmergeMergeToPlainValues(
2388 MachineInstr &MI, SmallVectorImpl<Register> &Operands) const {
2389 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2390 "Expected an unmerge");
2391 assert((MI.getNumOperands() - 1 == Operands.size()) &&
2392 "Not enough operands to replace all defs");
2393 unsigned NumElems = MI.getNumOperands() - 1;
2394
2395 LLT SrcTy = MRI.getType(Reg: Operands[0]);
2396 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2397 bool CanReuseInputDirectly = DstTy == SrcTy;
2398 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2399 Register DstReg = MI.getOperand(i: Idx).getReg();
2400 Register SrcReg = Operands[Idx];
2401
2402 // This combine may run after RegBankSelect, so we need to be aware of
2403 // register banks.
2404 const auto &DstCB = MRI.getRegClassOrRegBank(Reg: DstReg);
2405 if (!DstCB.isNull() && DstCB != MRI.getRegClassOrRegBank(Reg: SrcReg)) {
2406 SrcReg = Builder.buildCopy(Res: MRI.getType(Reg: SrcReg), Op: SrcReg).getReg(Idx: 0);
2407 MRI.setRegClassOrRegBank(Reg: SrcReg, RCOrRB: DstCB);
2408 }
2409
2410 if (CanReuseInputDirectly)
2411 replaceRegWith(MRI, FromReg: DstReg, ToReg: SrcReg);
2412 else
2413 Builder.buildCast(Dst: DstReg, Src: SrcReg);
2414 }
2415 MI.eraseFromParent();
2416}
2417
2418bool CombinerHelper::matchCombineUnmergeConstant(
2419 MachineInstr &MI, SmallVectorImpl<APInt> &Csts) const {
2420 unsigned SrcIdx = MI.getNumOperands() - 1;
2421 Register SrcReg = MI.getOperand(i: SrcIdx).getReg();
2422 // Break down the big constant in smaller ones.
2423 APInt Val;
2424 if (!mi_match(R: SrcReg, MRI, P: m_GConstantOrFConstantBits(Bits&: Val)))
2425 return false;
2426
2427 LLT Dst0Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2428 unsigned ShiftAmt = Dst0Ty.getSizeInBits();
2429 // Unmerge a constant.
2430 for (unsigned Idx = 0; Idx != SrcIdx; ++Idx) {
2431 Csts.emplace_back(Args: Val.trunc(width: ShiftAmt));
2432 Val = Val.lshr(shiftAmt: ShiftAmt);
2433 }
2434
2435 return true;
2436}
2437
2438void CombinerHelper::applyCombineUnmergeConstant(
2439 MachineInstr &MI, SmallVectorImpl<APInt> &Csts) const {
2440 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2441 "Expected an unmerge");
2442 assert((MI.getNumOperands() - 1 == Csts.size()) &&
2443 "Not enough operands to replace all defs");
2444 unsigned NumElems = MI.getNumOperands() - 1;
2445 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2446 Register DstReg = MI.getOperand(i: Idx).getReg();
2447 Builder.buildConstant(Res: DstReg, Val: Csts[Idx]);
2448 }
2449
2450 MI.eraseFromParent();
2451}
2452
2453bool CombinerHelper::matchCombineUnmergeUndef(
2454 MachineInstr &MI,
2455 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
2456 unsigned SrcIdx = MI.getNumOperands() - 1;
2457 Register SrcReg = MI.getOperand(i: SrcIdx).getReg();
2458 MatchInfo = [&MI](MachineIRBuilder &B) {
2459 unsigned NumElems = MI.getNumOperands() - 1;
2460 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2461 Register DstReg = MI.getOperand(i: Idx).getReg();
2462 B.buildUndef(Res: DstReg);
2463 }
2464 };
2465 return mi_match(R: SrcReg, MRI, P: m_GImplicitDef());
2466}
2467
2468bool CombinerHelper::matchCombineUnmergeWithDeadLanesToTrunc(
2469 MachineInstr &MI) const {
2470 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2471 "Expected an unmerge");
2472 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isScalar() ||
2473 !MRI.getType(Reg: MI.getOperand(i: MI.getNumDefs()).getReg()).isScalar())
2474 return false;
2475 // Check that all the lanes are dead except the first one.
2476 for (unsigned Idx = 1, EndIdx = MI.getNumDefs(); Idx != EndIdx; ++Idx) {
2477 if (!MRI.use_nodbg_empty(RegNo: MI.getOperand(i: Idx).getReg()))
2478 return false;
2479 }
2480 return true;
2481}
2482
2483void CombinerHelper::applyCombineUnmergeWithDeadLanesToTrunc(
2484 MachineInstr &MI) const {
2485 Register SrcReg = MI.getOperand(i: MI.getNumDefs()).getReg();
2486 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2487 Builder.buildTrunc(Res: Dst0Reg, Op: SrcReg);
2488 MI.eraseFromParent();
2489}
2490
2491bool CombinerHelper::matchCombineUnmergeZExtToZExt(MachineInstr &MI) const {
2492 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2493 "Expected an unmerge");
2494 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2495 LLT Dst0Ty = MRI.getType(Reg: Dst0Reg);
2496 // G_ZEXT on vector applies to each lane, so it will
2497 // affect all destinations. Therefore we won't be able
2498 // to simplify the unmerge to just the first definition.
2499 if (Dst0Ty.isVector())
2500 return false;
2501 Register SrcReg = MI.getOperand(i: MI.getNumDefs()).getReg();
2502 LLT SrcTy = MRI.getType(Reg: SrcReg);
2503 if (SrcTy.isVector())
2504 return false;
2505
2506 Register ZExtSrcReg;
2507 if (!mi_match(R: SrcReg, MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg))))
2508 return false;
2509
2510 // Finally we can replace the first definition with
2511 // a zext of the source if the definition is big enough to hold
2512 // all of ZExtSrc bits.
2513 LLT ZExtSrcTy = MRI.getType(Reg: ZExtSrcReg);
2514 return ZExtSrcTy.getSizeInBits() <= Dst0Ty.getSizeInBits();
2515}
2516
2517void CombinerHelper::applyCombineUnmergeZExtToZExt(MachineInstr &MI) const {
2518 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2519 "Expected an unmerge");
2520
2521 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2522
2523 GZext *ZExtInstr =
2524 cast<GZext>(Val: MRI.getVRegDef(Reg: MI.getOperand(i: MI.getNumDefs()).getReg()));
2525 Register ZExtSrcReg = ZExtInstr->getSrcReg();
2526 LLT Dst0Ty = MRI.getType(Reg: Dst0Reg);
2527 LLT ZExtSrcTy = MRI.getType(Reg: ZExtSrcReg);
2528
2529 if (Dst0Ty.getSizeInBits() > ZExtSrcTy.getSizeInBits()) {
2530 Builder.buildZExt(Res: Dst0Reg, Op: ZExtSrcReg);
2531 } else {
2532 assert(Dst0Ty.getSizeInBits() == ZExtSrcTy.getSizeInBits() &&
2533 "ZExt src doesn't fit in destination");
2534 replaceRegWith(MRI, FromReg: Dst0Reg, ToReg: ZExtSrcReg);
2535 }
2536
2537 Register ZeroReg;
2538 for (unsigned Idx = 1, EndIdx = MI.getNumDefs(); Idx != EndIdx; ++Idx) {
2539 if (!ZeroReg)
2540 ZeroReg = Builder.buildConstant(Res: Dst0Ty, Val: 0).getReg(Idx: 0);
2541 replaceRegWith(MRI, FromReg: MI.getOperand(i: Idx).getReg(), ToReg: ZeroReg);
2542 }
2543 MI.eraseFromParent();
2544}
2545
2546bool CombinerHelper::matchCombineShiftToUnmerge(MachineInstr &MI,
2547 unsigned TargetShiftSize,
2548 unsigned &ShiftVal) const {
2549 assert((MI.getOpcode() == TargetOpcode::G_SHL ||
2550 MI.getOpcode() == TargetOpcode::G_LSHR ||
2551 MI.getOpcode() == TargetOpcode::G_ASHR) && "Expected a shift");
2552
2553 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2554 if (Ty.isVector()) // TODO:
2555 return false;
2556
2557 // Don't narrow further than the requested size.
2558 unsigned Size = Ty.getSizeInBits();
2559 if (Size <= TargetShiftSize)
2560 return false;
2561
2562 auto MaybeImmVal =
2563 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
2564 if (!MaybeImmVal)
2565 return false;
2566
2567 ShiftVal = MaybeImmVal->Value.getSExtValue();
2568 return ShiftVal >= Size / 2 && ShiftVal < Size;
2569}
2570
2571void CombinerHelper::applyCombineShiftToUnmerge(
2572 MachineInstr &MI, const unsigned &ShiftVal) const {
2573 Register DstReg = MI.getOperand(i: 0).getReg();
2574 Register SrcReg = MI.getOperand(i: 1).getReg();
2575 LLT Ty = MRI.getType(Reg: SrcReg);
2576 unsigned Size = Ty.getSizeInBits();
2577 unsigned HalfSize = Size / 2;
2578 assert(ShiftVal >= HalfSize);
2579
2580 LLT HalfTy = Ty.changeElementSize(NewEltSize: HalfSize);
2581
2582 auto Unmerge = Builder.buildUnmerge(Res: HalfTy, Op: SrcReg);
2583 unsigned NarrowShiftAmt = ShiftVal - HalfSize;
2584
2585 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
2586 Register Narrowed = Unmerge.getReg(Idx: 1);
2587
2588 // dst = G_LSHR s64:x, C for C >= 32
2589 // =>
2590 // lo, hi = G_UNMERGE_VALUES x
2591 // dst = G_MERGE_VALUES (G_LSHR hi, C - 32), 0
2592
2593 if (NarrowShiftAmt != 0) {
2594 Narrowed = Builder.buildLShr(Dst: HalfTy, Src0: Narrowed,
2595 Src1: Builder.buildConstant(Res: HalfTy, Val: NarrowShiftAmt)).getReg(Idx: 0);
2596 }
2597
2598 auto Zero = Builder.buildConstant(Res: HalfTy, Val: 0);
2599 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Narrowed, Zero});
2600 } else if (MI.getOpcode() == TargetOpcode::G_SHL) {
2601 Register Narrowed = Unmerge.getReg(Idx: 0);
2602 // dst = G_SHL s64:x, C for C >= 32
2603 // =>
2604 // lo, hi = G_UNMERGE_VALUES x
2605 // dst = G_MERGE_VALUES 0, (G_SHL hi, C - 32)
2606 if (NarrowShiftAmt != 0) {
2607 Narrowed = Builder.buildShl(Dst: HalfTy, Src0: Narrowed,
2608 Src1: Builder.buildConstant(Res: HalfTy, Val: NarrowShiftAmt)).getReg(Idx: 0);
2609 }
2610
2611 auto Zero = Builder.buildConstant(Res: HalfTy, Val: 0);
2612 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Zero, Narrowed});
2613 } else {
2614 assert(MI.getOpcode() == TargetOpcode::G_ASHR);
2615 auto Hi = Builder.buildAShr(
2616 Dst: HalfTy, Src0: Unmerge.getReg(Idx: 1),
2617 Src1: Builder.buildConstant(Res: HalfTy, Val: HalfSize - 1));
2618
2619 if (ShiftVal == HalfSize) {
2620 // (G_ASHR i64:x, 32) ->
2621 // G_MERGE_VALUES hi_32(x), (G_ASHR hi_32(x), 31)
2622 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Unmerge.getReg(Idx: 1), Hi});
2623 } else if (ShiftVal == Size - 1) {
2624 // Don't need a second shift.
2625 // (G_ASHR i64:x, 63) ->
2626 // %narrowed = (G_ASHR hi_32(x), 31)
2627 // G_MERGE_VALUES %narrowed, %narrowed
2628 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Hi, Hi});
2629 } else {
2630 auto Lo = Builder.buildAShr(
2631 Dst: HalfTy, Src0: Unmerge.getReg(Idx: 1),
2632 Src1: Builder.buildConstant(Res: HalfTy, Val: ShiftVal - HalfSize));
2633
2634 // (G_ASHR i64:x, C) ->, for C >= 32
2635 // G_MERGE_VALUES (G_ASHR hi_32(x), C - 32), (G_ASHR hi_32(x), 31)
2636 Builder.buildMergeLikeInstr(Res: DstReg, Ops: {Lo, Hi});
2637 }
2638 }
2639
2640 MI.eraseFromParent();
2641}
2642
2643bool CombinerHelper::tryCombineShiftToUnmerge(
2644 MachineInstr &MI, unsigned TargetShiftAmount) const {
2645 unsigned ShiftAmt;
2646 if (matchCombineShiftToUnmerge(MI, TargetShiftSize: TargetShiftAmount, ShiftVal&: ShiftAmt)) {
2647 applyCombineShiftToUnmerge(MI, ShiftVal: ShiftAmt);
2648 return true;
2649 }
2650
2651 return false;
2652}
2653
2654bool CombinerHelper::matchCombineI2PToP2I(MachineInstr &MI,
2655 Register &Reg) const {
2656 assert(MI.getOpcode() == TargetOpcode::G_INTTOPTR && "Expected a G_INTTOPTR");
2657 Register DstReg = MI.getOperand(i: 0).getReg();
2658 LLT DstTy = MRI.getType(Reg: DstReg);
2659 Register SrcReg = MI.getOperand(i: 1).getReg();
2660 return mi_match(R: SrcReg, MRI,
2661 P: m_GPtrToInt(Src: m_all_of(preds: m_SpecificType(Ty: DstTy), preds: m_Reg(R&: Reg))));
2662}
2663
2664void CombinerHelper::applyCombineI2PToP2I(MachineInstr &MI,
2665 Register &Reg) const {
2666 assert(MI.getOpcode() == TargetOpcode::G_INTTOPTR && "Expected a G_INTTOPTR");
2667 Register DstReg = MI.getOperand(i: 0).getReg();
2668 Builder.buildCopy(Res: DstReg, Op: Reg);
2669 MI.eraseFromParent();
2670}
2671
2672void CombinerHelper::applyCombineP2IToI2P(MachineInstr &MI,
2673 Register &Reg) const {
2674 assert(MI.getOpcode() == TargetOpcode::G_PTRTOINT && "Expected a G_PTRTOINT");
2675 Register DstReg = MI.getOperand(i: 0).getReg();
2676 Builder.buildZExtOrTrunc(Res: DstReg, Op: Reg);
2677 MI.eraseFromParent();
2678}
2679
2680bool CombinerHelper::matchCombineAddP2IToPtrAdd(
2681 MachineInstr &MI, std::pair<Register, bool> &PtrReg) const {
2682 assert(MI.getOpcode() == TargetOpcode::G_ADD);
2683 Register LHS = MI.getOperand(i: 1).getReg();
2684 Register RHS = MI.getOperand(i: 2).getReg();
2685 LLT IntTy = MRI.getType(Reg: LHS);
2686
2687 // G_PTR_ADD always has the pointer in the LHS, so we may need to commute the
2688 // instruction.
2689 PtrReg.second = false;
2690 for (Register SrcReg : {LHS, RHS}) {
2691 if (mi_match(R: SrcReg, MRI, P: m_GPtrToInt(Src: m_Reg(R&: PtrReg.first)))) {
2692 // Don't handle cases where the integer is implicitly converted to the
2693 // pointer width.
2694 LLT PtrTy = MRI.getType(Reg: PtrReg.first);
2695 if (PtrTy.getScalarSizeInBits() == IntTy.getScalarSizeInBits())
2696 return true;
2697 }
2698
2699 PtrReg.second = true;
2700 }
2701
2702 return false;
2703}
2704
2705void CombinerHelper::applyCombineAddP2IToPtrAdd(
2706 MachineInstr &MI, std::pair<Register, bool> &PtrReg) const {
2707 Register Dst = MI.getOperand(i: 0).getReg();
2708 Register LHS = MI.getOperand(i: 1).getReg();
2709 Register RHS = MI.getOperand(i: 2).getReg();
2710
2711 const bool DoCommute = PtrReg.second;
2712 if (DoCommute)
2713 std::swap(a&: LHS, b&: RHS);
2714 LHS = PtrReg.first;
2715
2716 LLT PtrTy = MRI.getType(Reg: LHS);
2717
2718 auto PtrAdd = Builder.buildPtrAdd(Res: PtrTy, Op0: LHS, Op1: RHS);
2719 Builder.buildPtrToInt(Dst, Src: PtrAdd);
2720 MI.eraseFromParent();
2721}
2722
2723bool CombinerHelper::matchCombineConstPtrAddToI2P(MachineInstr &MI,
2724 APInt &NewCst) const {
2725 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
2726 Register LHS = PtrAdd.getBaseReg();
2727 Register RHS = PtrAdd.getOffsetReg();
2728 MachineRegisterInfo &MRI = Builder.getMF().getRegInfo();
2729
2730 if (auto RHSCst = getIConstantVRegVal(VReg: RHS, MRI)) {
2731 APInt Cst;
2732 if (mi_match(R: LHS, MRI, P: m_GIntToPtr(Src: m_ICst(Cst)))) {
2733 auto DstTy = MRI.getType(Reg: PtrAdd.getReg(Idx: 0));
2734 // G_INTTOPTR uses zero-extension
2735 NewCst = Cst.zextOrTrunc(width: DstTy.getSizeInBits());
2736 NewCst += RHSCst->sextOrTrunc(width: DstTy.getSizeInBits());
2737 return true;
2738 }
2739 }
2740
2741 return false;
2742}
2743
2744void CombinerHelper::applyCombineConstPtrAddToI2P(MachineInstr &MI,
2745 APInt &NewCst) const {
2746 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
2747 Register Dst = PtrAdd.getReg(Idx: 0);
2748
2749 Builder.buildConstant(Res: Dst, Val: NewCst);
2750 PtrAdd.eraseFromParent();
2751}
2752
2753bool CombinerHelper::matchCombineAnyExtTrunc(MachineInstr &MI,
2754 Register &Reg) const {
2755 assert(MI.getOpcode() == TargetOpcode::G_ANYEXT && "Expected a G_ANYEXT");
2756 Register DstReg = MI.getOperand(i: 0).getReg();
2757 Register SrcReg = MI.getOperand(i: 1).getReg();
2758 Register OriginalSrcReg = getSrcRegIgnoringCopies(Reg: SrcReg, MRI);
2759 if (OriginalSrcReg.isValid())
2760 SrcReg = OriginalSrcReg;
2761 LLT DstTy = MRI.getType(Reg: DstReg);
2762 return mi_match(R: SrcReg, MRI,
2763 P: m_GTrunc(Src: m_all_of(preds: m_Reg(R&: Reg), preds: m_SpecificType(Ty: DstTy)))) &&
2764 canReplaceReg(DstReg, SrcReg: Reg, MRI);
2765}
2766
2767bool CombinerHelper::matchCombineZextTrunc(MachineInstr &MI,
2768 Register &Reg) const {
2769 assert(MI.getOpcode() == TargetOpcode::G_ZEXT && "Expected a G_ZEXT");
2770 Register DstReg = MI.getOperand(i: 0).getReg();
2771 Register SrcReg = MI.getOperand(i: 1).getReg();
2772 LLT DstTy = MRI.getType(Reg: DstReg);
2773 if (mi_match(R: SrcReg, MRI,
2774 P: m_GTrunc(Src: m_all_of(preds: m_Reg(R&: Reg), preds: m_SpecificType(Ty: DstTy)))) &&
2775 canReplaceReg(DstReg, SrcReg: Reg, MRI)) {
2776 unsigned DstSize = DstTy.getScalarSizeInBits();
2777 unsigned SrcSize = MRI.getType(Reg: SrcReg).getScalarSizeInBits();
2778 return VT->getKnownBits(R: Reg).countMinLeadingZeros() >= DstSize - SrcSize;
2779 }
2780 return false;
2781}
2782
2783static LLT getMidVTForTruncRightShiftCombine(LLT ShiftTy, LLT TruncTy) {
2784 const unsigned ShiftSize = ShiftTy.getScalarSizeInBits();
2785 const unsigned TruncSize = TruncTy.getScalarSizeInBits();
2786
2787 // ShiftTy > 32 > TruncTy -> 32
2788 if (ShiftSize > 32 && TruncSize < 32)
2789 return ShiftTy.changeElementSize(NewEltSize: 32);
2790
2791 // TODO: We could also reduce to 16 bits, but that's more target-dependent.
2792 // Some targets like it, some don't, some only like it under certain
2793 // conditions/processor versions, etc.
2794 // A TL hook might be needed for this.
2795
2796 // Don't combine
2797 return ShiftTy;
2798}
2799
2800bool CombinerHelper::matchCombineTruncOfShift(
2801 MachineInstr &MI, std::pair<MachineInstr *, LLT> &MatchInfo) const {
2802 assert(MI.getOpcode() == TargetOpcode::G_TRUNC && "Expected a G_TRUNC");
2803 Register DstReg = MI.getOperand(i: 0).getReg();
2804 Register SrcReg = MI.getOperand(i: 1).getReg();
2805
2806 if (!MRI.hasOneNonDBGUse(RegNo: SrcReg))
2807 return false;
2808
2809 LLT SrcTy = MRI.getType(Reg: SrcReg);
2810 LLT DstTy = MRI.getType(Reg: DstReg);
2811
2812 MachineInstr *SrcMI = getDefIgnoringCopies(Reg: SrcReg, MRI);
2813 const auto &TL = getTargetLowering();
2814
2815 LLT NewShiftTy;
2816 switch (SrcMI->getOpcode()) {
2817 default:
2818 return false;
2819 case TargetOpcode::G_SHL: {
2820 NewShiftTy = DstTy;
2821
2822 // Make sure new shift amount is legal.
2823 KnownBits Known = VT->getKnownBits(R: SrcMI->getOperand(i: 2).getReg());
2824 if (Known.getMaxValue().uge(RHS: NewShiftTy.getScalarSizeInBits()))
2825 return false;
2826 break;
2827 }
2828 case TargetOpcode::G_LSHR:
2829 case TargetOpcode::G_ASHR: {
2830 // For right shifts, we conservatively do not do the transform if the TRUNC
2831 // has any STORE users. The reason is that if we change the type of the
2832 // shift, we may break the truncstore combine.
2833 //
2834 // TODO: Fix truncstore combine to handle (trunc(lshr (trunc x), k)).
2835 for (auto &User : MRI.use_instructions(Reg: DstReg))
2836 if (User.getOpcode() == TargetOpcode::G_STORE)
2837 return false;
2838
2839 NewShiftTy = getMidVTForTruncRightShiftCombine(ShiftTy: SrcTy, TruncTy: DstTy);
2840 if (NewShiftTy == SrcTy)
2841 return false;
2842
2843 // Make sure we won't lose information by truncating the high bits.
2844 KnownBits Known = VT->getKnownBits(R: SrcMI->getOperand(i: 2).getReg());
2845 if (Known.getMaxValue().ugt(RHS: NewShiftTy.getScalarSizeInBits() -
2846 DstTy.getScalarSizeInBits()))
2847 return false;
2848 break;
2849 }
2850 }
2851
2852 if (!isLegalOrBeforeLegalizer(
2853 Query: {SrcMI->getOpcode(),
2854 {NewShiftTy, TL.getPreferredShiftAmountTy(ShiftValueTy: NewShiftTy)}}))
2855 return false;
2856
2857 MatchInfo = std::make_pair(x&: SrcMI, y&: NewShiftTy);
2858 return true;
2859}
2860
2861void CombinerHelper::applyCombineTruncOfShift(
2862 MachineInstr &MI, std::pair<MachineInstr *, LLT> &MatchInfo) const {
2863 MachineInstr *ShiftMI = MatchInfo.first;
2864 LLT NewShiftTy = MatchInfo.second;
2865
2866 Register Dst = MI.getOperand(i: 0).getReg();
2867 LLT DstTy = MRI.getType(Reg: Dst);
2868
2869 Register ShiftAmt = ShiftMI->getOperand(i: 2).getReg();
2870 Register ShiftSrc = ShiftMI->getOperand(i: 1).getReg();
2871 ShiftSrc = Builder.buildTrunc(Res: NewShiftTy, Op: ShiftSrc).getReg(Idx: 0);
2872
2873 const auto &TL = getTargetLowering();
2874 LLT PrefShiftTy = TL.getPreferredShiftAmountTy(ShiftValueTy: NewShiftTy);
2875 if (MRI.getType(Reg: ShiftAmt) != PrefShiftTy)
2876 ShiftAmt = Builder.buildZExtOrTrunc(Res: PrefShiftTy, Op: ShiftAmt).getReg(Idx: 0);
2877
2878 Register NewShift =
2879 Builder
2880 .buildInstr(Opc: ShiftMI->getOpcode(), DstOps: {NewShiftTy}, SrcOps: {ShiftSrc, ShiftAmt})
2881 .getReg(Idx: 0);
2882
2883 if (NewShiftTy == DstTy)
2884 replaceRegWith(MRI, FromReg: Dst, ToReg: NewShift);
2885 else
2886 Builder.buildTrunc(Res: Dst, Op: NewShift);
2887
2888 eraseInst(MI);
2889}
2890
2891bool CombinerHelper::matchAnyExplicitUseIsUndef(MachineInstr &MI) const {
2892 return any_of(Range: MI.explicit_uses(), P: [this](const MachineOperand &MO) {
2893 return MO.isReg() &&
2894 getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: MO.getReg(), MRI);
2895 });
2896}
2897
2898bool CombinerHelper::matchAllExplicitUsesAreUndef(MachineInstr &MI) const {
2899 return all_of(Range: MI.explicit_uses(), P: [this](const MachineOperand &MO) {
2900 return !MO.isReg() ||
2901 getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: MO.getReg(), MRI);
2902 });
2903}
2904
2905bool CombinerHelper::matchUndefShuffleVectorMask(MachineInstr &MI) const {
2906 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
2907 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
2908 return all_of(Range&: Mask, P: [](int Elt) { return Elt < 0; });
2909}
2910
2911bool CombinerHelper::matchUndefStore(MachineInstr &MI) const {
2912 assert(MI.getOpcode() == TargetOpcode::G_STORE);
2913 return getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: MI.getOperand(i: 0).getReg(),
2914 MRI);
2915}
2916
2917bool CombinerHelper::matchUndefSelectCmp(MachineInstr &MI) const {
2918 assert(MI.getOpcode() == TargetOpcode::G_SELECT);
2919 return getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: MI.getOperand(i: 1).getReg(),
2920 MRI);
2921}
2922
2923bool CombinerHelper::matchInsertExtractVecEltOutOfBounds(
2924 MachineInstr &MI) const {
2925 assert((MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT ||
2926 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT) &&
2927 "Expected an insert/extract element op");
2928 LLT VecTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
2929 if (VecTy.isScalableVector())
2930 return false;
2931
2932 unsigned IdxIdx =
2933 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
2934 auto Idx = getIConstantVRegVal(VReg: MI.getOperand(i: IdxIdx).getReg(), MRI);
2935 if (!Idx)
2936 return false;
2937 return Idx->getZExtValue() >= VecTy.getNumElements();
2938}
2939
2940bool CombinerHelper::matchConstantSelectCmp(MachineInstr &MI,
2941 unsigned &OpIdx) const {
2942 GSelect &SelMI = cast<GSelect>(Val&: MI);
2943 auto Cst = isConstantOrConstantSplatVector(Def: SelMI.getCondReg(), MRI);
2944 if (!Cst)
2945 return false;
2946 OpIdx = Cst->isZero() ? 3 : 2;
2947 return true;
2948}
2949
2950void CombinerHelper::eraseInst(MachineInstr &MI) const { MI.eraseFromParent(); }
2951
2952bool CombinerHelper::matchEqualDefs(const MachineOperand &MOP1,
2953 const MachineOperand &MOP2) const {
2954 if (!MOP1.isReg() || !MOP2.isReg())
2955 return false;
2956 auto InstAndDef1 = getDefSrcRegIgnoringCopies(Reg: MOP1.getReg(), MRI);
2957 if (!InstAndDef1)
2958 return false;
2959 auto InstAndDef2 = getDefSrcRegIgnoringCopies(Reg: MOP2.getReg(), MRI);
2960 if (!InstAndDef2)
2961 return false;
2962 MachineInstr *I1 = InstAndDef1->MI;
2963 MachineInstr *I2 = InstAndDef2->MI;
2964
2965 // Handle a case like this:
2966 //
2967 // %0:_(s64), %1:_(s64) = G_UNMERGE_VALUES %2:_(<2 x s64>)
2968 //
2969 // Even though %0 and %1 are produced by the same instruction they are not
2970 // the same values.
2971 if (I1 == I2)
2972 return MOP1.getReg() == MOP2.getReg();
2973
2974 // If we have an instruction which loads or stores, we can't guarantee that
2975 // it is identical.
2976 //
2977 // For example, we may have
2978 //
2979 // %x1 = G_LOAD %addr (load N from @somewhere)
2980 // ...
2981 // call @foo
2982 // ...
2983 // %x2 = G_LOAD %addr (load N from @somewhere)
2984 // ...
2985 // %or = G_OR %x1, %x2
2986 //
2987 // It's possible that @foo will modify whatever lives at the address we're
2988 // loading from. To be safe, let's just assume that all loads and stores
2989 // are different (unless we have something which is guaranteed to not
2990 // change.)
2991 if (I1->mayLoadOrStore() && !I1->isDereferenceableInvariantLoad())
2992 return false;
2993
2994 // If both instructions are loads or stores, they are equal only if both
2995 // are dereferenceable invariant loads with the same number of bits.
2996 if (I1->mayLoadOrStore() && I2->mayLoadOrStore()) {
2997 GLoadStore *LS1 = dyn_cast<GLoadStore>(Val: I1);
2998 GLoadStore *LS2 = dyn_cast<GLoadStore>(Val: I2);
2999 if (!LS1 || !LS2)
3000 return false;
3001
3002 if (!I2->isDereferenceableInvariantLoad() ||
3003 (LS1->getMemSizeInBits() != LS2->getMemSizeInBits()))
3004 return false;
3005 }
3006
3007 // Check for physical registers on the instructions first to avoid cases
3008 // like this:
3009 //
3010 // %a = COPY $physreg
3011 // ...
3012 // SOMETHING implicit-def $physreg
3013 // ...
3014 // %b = COPY $physreg
3015 //
3016 // These copies are not equivalent.
3017 if (any_of(Range: I1->uses(), P: [](const MachineOperand &MO) {
3018 return MO.isReg() && MO.getReg().isPhysical();
3019 })) {
3020 // Check if we have a case like this:
3021 //
3022 // %a = COPY $physreg
3023 // %b = COPY %a
3024 //
3025 // In this case, I1 and I2 will both be equal to %a = COPY $physreg.
3026 // From that, we know that they must have the same value, since they must
3027 // have come from the same COPY.
3028 return I1->isIdenticalTo(Other: *I2);
3029 }
3030
3031 // We don't have any physical registers, so we don't necessarily need the
3032 // same vreg defs.
3033 //
3034 // On the off-chance that there's some target instruction feeding into the
3035 // instruction, let's use produceSameValue instead of isIdenticalTo.
3036 if (Builder.getTII().produceSameValue(MI0: *I1, MI1: *I2, MRI: &MRI)) {
3037 // Handle instructions with multiple defs that produce same values. Values
3038 // are same for operands with same index.
3039 // %0:_(s8), %1:_(s8), %2:_(s8), %3:_(s8) = G_UNMERGE_VALUES %4:_(<4 x s8>)
3040 // %5:_(s8), %6:_(s8), %7:_(s8), %8:_(s8) = G_UNMERGE_VALUES %4:_(<4 x s8>)
3041 // I1 and I2 are different instructions but produce same values,
3042 // %1 and %6 are same, %1 and %7 are not the same value.
3043 return I1->findRegisterDefOperandIdx(Reg: InstAndDef1->Reg, /*TRI=*/nullptr) ==
3044 I2->findRegisterDefOperandIdx(Reg: InstAndDef2->Reg, /*TRI=*/nullptr);
3045 }
3046 return false;
3047}
3048
3049bool CombinerHelper::matchConstantOp(const MachineOperand &MOP,
3050 int64_t C) const {
3051 if (!MOP.isReg())
3052 return false;
3053 auto MaybeCst = isConstantOrConstantSplatVector(Def: MOP.getReg(), MRI);
3054 return MaybeCst && MaybeCst->getBitWidth() <= 64 &&
3055 MaybeCst->getSExtValue() == C;
3056}
3057
3058bool CombinerHelper::matchConstantFPOp(const MachineOperand &MOP,
3059 double C) const {
3060 if (!MOP.isReg())
3061 return false;
3062 std::optional<FPValueAndVReg> MaybeCst;
3063 if (!mi_match(R: MOP.getReg(), MRI, P: m_GFCstOrSplat(FPValReg&: MaybeCst)))
3064 return false;
3065
3066 return MaybeCst->Value.isExactlyValue(V: C);
3067}
3068
3069void CombinerHelper::replaceSingleDefInstWithOperand(MachineInstr &MI,
3070 unsigned OpIdx) const {
3071 assert(MI.getNumExplicitDefs() == 1 && "Expected one explicit def?");
3072 Register OldReg = MI.getOperand(i: 0).getReg();
3073 Register Replacement = MI.getOperand(i: OpIdx).getReg();
3074 assert(canReplaceReg(OldReg, Replacement, MRI) && "Cannot replace register?");
3075 replaceRegWith(MRI, FromReg: OldReg, ToReg: Replacement);
3076 MI.eraseFromParent();
3077}
3078
3079void CombinerHelper::replaceSingleDefInstWithReg(MachineInstr &MI,
3080 Register Replacement) const {
3081 assert(MI.getNumExplicitDefs() == 1 && "Expected one explicit def?");
3082 Register OldReg = MI.getOperand(i: 0).getReg();
3083 assert(canReplaceReg(OldReg, Replacement, MRI) && "Cannot replace register?");
3084 replaceRegWith(MRI, FromReg: OldReg, ToReg: Replacement);
3085 MI.eraseFromParent();
3086}
3087
3088bool CombinerHelper::matchConstantLargerBitWidth(MachineInstr &MI,
3089 unsigned ConstIdx) const {
3090 Register ConstReg = MI.getOperand(i: ConstIdx).getReg();
3091 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3092
3093 // Get the shift amount
3094 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstReg, MRI);
3095 if (!VRegAndVal)
3096 return false;
3097
3098 // Return true of shift amount >= Bitwidth
3099 return (VRegAndVal->Value.uge(RHS: DstTy.getSizeInBits()));
3100}
3101
3102void CombinerHelper::applyFunnelShiftConstantModulo(MachineInstr &MI) const {
3103 assert((MI.getOpcode() == TargetOpcode::G_FSHL ||
3104 MI.getOpcode() == TargetOpcode::G_FSHR) &&
3105 "This is not a funnel shift operation");
3106
3107 Register ConstReg = MI.getOperand(i: 3).getReg();
3108 LLT ConstTy = MRI.getType(Reg: ConstReg);
3109 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3110
3111 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstReg, MRI);
3112 assert((VRegAndVal) && "Value is not a constant");
3113
3114 // Calculate the new Shift Amount = Old Shift Amount % BitWidth
3115 APInt NewConst = VRegAndVal->Value.urem(
3116 RHS: APInt(ConstTy.getSizeInBits(), DstTy.getScalarSizeInBits()));
3117
3118 auto NewConstInstr = Builder.buildConstant(Res: ConstTy, Val: NewConst.getZExtValue());
3119 Builder.buildInstr(
3120 Opc: MI.getOpcode(), DstOps: {MI.getOperand(i: 0)},
3121 SrcOps: {MI.getOperand(i: 1), MI.getOperand(i: 2), NewConstInstr.getReg(Idx: 0)});
3122
3123 MI.eraseFromParent();
3124}
3125
3126bool CombinerHelper::matchSelectSameVal(MachineInstr &MI) const {
3127 assert(MI.getOpcode() == TargetOpcode::G_SELECT);
3128 // Match (cond ? x : x)
3129 return matchEqualDefs(MOP1: MI.getOperand(i: 2), MOP2: MI.getOperand(i: 3)) &&
3130 canReplaceReg(DstReg: MI.getOperand(i: 0).getReg(), SrcReg: MI.getOperand(i: 2).getReg(),
3131 MRI);
3132}
3133
3134bool CombinerHelper::matchBinOpSameVal(MachineInstr &MI) const {
3135 return matchEqualDefs(MOP1: MI.getOperand(i: 1), MOP2: MI.getOperand(i: 2)) &&
3136 canReplaceReg(DstReg: MI.getOperand(i: 0).getReg(), SrcReg: MI.getOperand(i: 1).getReg(),
3137 MRI);
3138}
3139
3140bool CombinerHelper::matchOperandIsUndef(MachineInstr &MI,
3141 unsigned OpIdx) const {
3142 MachineOperand &MO = MI.getOperand(i: OpIdx);
3143 return MO.isReg() &&
3144 getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: MO.getReg(), MRI);
3145}
3146
3147bool CombinerHelper::matchOperandIsKnownToBeAPowerOfTwo(
3148 const MachineOperand &MO, bool OrNegative) const {
3149 return isKnownToBeAPowerOfTwo(Val: MO.getReg(), MRI, ValueTracking: VT, OrNegative);
3150}
3151
3152void CombinerHelper::replaceInstWithFConstant(MachineInstr &MI,
3153 double C) const {
3154 assert(MI.getNumDefs() == 1 && "Expected only one def?");
3155 Builder.buildFConstant(Res: MI.getOperand(i: 0), Val: C);
3156 MI.eraseFromParent();
3157}
3158
3159void CombinerHelper::replaceInstWithConstant(MachineInstr &MI,
3160 int64_t C) const {
3161 assert(MI.getNumDefs() == 1 && "Expected only one def?");
3162 Builder.buildConstant(Res: MI.getOperand(i: 0), Val: C);
3163 MI.eraseFromParent();
3164}
3165
3166void CombinerHelper::replaceInstWithConstant(MachineInstr &MI, APInt C) const {
3167 assert(MI.getNumDefs() == 1 && "Expected only one def?");
3168 Builder.buildConstant(Res: MI.getOperand(i: 0), Val: C);
3169 MI.eraseFromParent();
3170}
3171
3172void CombinerHelper::replaceInstWithFConstant(MachineInstr &MI,
3173 ConstantFP *CFP) const {
3174 assert(MI.getNumDefs() == 1 && "Expected only one def?");
3175 Builder.buildFConstant(Res: MI.getOperand(i: 0), Val: CFP->getValueAPF());
3176 MI.eraseFromParent();
3177}
3178
3179void CombinerHelper::replaceInstWithUndef(MachineInstr &MI) const {
3180 assert(MI.getNumDefs() == 1 && "Expected only one def?");
3181 Builder.buildUndef(Res: MI.getOperand(i: 0));
3182 MI.eraseFromParent();
3183}
3184
3185bool CombinerHelper::matchSimplifyAddToSub(
3186 MachineInstr &MI, std::tuple<Register, Register> &MatchInfo) const {
3187 Register LHS = MI.getOperand(i: 1).getReg();
3188 Register RHS = MI.getOperand(i: 2).getReg();
3189 Register &NewLHS = std::get<0>(t&: MatchInfo);
3190 Register &NewRHS = std::get<1>(t&: MatchInfo);
3191
3192 // Helper lambda to check for opportunities for
3193 // ((0-A) + B) -> B - A
3194 // (A + (0-B)) -> A - B
3195 auto CheckFold = [&](Register &MaybeSub, Register &MaybeNewLHS) {
3196 if (!mi_match(R: MaybeSub, MRI, P: m_Neg(Src: m_Reg(R&: NewRHS))))
3197 return false;
3198 NewLHS = MaybeNewLHS;
3199 return true;
3200 };
3201
3202 return CheckFold(LHS, RHS) || CheckFold(RHS, LHS);
3203}
3204
3205bool CombinerHelper::matchCombineInsertVecElts(
3206 MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo) const {
3207 assert(MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT &&
3208 "Invalid opcode");
3209 Register DstReg = MI.getOperand(i: 0).getReg();
3210 LLT DstTy = MRI.getType(Reg: DstReg);
3211 assert(DstTy.isVector() && "Invalid G_INSERT_VECTOR_ELT?");
3212
3213 if (DstTy.isScalableVector())
3214 return false;
3215
3216 unsigned NumElts = DstTy.getNumElements();
3217 // If this MI is part of a sequence of insert_vec_elts, then
3218 // don't do the combine in the middle of the sequence.
3219 if (MRI.hasOneUse(RegNo: DstReg) && MRI.use_instr_begin(RegNo: DstReg)->getOpcode() ==
3220 TargetOpcode::G_INSERT_VECTOR_ELT)
3221 return false;
3222 MachineInstr *CurrInst = &MI;
3223 MachineInstr *TmpInst;
3224 int64_t IntImm;
3225 Register TmpReg;
3226 MatchInfo.resize(N: NumElts);
3227 while (mi_match(
3228 MI&: *CurrInst, MRI,
3229 P: m_GInsertVecElt(Src0: m_MInstr(MI&: TmpInst), Src1: m_Reg(R&: TmpReg), Src2: m_ICst(Cst&: IntImm)))) {
3230 if (IntImm >= NumElts || IntImm < 0)
3231 return false;
3232 if (!MatchInfo[IntImm])
3233 MatchInfo[IntImm] = TmpReg;
3234 CurrInst = TmpInst;
3235 }
3236 // Variable index.
3237 if (CurrInst->getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
3238 return false;
3239 if (TmpInst->getOpcode() == TargetOpcode::G_BUILD_VECTOR) {
3240 for (unsigned I = 1; I < TmpInst->getNumOperands(); ++I) {
3241 if (!MatchInfo[I - 1].isValid())
3242 MatchInfo[I - 1] = TmpInst->getOperand(i: I).getReg();
3243 }
3244 return true;
3245 }
3246 // If we didn't end in a G_IMPLICIT_DEF and the source is not fully
3247 // overwritten, bail out.
3248 return TmpInst->getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
3249 all_of(Range&: MatchInfo, P: [](Register Reg) { return !!Reg; });
3250}
3251
3252void CombinerHelper::applyCombineInsertVecElts(
3253 MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo) const {
3254 Register UndefReg;
3255 auto GetUndef = [&]() {
3256 if (UndefReg)
3257 return UndefReg;
3258 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3259 UndefReg = Builder.buildUndef(Res: DstTy.getScalarType()).getReg(Idx: 0);
3260 return UndefReg;
3261 };
3262 for (Register &Reg : MatchInfo) {
3263 if (!Reg)
3264 Reg = GetUndef();
3265 }
3266 Builder.buildBuildVector(Res: MI.getOperand(i: 0).getReg(), Ops: MatchInfo);
3267 MI.eraseFromParent();
3268}
3269
3270void CombinerHelper::applySimplifyAddToSub(
3271 MachineInstr &MI, std::tuple<Register, Register> &MatchInfo) const {
3272 Register SubLHS, SubRHS;
3273 std::tie(args&: SubLHS, args&: SubRHS) = MatchInfo;
3274 Builder.buildSub(Dst: MI.getOperand(i: 0).getReg(), Src0: SubLHS, Src1: SubRHS);
3275 MI.eraseFromParent();
3276}
3277
3278bool CombinerHelper::matchBinopWithNegInner(Register MInner, Register Other,
3279 unsigned RootOpc, Register Dst,
3280 LLT Ty,
3281 BuildFnTy &MatchInfo) const {
3282 /// Helper function for matchBinopWithNeg: tries to match one commuted form
3283 /// of `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`.
3284 MachineInstr *InnerDef;
3285 if (!mi_match(R: MInner, MRI, P: m_MInstr(MI&: InnerDef)))
3286 return false;
3287
3288 unsigned InnerOpc = InnerDef->getOpcode();
3289 if (InnerOpc != TargetOpcode::G_ADD && InnerOpc != TargetOpcode::G_SUB)
3290 return false;
3291
3292 if (!MRI.hasOneNonDBGUse(RegNo: MInner))
3293 return false;
3294
3295 Register InnerLHS = InnerDef->getOperand(i: 1).getReg();
3296 Register InnerRHS = InnerDef->getOperand(i: 2).getReg();
3297 Register NotSrc;
3298 Register B, C;
3299
3300 // Check if either operand is ~b
3301 auto TryMatch = [&](Register MaybeNot, Register Other) {
3302 if (mi_match(R: MaybeNot, MRI, P: m_Not(Src: m_Reg(R&: NotSrc)))) {
3303 if (!MRI.hasOneNonDBGUse(RegNo: MaybeNot))
3304 return false;
3305 B = NotSrc;
3306 C = Other;
3307 return true;
3308 }
3309 return false;
3310 };
3311
3312 // For SUB, the not must be the LHS. For ADD, it can be either operand.
3313 if (!TryMatch(InnerLHS, InnerRHS) &&
3314 !(InnerOpc == TargetOpcode::G_ADD && TryMatch(InnerRHS, InnerLHS)))
3315 return false;
3316
3317 // Flip add/sub
3318 unsigned FlippedOpc = (InnerOpc == TargetOpcode::G_ADD) ? TargetOpcode::G_SUB
3319 : TargetOpcode::G_ADD;
3320
3321 Register A = Other;
3322 MatchInfo = [=](MachineIRBuilder &Builder) {
3323 auto NewInner = Builder.buildInstr(Opc: FlippedOpc, DstOps: {Ty}, SrcOps: {B, C});
3324 auto NewNot = Builder.buildNot(Dst: Ty, Src0: NewInner);
3325 Builder.buildInstr(Opc: RootOpc, DstOps: {Dst}, SrcOps: {A, NewNot});
3326 };
3327 return true;
3328}
3329
3330bool CombinerHelper::matchBinopWithNeg(MachineInstr &MI,
3331 BuildFnTy &MatchInfo) const {
3332 // Fold `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`
3333 // Root MI is one of G_AND, G_OR, G_XOR.
3334 // We also look for commuted forms of operations. Pattern shouldn't apply
3335 // if there are multiple reasons of inner operations.
3336
3337 unsigned RootOpc = MI.getOpcode();
3338 Register Dst = MI.getOperand(i: 0).getReg();
3339 LLT Ty = MRI.getType(Reg: Dst);
3340
3341 Register LHS = MI.getOperand(i: 1).getReg();
3342 Register RHS = MI.getOperand(i: 2).getReg();
3343 // Check the commuted and uncommuted forms of the operation.
3344 return matchBinopWithNegInner(MInner: LHS, Other: RHS, RootOpc, Dst, Ty, MatchInfo) ||
3345 matchBinopWithNegInner(MInner: RHS, Other: LHS, RootOpc, Dst, Ty, MatchInfo);
3346}
3347
3348bool CombinerHelper::matchHoistLogicOpWithSameOpcodeHands(
3349 MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) const {
3350 // Matches: logic (hand x, ...), (hand y, ...) -> hand (logic x, y), ...
3351 //
3352 // Creates the new hand + logic instruction (but does not insert them.)
3353 //
3354 // On success, MatchInfo is populated with the new instructions. These are
3355 // inserted in applyHoistLogicOpWithSameOpcodeHands.
3356 unsigned LogicOpcode = MI.getOpcode();
3357 assert(LogicOpcode == TargetOpcode::G_AND ||
3358 LogicOpcode == TargetOpcode::G_OR ||
3359 LogicOpcode == TargetOpcode::G_XOR);
3360 MachineIRBuilder MIB(MI);
3361 Register Dst = MI.getOperand(i: 0).getReg();
3362 Register LHSReg = MI.getOperand(i: 1).getReg();
3363 Register RHSReg = MI.getOperand(i: 2).getReg();
3364
3365 // Don't recompute anything.
3366 if (!MRI.hasOneNonDBGUse(RegNo: LHSReg) || !MRI.hasOneNonDBGUse(RegNo: RHSReg))
3367 return false;
3368
3369 // Make sure we have (hand x, ...), (hand y, ...)
3370 MachineInstr *LeftHandInst = getDefIgnoringCopies(Reg: LHSReg, MRI);
3371 MachineInstr *RightHandInst = getDefIgnoringCopies(Reg: RHSReg, MRI);
3372 if (!LeftHandInst || !RightHandInst)
3373 return false;
3374 unsigned HandOpcode = LeftHandInst->getOpcode();
3375 if (HandOpcode != RightHandInst->getOpcode())
3376 return false;
3377 if (LeftHandInst->getNumOperands() < 2 ||
3378 !LeftHandInst->getOperand(i: 1).isReg() ||
3379 RightHandInst->getNumOperands() < 2 ||
3380 !RightHandInst->getOperand(i: 1).isReg())
3381 return false;
3382
3383 // Make sure the types match up, and if we're doing this post-legalization,
3384 // we end up with legal types.
3385 Register X = LeftHandInst->getOperand(i: 1).getReg();
3386 Register Y = RightHandInst->getOperand(i: 1).getReg();
3387 LLT XTy = MRI.getType(Reg: X);
3388 LLT YTy = MRI.getType(Reg: Y);
3389 if (!XTy.isValid() || XTy != YTy)
3390 return false;
3391
3392 // Optional extra source register.
3393 Register ExtraHandOpSrcReg;
3394 switch (HandOpcode) {
3395 default:
3396 return false;
3397 case TargetOpcode::G_ANYEXT:
3398 case TargetOpcode::G_SEXT:
3399 case TargetOpcode::G_ZEXT: {
3400 // Match: logic (ext X), (ext Y) --> ext (logic X, Y)
3401 break;
3402 }
3403 case TargetOpcode::G_TRUNC: {
3404 // Match: logic (trunc X), (trunc Y) -> trunc (logic X, Y)
3405 const MachineFunction *MF = MI.getMF();
3406 LLVMContext &Ctx = MF->getFunction().getContext();
3407
3408 LLT DstTy = MRI.getType(Reg: Dst);
3409 const TargetLowering &TLI = getTargetLowering();
3410
3411 // Be extra careful sinking truncate. If it's free, there's no benefit in
3412 // widening a binop.
3413 if (TLI.isZExtFree(FromTy: DstTy, ToTy: XTy, Ctx) && TLI.isTruncateFree(FromTy: XTy, ToTy: DstTy, Ctx))
3414 return false;
3415 break;
3416 }
3417 case TargetOpcode::G_AND:
3418 case TargetOpcode::G_ASHR:
3419 case TargetOpcode::G_LSHR:
3420 case TargetOpcode::G_SHL: {
3421 // Match: logic (binop x, z), (binop y, z) -> binop (logic x, y), z
3422 MachineOperand &ZOp = LeftHandInst->getOperand(i: 2);
3423 if (!matchEqualDefs(MOP1: ZOp, MOP2: RightHandInst->getOperand(i: 2)))
3424 return false;
3425 ExtraHandOpSrcReg = ZOp.getReg();
3426 break;
3427 }
3428 }
3429
3430 if (!isLegalOrBeforeLegalizer(Query: {LogicOpcode, {XTy, YTy}}))
3431 return false;
3432
3433 // Record the steps to build the new instructions.
3434 //
3435 // Steps to build (logic x, y)
3436 auto NewLogicDst = MRI.createGenericVirtualRegister(Ty: XTy);
3437 OperandBuildSteps LogicBuildSteps = {
3438 [=](MachineInstrBuilder &MIB) { MIB.addDef(RegNo: NewLogicDst); },
3439 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: X); },
3440 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Y); }};
3441 InstructionBuildSteps LogicSteps(LogicOpcode, LogicBuildSteps);
3442
3443 // Steps to build hand (logic x, y), ...z
3444 OperandBuildSteps HandBuildSteps = {
3445 [=](MachineInstrBuilder &MIB) { MIB.addDef(RegNo: Dst); },
3446 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: NewLogicDst); }};
3447 if (ExtraHandOpSrcReg.isValid())
3448 HandBuildSteps.push_back(
3449 Elt: [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ExtraHandOpSrcReg); });
3450 InstructionBuildSteps HandSteps(HandOpcode, HandBuildSteps);
3451
3452 MatchInfo = InstructionStepsMatchInfo({LogicSteps, HandSteps});
3453 return true;
3454}
3455
3456void CombinerHelper::applyBuildInstructionSteps(
3457 MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) const {
3458 assert(MatchInfo.InstrsToBuild.size() &&
3459 "Expected at least one instr to build?");
3460 for (auto &InstrToBuild : MatchInfo.InstrsToBuild) {
3461 assert(InstrToBuild.Opcode && "Expected a valid opcode?");
3462 assert(InstrToBuild.OperandFns.size() && "Expected at least one operand?");
3463 MachineInstrBuilder Instr = Builder.buildInstr(Opcode: InstrToBuild.Opcode);
3464 for (auto &OperandFn : InstrToBuild.OperandFns)
3465 OperandFn(Instr);
3466 }
3467 MI.eraseFromParent();
3468}
3469
3470bool CombinerHelper::matchAshrShlToSextInreg(
3471 MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {
3472 assert(MI.getOpcode() == TargetOpcode::G_ASHR);
3473 int64_t ShlCst, AshrCst;
3474 Register Src;
3475 if (!mi_match(R: MI.getOperand(i: 0).getReg(), MRI,
3476 P: m_GAShr(L: m_GShl(L: m_Reg(R&: Src), R: m_ICstOrSplat(Cst&: ShlCst)),
3477 R: m_ICstOrSplat(Cst&: AshrCst))))
3478 return false;
3479 if (ShlCst != AshrCst)
3480 return false;
3481 if (!isLegalOrBeforeLegalizer(
3482 Query: {TargetOpcode::G_SEXT_INREG,
3483 {MRI.getType(Reg: Src)},
3484 {},
3485 {MRI.getType(Reg: Src).getScalarSizeInBits() - ShlCst}}))
3486 return false;
3487 MatchInfo = std::make_tuple(args&: Src, args&: ShlCst);
3488 return true;
3489}
3490
3491void CombinerHelper::applyAshShlToSextInreg(
3492 MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {
3493 assert(MI.getOpcode() == TargetOpcode::G_ASHR);
3494 Register Src;
3495 int64_t ShiftAmt;
3496 std::tie(args&: Src, args&: ShiftAmt) = MatchInfo;
3497 unsigned Size = MRI.getType(Reg: Src).getScalarSizeInBits();
3498 Builder.buildSExtInReg(Res: MI.getOperand(i: 0).getReg(), Op: Src, ImmOp: Size - ShiftAmt);
3499 MI.eraseFromParent();
3500}
3501
3502/// and(and(x, C1), C2) -> C1&C2 ? and(x, C1&C2) : 0
3503bool CombinerHelper::matchOverlappingAnd(
3504 MachineInstr &MI,
3505 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
3506 assert(MI.getOpcode() == TargetOpcode::G_AND);
3507
3508 Register Dst = MI.getOperand(i: 0).getReg();
3509 LLT Ty = MRI.getType(Reg: Dst);
3510
3511 Register R;
3512 int64_t C1;
3513 int64_t C2;
3514 if (!mi_match(
3515 R: Dst, MRI,
3516 P: m_GAnd(L: m_GAnd(L: m_Reg(R), R: m_ICst(Cst&: C1)), R: m_ICst(Cst&: C2))))
3517 return false;
3518
3519 MatchInfo = [=](MachineIRBuilder &B) {
3520 if (C1 & C2) {
3521 B.buildAnd(Dst, Src0: R, Src1: B.buildConstant(Res: Ty, Val: C1 & C2));
3522 return;
3523 }
3524 auto Zero = B.buildConstant(Res: Ty, Val: 0);
3525 replaceRegWith(MRI, FromReg: Dst, ToReg: Zero->getOperand(i: 0).getReg());
3526 };
3527 return true;
3528}
3529
3530bool CombinerHelper::matchRedundantAnd(MachineInstr &MI,
3531 Register &Replacement) const {
3532 // Given
3533 //
3534 // %y:_(sN) = G_SOMETHING
3535 // %x:_(sN) = G_SOMETHING
3536 // %res:_(sN) = G_AND %x, %y
3537 //
3538 // Eliminate the G_AND when it is known that x & y == x or x & y == y.
3539 //
3540 // Patterns like this can appear as a result of legalization. E.g.
3541 //
3542 // %cmp:_(s32) = G_ICMP intpred(pred), %x(s32), %y
3543 // %one:_(s32) = G_CONSTANT i32 1
3544 // %and:_(s32) = G_AND %cmp, %one
3545 //
3546 // In this case, G_ICMP only produces a single bit, so x & 1 == x.
3547 assert(MI.getOpcode() == TargetOpcode::G_AND);
3548 if (!VT)
3549 return false;
3550
3551 Register AndDst = MI.getOperand(i: 0).getReg();
3552 Register LHS = MI.getOperand(i: 1).getReg();
3553 Register RHS = MI.getOperand(i: 2).getReg();
3554
3555 // Check the RHS (maybe a constant) first, and if we have no KnownBits there,
3556 // we can't do anything. If we do, then it depends on whether we have
3557 // KnownBits on the LHS.
3558 KnownBits RHSBits = VT->getKnownBits(R: RHS);
3559 if (RHSBits.isUnknown())
3560 return false;
3561
3562 KnownBits LHSBits = VT->getKnownBits(R: LHS);
3563
3564 // Check that x & Mask == x.
3565 // x & 1 == x, always
3566 // x & 0 == x, only if x is also 0
3567 // Meaning Mask has no effect if every bit is either one in Mask or zero in x.
3568 //
3569 // Check if we can replace AndDst with the LHS of the G_AND
3570 if (canReplaceReg(DstReg: AndDst, SrcReg: LHS, MRI) &&
3571 (LHSBits.Zero | RHSBits.One).isAllOnes()) {
3572 Replacement = LHS;
3573 return true;
3574 }
3575
3576 // Check if we can replace AndDst with the RHS of the G_AND
3577 if (canReplaceReg(DstReg: AndDst, SrcReg: RHS, MRI) &&
3578 (LHSBits.One | RHSBits.Zero).isAllOnes()) {
3579 Replacement = RHS;
3580 return true;
3581 }
3582
3583 return false;
3584}
3585
3586bool CombinerHelper::matchRedundantOr(MachineInstr &MI,
3587 Register &Replacement) const {
3588 // Given
3589 //
3590 // %y:_(sN) = G_SOMETHING
3591 // %x:_(sN) = G_SOMETHING
3592 // %res:_(sN) = G_OR %x, %y
3593 //
3594 // Eliminate the G_OR when it is known that x | y == x or x | y == y.
3595 assert(MI.getOpcode() == TargetOpcode::G_OR);
3596 if (!VT)
3597 return false;
3598
3599 Register OrDst = MI.getOperand(i: 0).getReg();
3600 Register LHS = MI.getOperand(i: 1).getReg();
3601 Register RHS = MI.getOperand(i: 2).getReg();
3602
3603 KnownBits LHSBits = VT->getKnownBits(R: LHS);
3604 KnownBits RHSBits = VT->getKnownBits(R: RHS);
3605
3606 // Check that x | Mask == x.
3607 // x | 0 == x, always
3608 // x | 1 == x, only if x is also 1
3609 // Meaning Mask has no effect if every bit is either zero in Mask or one in x.
3610 //
3611 // Check if we can replace OrDst with the LHS of the G_OR
3612 if (canReplaceReg(DstReg: OrDst, SrcReg: LHS, MRI) &&
3613 (LHSBits.One | RHSBits.Zero).isAllOnes()) {
3614 Replacement = LHS;
3615 return true;
3616 }
3617
3618 // Check if we can replace OrDst with the RHS of the G_OR
3619 if (canReplaceReg(DstReg: OrDst, SrcReg: RHS, MRI) &&
3620 (LHSBits.Zero | RHSBits.One).isAllOnes()) {
3621 Replacement = RHS;
3622 return true;
3623 }
3624
3625 return false;
3626}
3627
3628bool CombinerHelper::matchRedundantSExtInReg(MachineInstr &MI) const {
3629 // If the input is already sign extended, just drop the extension.
3630 Register Src = MI.getOperand(i: 1).getReg();
3631 unsigned ExtBits = MI.getOperand(i: 2).getImm();
3632 unsigned TypeSize = MRI.getType(Reg: Src).getScalarSizeInBits();
3633 return VT->computeNumSignBits(R: Src) >= (TypeSize - ExtBits + 1);
3634}
3635
3636static bool isConstValidTrue(const TargetLowering &TLI, unsigned ScalarSizeBits,
3637 int64_t Cst, bool IsVector, bool IsFP) {
3638 // For i1, Cst will always be -1 regardless of boolean contents.
3639 return (ScalarSizeBits == 1 && Cst == -1) ||
3640 isConstTrueVal(TLI, Val: Cst, IsVector, IsFP);
3641}
3642
3643// This pattern aims to match the following shape to avoid extra mov
3644// instructions
3645// G_BUILD_VECTOR(
3646// G_UNMERGE_VALUES(src, 0)
3647// G_UNMERGE_VALUES(src, 1)
3648// G_IMPLICIT_DEF
3649// G_IMPLICIT_DEF
3650// )
3651// ->
3652// G_CONCAT_VECTORS(
3653// src,
3654// undef
3655// )
3656bool CombinerHelper::matchCombineBuildUnmerge(MachineInstr &MI,
3657 MachineRegisterInfo &MRI,
3658 Register &UnmergeSrc) const {
3659 auto &BV = cast<GBuildVector>(Val&: MI);
3660
3661 unsigned BuildUseCount = BV.getNumSources();
3662 if (BuildUseCount % 2 != 0)
3663 return false;
3664
3665 unsigned NumUnmerge = BuildUseCount / 2;
3666
3667 auto *Unmerge = getOpcodeDef<GUnmerge>(Reg: BV.getSourceReg(I: 0), MRI);
3668
3669 // Check the first operand is an unmerge and has the correct number of
3670 // operands
3671 if (!Unmerge || Unmerge->getNumDefs() != NumUnmerge)
3672 return false;
3673
3674 UnmergeSrc = Unmerge->getSourceReg();
3675
3676 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3677 LLT UnmergeSrcTy = MRI.getType(Reg: UnmergeSrc);
3678
3679 if (!UnmergeSrcTy.isVector())
3680 return false;
3681
3682 // Ensure we only generate legal instructions post-legalizer
3683 if (!IsPreLegalize &&
3684 !isLegal(Query: {TargetOpcode::G_CONCAT_VECTORS, {DstTy, UnmergeSrcTy}}))
3685 return false;
3686
3687 // Check that all of the operands before the midpoint come from the same
3688 // unmerge and are in the same order as they are used in the build_vector
3689 for (unsigned I = 0; I < NumUnmerge; ++I) {
3690 auto MaybeUnmergeReg = BV.getSourceReg(I);
3691 auto *LoopUnmerge = getOpcodeDef<GUnmerge>(Reg: MaybeUnmergeReg, MRI);
3692
3693 if (!LoopUnmerge || LoopUnmerge != Unmerge)
3694 return false;
3695
3696 if (LoopUnmerge->getOperand(i: I).getReg() != MaybeUnmergeReg)
3697 return false;
3698 }
3699
3700 // Check that all of the unmerged values are used
3701 if (Unmerge->getNumDefs() != NumUnmerge)
3702 return false;
3703
3704 // Check that all of the operands after the mid point are undefs.
3705 for (unsigned I = NumUnmerge; I < BuildUseCount; ++I) {
3706 auto *Undef = getDefIgnoringCopies(Reg: BV.getSourceReg(I), MRI);
3707
3708 if (Undef->getOpcode() != TargetOpcode::G_IMPLICIT_DEF)
3709 return false;
3710 }
3711
3712 return true;
3713}
3714
3715void CombinerHelper::applyCombineBuildUnmerge(MachineInstr &MI,
3716 MachineRegisterInfo &MRI,
3717 MachineIRBuilder &B,
3718 Register &UnmergeSrc) const {
3719 assert(UnmergeSrc && "Expected there to be one matching G_UNMERGE_VALUES");
3720 B.setInstrAndDebugLoc(MI);
3721
3722 Register UndefVec = B.buildUndef(Res: MRI.getType(Reg: UnmergeSrc)).getReg(Idx: 0);
3723 B.buildConcatVectors(Res: MI.getOperand(i: 0), Ops: {UnmergeSrc, UndefVec});
3724
3725 MI.eraseFromParent();
3726}
3727
3728// This combine tries to reduce the number of scalarised G_TRUNC instructions by
3729// using vector truncates instead
3730//
3731// EXAMPLE:
3732// %a(i32), %b(i32) = G_UNMERGE_VALUES %src(<2 x i32>)
3733// %T_a(i16) = G_TRUNC %a(i32)
3734// %T_b(i16) = G_TRUNC %b(i32)
3735// %Undef(i16) = G_IMPLICIT_DEF(i16)
3736// %dst(v4i16) = G_BUILD_VECTORS %T_a(i16), %T_b(i16), %Undef(i16), %Undef(i16)
3737//
3738// ===>
3739// %Undef(<2 x i32>) = G_IMPLICIT_DEF(<2 x i32>)
3740// %Mid(<4 x s32>) = G_CONCAT_VECTORS %src(<2 x i32>), %Undef(<2 x i32>)
3741// %dst(<4 x s16>) = G_TRUNC %Mid(<4 x s32>)
3742//
3743// Only matches sources made up of G_TRUNCs followed by G_IMPLICIT_DEFs
3744bool CombinerHelper::matchUseVectorTruncate(MachineInstr &MI,
3745 Register &MatchInfo) const {
3746 auto BuildMI = cast<GBuildVector>(Val: &MI);
3747 unsigned NumOperands = BuildMI->getNumSources();
3748 LLT DstTy = MRI.getType(Reg: BuildMI->getReg(Idx: 0));
3749
3750 // Check the G_BUILD_VECTOR sources
3751 unsigned I;
3752 GUnmerge *UnmergeMI = nullptr;
3753
3754 // Check all source TRUNCs come from the same UNMERGE instruction
3755 // and that the element order matches (BUILD_VECTOR position I
3756 // corresponds to UNMERGE result I)
3757 for (I = 0; I < NumOperands; ++I) {
3758 // Check if the G_TRUNC instructions all come from the same MI
3759 Register TruncSrcReg;
3760 if (!mi_match(R: BuildMI->getSourceReg(I), MRI, P: m_GTrunc(Src: m_Reg(R&: TruncSrcReg))))
3761 break;
3762
3763 if (!UnmergeMI) {
3764 if (!mi_match(R: TruncSrcReg, MRI, P: m_GUnmerge(Inst&: UnmergeMI)))
3765 return false;
3766 } else {
3767 MachineInstr *UnmergeSrcMI;
3768 if (!mi_match(R: TruncSrcReg, MRI, P: m_MInstr(MI&: UnmergeSrcMI)) ||
3769 UnmergeMI != UnmergeSrcMI)
3770 return false;
3771 }
3772 // Element order must match: position I must use UNMERGE result I.
3773 if (UnmergeMI->getOperand(i: I).getReg() != TruncSrcReg)
3774 return false;
3775 }
3776 if (I < 2)
3777 return false;
3778
3779 // Check the remaining source elements are only G_IMPLICIT_DEF
3780 for (; I < NumOperands; ++I) {
3781 if (!mi_match(R: BuildMI->getSourceReg(I), MRI, P: m_GImplicitDef()))
3782 return false;
3783 }
3784
3785 // Check the size of unmerge source
3786 MatchInfo = UnmergeMI->getSourceReg();
3787 LLT UnmergeSrcTy = MRI.getType(Reg: MatchInfo);
3788 if (!DstTy.getElementCount().isKnownMultipleOf(RHS: UnmergeSrcTy.getNumElements()))
3789 return false;
3790
3791 // Check the unmerge source and destination element types match
3792 LLT UnmergeSrcEltTy = UnmergeSrcTy.getElementType();
3793 Register UnmergeDstReg = UnmergeMI->getOperand(i: 0).getReg();
3794 LLT UnmergeDstEltTy = MRI.getType(Reg: UnmergeDstReg);
3795 if (UnmergeSrcEltTy != UnmergeDstEltTy)
3796 return false;
3797
3798 // Only generate legal instructions post-legalizer
3799 if (!IsPreLegalize) {
3800 LLT MidTy = DstTy.changeElementType(NewEltTy: UnmergeSrcTy.getScalarType());
3801
3802 if (DstTy.getElementCount() != UnmergeSrcTy.getElementCount() &&
3803 !isLegal(Query: {TargetOpcode::G_CONCAT_VECTORS, {MidTy, UnmergeSrcTy}}))
3804 return false;
3805
3806 if (!isLegal(Query: {TargetOpcode::G_TRUNC, {DstTy, MidTy}}))
3807 return false;
3808 }
3809
3810 return true;
3811}
3812
3813void CombinerHelper::applyUseVectorTruncate(MachineInstr &MI,
3814 Register &MatchInfo) const {
3815 Register MidReg;
3816 auto BuildMI = cast<GBuildVector>(Val: &MI);
3817 Register DstReg = BuildMI->getReg(Idx: 0);
3818 LLT DstTy = MRI.getType(Reg: DstReg);
3819 LLT UnmergeSrcTy = MRI.getType(Reg: MatchInfo);
3820 unsigned DstTyNumElt = DstTy.getNumElements();
3821 unsigned UnmergeSrcTyNumElt = UnmergeSrcTy.getNumElements();
3822
3823 // No need to pad vector if only G_TRUNC is needed
3824 if (DstTyNumElt / UnmergeSrcTyNumElt == 1) {
3825 MidReg = MatchInfo;
3826 } else {
3827 Register UndefReg = Builder.buildUndef(Res: UnmergeSrcTy).getReg(Idx: 0);
3828 SmallVector<Register> ConcatRegs = {MatchInfo};
3829 for (unsigned I = 1; I < DstTyNumElt / UnmergeSrcTyNumElt; ++I)
3830 ConcatRegs.push_back(Elt: UndefReg);
3831
3832 auto MidTy = DstTy.changeElementType(NewEltTy: UnmergeSrcTy.getScalarType());
3833 MidReg = Builder.buildConcatVectors(Res: MidTy, Ops: ConcatRegs).getReg(Idx: 0);
3834 }
3835
3836 Builder.buildTrunc(Res: DstReg, Op: MidReg);
3837 MI.eraseFromParent();
3838}
3839
3840bool CombinerHelper::matchNotCmp(
3841 MachineInstr &MI, SmallVectorImpl<Register> &RegsToNegate) const {
3842 assert(MI.getOpcode() == TargetOpcode::G_XOR);
3843 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3844 const auto &TLI = *Builder.getMF().getSubtarget().getTargetLowering();
3845 Register XorSrc;
3846 Register CstReg;
3847 // We match xor(src, true) here.
3848 if (!mi_match(R: MI.getOperand(i: 0).getReg(), MRI,
3849 P: m_GXor(L: m_Reg(R&: XorSrc), R: m_Reg(R&: CstReg))))
3850 return false;
3851
3852 if (!MRI.hasOneNonDBGUse(RegNo: XorSrc))
3853 return false;
3854
3855 // Check that XorSrc is the root of a tree of comparisons combined with ANDs
3856 // and ORs. The suffix of RegsToNegate starting from index I is used a work
3857 // list of tree nodes to visit.
3858 RegsToNegate.push_back(Elt: XorSrc);
3859 // Remember whether the comparisons are all integer or all floating point.
3860 bool IsInt = false;
3861 bool IsFP = false;
3862 for (unsigned I = 0; I < RegsToNegate.size(); ++I) {
3863 Register Reg = RegsToNegate[I];
3864 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
3865 return false;
3866 MachineInstr *Def;
3867 if (!mi_match(R: Reg, MRI, P: m_MInstr(MI&: Def)))
3868 return false;
3869 switch (Def->getOpcode()) {
3870 default:
3871 // Don't match if the tree contains anything other than ANDs, ORs and
3872 // comparisons.
3873 return false;
3874 case TargetOpcode::G_ICMP:
3875 if (IsFP)
3876 return false;
3877 IsInt = true;
3878 // When we apply the combine we will invert the predicate.
3879 break;
3880 case TargetOpcode::G_FCMP:
3881 if (IsInt)
3882 return false;
3883 IsFP = true;
3884 // When we apply the combine we will invert the predicate.
3885 break;
3886 case TargetOpcode::G_AND:
3887 case TargetOpcode::G_OR:
3888 // Implement De Morgan's laws:
3889 // ~(x & y) -> ~x | ~y
3890 // ~(x | y) -> ~x & ~y
3891 // When we apply the combine we will change the opcode and recursively
3892 // negate the operands.
3893 RegsToNegate.push_back(Elt: Def->getOperand(i: 1).getReg());
3894 RegsToNegate.push_back(Elt: Def->getOperand(i: 2).getReg());
3895 break;
3896 }
3897 }
3898
3899 // Now we know whether the comparisons are integer or floating point, check
3900 // the constant in the xor.
3901 int64_t Cst;
3902 if (Ty.isVector()) {
3903 int64_t SplatCst;
3904 if (!mi_match(R: CstReg, MRI, P: m_ICstOrSplat(Cst&: SplatCst)))
3905 return false;
3906 if (!isConstValidTrue(TLI, ScalarSizeBits: Ty.getScalarSizeInBits(), Cst: SplatCst, IsVector: true, IsFP))
3907 return false;
3908 } else {
3909 if (!mi_match(R: CstReg, MRI, P: m_ICst(Cst)))
3910 return false;
3911 if (!isConstValidTrue(TLI, ScalarSizeBits: Ty.getSizeInBits(), Cst, IsVector: false, IsFP))
3912 return false;
3913 }
3914
3915 return true;
3916}
3917
3918void CombinerHelper::applyNotCmp(
3919 MachineInstr &MI, SmallVectorImpl<Register> &RegsToNegate) const {
3920 for (Register Reg : RegsToNegate) {
3921 MachineInstr *Def = MRI.getVRegDef(Reg);
3922 Observer.changingInstr(MI&: *Def);
3923 // For each comparison, invert the opcode. For each AND and OR, change the
3924 // opcode.
3925 switch (Def->getOpcode()) {
3926 default:
3927 llvm_unreachable("Unexpected opcode");
3928 case TargetOpcode::G_ICMP:
3929 case TargetOpcode::G_FCMP: {
3930 MachineOperand &PredOp = Def->getOperand(i: 1);
3931 CmpInst::Predicate NewP = CmpInst::getInversePredicate(
3932 pred: (CmpInst::Predicate)PredOp.getPredicate());
3933 PredOp.setPredicate(NewP);
3934 break;
3935 }
3936 case TargetOpcode::G_AND:
3937 Def->setDesc(Builder.getTII().get(Opcode: TargetOpcode::G_OR));
3938 break;
3939 case TargetOpcode::G_OR:
3940 Def->setDesc(Builder.getTII().get(Opcode: TargetOpcode::G_AND));
3941 break;
3942 }
3943 Observer.changedInstr(MI&: *Def);
3944 }
3945
3946 replaceRegWith(MRI, FromReg: MI.getOperand(i: 0).getReg(), ToReg: MI.getOperand(i: 1).getReg());
3947 MI.eraseFromParent();
3948}
3949
3950bool CombinerHelper::matchXorOfAndWithSameReg(
3951 MachineInstr &MI, std::pair<Register, Register> &MatchInfo) const {
3952 // Match (xor (and x, y), y) (or any of its commuted cases)
3953 assert(MI.getOpcode() == TargetOpcode::G_XOR);
3954 Register &X = MatchInfo.first;
3955 Register &Y = MatchInfo.second;
3956 Register AndReg = MI.getOperand(i: 1).getReg();
3957 Register SharedReg = MI.getOperand(i: 2).getReg();
3958
3959 // Find a G_AND on either side of the G_XOR.
3960 // Look for one of
3961 //
3962 // (xor (and x, y), SharedReg)
3963 // (xor SharedReg, (and x, y))
3964 if (!mi_match(R: AndReg, MRI, P: m_GAnd(L: m_Reg(R&: X), R: m_Reg(R&: Y)))) {
3965 std::swap(a&: AndReg, b&: SharedReg);
3966 if (!mi_match(R: AndReg, MRI, P: m_GAnd(L: m_Reg(R&: X), R: m_Reg(R&: Y))))
3967 return false;
3968 }
3969
3970 // Only do this if we'll eliminate the G_AND.
3971 if (!MRI.hasOneNonDBGUse(RegNo: AndReg))
3972 return false;
3973
3974 // We can combine if SharedReg is the same as either the LHS or RHS of the
3975 // G_AND.
3976 if (Y != SharedReg)
3977 std::swap(a&: X, b&: Y);
3978 return Y == SharedReg;
3979}
3980
3981void CombinerHelper::applyXorOfAndWithSameReg(
3982 MachineInstr &MI, std::pair<Register, Register> &MatchInfo) const {
3983 // Fold (xor (and x, y), y) -> (and (not x), y)
3984 Register X, Y;
3985 std::tie(args&: X, args&: Y) = MatchInfo;
3986 auto Not = Builder.buildNot(Dst: MRI.getType(Reg: X), Src0: X);
3987 Observer.changingInstr(MI);
3988 MI.setDesc(Builder.getTII().get(Opcode: TargetOpcode::G_AND));
3989 MI.getOperand(i: 1).setReg(Not->getOperand(i: 0).getReg());
3990 MI.getOperand(i: 2).setReg(Y);
3991 Observer.changedInstr(MI);
3992}
3993
3994bool CombinerHelper::matchPtrAddZero(MachineInstr &MI) const {
3995 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
3996 Register DstReg = PtrAdd.getReg(Idx: 0);
3997 LLT Ty = MRI.getType(Reg: DstReg);
3998 const DataLayout &DL = Builder.getMF().getDataLayout();
3999
4000 if (DL.isNonIntegralAddressSpace(AddrSpace: Ty.getScalarType().getAddressSpace()))
4001 return false;
4002
4003 if (Ty.isPointer()) {
4004 auto ConstVal = getIConstantVRegVal(VReg: PtrAdd.getBaseReg(), MRI);
4005 return ConstVal && *ConstVal == 0;
4006 }
4007
4008 assert(Ty.isVector() && "Expecting a vector type");
4009 const MachineInstr *VecMI;
4010 if (!mi_match(R: PtrAdd.getBaseReg(), MRI, P: m_MInstr(MI&: VecMI)))
4011 return false;
4012 return isBuildVectorAllZeros(MI: *VecMI, MRI);
4013}
4014
4015void CombinerHelper::applyPtrAddZero(MachineInstr &MI) const {
4016 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
4017 Builder.buildIntToPtr(Dst: PtrAdd.getReg(Idx: 0), Src: PtrAdd.getOffsetReg());
4018 PtrAdd.eraseFromParent();
4019}
4020
4021/// The second source operand is known to be a power of 2.
4022void CombinerHelper::applySimplifyURemByPow2(MachineInstr &MI) const {
4023 Register DstReg = MI.getOperand(i: 0).getReg();
4024 Register Src0 = MI.getOperand(i: 1).getReg();
4025 Register Pow2Src1 = MI.getOperand(i: 2).getReg();
4026 LLT Ty = MRI.getType(Reg: DstReg);
4027
4028 // Fold (urem x, pow2) -> (and x, pow2-1)
4029 auto NegOne = Builder.buildConstant(Res: Ty, Val: -1);
4030 auto Add = Builder.buildAdd(Dst: Ty, Src0: Pow2Src1, Src1: NegOne);
4031 Builder.buildAnd(Dst: DstReg, Src0, Src1: Add);
4032 MI.eraseFromParent();
4033}
4034
4035bool CombinerHelper::matchFoldBinOpIntoSelect(MachineInstr &MI,
4036 unsigned &SelectOpNo) const {
4037 Register LHS = MI.getOperand(i: 1).getReg();
4038 Register RHS = MI.getOperand(i: 2).getReg();
4039
4040 Register OtherOperandReg = RHS;
4041 SelectOpNo = 1;
4042 Register SelectTrue, SelectFalse;
4043
4044 // Don't do this unless the old select is going away. We want to eliminate the
4045 // binary operator, not replace a binop with a select.
4046 if (!mi_match(R: LHS, MRI,
4047 P: m_GISelect(Src0: m_Reg(), Src1: m_Reg(R&: SelectTrue), Src2: m_Reg(R&: SelectFalse))) ||
4048 !MRI.hasOneNonDBGUse(RegNo: LHS)) {
4049 OtherOperandReg = LHS;
4050 SelectOpNo = 2;
4051 if (!mi_match(R: RHS, MRI,
4052 P: m_GISelect(Src0: m_Reg(), Src1: m_Reg(R&: SelectTrue), Src2: m_Reg(R&: SelectFalse))) ||
4053 !MRI.hasOneNonDBGUse(RegNo: RHS))
4054 return false;
4055 }
4056
4057 MachineInstr *SelectLHS, *SelectRHS;
4058 if (!mi_match(R: SelectTrue, MRI, P: m_MInstr(MI&: SelectLHS)) ||
4059 !mi_match(R: SelectFalse, MRI, P: m_MInstr(MI&: SelectRHS)))
4060 return false;
4061
4062 if (!isConstantOrConstantVector(MI: *SelectLHS, MRI,
4063 /*AllowFP*/ true,
4064 /*AllowOpaqueConstants*/ false))
4065 return false;
4066 if (!isConstantOrConstantVector(MI: *SelectRHS, MRI,
4067 /*AllowFP*/ true,
4068 /*AllowOpaqueConstants*/ false))
4069 return false;
4070
4071 unsigned BinOpcode = MI.getOpcode();
4072
4073 // We know that one of the operands is a select of constants. Now verify that
4074 // the other binary operator operand is either a constant, or we can handle a
4075 // variable.
4076 bool CanFoldNonConst =
4077 (BinOpcode == TargetOpcode::G_AND || BinOpcode == TargetOpcode::G_OR) &&
4078 (isNullOrNullSplat(MI: *SelectLHS, MRI) ||
4079 isAllOnesOrAllOnesSplat(MI: *SelectLHS, MRI)) &&
4080 (isNullOrNullSplat(MI: *SelectRHS, MRI) ||
4081 isAllOnesOrAllOnesSplat(MI: *SelectRHS, MRI));
4082 if (CanFoldNonConst)
4083 return true;
4084
4085 MachineInstr *OtherOperandDef;
4086 if (!mi_match(R: OtherOperandReg, MRI, P: m_MInstr(MI&: OtherOperandDef)))
4087 return false;
4088 return isConstantOrConstantVector(MI: *OtherOperandDef, MRI,
4089 /*AllowFP*/ true,
4090 /*AllowOpaqueConstants*/ false);
4091}
4092
4093/// \p SelectOperand is the operand in binary operator \p MI that is the select
4094/// to fold.
4095void CombinerHelper::applyFoldBinOpIntoSelect(
4096 MachineInstr &MI, const unsigned &SelectOperand) const {
4097 Register Dst = MI.getOperand(i: 0).getReg();
4098 Register LHS = MI.getOperand(i: 1).getReg();
4099 Register RHS = MI.getOperand(i: 2).getReg();
4100 GSelect *Select =
4101 cast<GSelect>(Val: MRI.getVRegDef(Reg: MI.getOperand(i: SelectOperand).getReg()));
4102
4103 Register SelectCond = Select->getCondReg();
4104 Register SelectTrue = Select->getTrueReg();
4105 Register SelectFalse = Select->getFalseReg();
4106
4107 LLT Ty = MRI.getType(Reg: Dst);
4108 unsigned BinOpcode = MI.getOpcode();
4109
4110 Register FoldTrue, FoldFalse;
4111
4112 // We have a select-of-constants followed by a binary operator with a
4113 // constant. Eliminate the binop by pulling the constant math into the select.
4114 // Example: add (select Cond, CT, CF), CBO --> select Cond, CT + CBO, CF + CBO
4115 if (SelectOperand == 1) {
4116 // TODO: SelectionDAG verifies this actually constant folds before
4117 // committing to the combine.
4118
4119 FoldTrue = Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {SelectTrue, RHS}).getReg(Idx: 0);
4120 FoldFalse =
4121 Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {SelectFalse, RHS}).getReg(Idx: 0);
4122 } else {
4123 FoldTrue = Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {LHS, SelectTrue}).getReg(Idx: 0);
4124 FoldFalse =
4125 Builder.buildInstr(Opc: BinOpcode, DstOps: {Ty}, SrcOps: {LHS, SelectFalse}).getReg(Idx: 0);
4126 }
4127
4128 Builder.buildSelect(Res: Dst, Tst: SelectCond, Op0: FoldTrue, Op1: FoldFalse, Flags: MI.getFlags());
4129 MI.eraseFromParent();
4130}
4131
4132std::optional<SmallVector<Register, 8>>
4133CombinerHelper::findCandidatesForLoadOrCombine(const MachineInstr *Root) const {
4134 assert(Root->getOpcode() == TargetOpcode::G_OR && "Expected G_OR only!");
4135 // We want to detect if Root is part of a tree which represents a bunch
4136 // of loads being merged into a larger load. We'll try to recognize patterns
4137 // like, for example:
4138 //
4139 // Reg Reg
4140 // \ /
4141 // OR_1 Reg
4142 // \ /
4143 // OR_2
4144 // \ Reg
4145 // .. /
4146 // Root
4147 //
4148 // Reg Reg Reg Reg
4149 // \ / \ /
4150 // OR_1 OR_2
4151 // \ /
4152 // \ /
4153 // ...
4154 // Root
4155 //
4156 // Each "Reg" may have been produced by a load + some arithmetic. This
4157 // function will save each of them.
4158 SmallVector<Register, 8> RegsToVisit;
4159 SmallVector<const MachineInstr *, 7> Ors = {Root};
4160
4161 // In the "worst" case, we're dealing with a load for each byte. So, there
4162 // are at most #bytes - 1 ORs.
4163 const unsigned MaxIter =
4164 MRI.getType(Reg: Root->getOperand(i: 0).getReg()).getSizeInBytes() - 1;
4165 for (unsigned Iter = 0; Iter < MaxIter; ++Iter) {
4166 if (Ors.empty())
4167 break;
4168 const MachineInstr *Curr = Ors.pop_back_val();
4169 Register OrLHS = Curr->getOperand(i: 1).getReg();
4170 Register OrRHS = Curr->getOperand(i: 2).getReg();
4171
4172 // In the combine, we want to elimate the entire tree.
4173 if (!MRI.hasOneNonDBGUse(RegNo: OrLHS) || !MRI.hasOneNonDBGUse(RegNo: OrRHS))
4174 return std::nullopt;
4175
4176 // If it's a G_OR, save it and continue to walk. If it's not, then it's
4177 // something that may be a load + arithmetic.
4178 if (const MachineInstr *Or = getOpcodeDef(Opcode: TargetOpcode::G_OR, Reg: OrLHS, MRI))
4179 Ors.push_back(Elt: Or);
4180 else
4181 RegsToVisit.push_back(Elt: OrLHS);
4182 if (const MachineInstr *Or = getOpcodeDef(Opcode: TargetOpcode::G_OR, Reg: OrRHS, MRI))
4183 Ors.push_back(Elt: Or);
4184 else
4185 RegsToVisit.push_back(Elt: OrRHS);
4186 }
4187
4188 // We're going to try and merge each register into a wider power-of-2 type,
4189 // so we ought to have an even number of registers.
4190 if (RegsToVisit.empty() || RegsToVisit.size() % 2 != 0)
4191 return std::nullopt;
4192 return RegsToVisit;
4193}
4194
4195/// Helper function for findLoadOffsetsForLoadOrCombine.
4196///
4197/// Check if \p Reg is the result of loading a \p MemSizeInBits wide value,
4198/// and then moving that value into a specific byte offset.
4199///
4200/// e.g. x[i] << 24
4201///
4202/// \returns The load instruction and the byte offset it is moved into.
4203static std::optional<std::pair<GZExtLoad *, int64_t>>
4204matchLoadAndBytePosition(Register Reg, unsigned MemSizeInBits,
4205 const MachineRegisterInfo &MRI) {
4206 assert(MRI.hasOneNonDBGUse(Reg) &&
4207 "Expected Reg to only have one non-debug use?");
4208 Register MaybeLoad;
4209 int64_t Shift;
4210 if (!mi_match(R: Reg, MRI,
4211 P: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: MaybeLoad), R: m_ICst(Cst&: Shift))))) {
4212 Shift = 0;
4213 MaybeLoad = Reg;
4214 }
4215
4216 if (Shift % MemSizeInBits != 0)
4217 return std::nullopt;
4218
4219 // TODO: Handle other types of loads.
4220 auto *Load = getOpcodeDef<GZExtLoad>(Reg: MaybeLoad, MRI);
4221 if (!Load)
4222 return std::nullopt;
4223
4224 if (!Load->isUnordered() || Load->getMemSizeInBits() != MemSizeInBits)
4225 return std::nullopt;
4226
4227 return std::make_pair(x&: Load, y: Shift / MemSizeInBits);
4228}
4229
4230std::optional<std::tuple<GZExtLoad *, int64_t, GZExtLoad *>>
4231CombinerHelper::findLoadOffsetsForLoadOrCombine(
4232 SmallDenseMap<int64_t, int64_t, 8> &MemOffset2Idx,
4233 const SmallVector<Register, 8> &RegsToVisit,
4234 const unsigned MemSizeInBits) const {
4235
4236 // Each load found for the pattern. There should be one for each RegsToVisit.
4237 SmallSetVector<const MachineInstr *, 8> Loads;
4238
4239 // The lowest index used in any load. (The lowest "i" for each x[i].)
4240 int64_t LowestIdx = INT64_MAX;
4241
4242 // The load which uses the lowest index.
4243 GZExtLoad *LowestIdxLoad = nullptr;
4244
4245 // Keeps track of the load indices we see. We shouldn't see any indices twice.
4246 SmallSet<int64_t, 8> SeenIdx;
4247
4248 // Ensure each load is in the same MBB.
4249 // TODO: Support multiple MachineBasicBlocks.
4250 MachineBasicBlock *MBB = nullptr;
4251 const MachineMemOperand *MMO = nullptr;
4252
4253 // Earliest instruction-order load in the pattern.
4254 GZExtLoad *EarliestLoad = nullptr;
4255
4256 // Latest instruction-order load in the pattern.
4257 GZExtLoad *LatestLoad = nullptr;
4258
4259 // Base pointer which every load should share.
4260 Register BasePtr;
4261
4262 // We want to find a load for each register. Each load should have some
4263 // appropriate bit twiddling arithmetic. During this loop, we will also keep
4264 // track of the load which uses the lowest index. Later, we will check if we
4265 // can use its pointer in the final, combined load.
4266 for (auto Reg : RegsToVisit) {
4267 // Find the load, and find the position that it will end up in (e.g. a
4268 // shifted) value.
4269 auto LoadAndPos = matchLoadAndBytePosition(Reg, MemSizeInBits, MRI);
4270 if (!LoadAndPos)
4271 return std::nullopt;
4272 GZExtLoad *Load;
4273 int64_t DstPos;
4274 std::tie(args&: Load, args&: DstPos) = *LoadAndPos;
4275
4276 // TODO: Handle multiple MachineBasicBlocks. Currently not handled because
4277 // it is difficult to check for stores/calls/etc between loads.
4278 MachineBasicBlock *LoadMBB = Load->getParent();
4279 if (!MBB)
4280 MBB = LoadMBB;
4281 if (LoadMBB != MBB)
4282 return std::nullopt;
4283
4284 // Make sure that the MachineMemOperands of every seen load are compatible.
4285 auto &LoadMMO = Load->getMMO();
4286 if (!MMO)
4287 MMO = &LoadMMO;
4288 if (MMO->getAddrSpace() != LoadMMO.getAddrSpace())
4289 return std::nullopt;
4290
4291 // Find out what the base pointer and index for the load is.
4292 Register LoadPtr;
4293 int64_t Idx;
4294 if (!mi_match(R: Load->getOperand(i: 1).getReg(), MRI,
4295 P: m_GPtrAdd(L: m_Reg(R&: LoadPtr), R: m_ICst(Cst&: Idx)))) {
4296 LoadPtr = Load->getOperand(i: 1).getReg();
4297 Idx = 0;
4298 }
4299
4300 // Don't combine things like a[i], a[i] -> a bigger load.
4301 if (!SeenIdx.insert(V: Idx).second)
4302 return std::nullopt;
4303
4304 // Every load must share the same base pointer; don't combine things like:
4305 //
4306 // a[i], b[i + 1] -> a bigger load.
4307 if (!BasePtr.isValid())
4308 BasePtr = LoadPtr;
4309 if (BasePtr != LoadPtr)
4310 return std::nullopt;
4311
4312 if (Idx < LowestIdx) {
4313 LowestIdx = Idx;
4314 LowestIdxLoad = Load;
4315 }
4316
4317 // Keep track of the byte offset that this load ends up at. If we have seen
4318 // the byte offset, then stop here. We do not want to combine:
4319 //
4320 // a[i] << 16, a[i + k] << 16 -> a bigger load.
4321 if (!MemOffset2Idx.try_emplace(Key: DstPos, Args&: Idx).second)
4322 return std::nullopt;
4323 Loads.insert(X: Load);
4324
4325 // Keep track of the position of the earliest/latest loads in the pattern.
4326 // We will check that there are no load fold barriers between them later
4327 // on.
4328 //
4329 // FIXME: Is there a better way to check for load fold barriers?
4330 if (!EarliestLoad || dominates(DefMI: *Load, UseMI: *EarliestLoad))
4331 EarliestLoad = Load;
4332 if (!LatestLoad || dominates(DefMI: *LatestLoad, UseMI: *Load))
4333 LatestLoad = Load;
4334 }
4335
4336 // We found a load for each register. Let's check if each load satisfies the
4337 // pattern.
4338 assert(Loads.size() == RegsToVisit.size() &&
4339 "Expected to find a load for each register?");
4340 assert(EarliestLoad != LatestLoad && EarliestLoad &&
4341 LatestLoad && "Expected at least two loads?");
4342
4343 // Check if there are any stores, calls, etc. between any of the loads. If
4344 // there are, then we can't safely perform the combine.
4345 //
4346 // MaxIter is chosen based off the (worst case) number of iterations it
4347 // typically takes to succeed in the LLVM test suite plus some padding.
4348 //
4349 // FIXME: Is there a better way to check for load fold barriers?
4350 const unsigned MaxIter = 20;
4351 unsigned Iter = 0;
4352 for (const auto &MI : instructionsWithoutDebug(It: EarliestLoad->getIterator(),
4353 End: LatestLoad->getIterator())) {
4354 if (Loads.count(key: &MI))
4355 continue;
4356 if (MI.isLoadFoldBarrier())
4357 return std::nullopt;
4358 if (Iter++ == MaxIter)
4359 return std::nullopt;
4360 }
4361
4362 return std::make_tuple(args&: LowestIdxLoad, args&: LowestIdx, args&: LatestLoad);
4363}
4364
4365bool CombinerHelper::matchLoadOrCombine(
4366 MachineInstr &MI,
4367 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4368 assert(MI.getOpcode() == TargetOpcode::G_OR);
4369 MachineFunction &MF = *MI.getMF();
4370 // Assuming a little-endian target, transform:
4371 // s8 *a = ...
4372 // s32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
4373 // =>
4374 // s32 val = *((i32)a)
4375 //
4376 // s8 *a = ...
4377 // s32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
4378 // =>
4379 // s32 val = BSWAP(*((s32)a))
4380 Register Dst = MI.getOperand(i: 0).getReg();
4381 LLT Ty = MRI.getType(Reg: Dst);
4382 if (Ty.isVector())
4383 return false;
4384
4385 // We need to combine at least two loads into this type. Since the smallest
4386 // possible load is into a byte, we need at least a 16-bit wide type.
4387 const unsigned WideMemSizeInBits = Ty.getSizeInBits();
4388 if (WideMemSizeInBits < 16 || WideMemSizeInBits % 8 != 0)
4389 return false;
4390
4391 // Match a collection of non-OR instructions in the pattern.
4392 auto RegsToVisit = findCandidatesForLoadOrCombine(Root: &MI);
4393 if (!RegsToVisit)
4394 return false;
4395
4396 // We have a collection of non-OR instructions. Figure out how wide each of
4397 // the small loads should be based off of the number of potential loads we
4398 // found.
4399 const unsigned NarrowMemSizeInBits = WideMemSizeInBits / RegsToVisit->size();
4400 if (NarrowMemSizeInBits % 8 != 0)
4401 return false;
4402
4403 // Check if each register feeding into each OR is a load from the same
4404 // base pointer + some arithmetic.
4405 //
4406 // e.g. a[0], a[1] << 8, a[2] << 16, etc.
4407 //
4408 // Also verify that each of these ends up putting a[i] into the same memory
4409 // offset as a load into a wide type would.
4410 SmallDenseMap<int64_t, int64_t, 8> MemOffset2Idx;
4411 GZExtLoad *LowestIdxLoad, *LatestLoad;
4412 int64_t LowestIdx;
4413 auto MaybeLoadInfo = findLoadOffsetsForLoadOrCombine(
4414 MemOffset2Idx, RegsToVisit: *RegsToVisit, MemSizeInBits: NarrowMemSizeInBits);
4415 if (!MaybeLoadInfo)
4416 return false;
4417 std::tie(args&: LowestIdxLoad, args&: LowestIdx, args&: LatestLoad) = *MaybeLoadInfo;
4418
4419 // We have a bunch of loads being OR'd together. Using the addresses + offsets
4420 // we found before, check if this corresponds to a big or little endian byte
4421 // pattern. If it does, then we can represent it using a load + possibly a
4422 // BSWAP.
4423 bool IsBigEndianTarget = MF.getDataLayout().isBigEndian();
4424 std::optional<bool> IsBigEndian = isBigEndian(MemOffset2Idx, LowestIdx);
4425 if (!IsBigEndian)
4426 return false;
4427 bool NeedsBSwap = IsBigEndianTarget != *IsBigEndian;
4428 if (NeedsBSwap && !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_BSWAP, {Ty}}))
4429 return false;
4430
4431 // Make sure that the load from the lowest index produces offset 0 in the
4432 // final value.
4433 //
4434 // This ensures that we won't combine something like this:
4435 //
4436 // load x[i] -> byte 2
4437 // load x[i+1] -> byte 0 ---> wide_load x[i]
4438 // load x[i+2] -> byte 1
4439 const unsigned NumLoadsInTy = WideMemSizeInBits / NarrowMemSizeInBits;
4440 const unsigned ZeroByteOffset =
4441 *IsBigEndian
4442 ? bigEndianByteAt(ByteWidth: NumLoadsInTy, I: 0)
4443 : littleEndianByteAt(ByteWidth: NumLoadsInTy, I: 0);
4444 auto ZeroOffsetIdx = MemOffset2Idx.find(Val: ZeroByteOffset);
4445 if (ZeroOffsetIdx == MemOffset2Idx.end() ||
4446 ZeroOffsetIdx->second != LowestIdx)
4447 return false;
4448
4449 // We wil reuse the pointer from the load which ends up at byte offset 0. It
4450 // may not use index 0.
4451 Register Ptr = LowestIdxLoad->getPointerReg();
4452 const MachineMemOperand &MMO = LowestIdxLoad->getMMO();
4453 LegalityQuery::MemDesc MMDesc(MMO);
4454 MMDesc.MemoryTy = Ty;
4455 if (!isLegalOrBeforeLegalizer(
4456 Query: {TargetOpcode::G_LOAD, {Ty, MRI.getType(Reg: Ptr)}, {MMDesc}}))
4457 return false;
4458 auto PtrInfo = MMO.getPointerInfo();
4459 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Size: WideMemSizeInBits / 8);
4460
4461 // Load must be allowed and fast on the target.
4462 LLVMContext &C = MF.getFunction().getContext();
4463 auto &DL = MF.getDataLayout();
4464 unsigned Fast = 0;
4465 if (!getTargetLowering().allowsMemoryAccess(Context&: C, DL, Ty, MMO: *NewMMO, Fast: &Fast) ||
4466 !Fast)
4467 return false;
4468
4469 MatchInfo = [=](MachineIRBuilder &MIB) {
4470 MIB.setInstrAndDebugLoc(*LatestLoad);
4471 Register LoadDst = NeedsBSwap ? MRI.cloneVirtualRegister(VReg: Dst) : Dst;
4472 MIB.buildLoad(Res: LoadDst, Addr: Ptr, MMO&: *NewMMO);
4473 if (NeedsBSwap)
4474 MIB.buildBSwap(Dst, Src0: LoadDst);
4475 };
4476 return true;
4477}
4478
4479bool CombinerHelper::matchExtendThroughPhis(MachineInstr &MI,
4480 MachineInstr *&ExtMI) const {
4481 auto &PHI = cast<GPhi>(Val&: MI);
4482 Register DstReg = PHI.getReg(Idx: 0);
4483
4484 // TODO: Extending a vector may be expensive, don't do this until heuristics
4485 // are better.
4486 if (MRI.getType(Reg: DstReg).isVector())
4487 return false;
4488
4489 // Try to match a phi, whose only use is an extend.
4490 if (!MRI.hasOneNonDBGUse(RegNo: DstReg))
4491 return false;
4492 ExtMI = &*MRI.use_instr_nodbg_begin(RegNo: DstReg);
4493 switch (ExtMI->getOpcode()) {
4494 case TargetOpcode::G_ANYEXT:
4495 return true; // G_ANYEXT is usually free.
4496 case TargetOpcode::G_ZEXT:
4497 case TargetOpcode::G_SEXT:
4498 break;
4499 default:
4500 return false;
4501 }
4502
4503 // If the target is likely to fold this extend away, don't propagate.
4504 if (Builder.getTII().isExtendLikelyToBeFolded(ExtMI&: *ExtMI, MRI))
4505 return false;
4506
4507 // We don't want to propagate the extends unless there's a good chance that
4508 // they'll be optimized in some way.
4509 // Collect the unique incoming values.
4510 SmallPtrSet<MachineInstr *, 4> InSrcs;
4511 for (unsigned I = 0; I < PHI.getNumIncomingValues(); ++I) {
4512 auto *DefMI = getDefIgnoringCopies(Reg: PHI.getIncomingValue(I), MRI);
4513 switch (DefMI->getOpcode()) {
4514 case TargetOpcode::G_LOAD:
4515 case TargetOpcode::G_TRUNC:
4516 case TargetOpcode::G_SEXT:
4517 case TargetOpcode::G_ZEXT:
4518 case TargetOpcode::G_ANYEXT:
4519 case TargetOpcode::G_CONSTANT:
4520 InSrcs.insert(Ptr: DefMI);
4521 // Don't try to propagate if there are too many places to create new
4522 // extends, chances are it'll increase code size.
4523 if (InSrcs.size() > 2)
4524 return false;
4525 break;
4526 default:
4527 return false;
4528 }
4529 }
4530 return true;
4531}
4532
4533void CombinerHelper::applyExtendThroughPhis(MachineInstr &MI,
4534 MachineInstr *&ExtMI) const {
4535 auto &PHI = cast<GPhi>(Val&: MI);
4536 Register DstReg = ExtMI->getOperand(i: 0).getReg();
4537 LLT ExtTy = MRI.getType(Reg: DstReg);
4538
4539 // Propagate the extension into the block of each incoming reg's block.
4540 // Use a SetVector here because PHIs can have duplicate edges, and we want
4541 // deterministic iteration order.
4542 SmallSetVector<MachineInstr *, 8> SrcMIs;
4543 SmallDenseMap<MachineInstr *, MachineInstr *, 8> OldToNewSrcMap;
4544 for (unsigned I = 0; I < PHI.getNumIncomingValues(); ++I) {
4545 auto SrcReg = PHI.getIncomingValue(I);
4546 MachineInstr *SrcMI;
4547 if (!mi_match(R: SrcReg, MRI, P: m_MInstr(MI&: SrcMI)))
4548 continue;
4549 if (!SrcMIs.insert(X: SrcMI))
4550 continue;
4551
4552 // Build an extend after each src inst.
4553 auto *MBB = SrcMI->getParent();
4554 MachineBasicBlock::iterator InsertPt = ++SrcMI->getIterator();
4555 if (InsertPt != MBB->end() && InsertPt->isPHI())
4556 InsertPt = MBB->getFirstNonPHI();
4557
4558 Builder.setInsertPt(MBB&: *SrcMI->getParent(), II: InsertPt);
4559 Builder.setDebugLoc(MI.getDebugLoc());
4560 auto NewExt = Builder.buildExtOrTrunc(ExtOpc: ExtMI->getOpcode(), Res: ExtTy, Op: SrcReg);
4561 OldToNewSrcMap[SrcMI] = NewExt;
4562 }
4563
4564 // Create a new phi with the extended inputs.
4565 Builder.setInstrAndDebugLoc(MI);
4566 auto NewPhi = Builder.buildInstrNoInsert(Opcode: TargetOpcode::G_PHI);
4567 NewPhi.addDef(RegNo: DstReg);
4568 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
4569 if (!MO.isReg()) {
4570 NewPhi.addMBB(MBB: MO.getMBB());
4571 continue;
4572 }
4573 auto *NewSrc = OldToNewSrcMap[MRI.getVRegDef(Reg: MO.getReg())];
4574 NewPhi.addUse(RegNo: NewSrc->getOperand(i: 0).getReg());
4575 }
4576 Builder.insertInstr(MIB: NewPhi);
4577 ExtMI->eraseFromParent();
4578}
4579
4580bool CombinerHelper::matchExtractVecEltBuildVec(MachineInstr &MI,
4581 Register &Reg) const {
4582 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
4583 // If we have a constant index, look for a G_BUILD_VECTOR source
4584 // and find the source register that the index maps to.
4585 Register SrcVec = MI.getOperand(i: 1).getReg();
4586 LLT SrcTy = MRI.getType(Reg: SrcVec);
4587 if (SrcTy.isScalableVector())
4588 return false;
4589
4590 auto Cst = getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
4591 if (!Cst || Cst->Value.getZExtValue() >= SrcTy.getNumElements())
4592 return false;
4593
4594 unsigned VecIdx = Cst->Value.getZExtValue();
4595
4596 // Check if we have a build_vector or build_vector_trunc with an optional
4597 // trunc in front.
4598 MachineInstr *SrcVecMI;
4599 Register TruncSrc;
4600 if (mi_match(R: SrcVec, MRI, P: m_GTrunc(Src: m_Reg(R&: TruncSrc)))) {
4601 if (!mi_match(R: TruncSrc, MRI, P: m_MInstr(MI&: SrcVecMI)))
4602 return false;
4603 } else if (!mi_match(R: SrcVec, MRI, P: m_MInstr(MI&: SrcVecMI)))
4604 return false;
4605
4606 if (SrcVecMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR &&
4607 SrcVecMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR_TRUNC)
4608 return false;
4609
4610 EVT Ty(getMVTForLLT(Ty: SrcTy));
4611 if (!MRI.hasOneNonDBGUse(RegNo: SrcVec) &&
4612 !getTargetLowering().aggressivelyPreferBuildVectorSources(VecVT: Ty))
4613 return false;
4614
4615 Reg = SrcVecMI->getOperand(i: VecIdx + 1).getReg();
4616 return true;
4617}
4618
4619void CombinerHelper::applyExtractVecEltBuildVec(MachineInstr &MI,
4620 Register &Reg) const {
4621 // Check the type of the register, since it may have come from a
4622 // G_BUILD_VECTOR_TRUNC.
4623 LLT ScalarTy = MRI.getType(Reg);
4624 Register DstReg = MI.getOperand(i: 0).getReg();
4625 LLT DstTy = MRI.getType(Reg: DstReg);
4626
4627 if (ScalarTy != DstTy) {
4628 assert(ScalarTy.getSizeInBits() > DstTy.getSizeInBits());
4629 Builder.buildTrunc(Res: DstReg, Op: Reg);
4630 MI.eraseFromParent();
4631 return;
4632 }
4633 replaceSingleDefInstWithReg(MI, Replacement: Reg);
4634}
4635
4636bool CombinerHelper::matchExtractAllEltsFromBuildVector(
4637 MachineInstr &MI,
4638 SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs) const {
4639 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4640 // This combine tries to find build_vector's which have every source element
4641 // extracted using G_EXTRACT_VECTOR_ELT. This can happen when transforms like
4642 // the masked load scalarization is run late in the pipeline. There's already
4643 // a combine for a similar pattern starting from the extract, but that
4644 // doesn't attempt to do it if there are multiple uses of the build_vector,
4645 // which in this case is true. Starting the combine from the build_vector
4646 // feels more natural than trying to find sibling nodes of extracts.
4647 // E.g.
4648 // %vec(<4 x s32>) = G_BUILD_VECTOR %s1(s32), %s2, %s3, %s4
4649 // %ext1 = G_EXTRACT_VECTOR_ELT %vec, 0
4650 // %ext2 = G_EXTRACT_VECTOR_ELT %vec, 1
4651 // %ext3 = G_EXTRACT_VECTOR_ELT %vec, 2
4652 // %ext4 = G_EXTRACT_VECTOR_ELT %vec, 3
4653 // ==>
4654 // replace ext{1,2,3,4} with %s{1,2,3,4}
4655
4656 Register DstReg = MI.getOperand(i: 0).getReg();
4657 LLT DstTy = MRI.getType(Reg: DstReg);
4658 unsigned NumElts = DstTy.getNumElements();
4659
4660 SmallBitVector ExtractedElts(NumElts);
4661 for (MachineInstr &II : MRI.use_nodbg_instructions(Reg: DstReg)) {
4662 if (II.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)
4663 return false;
4664 auto Cst = getIConstantVRegVal(VReg: II.getOperand(i: 2).getReg(), MRI);
4665 if (!Cst)
4666 return false;
4667 unsigned Idx = Cst->getZExtValue();
4668 if (Idx >= NumElts)
4669 return false; // Out of range.
4670 ExtractedElts.set(Idx);
4671 SrcDstPairs.emplace_back(
4672 Args: std::make_pair(x: MI.getOperand(i: Idx + 1).getReg(), y: &II));
4673 }
4674 // Match if every element was extracted.
4675 return ExtractedElts.all();
4676}
4677
4678void CombinerHelper::applyExtractAllEltsFromBuildVector(
4679 MachineInstr &MI,
4680 SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs) const {
4681 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4682 for (auto &Pair : SrcDstPairs) {
4683 auto *ExtMI = Pair.second;
4684 replaceRegWith(MRI, FromReg: ExtMI->getOperand(i: 0).getReg(), ToReg: Pair.first);
4685 ExtMI->eraseFromParent();
4686 }
4687 MI.eraseFromParent();
4688}
4689
4690void CombinerHelper::applyBuildFn(
4691 MachineInstr &MI,
4692 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4693 applyBuildFnNoErase(MI, MatchInfo);
4694 MI.eraseFromParent();
4695}
4696
4697void CombinerHelper::applyBuildFnNoErase(
4698 MachineInstr &MI,
4699 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4700 MatchInfo(Builder);
4701}
4702
4703bool CombinerHelper::matchOrShiftToFunnelShift(MachineInstr &MI,
4704 bool AllowScalarConstants,
4705 BuildFnTy &MatchInfo) const {
4706 assert(MI.getOpcode() == TargetOpcode::G_OR);
4707
4708 Register Dst = MI.getOperand(i: 0).getReg();
4709 LLT Ty = MRI.getType(Reg: Dst);
4710 unsigned BitWidth = Ty.getScalarSizeInBits();
4711
4712 Register ShlSrc, ShlAmt, LShrSrc, LShrAmt, Amt;
4713 unsigned FshOpc = 0;
4714
4715 // Match (or (shl ...), (lshr ...)).
4716 if (!mi_match(R: Dst, MRI,
4717 // m_GOr() handles the commuted version as well.
4718 P: m_GOr(L: m_GShl(L: m_Reg(R&: ShlSrc), R: m_Reg(R&: ShlAmt)),
4719 R: m_GLShr(L: m_Reg(R&: LShrSrc), R: m_Reg(R&: LShrAmt)))))
4720 return false;
4721
4722 // Given constants C0 and C1 such that C0 + C1 is bit-width:
4723 // (or (shl x, C0), (lshr y, C1)) -> (fshl x, y, C0) or (fshr x, y, C1)
4724 int64_t CstShlAmt = 0, CstLShrAmt;
4725 if (mi_match(R: ShlAmt, MRI, P: m_ICstOrSplat(Cst&: CstShlAmt)) &&
4726 mi_match(R: LShrAmt, MRI, P: m_ICstOrSplat(Cst&: CstLShrAmt)) &&
4727 CstShlAmt + CstLShrAmt == BitWidth) {
4728 FshOpc = TargetOpcode::G_FSHR;
4729 Amt = LShrAmt;
4730 } else if (mi_match(R: LShrAmt, MRI,
4731 P: m_GSub(L: m_SpecificICstOrSplat(RequestedValue: BitWidth), R: m_Reg(R&: Amt))) &&
4732 ShlAmt == Amt) {
4733 // (or (shl x, amt), (lshr y, (sub bw, amt))) -> (fshl x, y, amt)
4734 FshOpc = TargetOpcode::G_FSHL;
4735 } else if (mi_match(R: ShlAmt, MRI,
4736 P: m_GSub(L: m_SpecificICstOrSplat(RequestedValue: BitWidth), R: m_Reg(R&: Amt))) &&
4737 LShrAmt == Amt) {
4738 // (or (shl x, (sub bw, amt)), (lshr y, amt)) -> (fshr x, y, amt)
4739 FshOpc = TargetOpcode::G_FSHR;
4740 } else {
4741 return false;
4742 }
4743
4744 LLT AmtTy = MRI.getType(Reg: Amt);
4745 if (!isLegalOrBeforeLegalizer(Query: {FshOpc, {Ty, AmtTy}}) &&
4746 (!AllowScalarConstants || CstShlAmt == 0 || !Ty.isScalar()))
4747 return false;
4748
4749 MatchInfo = [=](MachineIRBuilder &B) {
4750 B.buildInstr(Opc: FshOpc, DstOps: {Dst}, SrcOps: {ShlSrc, LShrSrc, Amt});
4751 };
4752 return true;
4753}
4754
4755/// Match an FSHL or FSHR that can be combined to a ROTR or ROTL rotate.
4756bool CombinerHelper::matchFunnelShiftToRotate(MachineInstr &MI) const {
4757 unsigned Opc = MI.getOpcode();
4758 assert(Opc == TargetOpcode::G_FSHL || Opc == TargetOpcode::G_FSHR);
4759 Register X = MI.getOperand(i: 1).getReg();
4760 Register Y = MI.getOperand(i: 2).getReg();
4761 if (X != Y)
4762 return false;
4763 unsigned RotateOpc =
4764 Opc == TargetOpcode::G_FSHL ? TargetOpcode::G_ROTL : TargetOpcode::G_ROTR;
4765 return isLegalOrBeforeLegalizer(Query: {RotateOpc, {MRI.getType(Reg: X), MRI.getType(Reg: Y)}});
4766}
4767
4768void CombinerHelper::applyFunnelShiftToRotate(MachineInstr &MI) const {
4769 unsigned Opc = MI.getOpcode();
4770 assert(Opc == TargetOpcode::G_FSHL || Opc == TargetOpcode::G_FSHR);
4771 bool IsFSHL = Opc == TargetOpcode::G_FSHL;
4772 Observer.changingInstr(MI);
4773 MI.setDesc(Builder.getTII().get(Opcode: IsFSHL ? TargetOpcode::G_ROTL
4774 : TargetOpcode::G_ROTR));
4775 MI.removeOperand(OpNo: 2);
4776 Observer.changedInstr(MI);
4777}
4778
4779// Fold (rot x, c) -> (rot x, c % BitSize)
4780bool CombinerHelper::matchRotateOutOfRange(MachineInstr &MI) const {
4781 assert(MI.getOpcode() == TargetOpcode::G_ROTL ||
4782 MI.getOpcode() == TargetOpcode::G_ROTR);
4783 unsigned Bitsize =
4784 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getScalarSizeInBits();
4785 Register AmtReg = MI.getOperand(i: 2).getReg();
4786 bool OutOfRange = false;
4787 auto MatchOutOfRange = [Bitsize, &OutOfRange](const Constant *C) {
4788 if (auto *CI = dyn_cast<ConstantInt>(Val: C))
4789 OutOfRange |= CI->getValue().uge(RHS: Bitsize);
4790 return true;
4791 };
4792 return matchUnaryPredicate(MRI, Reg: AmtReg, Match: MatchOutOfRange) && OutOfRange;
4793}
4794
4795void CombinerHelper::applyRotateOutOfRange(MachineInstr &MI) const {
4796 assert(MI.getOpcode() == TargetOpcode::G_ROTL ||
4797 MI.getOpcode() == TargetOpcode::G_ROTR);
4798 unsigned Bitsize =
4799 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getScalarSizeInBits();
4800 Register Amt = MI.getOperand(i: 2).getReg();
4801 LLT AmtTy = MRI.getType(Reg: Amt);
4802 auto Bits = Builder.buildConstant(Res: AmtTy, Val: Bitsize);
4803 Amt = Builder.buildURem(Dst: AmtTy, Src0: MI.getOperand(i: 2).getReg(), Src1: Bits).getReg(Idx: 0);
4804 Observer.changingInstr(MI);
4805 MI.getOperand(i: 2).setReg(Amt);
4806 Observer.changedInstr(MI);
4807}
4808
4809bool CombinerHelper::matchICmpToTrueFalseKnownBits(MachineInstr &MI,
4810 int64_t &MatchInfo) const {
4811 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
4812 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
4813
4814 // We want to avoid calling KnownBits on the LHS if possible, as this combine
4815 // has no filter and runs on every G_ICMP instruction. We can avoid calling
4816 // KnownBits on the LHS in two cases:
4817 //
4818 // - The RHS is unknown: Constants are always on RHS. If the RHS is unknown
4819 // we cannot do any transforms so we can safely bail out early.
4820 // - The RHS is zero: we don't need to know the LHS to do unsigned <0 and
4821 // >=0.
4822 auto KnownRHS = VT->getKnownBits(R: MI.getOperand(i: 3).getReg());
4823 if (KnownRHS.isUnknown())
4824 return false;
4825
4826 std::optional<bool> KnownVal;
4827 if (KnownRHS.isZero()) {
4828 // ? uge 0 -> always true
4829 // ? ult 0 -> always false
4830 if (Pred == CmpInst::ICMP_UGE)
4831 KnownVal = true;
4832 else if (Pred == CmpInst::ICMP_ULT)
4833 KnownVal = false;
4834 }
4835
4836 if (!KnownVal) {
4837 auto KnownLHS = VT->getKnownBits(R: MI.getOperand(i: 2).getReg());
4838 KnownVal = ICmpInst::compare(LHS: KnownLHS, RHS: KnownRHS, Pred);
4839 }
4840
4841 if (!KnownVal)
4842 return false;
4843 MatchInfo =
4844 *KnownVal
4845 ? getICmpTrueVal(TLI: getTargetLowering(),
4846 /*IsVector = */
4847 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector(),
4848 /* IsFP = */ false)
4849 : 0;
4850 return true;
4851}
4852
4853bool CombinerHelper::matchICmpToLHSKnownBits(
4854 MachineInstr &MI,
4855 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4856 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
4857 // Given:
4858 //
4859 // %x = G_WHATEVER (... x is known to be 0 or 1 ...)
4860 // %cmp = G_ICMP ne %x, 0
4861 //
4862 // Or:
4863 //
4864 // %x = G_WHATEVER (... x is known to be 0 or 1 ...)
4865 // %cmp = G_ICMP eq %x, 1
4866 //
4867 // We can replace %cmp with %x assuming true is 1 on the target.
4868 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
4869 if (!CmpInst::isEquality(pred: Pred))
4870 return false;
4871 Register Dst = MI.getOperand(i: 0).getReg();
4872 LLT DstTy = MRI.getType(Reg: Dst);
4873 if (getICmpTrueVal(TLI: getTargetLowering(), IsVector: DstTy.isVector(),
4874 /* IsFP = */ false) != 1)
4875 return false;
4876 int64_t OneOrZero = Pred == CmpInst::ICMP_EQ;
4877 if (!mi_match(R: MI.getOperand(i: 3).getReg(), MRI, P: m_SpecificICst(RequestedValue: OneOrZero)))
4878 return false;
4879 Register LHS = MI.getOperand(i: 2).getReg();
4880 auto KnownLHS = VT->getKnownBits(R: LHS);
4881 if (KnownLHS.getMinValue() != 0 || KnownLHS.getMaxValue() != 1)
4882 return false;
4883 // Make sure replacing Dst with the LHS is a legal operation.
4884 LLT LHSTy = MRI.getType(Reg: LHS);
4885 unsigned LHSSize = LHSTy.getSizeInBits();
4886 unsigned DstSize = DstTy.getSizeInBits();
4887 unsigned Op = TargetOpcode::COPY;
4888 if (DstSize != LHSSize)
4889 Op = DstSize < LHSSize ? TargetOpcode::G_TRUNC : TargetOpcode::G_ZEXT;
4890 if (!isLegalOrBeforeLegalizer(Query: {Op, {DstTy, LHSTy}}))
4891 return false;
4892 MatchInfo = [=](MachineIRBuilder &B) { B.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {LHS}); };
4893 return true;
4894}
4895
4896// Replace (and (or x, c1), c2) with (and x, c2) iff c1 & c2 == 0
4897bool CombinerHelper::matchAndOrDisjointMask(
4898 MachineInstr &MI,
4899 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4900 assert(MI.getOpcode() == TargetOpcode::G_AND);
4901
4902 // Ignore vector types to simplify matching the two constants.
4903 // TODO: do this for vectors and scalars via a demanded bits analysis.
4904 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4905 if (Ty.isVector())
4906 return false;
4907
4908 Register Src;
4909 Register AndMaskReg;
4910 int64_t AndMaskBits;
4911 int64_t OrMaskBits;
4912 if (!mi_match(MI, MRI,
4913 P: m_GAnd(L: m_GOr(L: m_Reg(R&: Src), R: m_ICst(Cst&: OrMaskBits)),
4914 R: m_all_of(preds: m_ICst(Cst&: AndMaskBits), preds: m_Reg(R&: AndMaskReg)))))
4915 return false;
4916
4917 // Check if OrMask could turn on any bits in Src.
4918 if (AndMaskBits & OrMaskBits)
4919 return false;
4920
4921 MatchInfo = [=, &MI](MachineIRBuilder &B) {
4922 Observer.changingInstr(MI);
4923 // Canonicalize the result to have the constant on the RHS.
4924 if (MI.getOperand(i: 1).getReg() == AndMaskReg)
4925 MI.getOperand(i: 2).setReg(AndMaskReg);
4926 MI.getOperand(i: 1).setReg(Src);
4927 Observer.changedInstr(MI);
4928 };
4929 return true;
4930}
4931
4932/// Form a G_SBFX from a G_SEXT_INREG fed by a right shift.
4933bool CombinerHelper::matchBitfieldExtractFromSExtInReg(
4934 MachineInstr &MI,
4935 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4936 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
4937 Register Dst = MI.getOperand(i: 0).getReg();
4938 Register Src = MI.getOperand(i: 1).getReg();
4939 LLT Ty = MRI.getType(Reg: Src);
4940 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
4941 if (!LI || !LI->isLegalOrCustom(Query: {TargetOpcode::G_SBFX, {Ty, ExtractTy}}))
4942 return false;
4943 int64_t Width = MI.getOperand(i: 2).getImm();
4944 Register ShiftSrc;
4945 int64_t ShiftImm;
4946 if (!mi_match(
4947 R: Src, MRI,
4948 P: m_OneNonDBGUse(SP: m_any_of(preds: m_GAShr(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: ShiftImm)),
4949 preds: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: ShiftImm))))))
4950 return false;
4951 if (ShiftImm < 0 || ShiftImm + Width > Ty.getScalarSizeInBits())
4952 return false;
4953
4954 MatchInfo = [=](MachineIRBuilder &B) {
4955 auto Cst1 = B.buildConstant(Res: ExtractTy, Val: ShiftImm);
4956 auto Cst2 = B.buildConstant(Res: ExtractTy, Val: Width);
4957 B.buildSbfx(Dst, Src: ShiftSrc, LSB: Cst1, Width: Cst2);
4958 };
4959 return true;
4960}
4961
4962/// Form a G_UBFX from "(a srl b) & mask", where b and mask are constants.
4963bool CombinerHelper::matchBitfieldExtractFromAnd(MachineInstr &MI,
4964 BuildFnTy &MatchInfo) const {
4965 GAnd *And = cast<GAnd>(Val: &MI);
4966 Register Dst = And->getReg(Idx: 0);
4967 LLT Ty = MRI.getType(Reg: Dst);
4968 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
4969 // Note that isLegalOrBeforeLegalizer is stricter and does not take custom
4970 // into account.
4971 if (LI && !LI->isLegalOrCustom(Query: {TargetOpcode::G_UBFX, {Ty, ExtractTy}}))
4972 return false;
4973
4974 int64_t AndImm, LSBImm;
4975 Register ShiftSrc;
4976 const unsigned Size = Ty.getScalarSizeInBits();
4977 if (!mi_match(R: And->getReg(Idx: 0), MRI,
4978 P: m_GAnd(L: m_OneNonDBGUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: LSBImm))),
4979 R: m_ICst(Cst&: AndImm))))
4980 return false;
4981
4982 // The mask is a mask of the low bits iff imm & (imm+1) == 0.
4983 auto MaybeMask = static_cast<uint64_t>(AndImm);
4984 if (MaybeMask & (MaybeMask + 1))
4985 return false;
4986
4987 // LSB must fit within the register.
4988 if (static_cast<uint64_t>(LSBImm) >= Size)
4989 return false;
4990
4991 uint64_t Width = APInt(Size, AndImm).countr_one();
4992 MatchInfo = [=](MachineIRBuilder &B) {
4993 auto WidthCst = B.buildConstant(Res: ExtractTy, Val: Width);
4994 auto LSBCst = B.buildConstant(Res: ExtractTy, Val: LSBImm);
4995 B.buildInstr(Opc: TargetOpcode::G_UBFX, DstOps: {Dst}, SrcOps: {ShiftSrc, LSBCst, WidthCst});
4996 };
4997 return true;
4998}
4999
5000bool CombinerHelper::matchBitfieldExtractFromShr(
5001 MachineInstr &MI,
5002 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
5003 const unsigned Opcode = MI.getOpcode();
5004 assert(Opcode == TargetOpcode::G_ASHR || Opcode == TargetOpcode::G_LSHR);
5005
5006 const Register Dst = MI.getOperand(i: 0).getReg();
5007
5008 const unsigned ExtrOpcode = Opcode == TargetOpcode::G_ASHR
5009 ? TargetOpcode::G_SBFX
5010 : TargetOpcode::G_UBFX;
5011
5012 // Check if the type we would use for the extract is legal
5013 LLT Ty = MRI.getType(Reg: Dst);
5014 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5015 if (!LI || !LI->isLegalOrCustom(Query: {ExtrOpcode, {Ty, ExtractTy}}))
5016 return false;
5017
5018 Register ShlSrc;
5019 int64_t ShrAmt;
5020 int64_t ShlAmt;
5021 const unsigned Size = Ty.getScalarSizeInBits();
5022
5023 // Try to match shr (shl x, c1), c2
5024 if (!mi_match(R: Dst, MRI,
5025 P: m_BinOp(Opcode,
5026 L: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: ShlSrc), R: m_ICst(Cst&: ShlAmt))),
5027 R: m_ICst(Cst&: ShrAmt))))
5028 return false;
5029
5030 // Make sure that the shift sizes can fit a bitfield extract
5031 if (ShlAmt < 0 || ShlAmt > ShrAmt || ShrAmt >= Size)
5032 return false;
5033
5034 // Skip this combine if the G_SEXT_INREG combine could handle it
5035 if (Opcode == TargetOpcode::G_ASHR && ShlAmt == ShrAmt)
5036 return false;
5037
5038 // Calculate start position and width of the extract
5039 const int64_t Pos = ShrAmt - ShlAmt;
5040 const int64_t Width = Size - ShrAmt;
5041
5042 MatchInfo = [=](MachineIRBuilder &B) {
5043 auto WidthCst = B.buildConstant(Res: ExtractTy, Val: Width);
5044 auto PosCst = B.buildConstant(Res: ExtractTy, Val: Pos);
5045 B.buildInstr(Opc: ExtrOpcode, DstOps: {Dst}, SrcOps: {ShlSrc, PosCst, WidthCst});
5046 };
5047 return true;
5048}
5049
5050bool CombinerHelper::matchBitfieldExtractFromShrAnd(
5051 MachineInstr &MI,
5052 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
5053 const unsigned Opcode = MI.getOpcode();
5054 assert(Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_ASHR);
5055
5056 const Register Dst = MI.getOperand(i: 0).getReg();
5057 LLT Ty = MRI.getType(Reg: Dst);
5058 LLT ExtractTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5059 if (LI && !LI->isLegalOrCustom(Query: {TargetOpcode::G_UBFX, {Ty, ExtractTy}}))
5060 return false;
5061
5062 // Try to match shr (and x, c1), c2
5063 Register AndSrc;
5064 int64_t ShrAmt;
5065 int64_t SMask;
5066 if (!mi_match(R: Dst, MRI,
5067 P: m_BinOp(Opcode,
5068 L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: AndSrc), R: m_ICst(Cst&: SMask))),
5069 R: m_ICst(Cst&: ShrAmt))))
5070 return false;
5071
5072 const unsigned Size = Ty.getScalarSizeInBits();
5073 if (ShrAmt < 0 || ShrAmt >= Size)
5074 return false;
5075
5076 // If the shift subsumes the mask, emit the 0 directly.
5077 if (0 == (SMask >> ShrAmt)) {
5078 MatchInfo = [=](MachineIRBuilder &B) {
5079 B.buildConstant(Res: Dst, Val: 0);
5080 };
5081 return true;
5082 }
5083
5084 // Check that ubfx can do the extraction, with no holes in the mask.
5085 uint64_t UMask = SMask;
5086 UMask |= maskTrailingOnes<uint64_t>(N: ShrAmt);
5087 UMask &= maskTrailingOnes<uint64_t>(N: Size);
5088 if (!isMask_64(Value: UMask))
5089 return false;
5090
5091 // Calculate start position and width of the extract.
5092 const int64_t Pos = ShrAmt;
5093 const int64_t Width = llvm::countr_one(Value: UMask) - ShrAmt;
5094
5095 // It's preferable to keep the shift, rather than form G_SBFX.
5096 // TODO: remove the G_AND via demanded bits analysis.
5097 if (Opcode == TargetOpcode::G_ASHR && Width + ShrAmt == Size)
5098 return false;
5099
5100 MatchInfo = [=](MachineIRBuilder &B) {
5101 auto WidthCst = B.buildConstant(Res: ExtractTy, Val: Width);
5102 auto PosCst = B.buildConstant(Res: ExtractTy, Val: Pos);
5103 B.buildInstr(Opc: TargetOpcode::G_UBFX, DstOps: {Dst}, SrcOps: {AndSrc, PosCst, WidthCst});
5104 };
5105 return true;
5106}
5107
5108bool CombinerHelper::reassociationCanBreakAddressingModePattern(
5109 MachineInstr &MI) const {
5110 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
5111
5112 Register Src1Reg = PtrAdd.getBaseReg();
5113 auto *Src1Def = getOpcodeDef<GPtrAdd>(Reg: Src1Reg, MRI);
5114 if (!Src1Def)
5115 return false;
5116
5117 Register Src2Reg = PtrAdd.getOffsetReg();
5118
5119 if (MRI.hasOneNonDBGUse(RegNo: Src1Reg))
5120 return false;
5121
5122 auto C1 = getIConstantVRegVal(VReg: Src1Def->getOffsetReg(), MRI);
5123 if (!C1)
5124 return false;
5125 auto C2 = getIConstantVRegVal(VReg: Src2Reg, MRI);
5126 if (!C2)
5127 return false;
5128
5129 const APInt &C1APIntVal = *C1;
5130 const APInt &C2APIntVal = *C2;
5131 const int64_t CombinedValue = (C1APIntVal + C2APIntVal).getSExtValue();
5132
5133 for (auto &UseMI : MRI.use_nodbg_instructions(Reg: PtrAdd.getReg(Idx: 0))) {
5134 // This combine may end up running before ptrtoint/inttoptr combines
5135 // manage to eliminate redundant conversions, so try to look through them.
5136 MachineInstr *ConvUseMI = &UseMI;
5137 unsigned ConvUseOpc = ConvUseMI->getOpcode();
5138 while (ConvUseOpc == TargetOpcode::G_INTTOPTR ||
5139 ConvUseOpc == TargetOpcode::G_PTRTOINT) {
5140 Register DefReg = ConvUseMI->getOperand(i: 0).getReg();
5141 if (!MRI.hasOneNonDBGUse(RegNo: DefReg))
5142 break;
5143 ConvUseMI = &*MRI.use_instr_nodbg_begin(RegNo: DefReg);
5144 ConvUseOpc = ConvUseMI->getOpcode();
5145 }
5146 auto *LdStMI = dyn_cast<GLoadStore>(Val: ConvUseMI);
5147 if (!LdStMI)
5148 continue;
5149 // Is x[offset2] already not a legal addressing mode? If so then
5150 // reassociating the constants breaks nothing (we test offset2 because
5151 // that's the one we hope to fold into the load or store).
5152 TargetLoweringBase::AddrMode AM;
5153 AM.HasBaseReg = true;
5154 AM.BaseOffs = C2APIntVal.getSExtValue();
5155 unsigned AS = MRI.getType(Reg: LdStMI->getPointerReg()).getAddressSpace();
5156 Type *AccessTy = getTypeForLLT(Ty: LdStMI->getMMO().getMemoryType(),
5157 C&: PtrAdd.getMF()->getFunction().getContext());
5158 const auto &TLI = *PtrAdd.getMF()->getSubtarget().getTargetLowering();
5159 if (!TLI.isLegalAddressingMode(DL: PtrAdd.getMF()->getDataLayout(), AM,
5160 Ty: AccessTy, AddrSpace: AS))
5161 continue;
5162
5163 // Would x[offset1+offset2] still be a legal addressing mode?
5164 AM.BaseOffs = CombinedValue;
5165 if (!TLI.isLegalAddressingMode(DL: PtrAdd.getMF()->getDataLayout(), AM,
5166 Ty: AccessTy, AddrSpace: AS))
5167 return true;
5168 }
5169
5170 return false;
5171}
5172
5173bool CombinerHelper::matchReassocConstantInnerRHS(GPtrAdd &MI,
5174 MachineInstr *RHS,
5175 BuildFnTy &MatchInfo) const {
5176 // G_PTR_ADD(BASE, G_ADD(X, C)) -> G_PTR_ADD(G_PTR_ADD(BASE, X), C)
5177 Register Src1Reg = MI.getOperand(i: 1).getReg();
5178 if (RHS->getOpcode() != TargetOpcode::G_ADD)
5179 return false;
5180 auto C2 = getIConstantVRegVal(VReg: RHS->getOperand(i: 2).getReg(), MRI);
5181 if (!C2)
5182 return false;
5183
5184 // If both additions are nuw, the reassociated additions are also nuw.
5185 // If the original G_PTR_ADD is additionally nusw, X and C are both not
5186 // negative, so BASE+X is between BASE and BASE+(X+C). The new G_PTR_ADDs are
5187 // therefore also nusw.
5188 // If the original G_PTR_ADD is additionally inbounds (which implies nusw),
5189 // the new G_PTR_ADDs are then also inbounds.
5190 unsigned PtrAddFlags = MI.getFlags();
5191 unsigned AddFlags = RHS->getFlags();
5192 bool IsNoUWrap = PtrAddFlags & AddFlags & MachineInstr::MIFlag::NoUWrap;
5193 bool IsNoUSWrap = IsNoUWrap && (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap);
5194 bool IsInBounds = IsNoUWrap && (PtrAddFlags & MachineInstr::MIFlag::InBounds);
5195 unsigned Flags = 0;
5196 if (IsNoUWrap)
5197 Flags |= MachineInstr::MIFlag::NoUWrap;
5198 if (IsNoUSWrap)
5199 Flags |= MachineInstr::MIFlag::NoUSWrap;
5200 if (IsInBounds)
5201 Flags |= MachineInstr::MIFlag::InBounds;
5202
5203 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5204 LLT PtrTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5205
5206 auto NewBase =
5207 Builder.buildPtrAdd(Res: PtrTy, Op0: Src1Reg, Op1: RHS->getOperand(i: 1).getReg(), Flags);
5208 Observer.changingInstr(MI);
5209 MI.getOperand(i: 1).setReg(NewBase.getReg(Idx: 0));
5210 MI.getOperand(i: 2).setReg(RHS->getOperand(i: 2).getReg());
5211 MI.setFlags(Flags);
5212 Observer.changedInstr(MI);
5213 };
5214 return !reassociationCanBreakAddressingModePattern(MI);
5215}
5216
5217bool CombinerHelper::matchReassocConstantInnerLHS(GPtrAdd &MI,
5218 MachineInstr *LHS,
5219 MachineInstr *RHS,
5220 BuildFnTy &MatchInfo) const {
5221 // G_PTR_ADD (G_PTR_ADD X, C), Y) -> (G_PTR_ADD (G_PTR_ADD(X, Y), C)
5222 // if and only if (G_PTR_ADD X, C) has one use.
5223 Register LHSBase;
5224 std::optional<ValueAndVReg> LHSCstOff;
5225 if (!mi_match(R: MI.getBaseReg(), MRI,
5226 P: m_OneNonDBGUse(SP: m_GPtrAdd(L: m_Reg(R&: LHSBase), R: m_GCst(ValReg&: LHSCstOff)))))
5227 return false;
5228
5229 auto *LHSPtrAdd = cast<GPtrAdd>(Val: LHS);
5230
5231 // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
5232 // nuw and inbounds (which implies nusw), the offsets are both non-negative,
5233 // so the new G_PTR_ADDs are also inbounds.
5234 unsigned PtrAddFlags = MI.getFlags();
5235 unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();
5236 bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
5237 bool IsNoUSWrap = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &
5238 MachineInstr::MIFlag::NoUSWrap);
5239 bool IsInBounds = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &
5240 MachineInstr::MIFlag::InBounds);
5241 unsigned Flags = 0;
5242 if (IsNoUWrap)
5243 Flags |= MachineInstr::MIFlag::NoUWrap;
5244 if (IsNoUSWrap)
5245 Flags |= MachineInstr::MIFlag::NoUSWrap;
5246 if (IsInBounds)
5247 Flags |= MachineInstr::MIFlag::InBounds;
5248
5249 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5250 // When we change LHSPtrAdd's offset register we might cause it to use a reg
5251 // before its def. Sink the instruction so the outer PTR_ADD to ensure this
5252 // doesn't happen.
5253 LHSPtrAdd->moveBefore(MovePos: &MI);
5254 Register RHSReg = MI.getOffsetReg();
5255 // set VReg will cause type mismatch if it comes from extend/trunc
5256 auto NewCst = B.buildConstant(Res: MRI.getType(Reg: RHSReg), Val: LHSCstOff->Value);
5257 Observer.changingInstr(MI);
5258 MI.getOperand(i: 2).setReg(NewCst.getReg(Idx: 0));
5259 MI.setFlags(Flags);
5260 Observer.changedInstr(MI);
5261 Observer.changingInstr(MI&: *LHSPtrAdd);
5262 LHSPtrAdd->getOperand(i: 2).setReg(RHSReg);
5263 LHSPtrAdd->setFlags(Flags);
5264 Observer.changedInstr(MI&: *LHSPtrAdd);
5265 };
5266 return !reassociationCanBreakAddressingModePattern(MI);
5267}
5268
5269bool CombinerHelper::matchReassocFoldConstantsInSubTree(
5270 GPtrAdd &MI, MachineInstr *LHS, MachineInstr *RHS,
5271 BuildFnTy &MatchInfo) const {
5272 // G_PTR_ADD(G_PTR_ADD(BASE, C1), C2) -> G_PTR_ADD(BASE, C1+C2)
5273 auto *LHSPtrAdd = dyn_cast<GPtrAdd>(Val: LHS);
5274 if (!LHSPtrAdd)
5275 return false;
5276
5277 Register Src2Reg = MI.getOperand(i: 2).getReg();
5278 Register LHSSrc1 = LHSPtrAdd->getBaseReg();
5279 Register LHSSrc2 = LHSPtrAdd->getOffsetReg();
5280 auto C1 = getIConstantVRegVal(VReg: LHSSrc2, MRI);
5281 if (!C1)
5282 return false;
5283 auto C2 = getIConstantVRegVal(VReg: Src2Reg, MRI);
5284 if (!C2)
5285 return false;
5286
5287 // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
5288 // inbounds, reaching the same result in one G_PTR_ADD is also inbounds.
5289 // The nusw constraints are satisfied because imm1+imm2 cannot exceed the
5290 // largest signed integer that fits into the index type, which is the maximum
5291 // size of allocated objects according to the IR Language Reference.
5292 unsigned PtrAddFlags = MI.getFlags();
5293 unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();
5294 bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
5295 bool IsInBounds =
5296 PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::InBounds;
5297 unsigned Flags = 0;
5298 if (IsNoUWrap)
5299 Flags |= MachineInstr::MIFlag::NoUWrap;
5300 if (IsInBounds) {
5301 Flags |= MachineInstr::MIFlag::InBounds;
5302 Flags |= MachineInstr::MIFlag::NoUSWrap;
5303 }
5304
5305 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5306 auto NewCst = B.buildConstant(Res: MRI.getType(Reg: Src2Reg), Val: *C1 + *C2);
5307 Observer.changingInstr(MI);
5308 MI.getOperand(i: 1).setReg(LHSSrc1);
5309 MI.getOperand(i: 2).setReg(NewCst.getReg(Idx: 0));
5310 MI.setFlags(Flags);
5311 Observer.changedInstr(MI);
5312 };
5313 return !reassociationCanBreakAddressingModePattern(MI);
5314}
5315
5316bool CombinerHelper::matchReassocPtrAdd(MachineInstr &MI,
5317 BuildFnTy &MatchInfo) const {
5318 auto &PtrAdd = cast<GPtrAdd>(Val&: MI);
5319 // We're trying to match a few pointer computation patterns here for
5320 // re-association opportunities.
5321 // 1) Isolating a constant operand to be on the RHS, e.g.:
5322 // G_PTR_ADD(BASE, G_ADD(X, C)) -> G_PTR_ADD(G_PTR_ADD(BASE, X), C)
5323 //
5324 // 2) Folding two constants in each sub-tree as long as such folding
5325 // doesn't break a legal addressing mode.
5326 // G_PTR_ADD(G_PTR_ADD(BASE, C1), C2) -> G_PTR_ADD(BASE, C1+C2)
5327 //
5328 // 3) Move a constant from the LHS of an inner op to the RHS of the outer.
5329 // G_PTR_ADD (G_PTR_ADD X, C), Y) -> G_PTR_ADD (G_PTR_ADD(X, Y), C)
5330 // iif (G_PTR_ADD X, C) has one use.
5331 MachineInstr *LHS, *RHS;
5332 if (!mi_match(R: PtrAdd.getBaseReg(), MRI, P: m_MInstr(MI&: LHS)) ||
5333 !mi_match(R: PtrAdd.getOffsetReg(), MRI, P: m_MInstr(MI&: RHS)))
5334 return false;
5335
5336 // Try to match example 2.
5337 if (matchReassocFoldConstantsInSubTree(MI&: PtrAdd, LHS, RHS, MatchInfo))
5338 return true;
5339
5340 // Try to match example 3.
5341 if (matchReassocConstantInnerLHS(MI&: PtrAdd, LHS, RHS, MatchInfo))
5342 return true;
5343
5344 // Try to match example 1.
5345 if (matchReassocConstantInnerRHS(MI&: PtrAdd, RHS, MatchInfo))
5346 return true;
5347
5348 return false;
5349}
5350bool CombinerHelper::tryReassocBinOp(unsigned Opc, Register DstReg,
5351 Register OpLHS, Register OpRHS,
5352 BuildFnTy &MatchInfo) const {
5353 LLT OpRHSTy = MRI.getType(Reg: OpRHS);
5354 MachineInstr *OpLHSDef;
5355 if (!mi_match(R: OpLHS, MRI, P: m_MInstr(MI&: OpLHSDef)) || OpLHSDef->getOpcode() != Opc)
5356 return false;
5357
5358 Register OpLHSLHS = OpLHSDef->getOperand(i: 1).getReg();
5359 Register OpLHSRHS = OpLHSDef->getOperand(i: 2).getReg();
5360
5361 // If the inner op is (X op C), pull the constant out so it can be folded with
5362 // other constants in the expression tree. Folding is not guaranteed so we
5363 // might have (C1 op C2). In that case do not pull a constant out because it
5364 // won't help and can lead to infinite loops.
5365 if (isConstantOrConstantSplatVector(Def: OpLHSRHS, MRI) &&
5366 !isConstantOrConstantSplatVector(Def: OpLHSLHS, MRI)) {
5367 if (isConstantOrConstantSplatVector(Def: OpRHS, MRI)) {
5368 // (Opc (Opc X, C1), C2) -> (Opc X, (Opc C1, C2))
5369 MatchInfo = [=](MachineIRBuilder &B) {
5370 auto NewCst = B.buildInstr(Opc, DstOps: {OpRHSTy}, SrcOps: {OpLHSRHS, OpRHS});
5371 B.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {OpLHSLHS, NewCst});
5372 };
5373 return true;
5374 }
5375 if (getTargetLowering().isReassocProfitable(MRI, N0: OpLHS, N1: OpRHS)) {
5376 // Reassociate: (op (op x, c1), y) -> (op (op x, y), c1)
5377 // iff (op x, c1) has one use
5378 MatchInfo = [=](MachineIRBuilder &B) {
5379 auto NewLHSLHS = B.buildInstr(Opc, DstOps: {OpRHSTy}, SrcOps: {OpLHSLHS, OpRHS});
5380 B.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {NewLHSLHS, OpLHSRHS});
5381 };
5382 return true;
5383 }
5384 }
5385
5386 return false;
5387}
5388
5389bool CombinerHelper::matchReassocCommBinOp(MachineInstr &MI,
5390 BuildFnTy &MatchInfo) const {
5391 // We don't check if the reassociation will break a legal addressing mode
5392 // here since pointer arithmetic is handled by G_PTR_ADD.
5393 unsigned Opc = MI.getOpcode();
5394 Register DstReg = MI.getOperand(i: 0).getReg();
5395 Register LHSReg = MI.getOperand(i: 1).getReg();
5396 Register RHSReg = MI.getOperand(i: 2).getReg();
5397
5398 if (tryReassocBinOp(Opc, DstReg, OpLHS: LHSReg, OpRHS: RHSReg, MatchInfo))
5399 return true;
5400 if (tryReassocBinOp(Opc, DstReg, OpLHS: RHSReg, OpRHS: LHSReg, MatchInfo))
5401 return true;
5402 return false;
5403}
5404
5405bool CombinerHelper::matchConstantFoldCastOp(MachineInstr &MI,
5406 APInt &MatchInfo) const {
5407 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5408 Register SrcOp = MI.getOperand(i: 1).getReg();
5409
5410 if (auto MaybeCst = ConstantFoldCastOp(Opcode: MI.getOpcode(), DstTy, Op0: SrcOp, MRI)) {
5411 MatchInfo = *MaybeCst;
5412 return true;
5413 }
5414
5415 return false;
5416}
5417
5418bool CombinerHelper::matchConstantFoldUnaryIntOp(MachineInstr &MI,
5419 BuildFnTy &MatchInfo) const {
5420 Register Dst = MI.getOperand(i: 0).getReg();
5421 auto Csts = ConstantFoldUnaryIntOp(Opcode: MI.getOpcode(), DstTy: MRI.getType(Reg: Dst),
5422 Src: MI.getOperand(i: 1).getReg(), MRI);
5423 if (Csts.empty())
5424 return false;
5425
5426 MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
5427 if (Csts.size() == 1)
5428 B.buildConstant(Res: Dst, Val: Csts[0]);
5429 else
5430 B.buildBuildVectorConstant(Res: Dst, Ops: Csts);
5431 };
5432 return true;
5433}
5434
5435bool CombinerHelper::matchConstantFoldBinOp(MachineInstr &MI,
5436 APInt &MatchInfo) const {
5437 Register Op1 = MI.getOperand(i: 1).getReg();
5438 Register Op2 = MI.getOperand(i: 2).getReg();
5439 auto MaybeCst = ConstantFoldBinOp(Opcode: MI.getOpcode(), Op1, Op2, MRI);
5440 if (!MaybeCst)
5441 return false;
5442 MatchInfo = *MaybeCst;
5443 return true;
5444}
5445
5446bool CombinerHelper::matchConstantFoldFPBinOp(MachineInstr &MI,
5447 ConstantFP *&MatchInfo) const {
5448 Register Op1 = MI.getOperand(i: 1).getReg();
5449 Register Op2 = MI.getOperand(i: 2).getReg();
5450 auto MaybeCst = ConstantFoldFPBinOp(Opcode: MI.getOpcode(), Op1, Op2, MRI);
5451 if (!MaybeCst)
5452 return false;
5453 MatchInfo =
5454 ConstantFP::get(Context&: MI.getMF()->getFunction().getContext(), V: *MaybeCst);
5455 return true;
5456}
5457
5458bool CombinerHelper::matchConstantFoldFMA(MachineInstr &MI,
5459 ConstantFP *&MatchInfo) const {
5460 assert(MI.getOpcode() == TargetOpcode::G_FMA ||
5461 MI.getOpcode() == TargetOpcode::G_FMAD);
5462 auto [_, Op1, Op2, Op3] = MI.getFirst4Regs();
5463
5464 const ConstantFP *Op3Cst = getConstantFPVRegVal(VReg: Op3, MRI);
5465 if (!Op3Cst)
5466 return false;
5467
5468 const ConstantFP *Op2Cst = getConstantFPVRegVal(VReg: Op2, MRI);
5469 if (!Op2Cst)
5470 return false;
5471
5472 const ConstantFP *Op1Cst = getConstantFPVRegVal(VReg: Op1, MRI);
5473 if (!Op1Cst)
5474 return false;
5475
5476 APFloat Op1F = Op1Cst->getValueAPF();
5477 Op1F.fusedMultiplyAdd(Multiplicand: Op2Cst->getValueAPF(), Addend: Op3Cst->getValueAPF(),
5478 RM: APFloat::rmNearestTiesToEven);
5479 MatchInfo = ConstantFP::get(Context&: MI.getMF()->getFunction().getContext(), V: Op1F);
5480 return true;
5481}
5482
5483bool CombinerHelper::matchNarrowBinopFeedingAnd(
5484 MachineInstr &MI,
5485 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
5486 // Look for a binop feeding into an AND with a mask:
5487 //
5488 // %add = G_ADD %lhs, %rhs
5489 // %and = G_AND %add, 000...11111111
5490 //
5491 // Check if it's possible to perform the binop at a narrower width and zext
5492 // back to the original width like so:
5493 //
5494 // %narrow_lhs = G_TRUNC %lhs
5495 // %narrow_rhs = G_TRUNC %rhs
5496 // %narrow_add = G_ADD %narrow_lhs, %narrow_rhs
5497 // %new_add = G_ZEXT %narrow_add
5498 // %and = G_AND %new_add, 000...11111111
5499 //
5500 // This can allow later combines to eliminate the G_AND if it turns out
5501 // that the mask is irrelevant.
5502 assert(MI.getOpcode() == TargetOpcode::G_AND);
5503 Register Dst = MI.getOperand(i: 0).getReg();
5504 Register AndLHS = MI.getOperand(i: 1).getReg();
5505 Register AndRHS = MI.getOperand(i: 2).getReg();
5506 LLT WideTy = MRI.getType(Reg: Dst);
5507
5508 // If the potential binop has more than one use, then it's possible that one
5509 // of those uses will need its full width.
5510 if (!WideTy.isScalar() || !MRI.hasOneNonDBGUse(RegNo: AndLHS))
5511 return false;
5512
5513 // Check if the LHS feeding the AND is impacted by the high bits that we're
5514 // masking out.
5515 //
5516 // e.g. for 64-bit x, y:
5517 //
5518 // add_64(x, y) & 65535 == zext(add_16(trunc(x), trunc(y))) & 65535
5519 MachineInstr *LHSInst = getDefIgnoringCopies(Reg: AndLHS, MRI);
5520 if (!LHSInst)
5521 return false;
5522 unsigned LHSOpc = LHSInst->getOpcode();
5523 switch (LHSOpc) {
5524 default:
5525 return false;
5526 case TargetOpcode::G_ADD:
5527 case TargetOpcode::G_SUB:
5528 case TargetOpcode::G_MUL:
5529 case TargetOpcode::G_AND:
5530 case TargetOpcode::G_OR:
5531 case TargetOpcode::G_XOR:
5532 break;
5533 }
5534
5535 // Find the mask on the RHS.
5536 auto Cst = getIConstantVRegValWithLookThrough(VReg: AndRHS, MRI);
5537 if (!Cst)
5538 return false;
5539 auto Mask = Cst->Value;
5540 if (!Mask.isMask())
5541 return false;
5542
5543 // No point in combining if there's nothing to truncate.
5544 unsigned NarrowWidth = Mask.countr_one();
5545 if (NarrowWidth == WideTy.getSizeInBits())
5546 return false;
5547 LLT NarrowTy = LLT::integer(SizeInBits: NarrowWidth);
5548
5549 // Check if adding the zext + truncates could be harmful.
5550 auto &MF = *MI.getMF();
5551 const auto &TLI = getTargetLowering();
5552 LLVMContext &Ctx = MF.getFunction().getContext();
5553 if (!TLI.isTruncateFree(FromTy: WideTy, ToTy: NarrowTy, Ctx) ||
5554 !TLI.isZExtFree(FromTy: NarrowTy, ToTy: WideTy, Ctx))
5555 return false;
5556 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_TRUNC, {NarrowTy, WideTy}}) ||
5557 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ZEXT, {WideTy, NarrowTy}}))
5558 return false;
5559 Register BinOpLHS = LHSInst->getOperand(i: 1).getReg();
5560 Register BinOpRHS = LHSInst->getOperand(i: 2).getReg();
5561 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5562 auto NarrowLHS = Builder.buildTrunc(Res: NarrowTy, Op: BinOpLHS);
5563 auto NarrowRHS = Builder.buildTrunc(Res: NarrowTy, Op: BinOpRHS);
5564 auto NarrowBinOp =
5565 Builder.buildInstr(Opc: LHSOpc, DstOps: {NarrowTy}, SrcOps: {NarrowLHS, NarrowRHS});
5566 auto Ext = Builder.buildZExt(Res: WideTy, Op: NarrowBinOp);
5567 Observer.changingInstr(MI);
5568 MI.getOperand(i: 1).setReg(Ext.getReg(Idx: 0));
5569 Observer.changedInstr(MI);
5570 };
5571 return true;
5572}
5573
5574bool CombinerHelper::matchMulOBy2(MachineInstr &MI,
5575 BuildFnTy &MatchInfo) const {
5576 unsigned Opc = MI.getOpcode();
5577 assert(Opc == TargetOpcode::G_UMULO || Opc == TargetOpcode::G_SMULO);
5578
5579 if (!mi_match(R: MI.getOperand(i: 3).getReg(), MRI, P: m_SpecificICstOrSplat(RequestedValue: 2)))
5580 return false;
5581
5582 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5583 Observer.changingInstr(MI);
5584 unsigned NewOpc = Opc == TargetOpcode::G_UMULO ? TargetOpcode::G_UADDO
5585 : TargetOpcode::G_SADDO;
5586 MI.setDesc(Builder.getTII().get(Opcode: NewOpc));
5587 MI.getOperand(i: 3).setReg(MI.getOperand(i: 2).getReg());
5588 Observer.changedInstr(MI);
5589 };
5590 return true;
5591}
5592
5593bool CombinerHelper::matchMulOBy0(MachineInstr &MI,
5594 BuildFnTy &MatchInfo) const {
5595 // (G_*MULO x, 0) -> 0 + no carry out
5596 assert(MI.getOpcode() == TargetOpcode::G_UMULO ||
5597 MI.getOpcode() == TargetOpcode::G_SMULO);
5598 if (!mi_match(R: MI.getOperand(i: 3).getReg(), MRI, P: m_SpecificICstOrSplat(RequestedValue: 0)))
5599 return false;
5600 Register Dst = MI.getOperand(i: 0).getReg();
5601 Register Carry = MI.getOperand(i: 1).getReg();
5602 if (!isConstantLegalOrBeforeLegalizer(Ty: MRI.getType(Reg: Dst)) ||
5603 !isConstantLegalOrBeforeLegalizer(Ty: MRI.getType(Reg: Carry)))
5604 return false;
5605 MatchInfo = [=](MachineIRBuilder &B) {
5606 B.buildConstant(Res: Dst, Val: 0);
5607 B.buildConstant(Res: Carry, Val: 0);
5608 };
5609 return true;
5610}
5611
5612bool CombinerHelper::matchAddEToAddO(MachineInstr &MI,
5613 BuildFnTy &MatchInfo) const {
5614 // (G_*ADDE x, y, 0) -> (G_*ADDO x, y)
5615 // (G_*SUBE x, y, 0) -> (G_*SUBO x, y)
5616 assert(MI.getOpcode() == TargetOpcode::G_UADDE ||
5617 MI.getOpcode() == TargetOpcode::G_SADDE ||
5618 MI.getOpcode() == TargetOpcode::G_USUBE ||
5619 MI.getOpcode() == TargetOpcode::G_SSUBE);
5620 if (!mi_match(R: MI.getOperand(i: 4).getReg(), MRI, P: m_SpecificICstOrSplat(RequestedValue: 0)))
5621 return false;
5622 MatchInfo = [&](MachineIRBuilder &B) {
5623 unsigned NewOpcode;
5624 switch (MI.getOpcode()) {
5625 case TargetOpcode::G_UADDE:
5626 NewOpcode = TargetOpcode::G_UADDO;
5627 break;
5628 case TargetOpcode::G_SADDE:
5629 NewOpcode = TargetOpcode::G_SADDO;
5630 break;
5631 case TargetOpcode::G_USUBE:
5632 NewOpcode = TargetOpcode::G_USUBO;
5633 break;
5634 case TargetOpcode::G_SSUBE:
5635 NewOpcode = TargetOpcode::G_SSUBO;
5636 break;
5637 }
5638 Observer.changingInstr(MI);
5639 MI.setDesc(B.getTII().get(Opcode: NewOpcode));
5640 MI.removeOperand(OpNo: 4);
5641 Observer.changedInstr(MI);
5642 };
5643 return true;
5644}
5645
5646bool CombinerHelper::matchSubAddSameReg(MachineInstr &MI,
5647 BuildFnTy &MatchInfo) const {
5648 assert(MI.getOpcode() == TargetOpcode::G_SUB);
5649 Register Dst = MI.getOperand(i: 0).getReg();
5650 // (x + y) - z -> x (if y == z)
5651 // (x + y) - z -> y (if x == z)
5652 Register X, Y, Z;
5653 if (mi_match(R: Dst, MRI, P: m_GSub(L: m_GAdd(L: m_Reg(R&: X), R: m_Reg(R&: Y)), R: m_Reg(R&: Z)))) {
5654 Register ReplaceReg;
5655 int64_t CstX, CstY;
5656 if (Y == Z || (mi_match(R: Y, MRI, P: m_ICstOrSplat(Cst&: CstY)) &&
5657 mi_match(R: Z, MRI, P: m_SpecificICstOrSplat(RequestedValue: CstY))))
5658 ReplaceReg = X;
5659 else if (X == Z || (mi_match(R: X, MRI, P: m_ICstOrSplat(Cst&: CstX)) &&
5660 mi_match(R: Z, MRI, P: m_SpecificICstOrSplat(RequestedValue: CstX))))
5661 ReplaceReg = Y;
5662 if (ReplaceReg) {
5663 MatchInfo = [=](MachineIRBuilder &B) { B.buildCopy(Res: Dst, Op: ReplaceReg); };
5664 return true;
5665 }
5666 }
5667
5668 // x - (y + z) -> 0 - y (if x == z)
5669 // x - (y + z) -> 0 - z (if x == y)
5670 if (mi_match(R: Dst, MRI, P: m_GSub(L: m_Reg(R&: X), R: m_GAdd(L: m_Reg(R&: Y), R: m_Reg(R&: Z))))) {
5671 Register ReplaceReg;
5672 int64_t CstX;
5673 if (X == Z || (mi_match(R: X, MRI, P: m_ICstOrSplat(Cst&: CstX)) &&
5674 mi_match(R: Z, MRI, P: m_SpecificICstOrSplat(RequestedValue: CstX))))
5675 ReplaceReg = Y;
5676 else if (X == Y || (mi_match(R: X, MRI, P: m_ICstOrSplat(Cst&: CstX)) &&
5677 mi_match(R: Y, MRI, P: m_SpecificICstOrSplat(RequestedValue: CstX))))
5678 ReplaceReg = Z;
5679 if (ReplaceReg) {
5680 MatchInfo = [=](MachineIRBuilder &B) {
5681 auto Zero = B.buildConstant(Res: MRI.getType(Reg: Dst), Val: 0);
5682 B.buildSub(Dst, Src0: Zero, Src1: ReplaceReg);
5683 };
5684 return true;
5685 }
5686 }
5687 return false;
5688}
5689
5690MachineInstr *CombinerHelper::buildUDivOrURemUsingMul(MachineInstr &MI) const {
5691 unsigned Opcode = MI.getOpcode();
5692 assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
5693 auto &UDivorRem = cast<GenericMachineInstr>(Val&: MI);
5694 Register Dst = UDivorRem.getReg(Idx: 0);
5695 Register LHS = UDivorRem.getReg(Idx: 1);
5696 Register RHS = UDivorRem.getReg(Idx: 2);
5697 LLT Ty = MRI.getType(Reg: Dst);
5698 LLT ScalarTy = Ty.getScalarType();
5699 const unsigned EltBits = ScalarTy.getScalarSizeInBits();
5700 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5701 LLT ScalarShiftAmtTy = ShiftAmtTy.getScalarType();
5702
5703 auto &MIB = Builder;
5704
5705 bool UseSRL = false;
5706 SmallVector<Register, 16> Shifts, Factors;
5707 auto *RHSDefInstr = cast<GenericMachineInstr>(Val: getDefIgnoringCopies(Reg: RHS, MRI));
5708 bool IsSplat = getIConstantSplatVal(MI: *RHSDefInstr, MRI).has_value();
5709
5710 auto BuildExactUDIVPattern = [&](const Constant *C) {
5711 // Don't recompute inverses for each splat element.
5712 if (IsSplat && !Factors.empty()) {
5713 Shifts.push_back(Elt: Shifts[0]);
5714 Factors.push_back(Elt: Factors[0]);
5715 return true;
5716 }
5717
5718 auto *CI = cast<ConstantInt>(Val: C);
5719 APInt Divisor = CI->getValue();
5720 unsigned Shift = Divisor.countr_zero();
5721 if (Shift) {
5722 Divisor.lshrInPlace(ShiftAmt: Shift);
5723 UseSRL = true;
5724 }
5725
5726 // Calculate the multiplicative inverse modulo BW.
5727 APInt Factor = Divisor.multiplicativeInverse();
5728 Shifts.push_back(Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: Shift).getReg(Idx: 0));
5729 Factors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Factor).getReg(Idx: 0));
5730 return true;
5731 };
5732
5733 if (MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
5734 // Collect all magic values from the build vector.
5735 if (!matchUnaryPredicate(MRI, Reg: RHS, Match: BuildExactUDIVPattern))
5736 llvm_unreachable("Expected unary predicate match to succeed");
5737
5738 Register Shift, Factor;
5739 if (Ty.isVector()) {
5740 Shift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: Shifts).getReg(Idx: 0);
5741 Factor = MIB.buildBuildVector(Res: Ty, Ops: Factors).getReg(Idx: 0);
5742 } else {
5743 Shift = Shifts[0];
5744 Factor = Factors[0];
5745 }
5746
5747 Register Res = LHS;
5748
5749 if (UseSRL)
5750 Res = MIB.buildLShr(Dst: Ty, Src0: Res, Src1: Shift, Flags: MachineInstr::IsExact).getReg(Idx: 0);
5751
5752 return MIB.buildMul(Dst: Ty, Src0: Res, Src1: Factor);
5753 }
5754
5755 unsigned KnownLeadingZeros =
5756 VT ? VT->getKnownBits(R: LHS).countMinLeadingZeros() : 0;
5757
5758 bool UseNPQ = false;
5759 SmallVector<Register, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
5760 auto BuildUDIVPattern = [&](const Constant *C) {
5761 auto *CI = cast<ConstantInt>(Val: C);
5762 const APInt &Divisor = CI->getValue();
5763
5764 bool SelNPQ = false;
5765 APInt Magic(Divisor.getBitWidth(), 0);
5766 unsigned PreShift = 0, PostShift = 0;
5767
5768 // Magic algorithm doesn't work for division by 1. We need to emit a select
5769 // at the end.
5770 // TODO: Use undef values for divisor of 1.
5771 if (!Divisor.isOne()) {
5772
5773 // UnsignedDivisionByConstantInfo doesn't work correctly if leading zeros
5774 // in the dividend exceeds the leading zeros for the divisor.
5775 UnsignedDivisionByConstantInfo magics =
5776 UnsignedDivisionByConstantInfo::get(
5777 D: Divisor, LeadingZeros: std::min(a: KnownLeadingZeros, b: Divisor.countl_zero()));
5778
5779 Magic = std::move(magics.Magic);
5780
5781 assert(magics.PreShift < Divisor.getBitWidth() &&
5782 "We shouldn't generate an undefined shift!");
5783 assert(magics.PostShift < Divisor.getBitWidth() &&
5784 "We shouldn't generate an undefined shift!");
5785 assert((!magics.IsAdd || magics.PreShift == 0) && "Unexpected pre-shift");
5786 PreShift = magics.PreShift;
5787 PostShift = magics.PostShift;
5788 SelNPQ = magics.IsAdd;
5789 }
5790
5791 PreShifts.push_back(
5792 Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: PreShift).getReg(Idx: 0));
5793 MagicFactors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Magic).getReg(Idx: 0));
5794 NPQFactors.push_back(
5795 Elt: MIB.buildConstant(Res: ScalarTy,
5796 Val: SelNPQ ? APInt::getOneBitSet(numBits: EltBits, BitNo: EltBits - 1)
5797 : APInt::getZero(numBits: EltBits))
5798 .getReg(Idx: 0));
5799 PostShifts.push_back(
5800 Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: PostShift).getReg(Idx: 0));
5801 UseNPQ |= SelNPQ;
5802 return true;
5803 };
5804
5805 // Collect the shifts/magic values from each element.
5806 bool Matched = matchUnaryPredicate(MRI, Reg: RHS, Match: BuildUDIVPattern);
5807 (void)Matched;
5808 assert(Matched && "Expected unary predicate match to succeed");
5809
5810 Register PreShift, PostShift, MagicFactor, NPQFactor;
5811 auto *RHSDef = getOpcodeDef<GBuildVector>(Reg: RHS, MRI);
5812 if (RHSDef) {
5813 PreShift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: PreShifts).getReg(Idx: 0);
5814 MagicFactor = MIB.buildBuildVector(Res: Ty, Ops: MagicFactors).getReg(Idx: 0);
5815 NPQFactor = MIB.buildBuildVector(Res: Ty, Ops: NPQFactors).getReg(Idx: 0);
5816 PostShift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: PostShifts).getReg(Idx: 0);
5817 } else {
5818 assert(MRI.getType(RHS).isScalar() &&
5819 "Non-build_vector operation should have been a scalar");
5820 PreShift = PreShifts[0];
5821 MagicFactor = MagicFactors[0];
5822 PostShift = PostShifts[0];
5823 }
5824
5825 Register Q = LHS;
5826 Q = MIB.buildLShr(Dst: Ty, Src0: Q, Src1: PreShift).getReg(Idx: 0);
5827
5828 // Multiply the numerator (operand 0) by the magic value.
5829 Q = MIB.buildUMulH(Dst: Ty, Src0: Q, Src1: MagicFactor).getReg(Idx: 0);
5830
5831 if (UseNPQ) {
5832 Register NPQ = MIB.buildSub(Dst: Ty, Src0: LHS, Src1: Q).getReg(Idx: 0);
5833
5834 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
5835 // G_UMULH to act as a SRL-by-1 for NPQ, else multiply by zero.
5836 if (Ty.isVector())
5837 NPQ = MIB.buildUMulH(Dst: Ty, Src0: NPQ, Src1: NPQFactor).getReg(Idx: 0);
5838 else
5839 NPQ = MIB.buildLShr(Dst: Ty, Src0: NPQ, Src1: MIB.buildConstant(Res: ShiftAmtTy, Val: 1)).getReg(Idx: 0);
5840
5841 Q = MIB.buildAdd(Dst: Ty, Src0: NPQ, Src1: Q).getReg(Idx: 0);
5842 }
5843
5844 Q = MIB.buildLShr(Dst: Ty, Src0: Q, Src1: PostShift).getReg(Idx: 0);
5845 auto One = MIB.buildConstant(Res: Ty, Val: 1);
5846 auto IsOne = MIB.buildICmp(
5847 Pred: CmpInst::Predicate::ICMP_EQ,
5848 Res: Ty.isScalar() ? LLT::integer(SizeInBits: 1) : Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1)),
5849 Op0: RHS, Op1: One);
5850 auto ret = MIB.buildSelect(Res: Ty, Tst: IsOne, Op0: LHS, Op1: Q);
5851
5852 if (Opcode == TargetOpcode::G_UREM) {
5853 auto Prod = MIB.buildMul(Dst: Ty, Src0: ret, Src1: RHS);
5854 return MIB.buildSub(Dst: Ty, Src0: LHS, Src1: Prod);
5855 }
5856 return ret;
5857}
5858
5859bool CombinerHelper::matchUDivOrURemByConst(MachineInstr &MI) const {
5860 unsigned Opcode = MI.getOpcode();
5861 assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
5862 Register Dst = MI.getOperand(i: 0).getReg();
5863 Register RHS = MI.getOperand(i: 2).getReg();
5864 LLT DstTy = MRI.getType(Reg: Dst);
5865
5866 auto &MF = *MI.getMF();
5867 AttributeList Attr = MF.getFunction().getAttributes();
5868 const auto &TLI = getTargetLowering();
5869 LLVMContext &Ctx = MF.getFunction().getContext();
5870 if (DstTy.getScalarSizeInBits() == 1 ||
5871 TLI.isIntDivCheap(VT: getApproximateEVTForLLT(Ty: DstTy, Ctx), Attr))
5872 return false;
5873
5874 // Don't do this for minsize because the instruction sequence is usually
5875 // larger.
5876 if (MF.getFunction().hasMinSize())
5877 return false;
5878
5879 if (Opcode == TargetOpcode::G_UDIV &&
5880 MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
5881 return matchUnaryPredicate(
5882 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5883 }
5884
5885 MachineInstr *RHSDef;
5886 if (!mi_match(R: RHS, MRI, P: m_MInstr(MI&: RHSDef)) ||
5887 !isConstantOrConstantVector(MI&: *RHSDef, MRI))
5888 return false;
5889
5890 // Don't do this if the types are not going to be legal.
5891 if (LI) {
5892 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_MUL, {DstTy, DstTy}}))
5893 return false;
5894 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_UMULH, {DstTy}}))
5895 return false;
5896 if (!isLegalOrBeforeLegalizer(
5897 Query: {TargetOpcode::G_ICMP,
5898 {DstTy.isVector() ? DstTy.changeElementSize(NewEltSize: 1) : LLT::scalar(SizeInBits: 1),
5899 DstTy}}))
5900 return false;
5901 if (Opcode == TargetOpcode::G_UREM &&
5902 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SUB, {DstTy, DstTy}}))
5903 return false;
5904 }
5905
5906 return matchUnaryPredicate(
5907 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5908}
5909
5910void CombinerHelper::applyUDivOrURemByConst(MachineInstr &MI) const {
5911 auto *NewMI = buildUDivOrURemUsingMul(MI);
5912 replaceSingleDefInstWithReg(MI, Replacement: NewMI->getOperand(i: 0).getReg());
5913}
5914
5915bool CombinerHelper::matchSDivOrSRemByConst(MachineInstr &MI) const {
5916 unsigned Opcode = MI.getOpcode();
5917 assert(Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM);
5918 Register Dst = MI.getOperand(i: 0).getReg();
5919 Register RHS = MI.getOperand(i: 2).getReg();
5920 LLT DstTy = MRI.getType(Reg: Dst);
5921 auto SizeInBits = DstTy.getScalarSizeInBits();
5922 LLT WideTy = DstTy.changeElementSize(NewEltSize: SizeInBits * 2);
5923
5924 auto &MF = *MI.getMF();
5925 AttributeList Attr = MF.getFunction().getAttributes();
5926 const auto &TLI = getTargetLowering();
5927 LLVMContext &Ctx = MF.getFunction().getContext();
5928 if (DstTy.getScalarSizeInBits() < 3 ||
5929 TLI.isIntDivCheap(VT: getApproximateEVTForLLT(Ty: DstTy, Ctx), Attr))
5930 return false;
5931
5932 // Don't do this for minsize because the instruction sequence is usually
5933 // larger.
5934 if (MF.getFunction().hasMinSize())
5935 return false;
5936
5937 // If the sdiv has an 'exact' flag we can use a simpler lowering.
5938 if (Opcode == TargetOpcode::G_SDIV &&
5939 MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
5940 return matchUnaryPredicate(
5941 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5942 }
5943
5944 MachineInstr *RHSDef;
5945 if (!mi_match(R: RHS, MRI, P: m_MInstr(MI&: RHSDef)) ||
5946 !isConstantOrConstantVector(MI&: *RHSDef, MRI))
5947 return false;
5948
5949 // Don't do this if the types are not going to be legal.
5950 if (LI) {
5951 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_MUL, {DstTy, DstTy}}))
5952 return false;
5953 if (!isLegal(Query: {TargetOpcode::G_SMULH, {DstTy}}) &&
5954 !isLegalOrHasWidenScalar(Query: {TargetOpcode::G_MUL, {WideTy, WideTy}}))
5955 return false;
5956 if (Opcode == TargetOpcode::G_SREM &&
5957 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SUB, {DstTy, DstTy}}))
5958 return false;
5959 }
5960
5961 return matchUnaryPredicate(
5962 MRI, Reg: RHS, Match: [](const Constant *C) { return C && !C->isNullValue(); });
5963}
5964
5965void CombinerHelper::applySDivOrSRemByConst(MachineInstr &MI) const {
5966 auto *NewMI = buildSDivOrSRemUsingMul(MI);
5967 replaceSingleDefInstWithReg(MI, Replacement: NewMI->getOperand(i: 0).getReg());
5968}
5969
5970MachineInstr *CombinerHelper::buildSDivOrSRemUsingMul(MachineInstr &MI) const {
5971 unsigned Opcode = MI.getOpcode();
5972 assert(MI.getOpcode() == TargetOpcode::G_SDIV ||
5973 Opcode == TargetOpcode::G_SREM);
5974 auto &SDivorRem = cast<GenericMachineInstr>(Val&: MI);
5975 Register Dst = SDivorRem.getReg(Idx: 0);
5976 Register LHS = SDivorRem.getReg(Idx: 1);
5977 Register RHS = SDivorRem.getReg(Idx: 2);
5978 LLT Ty = MRI.getType(Reg: Dst);
5979 LLT ScalarTy = Ty.getScalarType();
5980 const unsigned EltBits = ScalarTy.getScalarSizeInBits();
5981 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
5982 LLT ScalarShiftAmtTy = ShiftAmtTy.getScalarType();
5983 auto &MIB = Builder;
5984
5985 bool UseSRA = false;
5986 SmallVector<Register, 16> ExactShifts, ExactFactors;
5987
5988 auto *RHSDefInstr = cast<GenericMachineInstr>(Val: getDefIgnoringCopies(Reg: RHS, MRI));
5989 bool IsSplat = getIConstantSplatVal(MI: *RHSDefInstr, MRI).has_value();
5990
5991 auto BuildExactSDIVPattern = [&](const Constant *C) {
5992 // Don't recompute inverses for each splat element.
5993 if (IsSplat && !ExactFactors.empty()) {
5994 ExactShifts.push_back(Elt: ExactShifts[0]);
5995 ExactFactors.push_back(Elt: ExactFactors[0]);
5996 return true;
5997 }
5998
5999 auto *CI = cast<ConstantInt>(Val: C);
6000 APInt Divisor = CI->getValue();
6001 unsigned Shift = Divisor.countr_zero();
6002 if (Shift) {
6003 Divisor.ashrInPlace(ShiftAmt: Shift);
6004 UseSRA = true;
6005 }
6006
6007 // Calculate the multiplicative inverse modulo BW.
6008 // 2^W requires W + 1 bits, so we have to extend and then truncate.
6009 APInt Factor = Divisor.multiplicativeInverse();
6010 ExactShifts.push_back(Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: Shift).getReg(Idx: 0));
6011 ExactFactors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Factor).getReg(Idx: 0));
6012 return true;
6013 };
6014
6015 if (MI.getFlag(Flag: MachineInstr::MIFlag::IsExact)) {
6016 // Collect all magic values from the build vector.
6017 bool Matched = matchUnaryPredicate(MRI, Reg: RHS, Match: BuildExactSDIVPattern);
6018 (void)Matched;
6019 assert(Matched && "Expected unary predicate match to succeed");
6020
6021 Register Shift, Factor;
6022 if (Ty.isVector()) {
6023 Shift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: ExactShifts).getReg(Idx: 0);
6024 Factor = MIB.buildBuildVector(Res: Ty, Ops: ExactFactors).getReg(Idx: 0);
6025 } else {
6026 Shift = ExactShifts[0];
6027 Factor = ExactFactors[0];
6028 }
6029
6030 Register Res = LHS;
6031
6032 if (UseSRA)
6033 Res = MIB.buildAShr(Dst: Ty, Src0: Res, Src1: Shift, Flags: MachineInstr::IsExact).getReg(Idx: 0);
6034
6035 return MIB.buildMul(Dst: Ty, Src0: Res, Src1: Factor);
6036 }
6037
6038 SmallVector<Register, 16> MagicFactors, Factors, Shifts, ShiftMasks;
6039
6040 auto BuildSDIVPattern = [&](const Constant *C) {
6041 auto *CI = cast<ConstantInt>(Val: C);
6042 const APInt &Divisor = CI->getValue();
6043
6044 SignedDivisionByConstantInfo Magics =
6045 SignedDivisionByConstantInfo::get(D: Divisor);
6046 int NumeratorFactor = 0;
6047 int ShiftMask = -1;
6048
6049 if (Divisor.isOne() || Divisor.isAllOnes()) {
6050 // If d is +1/-1, we just multiply the numerator by +1/-1.
6051 NumeratorFactor = Divisor.getSExtValue();
6052 Magics.Magic = 0;
6053 Magics.ShiftAmount = 0;
6054 ShiftMask = 0;
6055 } else if (Divisor.isStrictlyPositive() && Magics.Magic.isNegative()) {
6056 // If d > 0 and m < 0, add the numerator.
6057 NumeratorFactor = 1;
6058 } else if (Divisor.isNegative() && Magics.Magic.isStrictlyPositive()) {
6059 // If d < 0 and m > 0, subtract the numerator.
6060 NumeratorFactor = -1;
6061 }
6062
6063 MagicFactors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: Magics.Magic).getReg(Idx: 0));
6064 Factors.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: NumeratorFactor).getReg(Idx: 0));
6065 Shifts.push_back(
6066 Elt: MIB.buildConstant(Res: ScalarShiftAmtTy, Val: Magics.ShiftAmount).getReg(Idx: 0));
6067 ShiftMasks.push_back(Elt: MIB.buildConstant(Res: ScalarTy, Val: ShiftMask).getReg(Idx: 0));
6068
6069 return true;
6070 };
6071
6072 // Collect the shifts/magic values from each element.
6073 bool Matched = matchUnaryPredicate(MRI, Reg: RHS, Match: BuildSDIVPattern);
6074 (void)Matched;
6075 assert(Matched && "Expected unary predicate match to succeed");
6076
6077 Register MagicFactor, Factor, Shift, ShiftMask;
6078 auto *RHSDef = getOpcodeDef<GBuildVector>(Reg: RHS, MRI);
6079 if (RHSDef) {
6080 MagicFactor = MIB.buildBuildVector(Res: Ty, Ops: MagicFactors).getReg(Idx: 0);
6081 Factor = MIB.buildBuildVector(Res: Ty, Ops: Factors).getReg(Idx: 0);
6082 Shift = MIB.buildBuildVector(Res: ShiftAmtTy, Ops: Shifts).getReg(Idx: 0);
6083 ShiftMask = MIB.buildBuildVector(Res: Ty, Ops: ShiftMasks).getReg(Idx: 0);
6084 } else {
6085 assert(MRI.getType(RHS).isScalar() &&
6086 "Non-build_vector operation should have been a scalar");
6087 MagicFactor = MagicFactors[0];
6088 Factor = Factors[0];
6089 Shift = Shifts[0];
6090 ShiftMask = ShiftMasks[0];
6091 }
6092
6093 Register Q = LHS;
6094 Q = MIB.buildSMulH(Dst: Ty, Src0: LHS, Src1: MagicFactor).getReg(Idx: 0);
6095
6096 // (Optionally) Add/subtract the numerator using Factor.
6097 Factor = MIB.buildMul(Dst: Ty, Src0: LHS, Src1: Factor).getReg(Idx: 0);
6098 Q = MIB.buildAdd(Dst: Ty, Src0: Q, Src1: Factor).getReg(Idx: 0);
6099
6100 // Shift right algebraic by shift value.
6101 Q = MIB.buildAShr(Dst: Ty, Src0: Q, Src1: Shift).getReg(Idx: 0);
6102
6103 // Extract the sign bit, mask it and add it to the quotient.
6104 auto SignShift = MIB.buildConstant(Res: ShiftAmtTy, Val: EltBits - 1);
6105 auto T = MIB.buildLShr(Dst: Ty, Src0: Q, Src1: SignShift);
6106 T = MIB.buildAnd(Dst: Ty, Src0: T, Src1: ShiftMask);
6107 auto ret = MIB.buildAdd(Dst: Ty, Src0: Q, Src1: T);
6108
6109 if (Opcode == TargetOpcode::G_SREM) {
6110 auto Prod = MIB.buildMul(Dst: Ty, Src0: ret, Src1: RHS);
6111 return MIB.buildSub(Dst: Ty, Src0: LHS, Src1: Prod);
6112 }
6113 return ret;
6114}
6115
6116bool CombinerHelper::matchDivByPow2(MachineInstr &MI, bool IsSigned) const {
6117 assert((MI.getOpcode() == TargetOpcode::G_SDIV ||
6118 MI.getOpcode() == TargetOpcode::G_UDIV) &&
6119 "Expected SDIV or UDIV");
6120 auto &Div = cast<GenericMachineInstr>(Val&: MI);
6121 Register RHS = Div.getReg(Idx: 2);
6122 auto MatchPow2 = [&](const Constant *C) {
6123 auto *CI = dyn_cast<ConstantInt>(Val: C);
6124 return CI && (CI->getValue().isPowerOf2() ||
6125 (IsSigned && CI->getValue().isNegatedPowerOf2()));
6126 };
6127 return matchUnaryPredicate(MRI, Reg: RHS, Match: MatchPow2, /*AllowUndefs=*/false);
6128}
6129
6130void CombinerHelper::applySDivByPow2(MachineInstr &MI) const {
6131 assert(MI.getOpcode() == TargetOpcode::G_SDIV && "Expected SDIV");
6132 auto &SDiv = cast<GenericMachineInstr>(Val&: MI);
6133 Register Dst = SDiv.getReg(Idx: 0);
6134 Register LHS = SDiv.getReg(Idx: 1);
6135 Register RHS = SDiv.getReg(Idx: 2);
6136 LLT Ty = MRI.getType(Reg: Dst);
6137 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
6138 LLT CCVT = Ty.isVector() ? LLT::vector(EC: Ty.getElementCount(), ScalarTy: LLT::integer(SizeInBits: 1))
6139 : LLT::integer(SizeInBits: 1);
6140
6141 // Effectively we want to lower G_SDIV %lhs, %rhs, where %rhs is a power of 2,
6142 // to the following version:
6143 //
6144 // %c1 = G_CTTZ %rhs
6145 // %inexact = G_SUB $bitwidth, %c1
6146 // %sign = %G_ASHR %lhs, $(bitwidth - 1)
6147 // %lshr = G_LSHR %sign, %inexact
6148 // %add = G_ADD %lhs, %lshr
6149 // %ashr = G_ASHR %add, %c1
6150 // %ashr = G_SELECT, %isoneorallones, %lhs, %ashr
6151 // %zero = G_CONSTANT $0
6152 // %neg = G_NEG %ashr
6153 // %isneg = G_ICMP SLT %rhs, %zero
6154 // %res = G_SELECT %isneg, %neg, %ashr
6155
6156 unsigned BitWidth = Ty.getScalarSizeInBits();
6157 auto Zero = Builder.buildConstant(Res: Ty, Val: 0);
6158
6159 auto Bits = Builder.buildConstant(Res: ShiftAmtTy, Val: BitWidth);
6160 auto C1 = Builder.buildCTTZ(Dst: ShiftAmtTy, Src0: RHS);
6161 auto Inexact = Builder.buildSub(Dst: ShiftAmtTy, Src0: Bits, Src1: C1);
6162 // Splat the sign bit into the register
6163 auto Sign = Builder.buildAShr(
6164 Dst: Ty, Src0: LHS, Src1: Builder.buildConstant(Res: ShiftAmtTy, Val: BitWidth - 1));
6165
6166 // Add (LHS < 0) ? abs2 - 1 : 0;
6167 auto LSrl = Builder.buildLShr(Dst: Ty, Src0: Sign, Src1: Inexact);
6168 auto Add = Builder.buildAdd(Dst: Ty, Src0: LHS, Src1: LSrl);
6169 auto AShr = Builder.buildAShr(Dst: Ty, Src0: Add, Src1: C1);
6170
6171 // Special case: (sdiv X, 1) -> X
6172 // Special Case: (sdiv X, -1) -> 0-X
6173 auto One = Builder.buildConstant(Res: Ty, Val: 1);
6174 auto MinusOne = Builder.buildConstant(Res: Ty, Val: -1);
6175 auto IsOne = Builder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: CCVT, Op0: RHS, Op1: One);
6176 auto IsMinusOne =
6177 Builder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: CCVT, Op0: RHS, Op1: MinusOne);
6178 auto IsOneOrMinusOne = Builder.buildOr(Dst: CCVT, Src0: IsOne, Src1: IsMinusOne);
6179 AShr = Builder.buildSelect(Res: Ty, Tst: IsOneOrMinusOne, Op0: LHS, Op1: AShr);
6180
6181 // If divided by a positive value, we're done. Otherwise, the result must be
6182 // negated.
6183 auto Neg = Builder.buildNeg(Dst: Ty, Src0: AShr);
6184 auto IsNeg = Builder.buildICmp(Pred: CmpInst::Predicate::ICMP_SLT, Res: CCVT, Op0: RHS, Op1: Zero);
6185 Builder.buildSelect(Res: MI.getOperand(i: 0).getReg(), Tst: IsNeg, Op0: Neg, Op1: AShr);
6186 MI.eraseFromParent();
6187}
6188
6189void CombinerHelper::applyUDivByPow2(MachineInstr &MI) const {
6190 assert(MI.getOpcode() == TargetOpcode::G_UDIV && "Expected UDIV");
6191 auto &UDiv = cast<GenericMachineInstr>(Val&: MI);
6192 Register Dst = UDiv.getReg(Idx: 0);
6193 Register LHS = UDiv.getReg(Idx: 1);
6194 Register RHS = UDiv.getReg(Idx: 2);
6195 LLT Ty = MRI.getType(Reg: Dst);
6196 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
6197
6198 auto C1 = Builder.buildCTTZ(Dst: ShiftAmtTy, Src0: RHS);
6199 Builder.buildLShr(Dst: MI.getOperand(i: 0).getReg(), Src0: LHS, Src1: C1);
6200 MI.eraseFromParent();
6201}
6202
6203void CombinerHelper::applySimplifySRemByPow2(MachineInstr &MI) const {
6204 assert(MI.getOpcode() == TargetOpcode::G_SREM && "Expected SREM");
6205 auto &SRem = cast<GBinOp>(Val&: MI);
6206 Register Dst = SRem.getReg(Idx: 0);
6207 Register LHS = SRem.getLHSReg();
6208 Register RHS = SRem.getRHSReg();
6209 LLT Ty = MRI.getType(Reg: Dst);
6210 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
6211
6212 // Effectively we want to lower G_SREM %lhs, %rhs, where %rhs is +/- a power
6213 // of 2, to the following branch-free bias-and-mask version:
6214 //
6215 // %abs = G_ABS %rhs
6216 // %mask = G_SUB %abs, 1
6217 // %sign = G_ASHR %lhs, $(bitwidth - 1)
6218 // %bias = G_AND %sign, %mask
6219 // %biased = G_ADD %lhs, %bias
6220 // %masked = G_AND %biased, %mask
6221 // %res = G_SUB %masked, %bias
6222 //
6223 // The bias adds (|%rhs| - 1) for negative %lhs, correcting rounding towards
6224 // zero (instead of towards -inf that a plain mask would give). Constant
6225 // divisors collapse %mask to a single G_CONSTANT via the CSEMIRBuilder folds
6226 // for G_ABS and G_SUB.
6227
6228 unsigned BitWidth = Ty.getScalarSizeInBits();
6229 auto AbsRHS = Builder.buildAbs(Dst: Ty, Src: RHS);
6230 auto Mask = Builder.buildSub(Dst: Ty, Src0: AbsRHS, Src1: Builder.buildConstant(Res: Ty, Val: 1));
6231 auto BWMinusOne = Builder.buildConstant(Res: ShiftAmtTy, Val: BitWidth - 1);
6232 auto Sign = Builder.buildAShr(Dst: Ty, Src0: LHS, Src1: BWMinusOne);
6233 auto Bias = Builder.buildAnd(Dst: Ty, Src0: Sign, Src1: Mask);
6234 auto Biased = Builder.buildAdd(Dst: Ty, Src0: LHS, Src1: Bias);
6235 auto Masked = Builder.buildAnd(Dst: Ty, Src0: Biased, Src1: Mask);
6236 Builder.buildSub(Dst, Src0: Masked, Src1: Bias);
6237 MI.eraseFromParent();
6238}
6239
6240bool CombinerHelper::matchUMulHToLShr(MachineInstr &MI) const {
6241 assert(MI.getOpcode() == TargetOpcode::G_UMULH);
6242 Register RHS = MI.getOperand(i: 2).getReg();
6243 Register Dst = MI.getOperand(i: 0).getReg();
6244 LLT Ty = MRI.getType(Reg: Dst);
6245 LLT RHSTy = MRI.getType(Reg: RHS);
6246 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
6247 auto MatchPow2ExceptOne = [&](const Constant *C) {
6248 if (auto *CI = dyn_cast<ConstantInt>(Val: C))
6249 return CI->getValue().isPowerOf2() && !CI->getValue().isOne();
6250 return false;
6251 };
6252 if (!matchUnaryPredicate(MRI, Reg: RHS, Match: MatchPow2ExceptOne, AllowUndefs: false))
6253 return false;
6254 // We need to check both G_LSHR and G_CTLZ because the combine uses G_CTLZ to
6255 // get log base 2, and it is not always legal for on a target.
6256 return isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_LSHR, {Ty, ShiftAmtTy}}) &&
6257 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_CTLZ, {RHSTy, RHSTy}});
6258}
6259
6260void CombinerHelper::applyUMulHToLShr(MachineInstr &MI) const {
6261 Register LHS = MI.getOperand(i: 1).getReg();
6262 Register RHS = MI.getOperand(i: 2).getReg();
6263 Register Dst = MI.getOperand(i: 0).getReg();
6264 LLT Ty = MRI.getType(Reg: Dst);
6265 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(ShiftValueTy: Ty);
6266 unsigned NumEltBits = Ty.getScalarSizeInBits();
6267
6268 auto LogBase2 = buildLogBase2(V: RHS, MIB&: Builder);
6269 auto ShiftAmt =
6270 Builder.buildSub(Dst: Ty, Src0: Builder.buildConstant(Res: Ty, Val: NumEltBits), Src1: LogBase2);
6271 auto Trunc = Builder.buildZExtOrTrunc(Res: ShiftAmtTy, Op: ShiftAmt);
6272 Builder.buildLShr(Dst, Src0: LHS, Src1: Trunc);
6273 MI.eraseFromParent();
6274}
6275
6276bool CombinerHelper::matchTruncSSatS(MachineInstr &MI,
6277 Register &MatchInfo) const {
6278 Register Dst = MI.getOperand(i: 0).getReg();
6279 Register Src = MI.getOperand(i: 1).getReg();
6280 LLT DstTy = MRI.getType(Reg: Dst);
6281 LLT SrcTy = MRI.getType(Reg: Src);
6282 unsigned NumDstBits = DstTy.getScalarSizeInBits();
6283 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6284 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
6285
6286 if (!LI || !isLegalOrHasFewerElements(
6287 Query: {TargetOpcode::G_TRUNC_SSAT_S, {DstTy, SrcTy}}))
6288 return false;
6289
6290 APInt SignedMax = APInt::getSignedMaxValue(numBits: NumDstBits).sext(width: NumSrcBits);
6291 APInt SignedMin = APInt::getSignedMinValue(numBits: NumDstBits).sext(width: NumSrcBits);
6292 if (mi_match(
6293 R: Src, MRI,
6294 P: m_GSMin(L: m_GSMax(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: SignedMin)),
6295 R: m_SpecificICstOrSplat(RequestedValue: SignedMax))))
6296 return true;
6297 if (mi_match(
6298 R: Src, MRI,
6299 P: m_GSMax(L: m_GSMin(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: SignedMax)),
6300 R: m_SpecificICstOrSplat(RequestedValue: SignedMin))))
6301 return true;
6302
6303 // CVP in the midend will often transform trunc(smin(smax(..)) into
6304 // trunc nsw(smin(..)) as the smax against INT_MIN never saturates.
6305 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoSWrap) &&
6306 mi_match(R: Src, MRI,
6307 P: m_GSMin(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: SignedMax))))
6308 return true;
6309
6310 return false;
6311}
6312
6313void CombinerHelper::applyTruncSSatS(MachineInstr &MI,
6314 Register &MatchInfo) const {
6315 Register Dst = MI.getOperand(i: 0).getReg();
6316 Builder.buildTruncSSatS(Res: Dst, Op: MatchInfo);
6317 MI.eraseFromParent();
6318}
6319
6320bool CombinerHelper::matchTruncSSatU(MachineInstr &MI,
6321 Register &MatchInfo) const {
6322 Register Dst = MI.getOperand(i: 0).getReg();
6323 Register Src = MI.getOperand(i: 1).getReg();
6324 LLT DstTy = MRI.getType(Reg: Dst);
6325 LLT SrcTy = MRI.getType(Reg: Src);
6326 unsigned NumDstBits = DstTy.getScalarSizeInBits();
6327 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6328 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
6329
6330 if (!LI || !isLegalOrHasFewerElements(
6331 Query: {TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))
6332 return false;
6333 APInt UnsignedMax = APInt::getMaxValue(numBits: NumDstBits).zext(width: NumSrcBits);
6334 return mi_match(R: Src, MRI,
6335 P: m_GSMin(L: m_GSMax(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: 0)),
6336 R: m_SpecificICstOrSplat(RequestedValue: UnsignedMax))) ||
6337 mi_match(R: Src, MRI,
6338 P: m_GSMax(L: m_GSMin(L: m_Reg(R&: MatchInfo),
6339 R: m_SpecificICstOrSplat(RequestedValue: UnsignedMax)),
6340 R: m_SpecificICstOrSplat(RequestedValue: 0))) ||
6341 mi_match(R: Src, MRI,
6342 P: m_GUMin(L: m_GSMax(L: m_Reg(R&: MatchInfo), R: m_SpecificICstOrSplat(RequestedValue: 0)),
6343 R: m_SpecificICstOrSplat(RequestedValue: UnsignedMax)));
6344}
6345
6346void CombinerHelper::applyTruncSSatU(MachineInstr &MI,
6347 Register &MatchInfo) const {
6348 Register Dst = MI.getOperand(i: 0).getReg();
6349 Builder.buildTruncSSatU(Res: Dst, Op: MatchInfo);
6350 MI.eraseFromParent();
6351}
6352
6353bool CombinerHelper::matchTruncUSatU(MachineInstr &MI,
6354 MachineInstr &MinMI) const {
6355 Register Min = MinMI.getOperand(i: 2).getReg();
6356 Register Val = MinMI.getOperand(i: 1).getReg();
6357 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6358 LLT SrcTy = MRI.getType(Reg: Val);
6359 unsigned NumDstBits = DstTy.getScalarSizeInBits();
6360 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6361 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
6362
6363 if (!LI || !isLegalOrHasFewerElements(
6364 Query: {TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))
6365 return false;
6366 APInt UnsignedMax = APInt::getMaxValue(numBits: NumDstBits).zext(width: NumSrcBits);
6367 return mi_match(R: Min, MRI, P: m_SpecificICstOrSplat(RequestedValue: UnsignedMax)) &&
6368 !mi_match(R: Val, MRI, P: m_GSMax(L: m_Reg(), R: m_Reg()));
6369}
6370
6371bool CombinerHelper::matchTruncUSatUToFPTOUISat(MachineInstr &MI,
6372 MachineInstr &SrcMI) const {
6373 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6374 LLT SrcTy = MRI.getType(Reg: SrcMI.getOperand(i: 1).getReg());
6375
6376 return LI &&
6377 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FPTOUI_SAT, {DstTy, SrcTy}});
6378}
6379
6380bool CombinerHelper::matchRedundantNegOperands(MachineInstr &MI,
6381 BuildFnTy &MatchInfo) const {
6382 unsigned Opc = MI.getOpcode();
6383 assert(Opc == TargetOpcode::G_FADD || Opc == TargetOpcode::G_FSUB ||
6384 Opc == TargetOpcode::G_FMUL || Opc == TargetOpcode::G_FDIV ||
6385 Opc == TargetOpcode::G_FMAD || Opc == TargetOpcode::G_FMA);
6386
6387 Register Dst = MI.getOperand(i: 0).getReg();
6388 Register X = MI.getOperand(i: 1).getReg();
6389 Register Y = MI.getOperand(i: 2).getReg();
6390 LLT Type = MRI.getType(Reg: Dst);
6391
6392 // fold (fadd x, fneg(y)) -> (fsub x, y)
6393 // fold (fadd fneg(y), x) -> (fsub x, y)
6394 // G_ADD is commutative so both cases are checked by m_GFAdd
6395 if (mi_match(R: Dst, MRI, P: m_GFAdd(L: m_Reg(R&: X), R: m_GFNeg(Src: m_Reg(R&: Y)))) &&
6396 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FSUB, {Type}})) {
6397 Opc = TargetOpcode::G_FSUB;
6398 }
6399 /// fold (fsub x, fneg(y)) -> (fadd x, y)
6400 else if (mi_match(R: Dst, MRI, P: m_GFSub(L: m_Reg(R&: X), R: m_GFNeg(Src: m_Reg(R&: Y)))) &&
6401 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FADD, {Type}})) {
6402 Opc = TargetOpcode::G_FADD;
6403 }
6404 // fold (fmul fneg(x), fneg(y)) -> (fmul x, y)
6405 // fold (fdiv fneg(x), fneg(y)) -> (fdiv x, y)
6406 // fold (fmad fneg(x), fneg(y), z) -> (fmad x, y, z)
6407 // fold (fma fneg(x), fneg(y), z) -> (fma x, y, z)
6408 else if ((Opc == TargetOpcode::G_FMUL || Opc == TargetOpcode::G_FDIV ||
6409 Opc == TargetOpcode::G_FMAD || Opc == TargetOpcode::G_FMA) &&
6410 mi_match(R: X, MRI, P: m_GFNeg(Src: m_Reg(R&: X))) &&
6411 mi_match(R: Y, MRI, P: m_GFNeg(Src: m_Reg(R&: Y)))) {
6412 // no opcode change
6413 } else
6414 return false;
6415
6416 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6417 Observer.changingInstr(MI);
6418 MI.setDesc(B.getTII().get(Opcode: Opc));
6419 MI.getOperand(i: 1).setReg(X);
6420 MI.getOperand(i: 2).setReg(Y);
6421 Observer.changedInstr(MI);
6422 };
6423 return true;
6424}
6425
6426bool CombinerHelper::matchFsubToFneg(MachineInstr &MI,
6427 Register &MatchInfo) const {
6428 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6429
6430 Register LHS = MI.getOperand(i: 1).getReg();
6431 MatchInfo = MI.getOperand(i: 2).getReg();
6432 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6433
6434 const auto LHSCst = Ty.isVector()
6435 ? getFConstantSplat(VReg: LHS, MRI, /* allowUndef */ AllowUndef: true)
6436 : getFConstantVRegValWithLookThrough(VReg: LHS, MRI);
6437 if (!LHSCst)
6438 return false;
6439
6440 // -0.0 is always allowed
6441 if (LHSCst->Value.isNegZero())
6442 return true;
6443
6444 // +0.0 is only allowed if nsz is set.
6445 if (LHSCst->Value.isPosZero())
6446 return MI.getFlag(Flag: MachineInstr::FmNsz);
6447
6448 return false;
6449}
6450
6451void CombinerHelper::applyFsubToFneg(MachineInstr &MI,
6452 Register &MatchInfo) const {
6453 Register Dst = MI.getOperand(i: 0).getReg();
6454 Builder.buildFNeg(
6455 Dst, Src0: Builder.buildFCanonicalize(Dst: MRI.getType(Reg: Dst), Src0: MatchInfo).getReg(Idx: 0));
6456 eraseInst(MI);
6457}
6458
6459/// Checks if \p MI is TargetOpcode::G_FMUL and contractable either
6460/// due to global flags or MachineInstr flags.
6461static bool isContractableFMul(MachineInstr &MI, bool AllowFusionGlobally) {
6462 if (MI.getOpcode() != TargetOpcode::G_FMUL)
6463 return false;
6464 return AllowFusionGlobally || MI.getFlag(Flag: MachineInstr::MIFlag::FmContract);
6465}
6466
6467static bool hasMoreUses(const MachineInstr &MI0, const MachineInstr &MI1,
6468 const MachineRegisterInfo &MRI) {
6469 return std::distance(first: MRI.use_instr_nodbg_begin(RegNo: MI0.getOperand(i: 0).getReg()),
6470 last: MRI.use_instr_nodbg_end()) >
6471 std::distance(first: MRI.use_instr_nodbg_begin(RegNo: MI1.getOperand(i: 0).getReg()),
6472 last: MRI.use_instr_nodbg_end());
6473}
6474
6475bool CombinerHelper::canCombineFMadOrFMA(MachineInstr &MI,
6476 bool &AllowFusionGlobally,
6477 bool &HasFMAD, bool &Aggressive,
6478 bool CanReassociate) const {
6479
6480 auto *MF = MI.getMF();
6481 const auto &TLI = *MF->getSubtarget().getTargetLowering();
6482 const TargetOptions &Options = MF->getTarget().Options;
6483 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6484
6485 if (CanReassociate && !MI.getFlag(Flag: MachineInstr::MIFlag::FmReassoc))
6486 return false;
6487
6488 // Floating-point multiply-add with intermediate rounding.
6489 HasFMAD = (!isPreLegalize() && TLI.isFMADLegal(MI, Ty: DstType));
6490 // Floating-point multiply-add without intermediate rounding.
6491 bool HasFMA = TLI.isFMAFasterThanFMulAndFAdd(MF: *MF, DstType) &&
6492 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FMA, {DstType}});
6493 // No valid opcode, do not combine.
6494 if (!HasFMAD && !HasFMA)
6495 return false;
6496
6497 AllowFusionGlobally = Options.AllowFPOpFusion == FPOpFusion::Fast || HasFMAD;
6498 // If the addition is not contractable, do not combine.
6499 if (!AllowFusionGlobally && !MI.getFlag(Flag: MachineInstr::MIFlag::FmContract))
6500 return false;
6501
6502 Aggressive = TLI.enableAggressiveFMAFusion(Ty: DstType);
6503 return true;
6504}
6505
6506bool CombinerHelper::matchCombineFAddFMulToFMadOrFMA(
6507 MachineInstr &MI,
6508 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6509 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6510
6511 bool AllowFusionGlobally, HasFMAD, Aggressive;
6512 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6513 return false;
6514
6515 Register Op1 = MI.getOperand(i: 1).getReg();
6516 Register Op2 = MI.getOperand(i: 2).getReg();
6517 MachineInstr *Op1Def, *Op2Def;
6518 if (!mi_match(R: Op1, MRI, P: m_MInstr(MI&: Op1Def)) ||
6519 !mi_match(R: Op2, MRI, P: m_MInstr(MI&: Op2Def)))
6520 return false;
6521 DefinitionAndSourceRegister LHS = {.MI: Op1Def, .Reg: Op1};
6522 DefinitionAndSourceRegister RHS = {.MI: Op2Def, .Reg: Op2};
6523 unsigned PreferredFusedOpcode =
6524 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6525
6526 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
6527 // prefer to fold the multiply with fewer uses.
6528 if (Aggressive && isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6529 isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally)) {
6530 if (hasMoreUses(MI0: *LHS.MI, MI1: *RHS.MI, MRI))
6531 std::swap(a&: LHS, b&: RHS);
6532 }
6533
6534 // fold (fadd (fmul x, y), z) -> (fma x, y, z)
6535 if (isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6536 (Aggressive || MRI.hasOneNonDBGUse(RegNo: LHS.Reg))) {
6537 unsigned Flags = MI.getFlags() & LHS.MI->getFlags();
6538 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6539 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6540 SrcOps: {LHS.MI->getOperand(i: 1).getReg(),
6541 LHS.MI->getOperand(i: 2).getReg(), RHS.Reg},
6542 Flags);
6543 };
6544 return true;
6545 }
6546
6547 // fold (fadd x, (fmul y, z)) -> (fma y, z, x)
6548 if (isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally) &&
6549 (Aggressive || MRI.hasOneNonDBGUse(RegNo: RHS.Reg))) {
6550 unsigned Flags = MI.getFlags() & RHS.MI->getFlags();
6551 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6552 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6553 SrcOps: {RHS.MI->getOperand(i: 1).getReg(),
6554 RHS.MI->getOperand(i: 2).getReg(), LHS.Reg},
6555 Flags);
6556 };
6557 return true;
6558 }
6559
6560 return false;
6561}
6562
6563bool CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMA(
6564 MachineInstr &MI,
6565 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6566 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6567
6568 bool AllowFusionGlobally, HasFMAD, Aggressive;
6569 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6570 return false;
6571
6572 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
6573 Register Op1 = MI.getOperand(i: 1).getReg();
6574 Register Op2 = MI.getOperand(i: 2).getReg();
6575 MachineInstr *Op1Def, *Op2Def;
6576 if (!mi_match(R: Op1, MRI, P: m_MInstr(MI&: Op1Def)) ||
6577 !mi_match(R: Op2, MRI, P: m_MInstr(MI&: Op2Def)))
6578 return false;
6579 DefinitionAndSourceRegister LHS = {.MI: Op1Def, .Reg: Op1};
6580 DefinitionAndSourceRegister RHS = {.MI: Op2Def, .Reg: Op2};
6581 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6582
6583 unsigned PreferredFusedOpcode =
6584 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6585
6586 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
6587 // prefer to fold the multiply with fewer uses.
6588 if (Aggressive && isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6589 isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally)) {
6590 if (hasMoreUses(MI0: *LHS.MI, MI1: *RHS.MI, MRI))
6591 std::swap(a&: LHS, b&: RHS);
6592 }
6593
6594 // fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
6595 MachineInstr *FpExtSrc;
6596 if (mi_match(R: LHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FpExtSrc))) &&
6597 isContractableFMul(MI&: *FpExtSrc, AllowFusionGlobally) &&
6598 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6599 SrcTy: MRI.getType(Reg: FpExtSrc->getOperand(i: 1).getReg()))) {
6600 unsigned Flags = MI.getFlags() & FpExtSrc->getFlags();
6601 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6602 auto FpExtX = B.buildFPExt(Res: DstType, Op: FpExtSrc->getOperand(i: 1).getReg());
6603 auto FpExtY = B.buildFPExt(Res: DstType, Op: FpExtSrc->getOperand(i: 2).getReg());
6604 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6605 SrcOps: {FpExtX.getReg(Idx: 0), FpExtY.getReg(Idx: 0), RHS.Reg}, Flags);
6606 };
6607 return true;
6608 }
6609
6610 // fold (fadd z, (fpext (fmul x, y))) -> (fma (fpext x), (fpext y), z)
6611 // Note: Commutes FADD operands.
6612 if (mi_match(R: RHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FpExtSrc))) &&
6613 isContractableFMul(MI&: *FpExtSrc, AllowFusionGlobally) &&
6614 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6615 SrcTy: MRI.getType(Reg: FpExtSrc->getOperand(i: 1).getReg()))) {
6616 unsigned Flags = MI.getFlags() & FpExtSrc->getFlags();
6617 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6618 auto FpExtX = B.buildFPExt(Res: DstType, Op: FpExtSrc->getOperand(i: 1).getReg());
6619 auto FpExtY = B.buildFPExt(Res: DstType, Op: FpExtSrc->getOperand(i: 2).getReg());
6620 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6621 SrcOps: {FpExtX.getReg(Idx: 0), FpExtY.getReg(Idx: 0), LHS.Reg}, Flags);
6622 };
6623 return true;
6624 }
6625
6626 return false;
6627}
6628
6629bool CombinerHelper::matchCombineFAddFMAFMulToFMadOrFMA(
6630 MachineInstr &MI,
6631 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6632 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6633
6634 bool AllowFusionGlobally, HasFMAD, Aggressive;
6635 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive, CanReassociate: true))
6636 return false;
6637
6638 Register Op1 = MI.getOperand(i: 1).getReg();
6639 Register Op2 = MI.getOperand(i: 2).getReg();
6640 MachineInstr *Op1Def, *Op2Def;
6641 if (!mi_match(R: Op1, MRI, P: m_MInstr(MI&: Op1Def)) ||
6642 !mi_match(R: Op2, MRI, P: m_MInstr(MI&: Op2Def)))
6643 return false;
6644 DefinitionAndSourceRegister LHS = {.MI: Op1Def, .Reg: Op1};
6645 DefinitionAndSourceRegister RHS = {.MI: Op2Def, .Reg: Op2};
6646 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6647
6648 unsigned PreferredFusedOpcode =
6649 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6650
6651 MachineInstr *FMA = nullptr;
6652 Register Z;
6653 // fold (fadd (fma x, y, (fmul u, v)), z) -> (fma x, y, (fma u, v, z))
6654 if (LHS.MI->getOpcode() == PreferredFusedOpcode &&
6655 mi_match(R: LHS.MI->getOperand(i: 3).getReg(), MRI,
6656 P: m_GFMul(L: m_Reg(), R: m_Reg())) &&
6657 MRI.hasOneNonDBGUse(RegNo: LHS.MI->getOperand(i: 0).getReg()) &&
6658 MRI.hasOneNonDBGUse(RegNo: LHS.MI->getOperand(i: 3).getReg())) {
6659 FMA = LHS.MI;
6660 Z = RHS.Reg;
6661 }
6662 // fold (fadd z, (fma x, y, (fmul u, v))) -> (fma x, y, (fma u, v, z))
6663 else if (RHS.MI->getOpcode() == PreferredFusedOpcode &&
6664 mi_match(R: RHS.MI->getOperand(i: 3).getReg(), MRI,
6665 P: m_GFMul(L: m_Reg(), R: m_Reg())) &&
6666 MRI.hasOneNonDBGUse(RegNo: RHS.MI->getOperand(i: 0).getReg()) &&
6667 MRI.hasOneNonDBGUse(RegNo: RHS.MI->getOperand(i: 3).getReg())) {
6668 Z = LHS.Reg;
6669 FMA = RHS.MI;
6670 }
6671
6672 if (FMA) {
6673 MachineInstr *FMulMI;
6674 if (!mi_match(R: FMA->getOperand(i: 3).getReg(), MRI, P: m_MInstr(MI&: FMulMI)))
6675 return false;
6676 Register X = FMA->getOperand(i: 1).getReg();
6677 Register Y = FMA->getOperand(i: 2).getReg();
6678 Register U = FMulMI->getOperand(i: 1).getReg();
6679 Register V = FMulMI->getOperand(i: 2).getReg();
6680 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6681 unsigned OuterFlags = MI.getFlags() & FMA->getFlags();
6682
6683 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6684 Register InnerFMA = MRI.createGenericVirtualRegister(Ty: DstTy);
6685 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {InnerFMA}, SrcOps: {U, V, Z}, Flags: InnerFlags);
6686 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6687 SrcOps: {X, Y, InnerFMA}, Flags: OuterFlags);
6688 };
6689 return true;
6690 }
6691
6692 return false;
6693}
6694
6695bool CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMAAggressive(
6696 MachineInstr &MI,
6697 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6698 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6699
6700 bool AllowFusionGlobally, HasFMAD, Aggressive;
6701 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6702 return false;
6703
6704 if (!Aggressive)
6705 return false;
6706
6707 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
6708 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6709 Register Op1 = MI.getOperand(i: 1).getReg();
6710 Register Op2 = MI.getOperand(i: 2).getReg();
6711 MachineInstr *Op1Def, *Op2Def;
6712 if (!mi_match(R: Op1, MRI, P: m_MInstr(MI&: Op1Def)) ||
6713 !mi_match(R: Op2, MRI, P: m_MInstr(MI&: Op2Def)))
6714 return false;
6715 DefinitionAndSourceRegister LHS = {.MI: Op1Def, .Reg: Op1};
6716 DefinitionAndSourceRegister RHS = {.MI: Op2Def, .Reg: Op2};
6717
6718 unsigned PreferredFusedOpcode =
6719 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6720
6721 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
6722 // prefer to fold the multiply with fewer uses.
6723 if (Aggressive && isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6724 isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally)) {
6725 if (hasMoreUses(MI0: *LHS.MI, MI1: *RHS.MI, MRI))
6726 std::swap(a&: LHS, b&: RHS);
6727 }
6728
6729 // Builds: (fma x, y, (fma (fpext u), (fpext v), z))
6730 auto buildMatchInfo = [=, &MI](Register U, Register V, Register Z, Register X,
6731 Register Y, unsigned InnerFlags,
6732 unsigned OuterFlags, MachineIRBuilder &B) {
6733 Register FpExtU = B.buildFPExt(Res: DstType, Op: U).getReg(Idx: 0);
6734 Register FpExtV = B.buildFPExt(Res: DstType, Op: V).getReg(Idx: 0);
6735 Register InnerFMA = B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {DstType},
6736 SrcOps: {FpExtU, FpExtV, Z}, Flags: InnerFlags)
6737 .getReg(Idx: 0);
6738 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6739 SrcOps: {X, Y, InnerFMA}, Flags: OuterFlags);
6740 };
6741
6742 MachineInstr *FMulMI, *FMAMI;
6743 // fold (fadd (fma x, y, (fpext (fmul u, v))), z)
6744 // -> (fma x, y, (fma (fpext u), (fpext v), z))
6745 if (LHS.MI->getOpcode() == PreferredFusedOpcode &&
6746 mi_match(R: LHS.MI->getOperand(i: 3).getReg(), MRI,
6747 P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6748 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6749 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6750 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
6751 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6752 unsigned OuterFlags = MI.getFlags() & LHS.MI->getFlags();
6753 MatchInfo = [=](MachineIRBuilder &B) {
6754 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6755 FMulMI->getOperand(i: 2).getReg(), RHS.Reg,
6756 LHS.MI->getOperand(i: 1).getReg(),
6757 LHS.MI->getOperand(i: 2).getReg(), InnerFlags, OuterFlags, B);
6758 };
6759 return true;
6760 }
6761
6762 // fold (fadd (fpext (fma x, y, (fmul u, v))), z)
6763 // -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z))
6764 // FIXME: This turns two single-precision and one double-precision
6765 // operation into two double-precision operations, which might not be
6766 // interesting for all targets, especially GPUs.
6767 if (mi_match(R: LHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMAMI))) &&
6768 FMAMI->getOpcode() == PreferredFusedOpcode) {
6769 MachineInstr *FMulMI;
6770 if (!mi_match(R: FMAMI->getOperand(i: 3).getReg(), MRI, P: m_MInstr(MI&: FMulMI)))
6771 return false;
6772 if (isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6773 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6774 SrcTy: MRI.getType(Reg: FMAMI->getOperand(i: 0).getReg()))) {
6775 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6776 unsigned OuterFlags = MI.getFlags() & FMAMI->getFlags();
6777 MatchInfo = [=](MachineIRBuilder &B) {
6778 Register X = FMAMI->getOperand(i: 1).getReg();
6779 Register Y = FMAMI->getOperand(i: 2).getReg();
6780 X = B.buildFPExt(Res: DstType, Op: X).getReg(Idx: 0);
6781 Y = B.buildFPExt(Res: DstType, Op: Y).getReg(Idx: 0);
6782 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6783 FMulMI->getOperand(i: 2).getReg(), RHS.Reg, X, Y,
6784 InnerFlags, OuterFlags, B);
6785 };
6786
6787 return true;
6788 }
6789 }
6790
6791 // fold (fadd z, (fma x, y, (fpext (fmul u, v)))
6792 // -> (fma x, y, (fma (fpext u), (fpext v), z))
6793 if (RHS.MI->getOpcode() == PreferredFusedOpcode &&
6794 mi_match(R: RHS.MI->getOperand(i: 3).getReg(), MRI,
6795 P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6796 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6797 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6798 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
6799 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6800 unsigned OuterFlags = MI.getFlags() & RHS.MI->getFlags();
6801 MatchInfo = [=](MachineIRBuilder &B) {
6802 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6803 FMulMI->getOperand(i: 2).getReg(), LHS.Reg,
6804 RHS.MI->getOperand(i: 1).getReg(),
6805 RHS.MI->getOperand(i: 2).getReg(), InnerFlags, OuterFlags, B);
6806 };
6807 return true;
6808 }
6809
6810 // fold (fadd z, (fpext (fma x, y, (fmul u, v)))
6811 // -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z))
6812 // FIXME: This turns two single-precision and one double-precision
6813 // operation into two double-precision operations, which might not be
6814 // interesting for all targets, especially GPUs.
6815 if (mi_match(R: RHS.Reg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMAMI))) &&
6816 FMAMI->getOpcode() == PreferredFusedOpcode) {
6817 MachineInstr *FMulMI;
6818 if (!mi_match(R: FMAMI->getOperand(i: 3).getReg(), MRI, P: m_MInstr(MI&: FMulMI)))
6819 return false;
6820 if (isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6821 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstType,
6822 SrcTy: MRI.getType(Reg: FMAMI->getOperand(i: 0).getReg()))) {
6823 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6824 unsigned OuterFlags = MI.getFlags() & FMAMI->getFlags();
6825 MatchInfo = [=](MachineIRBuilder &B) {
6826 Register X = FMAMI->getOperand(i: 1).getReg();
6827 Register Y = FMAMI->getOperand(i: 2).getReg();
6828 X = B.buildFPExt(Res: DstType, Op: X).getReg(Idx: 0);
6829 Y = B.buildFPExt(Res: DstType, Op: Y).getReg(Idx: 0);
6830 buildMatchInfo(FMulMI->getOperand(i: 1).getReg(),
6831 FMulMI->getOperand(i: 2).getReg(), LHS.Reg, X, Y,
6832 InnerFlags, OuterFlags, B);
6833 };
6834 return true;
6835 }
6836 }
6837
6838 return false;
6839}
6840
6841bool CombinerHelper::matchCombineFSubFMulToFMadOrFMA(
6842 MachineInstr &MI,
6843 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6844 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6845
6846 bool AllowFusionGlobally, HasFMAD, Aggressive;
6847 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6848 return false;
6849
6850 Register Op1 = MI.getOperand(i: 1).getReg();
6851 Register Op2 = MI.getOperand(i: 2).getReg();
6852 MachineInstr *Op1Def, *Op2Def;
6853 if (!mi_match(R: Op1, MRI, P: m_MInstr(MI&: Op1Def)) ||
6854 !mi_match(R: Op2, MRI, P: m_MInstr(MI&: Op2Def)))
6855 return false;
6856 DefinitionAndSourceRegister LHS = {.MI: Op1Def, .Reg: Op1};
6857 DefinitionAndSourceRegister RHS = {.MI: Op2Def, .Reg: Op2};
6858 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6859
6860 // If we have two choices trying to fold (fsub (fmul u, v), (fmul x, y)),
6861 // prefer to fold the multiply with fewer uses.
6862 int FirstMulHasFewerUses = true;
6863 if (isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6864 isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally) &&
6865 hasMoreUses(MI0: *LHS.MI, MI1: *RHS.MI, MRI))
6866 FirstMulHasFewerUses = false;
6867
6868 unsigned PreferredFusedOpcode =
6869 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6870
6871 // fold (fsub (fmul x, y), z) -> (fma x, y, -z)
6872 if (FirstMulHasFewerUses &&
6873 (isContractableFMul(MI&: *LHS.MI, AllowFusionGlobally) &&
6874 (Aggressive || MRI.hasOneNonDBGUse(RegNo: LHS.Reg)))) {
6875 unsigned Flags = MI.getFlags() & LHS.MI->getFlags();
6876 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6877 Register NegZ = B.buildFNeg(Dst: DstTy, Src0: RHS.Reg).getReg(Idx: 0);
6878 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6879 SrcOps: {LHS.MI->getOperand(i: 1).getReg(),
6880 LHS.MI->getOperand(i: 2).getReg(), NegZ},
6881 Flags);
6882 };
6883 return true;
6884 }
6885 // fold (fsub x, (fmul y, z)) -> (fma -y, z, x)
6886 else if ((isContractableFMul(MI&: *RHS.MI, AllowFusionGlobally) &&
6887 (Aggressive || MRI.hasOneNonDBGUse(RegNo: RHS.Reg)))) {
6888 unsigned Flags = MI.getFlags() & RHS.MI->getFlags();
6889 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6890 Register NegY =
6891 B.buildFNeg(Dst: DstTy, Src0: RHS.MI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6892 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6893 SrcOps: {NegY, RHS.MI->getOperand(i: 2).getReg(), LHS.Reg}, Flags);
6894 };
6895 return true;
6896 }
6897
6898 return false;
6899}
6900
6901bool CombinerHelper::matchCombineFSubFNegFMulToFMadOrFMA(
6902 MachineInstr &MI,
6903 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6904 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6905
6906 bool AllowFusionGlobally, HasFMAD, Aggressive;
6907 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6908 return false;
6909
6910 Register LHSReg = MI.getOperand(i: 1).getReg();
6911 Register RHSReg = MI.getOperand(i: 2).getReg();
6912 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6913
6914 unsigned PreferredFusedOpcode =
6915 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6916
6917 MachineInstr *FMulMI;
6918 // fold (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
6919 if (mi_match(R: LHSReg, MRI, P: m_GFNeg(Src: m_MInstr(MI&: FMulMI))) &&
6920 (Aggressive || (MRI.hasOneNonDBGUse(RegNo: LHSReg) &&
6921 MRI.hasOneNonDBGUse(RegNo: FMulMI->getOperand(i: 0).getReg()))) &&
6922 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally)) {
6923 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6924 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6925 Register NegX =
6926 B.buildFNeg(Dst: DstTy, Src0: FMulMI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6927 Register NegZ = B.buildFNeg(Dst: DstTy, Src0: RHSReg).getReg(Idx: 0);
6928 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6929 SrcOps: {NegX, FMulMI->getOperand(i: 2).getReg(), NegZ}, Flags);
6930 };
6931 return true;
6932 }
6933
6934 // fold (fsub x, (fneg (fmul, y, z))) -> (fma y, z, x)
6935 if (mi_match(R: RHSReg, MRI, P: m_GFNeg(Src: m_MInstr(MI&: FMulMI))) &&
6936 (Aggressive || (MRI.hasOneNonDBGUse(RegNo: RHSReg) &&
6937 MRI.hasOneNonDBGUse(RegNo: FMulMI->getOperand(i: 0).getReg()))) &&
6938 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally)) {
6939 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6940 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6941 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6942 SrcOps: {FMulMI->getOperand(i: 1).getReg(),
6943 FMulMI->getOperand(i: 2).getReg(), LHSReg},
6944 Flags);
6945 };
6946 return true;
6947 }
6948
6949 return false;
6950}
6951
6952bool CombinerHelper::matchCombineFSubFpExtFMulToFMadOrFMA(
6953 MachineInstr &MI,
6954 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6955 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6956
6957 bool AllowFusionGlobally, HasFMAD, Aggressive;
6958 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6959 return false;
6960
6961 Register LHSReg = MI.getOperand(i: 1).getReg();
6962 Register RHSReg = MI.getOperand(i: 2).getReg();
6963 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6964
6965 unsigned PreferredFusedOpcode =
6966 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6967
6968 MachineInstr *FMulMI;
6969 // fold (fsub (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), (fneg z))
6970 if (mi_match(R: LHSReg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6971 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6972 (Aggressive || MRI.hasOneNonDBGUse(RegNo: LHSReg))) {
6973 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6974 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6975 Register FpExtX =
6976 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6977 Register FpExtY =
6978 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 2).getReg()).getReg(Idx: 0);
6979 Register NegZ = B.buildFNeg(Dst: DstTy, Src0: RHSReg).getReg(Idx: 0);
6980 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6981 SrcOps: {FpExtX, FpExtY, NegZ}, Flags);
6982 };
6983 return true;
6984 }
6985
6986 // fold (fsub x, (fpext (fmul y, z))) -> (fma (fneg (fpext y)), (fpext z), x)
6987 if (mi_match(R: RHSReg, MRI, P: m_GFPExt(Src: m_MInstr(MI&: FMulMI))) &&
6988 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
6989 (Aggressive || MRI.hasOneNonDBGUse(RegNo: RHSReg))) {
6990 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6991 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6992 Register FpExtY =
6993 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 1).getReg()).getReg(Idx: 0);
6994 Register NegY = B.buildFNeg(Dst: DstTy, Src0: FpExtY).getReg(Idx: 0);
6995 Register FpExtZ =
6996 B.buildFPExt(Res: DstTy, Op: FMulMI->getOperand(i: 2).getReg()).getReg(Idx: 0);
6997 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {MI.getOperand(i: 0).getReg()},
6998 SrcOps: {NegY, FpExtZ, LHSReg}, Flags);
6999 };
7000 return true;
7001 }
7002
7003 return false;
7004}
7005
7006bool CombinerHelper::matchCombineFSubFpExtFNegFMulToFMadOrFMA(
7007 MachineInstr &MI,
7008 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
7009 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
7010
7011 bool AllowFusionGlobally, HasFMAD, Aggressive;
7012 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
7013 return false;
7014
7015 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
7016 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7017 Register LHSReg = MI.getOperand(i: 1).getReg();
7018 Register RHSReg = MI.getOperand(i: 2).getReg();
7019
7020 unsigned PreferredFusedOpcode =
7021 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
7022
7023 auto buildMatchInfo = [=](Register Dst, Register X, Register Y, Register Z,
7024 unsigned Flags, MachineIRBuilder &B) {
7025 Register FpExtX = B.buildFPExt(Res: DstTy, Op: X).getReg(Idx: 0);
7026 Register FpExtY = B.buildFPExt(Res: DstTy, Op: Y).getReg(Idx: 0);
7027 B.buildInstr(Opc: PreferredFusedOpcode, DstOps: {Dst}, SrcOps: {FpExtX, FpExtY, Z}, Flags);
7028 };
7029
7030 MachineInstr *FMulMI;
7031 // fold (fsub (fpext (fneg (fmul x, y))), z) ->
7032 // (fneg (fma (fpext x), (fpext y), z))
7033 // fold (fsub (fneg (fpext (fmul x, y))), z) ->
7034 // (fneg (fma (fpext x), (fpext y), z))
7035 if ((mi_match(R: LHSReg, MRI, P: m_GFPExt(Src: m_GFNeg(Src: m_MInstr(MI&: FMulMI)))) ||
7036 mi_match(R: LHSReg, MRI, P: m_GFNeg(Src: m_GFPExt(Src: m_MInstr(MI&: FMulMI))))) &&
7037 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
7038 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstTy,
7039 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
7040 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
7041 MatchInfo = [=, &MI](MachineIRBuilder &B) {
7042 Register FMAReg = MRI.createGenericVirtualRegister(Ty: DstTy);
7043 buildMatchInfo(FMAReg, FMulMI->getOperand(i: 1).getReg(),
7044 FMulMI->getOperand(i: 2).getReg(), RHSReg, Flags, B);
7045 B.buildFNeg(Dst: MI.getOperand(i: 0).getReg(), Src0: FMAReg);
7046 };
7047 return true;
7048 }
7049
7050 // fold (fsub x, (fpext (fneg (fmul y, z)))) -> (fma (fpext y), (fpext z), x)
7051 // fold (fsub x, (fneg (fpext (fmul y, z)))) -> (fma (fpext y), (fpext z), x)
7052 if ((mi_match(R: RHSReg, MRI, P: m_GFPExt(Src: m_GFNeg(Src: m_MInstr(MI&: FMulMI)))) ||
7053 mi_match(R: RHSReg, MRI, P: m_GFNeg(Src: m_GFPExt(Src: m_MInstr(MI&: FMulMI))))) &&
7054 isContractableFMul(MI&: *FMulMI, AllowFusionGlobally) &&
7055 TLI.isFPExtFoldable(MI, Opcode: PreferredFusedOpcode, DestTy: DstTy,
7056 SrcTy: MRI.getType(Reg: FMulMI->getOperand(i: 0).getReg()))) {
7057 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
7058 MatchInfo = [=, &MI](MachineIRBuilder &B) {
7059 buildMatchInfo(MI.getOperand(i: 0).getReg(), FMulMI->getOperand(i: 1).getReg(),
7060 FMulMI->getOperand(i: 2).getReg(), LHSReg, Flags, B);
7061 };
7062 return true;
7063 }
7064
7065 return false;
7066}
7067
7068bool CombinerHelper::matchCombineFMinMaxNaN(MachineInstr &MI,
7069 unsigned &IdxToPropagate) const {
7070 bool PropagateNaN;
7071 switch (MI.getOpcode()) {
7072 default:
7073 return false;
7074 case TargetOpcode::G_FMINNUM:
7075 case TargetOpcode::G_FMAXNUM:
7076 PropagateNaN = false;
7077 break;
7078 case TargetOpcode::G_FMINIMUM:
7079 case TargetOpcode::G_FMAXIMUM:
7080 PropagateNaN = true;
7081 break;
7082 }
7083
7084 auto MatchNaN = [&](unsigned Idx) {
7085 Register MaybeNaNReg = MI.getOperand(i: Idx).getReg();
7086 const ConstantFP *MaybeCst = getConstantFPVRegVal(VReg: MaybeNaNReg, MRI);
7087 if (!MaybeCst || !MaybeCst->getValueAPF().isNaN())
7088 return false;
7089 IdxToPropagate = PropagateNaN ? Idx : (Idx == 1 ? 2 : 1);
7090 return true;
7091 };
7092
7093 return MatchNaN(1) || MatchNaN(2);
7094}
7095
7096// Combine multiple FDIVs with the same divisor into multiple FMULs by the
7097// reciprocal.
7098// E.g., (a / Y; b / Y;) -> (recip = 1.0 / Y; a * recip; b * recip)
7099bool CombinerHelper::matchRepeatedFPDivisor(
7100 MachineInstr &MI, SmallVector<MachineInstr *> &MatchInfo) const {
7101 assert(MI.getOpcode() == TargetOpcode::G_FDIV);
7102
7103 Register X = MI.getOperand(i: 1).getReg();
7104 Register Y = MI.getOperand(i: 2).getReg();
7105
7106 if (!MI.getFlag(Flag: MachineInstr::MIFlag::FmArcp))
7107 return false;
7108
7109 auto IsOne = [this](Register X) {
7110 auto N0CFP = isConstantOrConstantSplatVectorFP(Def: X, MRI);
7111 return N0CFP && (N0CFP->isOne() || N0CFP->isMinusOne());
7112 };
7113
7114 // Skip if current node is a reciprocal/fneg-reciprocal.
7115 if (IsOne(X))
7116 return false;
7117
7118 // Exit early if the target does not want this transform or if there can't
7119 // possibly be enough uses of the divisor to make the transform worthwhile.
7120 unsigned MinUses = getTargetLowering().combineRepeatedFPDivisors();
7121 if (!MinUses)
7122 return false;
7123
7124 // Find all FDIV users of the same divisor. For the moment we limit all
7125 // instructions to a single BB and use the first Instr in MatchInfo as the
7126 // dominating position.
7127 MatchInfo.push_back(Elt: &MI);
7128 for (auto &U : MRI.use_nodbg_instructions(Reg: Y)) {
7129 if (&U == &MI || U.getParent() != MI.getParent())
7130 continue;
7131 if (U.getOpcode() == TargetOpcode::G_FDIV &&
7132 U.getOperand(i: 2).getReg() == Y && U.getOperand(i: 1).getReg() != Y &&
7133 !IsOne(U.getOperand(i: 1).getReg())) {
7134 // This division is eligible for optimization only if global unsafe math
7135 // is enabled or if this division allows reciprocal formation.
7136 if (U.getFlag(Flag: MachineInstr::MIFlag::FmArcp)) {
7137 MatchInfo.push_back(Elt: &U);
7138 if (dominates(DefMI: U, UseMI: *MatchInfo[0]))
7139 std::swap(a&: MatchInfo[0], b&: MatchInfo.back());
7140 }
7141 }
7142 }
7143
7144 // Now that we have the actual number of divisor uses, make sure it meets
7145 // the minimum threshold specified by the target.
7146 return MatchInfo.size() >= MinUses;
7147}
7148
7149void CombinerHelper::applyRepeatedFPDivisor(
7150 SmallVector<MachineInstr *> &MatchInfo) const {
7151 // Generate the new div at the position of the first instruction, that we have
7152 // ensured will dominate all other instructions.
7153 Builder.setInsertPt(MBB&: *MatchInfo[0]->getParent(), II: MatchInfo[0]);
7154 LLT Ty = MRI.getType(Reg: MatchInfo[0]->getOperand(i: 0).getReg());
7155 auto Div = Builder.buildFDiv(Dst: Ty, Src0: Builder.buildFConstant(Res: Ty, Val: 1.0),
7156 Src1: MatchInfo[0]->getOperand(i: 2).getReg(),
7157 Flags: MatchInfo[0]->getFlags());
7158
7159 // Replace all found div's with fmul instructions.
7160 for (MachineInstr *MI : MatchInfo) {
7161 Builder.setInsertPt(MBB&: *MI->getParent(), II: MI);
7162 Builder.buildFMul(Dst: MI->getOperand(i: 0).getReg(), Src0: MI->getOperand(i: 1).getReg(),
7163 Src1: Div->getOperand(i: 0).getReg(), Flags: MI->getFlags());
7164 MI->eraseFromParent();
7165 }
7166}
7167
7168bool CombinerHelper::matchAddSubSameReg(MachineInstr &MI, Register &Src) const {
7169 assert(MI.getOpcode() == TargetOpcode::G_ADD && "Expected a G_ADD");
7170 Register LHS = MI.getOperand(i: 1).getReg();
7171 Register RHS = MI.getOperand(i: 2).getReg();
7172
7173 // Helper lambda to check for opportunities for
7174 // A + (B - A) -> B
7175 // (B - A) + A -> B
7176 auto CheckFold = [&](Register MaybeSub, Register MaybeSameReg) {
7177 Register Reg;
7178 return mi_match(R: MaybeSub, MRI, P: m_GSub(L: m_Reg(R&: Src), R: m_Reg(R&: Reg))) &&
7179 Reg == MaybeSameReg;
7180 };
7181 return CheckFold(LHS, RHS) || CheckFold(RHS, LHS);
7182}
7183
7184bool CombinerHelper::matchBuildVectorIdentityFold(MachineInstr &MI,
7185 Register &MatchInfo) const {
7186 // This combine folds the following patterns:
7187 //
7188 // G_BUILD_VECTOR_TRUNC (G_BITCAST(x), G_LSHR(G_BITCAST(x), k))
7189 // G_BUILD_VECTOR(G_TRUNC(G_BITCAST(x)), G_TRUNC(G_LSHR(G_BITCAST(x), k)))
7190 // into
7191 // x
7192 // if
7193 // k == sizeof(VecEltTy)/2
7194 // type(x) == type(dst)
7195 //
7196 // G_BUILD_VECTOR(G_TRUNC(G_BITCAST(x)), undef)
7197 // into
7198 // x
7199 // if
7200 // type(x) == type(dst)
7201
7202 LLT DstVecTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7203 LLT DstEltTy = DstVecTy.getElementType();
7204
7205 Register Lo, Hi;
7206
7207 if (mi_match(
7208 MI, MRI,
7209 P: m_GBuildVector(L: m_GTrunc(Src: m_GBitcast(Src: m_Reg(R&: Lo))), R: m_GImplicitDef()))) {
7210 MatchInfo = Lo;
7211 return MRI.getType(Reg: MatchInfo) == DstVecTy;
7212 }
7213
7214 std::optional<ValueAndVReg> ShiftAmount;
7215 const auto LoPattern = m_GBitcast(Src: m_Reg(R&: Lo));
7216 const auto HiPattern = m_GLShr(L: m_GBitcast(Src: m_Reg(R&: Hi)), R: m_GCst(ValReg&: ShiftAmount));
7217 if (mi_match(
7218 MI, MRI,
7219 P: m_any_of(preds: m_GBuildVectorTrunc(L: LoPattern, R: HiPattern),
7220 preds: m_GBuildVector(L: m_GTrunc(Src: LoPattern), R: m_GTrunc(Src: HiPattern))))) {
7221 if (Lo == Hi && ShiftAmount->Value == DstEltTy.getSizeInBits()) {
7222 MatchInfo = Lo;
7223 return MRI.getType(Reg: MatchInfo) == DstVecTy;
7224 }
7225 }
7226
7227 return false;
7228}
7229
7230bool CombinerHelper::matchTruncBuildVectorFold(MachineInstr &MI,
7231 Register &MatchInfo) const {
7232 // Replace (G_TRUNC (G_BITCAST (G_BUILD_VECTOR x, y)) with just x
7233 // if type(x) == type(G_TRUNC)
7234 if (!mi_match(R: MI.getOperand(i: 1).getReg(), MRI,
7235 P: m_GBitcast(Src: m_GBuildVector(L: m_Reg(R&: MatchInfo), R: m_Reg()))))
7236 return false;
7237
7238 return MRI.getType(Reg: MatchInfo) == MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7239}
7240
7241bool CombinerHelper::matchTruncLshrBuildVectorFold(MachineInstr &MI,
7242 Register &MatchInfo) const {
7243 // Replace (G_TRUNC (G_LSHR (G_BITCAST (G_BUILD_VECTOR x, y)), K)) with
7244 // y if K == size of vector element type
7245 std::optional<ValueAndVReg> ShiftAmt;
7246 if (!mi_match(R: MI.getOperand(i: 1).getReg(), MRI,
7247 P: m_GLShr(L: m_GBitcast(Src: m_GBuildVector(L: m_Reg(), R: m_Reg(R&: MatchInfo))),
7248 R: m_GCst(ValReg&: ShiftAmt))))
7249 return false;
7250
7251 LLT MatchTy = MRI.getType(Reg: MatchInfo);
7252 return ShiftAmt->Value.getZExtValue() == MatchTy.getSizeInBits() &&
7253 MatchTy == MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7254}
7255
7256unsigned CombinerHelper::getFPMinMaxOpcForSelect(
7257 CmpInst::Predicate Pred, LLT DstTy,
7258 SelectPatternNaNBehaviour VsNaNRetVal) const {
7259 assert(VsNaNRetVal != SelectPatternNaNBehaviour::NOT_APPLICABLE &&
7260 "Expected a NaN behaviour?");
7261 // Choose an opcode based off of legality or the behaviour when one of the
7262 // LHS/RHS may be NaN.
7263 switch (Pred) {
7264 default:
7265 return 0;
7266 case CmpInst::FCMP_UGT:
7267 case CmpInst::FCMP_UGE:
7268 case CmpInst::FCMP_OGT:
7269 case CmpInst::FCMP_OGE:
7270 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
7271 return TargetOpcode::G_FMAXNUM;
7272 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
7273 return TargetOpcode::G_FMAXIMUM;
7274 if (isLegal(Query: {TargetOpcode::G_FMAXNUM, {DstTy}}))
7275 return TargetOpcode::G_FMAXNUM;
7276 if (isLegal(Query: {TargetOpcode::G_FMAXIMUM, {DstTy}}))
7277 return TargetOpcode::G_FMAXIMUM;
7278 return 0;
7279 case CmpInst::FCMP_ULT:
7280 case CmpInst::FCMP_ULE:
7281 case CmpInst::FCMP_OLT:
7282 case CmpInst::FCMP_OLE:
7283 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
7284 return TargetOpcode::G_FMINNUM;
7285 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
7286 return TargetOpcode::G_FMINIMUM;
7287 if (isLegal(Query: {TargetOpcode::G_FMINNUM, {DstTy}}))
7288 return TargetOpcode::G_FMINNUM;
7289 if (!isLegal(Query: {TargetOpcode::G_FMINIMUM, {DstTy}}))
7290 return 0;
7291 return TargetOpcode::G_FMINIMUM;
7292 }
7293}
7294
7295CombinerHelper::SelectPatternNaNBehaviour
7296CombinerHelper::computeRetValAgainstNaN(Register LHS, Register RHS,
7297 bool IsOrderedComparison) const {
7298 bool LHSSafe = VT->isKnownNeverNaN(Val: LHS);
7299 bool RHSSafe = VT->isKnownNeverNaN(Val: RHS);
7300 // Completely unsafe.
7301 if (!LHSSafe && !RHSSafe)
7302 return SelectPatternNaNBehaviour::NOT_APPLICABLE;
7303 if (LHSSafe && RHSSafe)
7304 return SelectPatternNaNBehaviour::RETURNS_ANY;
7305 // An ordered comparison will return false when given a NaN, so it
7306 // returns the RHS.
7307 if (IsOrderedComparison)
7308 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_NAN
7309 : SelectPatternNaNBehaviour::RETURNS_OTHER;
7310 // An unordered comparison will return true when given a NaN, so it
7311 // returns the LHS.
7312 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_OTHER
7313 : SelectPatternNaNBehaviour::RETURNS_NAN;
7314}
7315
7316bool CombinerHelper::matchFPSelectToMinMax(Register Dst, Register Cond,
7317 Register TrueVal, Register FalseVal,
7318 BuildFnTy &MatchInfo) const {
7319 // Match: select (fcmp cond x, y) x, y
7320 // select (fcmp cond x, y) y, x
7321 // And turn it into fminnum/fmaxnum or fmin/fmax based off of the condition.
7322 LLT DstTy = MRI.getType(Reg: Dst);
7323 // Bail out early on pointers, since we'll never want to fold to a min/max.
7324 if (DstTy.isPointer())
7325 return false;
7326 // Match a floating point compare with a less-than/greater-than predicate.
7327 // TODO: Allow multiple users of the compare if they are all selects.
7328 CmpInst::Predicate Pred;
7329 Register CmpLHS, CmpRHS;
7330 if (!mi_match(R: Cond, MRI,
7331 P: m_OneNonDBGUse(
7332 SP: m_GFCmp(P: m_Pred(P&: Pred), L: m_Reg(R&: CmpLHS), R: m_Reg(R&: CmpRHS)))) ||
7333 CmpInst::isEquality(pred: Pred))
7334 return false;
7335 SelectPatternNaNBehaviour ResWithKnownNaNInfo =
7336 computeRetValAgainstNaN(LHS: CmpLHS, RHS: CmpRHS, IsOrderedComparison: CmpInst::isOrdered(predicate: Pred));
7337 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::NOT_APPLICABLE)
7338 return false;
7339 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
7340 std::swap(a&: CmpLHS, b&: CmpRHS);
7341 Pred = CmpInst::getSwappedPredicate(pred: Pred);
7342 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_NAN)
7343 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_OTHER;
7344 else if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_OTHER)
7345 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_NAN;
7346 }
7347 if (TrueVal != CmpLHS || FalseVal != CmpRHS)
7348 return false;
7349 // Decide what type of max/min this should be based off of the predicate.
7350 unsigned Opc = getFPMinMaxOpcForSelect(Pred, DstTy, VsNaNRetVal: ResWithKnownNaNInfo);
7351 if (!Opc || !isLegal(Query: {Opc, {DstTy}}))
7352 return false;
7353 // Comparisons between signed zero and zero may have different results...
7354 // unless we have fmaximum/fminimum. In that case, we know -0 < 0.
7355 if (Opc != TargetOpcode::G_FMAXIMUM && Opc != TargetOpcode::G_FMINIMUM) {
7356 // We don't know if a comparison between two 0s will give us a consistent
7357 // result. Be conservative and only proceed if at least one side is
7358 // non-zero.
7359 auto KnownNonZeroSide = getFConstantVRegValWithLookThrough(VReg: CmpLHS, MRI);
7360 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero()) {
7361 KnownNonZeroSide = getFConstantVRegValWithLookThrough(VReg: CmpRHS, MRI);
7362 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero())
7363 return false;
7364 }
7365 }
7366 MatchInfo = [=](MachineIRBuilder &B) {
7367 B.buildInstr(Opc, DstOps: {Dst}, SrcOps: {CmpLHS, CmpRHS});
7368 };
7369 return true;
7370}
7371
7372bool CombinerHelper::matchSimplifySelectToMinMax(MachineInstr &MI,
7373 BuildFnTy &MatchInfo) const {
7374 // TODO: Handle integer cases.
7375 assert(MI.getOpcode() == TargetOpcode::G_SELECT);
7376 // Condition may be fed by a truncated compare.
7377 Register Cond = MI.getOperand(i: 1).getReg();
7378 Register MaybeTrunc;
7379 if (mi_match(R: Cond, MRI, P: m_OneNonDBGUse(SP: m_GTrunc(Src: m_Reg(R&: MaybeTrunc)))))
7380 Cond = MaybeTrunc;
7381 Register Dst = MI.getOperand(i: 0).getReg();
7382 Register TrueVal = MI.getOperand(i: 2).getReg();
7383 Register FalseVal = MI.getOperand(i: 3).getReg();
7384 return matchFPSelectToMinMax(Dst, Cond, TrueVal, FalseVal, MatchInfo);
7385}
7386
7387bool CombinerHelper::matchRedundantBinOpInEquality(MachineInstr &MI,
7388 BuildFnTy &MatchInfo) const {
7389 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
7390 // (X + Y) == X --> Y == 0
7391 // (X + Y) != X --> Y != 0
7392 // (X - Y) == X --> Y == 0
7393 // (X - Y) != X --> Y != 0
7394 // (X ^ Y) == X --> Y == 0
7395 // (X ^ Y) != X --> Y != 0
7396 Register Dst = MI.getOperand(i: 0).getReg();
7397 CmpInst::Predicate Pred;
7398 Register X, Y, OpLHS, OpRHS;
7399 bool MatchedSub = mi_match(
7400 R: Dst, MRI,
7401 P: m_c_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: X), R: m_GSub(L: m_Reg(R&: OpLHS), R: m_Reg(R&: Y))));
7402 if (MatchedSub && X != OpLHS)
7403 return false;
7404 if (!MatchedSub) {
7405 if (!mi_match(R: Dst, MRI,
7406 P: m_c_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: X),
7407 R: m_any_of(preds: m_GAdd(L: m_Reg(R&: OpLHS), R: m_Reg(R&: OpRHS)),
7408 preds: m_GXor(L: m_Reg(R&: OpLHS), R: m_Reg(R&: OpRHS))))))
7409 return false;
7410 Y = X == OpLHS ? OpRHS : X == OpRHS ? OpLHS : Register();
7411 }
7412 MatchInfo = [=](MachineIRBuilder &B) {
7413 auto Zero = B.buildConstant(Res: MRI.getType(Reg: Y), Val: 0);
7414 B.buildICmp(Pred, Res: Dst, Op0: Y, Op1: Zero);
7415 };
7416 return CmpInst::isEquality(pred: Pred) && Y.isValid();
7417}
7418
7419/// Return the minimum useless shift amount that results in complete loss of the
7420/// source value. Return std::nullopt when it cannot determine a value.
7421static std::optional<unsigned>
7422getMinUselessShift(KnownBits ValueKB, unsigned Opcode,
7423 std::optional<int64_t> &Result) {
7424 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR ||
7425 Opcode == TargetOpcode::G_ASHR) &&
7426 "Expect G_SHL, G_LSHR or G_ASHR.");
7427 auto SignificantBits = 0;
7428 switch (Opcode) {
7429 case TargetOpcode::G_SHL:
7430 SignificantBits = ValueKB.countMinTrailingZeros();
7431 Result = 0;
7432 break;
7433 case TargetOpcode::G_LSHR:
7434 Result = 0;
7435 SignificantBits = ValueKB.countMinLeadingZeros();
7436 break;
7437 case TargetOpcode::G_ASHR:
7438 if (ValueKB.isNonNegative()) {
7439 SignificantBits = ValueKB.countMinLeadingZeros();
7440 Result = 0;
7441 } else if (ValueKB.isNegative()) {
7442 SignificantBits = ValueKB.countMinLeadingOnes();
7443 Result = -1;
7444 } else {
7445 // Cannot determine shift result.
7446 Result = std::nullopt;
7447 }
7448 break;
7449 default:
7450 break;
7451 }
7452 return ValueKB.getBitWidth() - SignificantBits;
7453}
7454
7455bool CombinerHelper::matchShiftsTooBig(
7456 MachineInstr &MI, std::optional<int64_t> &MatchInfo) const {
7457 Register ShiftVal = MI.getOperand(i: 1).getReg();
7458 Register ShiftReg = MI.getOperand(i: 2).getReg();
7459 LLT ResTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7460 auto IsShiftTooBig = [&](const Constant *C) {
7461 auto *CI = dyn_cast<ConstantInt>(Val: C);
7462 if (!CI)
7463 return false;
7464 if (CI->uge(Num: ResTy.getScalarSizeInBits())) {
7465 MatchInfo = std::nullopt;
7466 return true;
7467 }
7468 auto OptMaxUsefulShift = getMinUselessShift(ValueKB: VT->getKnownBits(R: ShiftVal),
7469 Opcode: MI.getOpcode(), Result&: MatchInfo);
7470 return OptMaxUsefulShift && CI->uge(Num: *OptMaxUsefulShift);
7471 };
7472 return matchUnaryPredicate(MRI, Reg: ShiftReg, Match: IsShiftTooBig);
7473}
7474
7475bool CombinerHelper::matchCommuteConstantToRHS(MachineInstr &MI) const {
7476 unsigned LHSOpndIdx = 1;
7477 unsigned RHSOpndIdx = 2;
7478 switch (MI.getOpcode()) {
7479 case TargetOpcode::G_UADDO:
7480 case TargetOpcode::G_SADDO:
7481 case TargetOpcode::G_UMULO:
7482 case TargetOpcode::G_SMULO:
7483 LHSOpndIdx = 2;
7484 RHSOpndIdx = 3;
7485 break;
7486 default:
7487 break;
7488 }
7489 Register LHS = MI.getOperand(i: LHSOpndIdx).getReg();
7490 Register RHS = MI.getOperand(i: RHSOpndIdx).getReg();
7491 MachineInstr *LHSDef, *RHSDef;
7492 if (!mi_match(R: LHS, MRI, P: m_MInstr(MI&: LHSDef)) ||
7493 !mi_match(R: RHS, MRI, P: m_MInstr(MI&: RHSDef)))
7494 return false;
7495
7496 if (!getIConstantVRegVal(VReg: LHS, MRI)) {
7497 // Skip commuting if LHS is not a constant. But, LHS may be a
7498 // G_CONSTANT_FOLD_BARRIER. If so we commute as long as we don't already
7499 // have a constant on the RHS.
7500 if (LHSDef->getOpcode() != TargetOpcode::G_CONSTANT_FOLD_BARRIER)
7501 return false;
7502 }
7503 // Commute as long as RHS is not a constant or G_CONSTANT_FOLD_BARRIER.
7504 return RHSDef->getOpcode() != TargetOpcode::G_CONSTANT_FOLD_BARRIER &&
7505 !getIConstantVRegVal(VReg: RHS, MRI);
7506}
7507
7508bool CombinerHelper::matchCommuteFPConstantToRHS(MachineInstr &MI) const {
7509 Register LHS = MI.getOperand(i: 1).getReg();
7510 Register RHS = MI.getOperand(i: 2).getReg();
7511 std::optional<FPValueAndVReg> ValAndVReg;
7512 if (!mi_match(R: LHS, MRI, P: m_GFCstOrSplat(FPValReg&: ValAndVReg)))
7513 return false;
7514 return !mi_match(R: RHS, MRI, P: m_GFCstOrSplat(FPValReg&: ValAndVReg));
7515}
7516
7517void CombinerHelper::applyCommuteBinOpOperands(MachineInstr &MI) const {
7518 Observer.changingInstr(MI);
7519 unsigned LHSOpndIdx = 1;
7520 unsigned RHSOpndIdx = 2;
7521 switch (MI.getOpcode()) {
7522 case TargetOpcode::G_UADDO:
7523 case TargetOpcode::G_SADDO:
7524 case TargetOpcode::G_UMULO:
7525 case TargetOpcode::G_SMULO:
7526 LHSOpndIdx = 2;
7527 RHSOpndIdx = 3;
7528 break;
7529 default:
7530 break;
7531 }
7532 Register LHSReg = MI.getOperand(i: LHSOpndIdx).getReg();
7533 Register RHSReg = MI.getOperand(i: RHSOpndIdx).getReg();
7534 MI.getOperand(i: LHSOpndIdx).setReg(RHSReg);
7535 MI.getOperand(i: RHSOpndIdx).setReg(LHSReg);
7536 Observer.changedInstr(MI);
7537}
7538
7539bool CombinerHelper::isOneOrOneSplat(Register Src, bool AllowUndefs) const {
7540 LLT SrcTy = MRI.getType(Reg: Src);
7541 if (SrcTy.isFixedVector())
7542 return isConstantSplatVector(Src, SplatValue: 1, AllowUndefs);
7543 if (SrcTy.isScalar()) {
7544 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Reg: Src, MRI) != nullptr)
7545 return true;
7546 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7547 return IConstant && IConstant->Value == 1;
7548 }
7549 return false; // scalable vector
7550}
7551
7552bool CombinerHelper::isZeroOrZeroSplat(Register Src, bool AllowUndefs) const {
7553 LLT SrcTy = MRI.getType(Reg: Src);
7554 if (SrcTy.isFixedVector())
7555 return isConstantSplatVector(Src, SplatValue: 0, AllowUndefs);
7556 if (SrcTy.isScalar()) {
7557 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Reg: Src, MRI) != nullptr)
7558 return true;
7559 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7560 return IConstant && IConstant->Value == 0;
7561 }
7562 return false; // scalable vector
7563}
7564
7565// Ignores COPYs during conformance checks.
7566// FIXME scalable vectors.
7567bool CombinerHelper::isConstantSplatVector(Register Src, int64_t SplatValue,
7568 bool AllowUndefs) const {
7569 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
7570 if (!BuildVector)
7571 return false;
7572 unsigned NumSources = BuildVector->getNumSources();
7573
7574 for (unsigned I = 0; I < NumSources; ++I) {
7575 GImplicitDef *ImplicitDef =
7576 getOpcodeDef<GImplicitDef>(Reg: BuildVector->getSourceReg(I), MRI);
7577 if (ImplicitDef && AllowUndefs)
7578 continue;
7579 if (ImplicitDef && !AllowUndefs)
7580 return false;
7581 std::optional<ValueAndVReg> IConstant =
7582 getIConstantVRegValWithLookThrough(VReg: BuildVector->getSourceReg(I), MRI);
7583 if (IConstant && IConstant->Value == SplatValue)
7584 continue;
7585 return false;
7586 }
7587 return true;
7588}
7589
7590// Ignores COPYs during lookups.
7591// FIXME scalable vectors
7592std::optional<APInt>
7593CombinerHelper::getConstantOrConstantSplatVector(Register Src) const {
7594 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7595 if (IConstant)
7596 return IConstant->Value;
7597
7598 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
7599 if (!BuildVector)
7600 return std::nullopt;
7601 unsigned NumSources = BuildVector->getNumSources();
7602
7603 std::optional<APInt> Value = std::nullopt;
7604 for (unsigned I = 0; I < NumSources; ++I) {
7605 std::optional<ValueAndVReg> IConstant =
7606 getIConstantVRegValWithLookThrough(VReg: BuildVector->getSourceReg(I), MRI);
7607 if (!IConstant)
7608 return std::nullopt;
7609 if (!Value)
7610 Value = IConstant->Value;
7611 else if (*Value != IConstant->Value)
7612 return std::nullopt;
7613 }
7614 return Value;
7615}
7616
7617// FIXME G_SPLAT_VECTOR
7618bool CombinerHelper::isConstantOrConstantVectorI(Register Src) const {
7619 auto IConstant = getIConstantVRegValWithLookThrough(VReg: Src, MRI);
7620 if (IConstant)
7621 return true;
7622
7623 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Reg: Src, MRI);
7624 if (!BuildVector)
7625 return false;
7626
7627 unsigned NumSources = BuildVector->getNumSources();
7628 for (unsigned I = 0; I < NumSources; ++I) {
7629 std::optional<ValueAndVReg> IConstant =
7630 getIConstantVRegValWithLookThrough(VReg: BuildVector->getSourceReg(I), MRI);
7631 if (!IConstant)
7632 return false;
7633 }
7634 return true;
7635}
7636
7637// TODO: use knownbits to determine zeros
7638bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select,
7639 BuildFnTy &MatchInfo) const {
7640 uint32_t Flags = Select->getFlags();
7641 Register Dest = Select->getReg(Idx: 0);
7642 Register Cond = Select->getCondReg();
7643 Register True = Select->getTrueReg();
7644 Register False = Select->getFalseReg();
7645 LLT CondTy = MRI.getType(Reg: Select->getCondReg());
7646 LLT TrueTy = MRI.getType(Reg: Select->getTrueReg());
7647
7648 // We only do this combine for scalar boolean conditions.
7649 if (CondTy != LLT::scalar(SizeInBits: 1))
7650 return false;
7651
7652 if (TrueTy.isPointer())
7653 return false;
7654
7655 // Both are scalars.
7656 std::optional<ValueAndVReg> TrueOpt =
7657 getIConstantVRegValWithLookThrough(VReg: True, MRI);
7658 std::optional<ValueAndVReg> FalseOpt =
7659 getIConstantVRegValWithLookThrough(VReg: False, MRI);
7660
7661 if (!TrueOpt || !FalseOpt)
7662 return false;
7663
7664 APInt TrueValue = TrueOpt->Value;
7665 APInt FalseValue = FalseOpt->Value;
7666
7667 // select Cond, 1, 0 --> zext (Cond)
7668 if (TrueValue.isOne() && FalseValue.isZero()) {
7669 MatchInfo = [=](MachineIRBuilder &B) {
7670 B.setInstrAndDebugLoc(*Select);
7671 B.buildZExtOrTrunc(Res: Dest, Op: Cond);
7672 };
7673 return true;
7674 }
7675
7676 // select Cond, -1, 0 --> sext (Cond)
7677 if (TrueValue.isAllOnes() && FalseValue.isZero()) {
7678 MatchInfo = [=](MachineIRBuilder &B) {
7679 B.setInstrAndDebugLoc(*Select);
7680 B.buildSExtOrTrunc(Res: Dest, Op: Cond);
7681 };
7682 return true;
7683 }
7684
7685 // select Cond, 0, 1 --> zext (!Cond)
7686 if (TrueValue.isZero() && FalseValue.isOne()) {
7687 MatchInfo = [=](MachineIRBuilder &B) {
7688 B.setInstrAndDebugLoc(*Select);
7689 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7690 B.buildNot(Dst: Inner, Src0: Cond);
7691 B.buildZExtOrTrunc(Res: Dest, Op: Inner);
7692 };
7693 return true;
7694 }
7695
7696 // select Cond, 0, -1 --> sext (!Cond)
7697 if (TrueValue.isZero() && FalseValue.isAllOnes()) {
7698 MatchInfo = [=](MachineIRBuilder &B) {
7699 B.setInstrAndDebugLoc(*Select);
7700 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7701 B.buildNot(Dst: Inner, Src0: Cond);
7702 B.buildSExtOrTrunc(Res: Dest, Op: Inner);
7703 };
7704 return true;
7705 }
7706
7707 // select Cond, C1, C1-1 --> add (zext Cond), C1-1
7708 if (TrueValue - 1 == FalseValue) {
7709 MatchInfo = [=](MachineIRBuilder &B) {
7710 B.setInstrAndDebugLoc(*Select);
7711 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7712 B.buildZExtOrTrunc(Res: Inner, Op: Cond);
7713 B.buildAdd(Dst: Dest, Src0: Inner, Src1: False);
7714 };
7715 return true;
7716 }
7717
7718 // select Cond, C1, C1+1 --> add (sext Cond), C1+1
7719 if (TrueValue + 1 == FalseValue) {
7720 MatchInfo = [=](MachineIRBuilder &B) {
7721 B.setInstrAndDebugLoc(*Select);
7722 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7723 B.buildSExtOrTrunc(Res: Inner, Op: Cond);
7724 B.buildAdd(Dst: Dest, Src0: Inner, Src1: False);
7725 };
7726 return true;
7727 }
7728
7729 // select Cond, Pow2, 0 --> (zext Cond) << log2(Pow2)
7730 if (TrueValue.isPowerOf2() && FalseValue.isZero()) {
7731 MatchInfo = [=](MachineIRBuilder &B) {
7732 B.setInstrAndDebugLoc(*Select);
7733 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7734 B.buildZExtOrTrunc(Res: Inner, Op: Cond);
7735 // The shift amount must be scalar.
7736 LLT ShiftTy = TrueTy.isVector() ? TrueTy.getElementType() : TrueTy;
7737 auto ShAmtC = B.buildConstant(Res: ShiftTy, Val: TrueValue.exactLogBase2());
7738 B.buildShl(Dst: Dest, Src0: Inner, Src1: ShAmtC, Flags);
7739 };
7740 return true;
7741 }
7742
7743 // select Cond, 0, Pow2 --> (zext (!Cond)) << log2(Pow2)
7744 if (FalseValue.isPowerOf2() && TrueValue.isZero()) {
7745 MatchInfo = [=](MachineIRBuilder &B) {
7746 B.setInstrAndDebugLoc(*Select);
7747 Register Not = MRI.createGenericVirtualRegister(Ty: CondTy);
7748 B.buildNot(Dst: Not, Src0: Cond);
7749 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7750 B.buildZExtOrTrunc(Res: Inner, Op: Not);
7751 // The shift amount must be scalar.
7752 LLT ShiftTy = TrueTy.isVector() ? TrueTy.getElementType() : TrueTy;
7753 auto ShAmtC = B.buildConstant(Res: ShiftTy, Val: FalseValue.exactLogBase2());
7754 B.buildShl(Dst: Dest, Src0: Inner, Src1: ShAmtC, Flags);
7755 };
7756 return true;
7757 }
7758
7759 // select Cond, -1, C --> or (sext Cond), C
7760 if (TrueValue.isAllOnes()) {
7761 MatchInfo = [=](MachineIRBuilder &B) {
7762 B.setInstrAndDebugLoc(*Select);
7763 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7764 B.buildSExtOrTrunc(Res: Inner, Op: Cond);
7765 B.buildOr(Dst: Dest, Src0: Inner, Src1: False, Flags);
7766 };
7767 return true;
7768 }
7769
7770 // select Cond, C, -1 --> or (sext (not Cond)), C
7771 if (FalseValue.isAllOnes()) {
7772 MatchInfo = [=](MachineIRBuilder &B) {
7773 B.setInstrAndDebugLoc(*Select);
7774 Register Not = MRI.createGenericVirtualRegister(Ty: CondTy);
7775 B.buildNot(Dst: Not, Src0: Cond);
7776 Register Inner = MRI.createGenericVirtualRegister(Ty: TrueTy);
7777 B.buildSExtOrTrunc(Res: Inner, Op: Not);
7778 B.buildOr(Dst: Dest, Src0: Inner, Src1: True, Flags);
7779 };
7780 return true;
7781 }
7782
7783 return false;
7784}
7785
7786// TODO: use knownbits to determine zeros
7787bool CombinerHelper::tryFoldBoolSelectToLogic(GSelect *Select,
7788 BuildFnTy &MatchInfo) const {
7789 uint32_t Flags = Select->getFlags();
7790 Register DstReg = Select->getReg(Idx: 0);
7791 Register Cond = Select->getCondReg();
7792 Register True = Select->getTrueReg();
7793 Register False = Select->getFalseReg();
7794 LLT CondTy = MRI.getType(Reg: Select->getCondReg());
7795 LLT TrueTy = MRI.getType(Reg: Select->getTrueReg());
7796
7797 // Boolean or fixed vector of booleans.
7798 if (CondTy.isScalableVector() ||
7799 (CondTy.isFixedVector() &&
7800 CondTy.getElementType().getScalarSizeInBits() != 1) ||
7801 CondTy.getScalarSizeInBits() != 1)
7802 return false;
7803
7804 if (CondTy != TrueTy)
7805 return false;
7806
7807 // select Cond, Cond, F --> or Cond, F
7808 // select Cond, 1, F --> or Cond, F
7809 if ((Cond == True) || isOneOrOneSplat(Src: True, /* AllowUndefs */ true)) {
7810 MatchInfo = [=](MachineIRBuilder &B) {
7811 B.setInstrAndDebugLoc(*Select);
7812 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7813 B.buildZExtOrTrunc(Res: Ext, Op: Cond);
7814 auto FreezeFalse = B.buildFreeze(Dst: TrueTy, Src: False);
7815 B.buildOr(Dst: DstReg, Src0: Ext, Src1: FreezeFalse, Flags);
7816 };
7817 return true;
7818 }
7819
7820 // select Cond, T, Cond --> and Cond, T
7821 // select Cond, T, 0 --> and Cond, T
7822 if ((Cond == False) || isZeroOrZeroSplat(Src: False, /* AllowUndefs */ true)) {
7823 MatchInfo = [=](MachineIRBuilder &B) {
7824 B.setInstrAndDebugLoc(*Select);
7825 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7826 B.buildZExtOrTrunc(Res: Ext, Op: Cond);
7827 auto FreezeTrue = B.buildFreeze(Dst: TrueTy, Src: True);
7828 B.buildAnd(Dst: DstReg, Src0: Ext, Src1: FreezeTrue);
7829 };
7830 return true;
7831 }
7832
7833 // select Cond, T, 1 --> or (not Cond), T
7834 if (isOneOrOneSplat(Src: False, /* AllowUndefs */ true)) {
7835 MatchInfo = [=](MachineIRBuilder &B) {
7836 B.setInstrAndDebugLoc(*Select);
7837 // First the not.
7838 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7839 B.buildNot(Dst: Inner, Src0: Cond);
7840 // Then an ext to match the destination register.
7841 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7842 B.buildZExtOrTrunc(Res: Ext, Op: Inner);
7843 auto FreezeTrue = B.buildFreeze(Dst: TrueTy, Src: True);
7844 B.buildOr(Dst: DstReg, Src0: Ext, Src1: FreezeTrue, Flags);
7845 };
7846 return true;
7847 }
7848
7849 // select Cond, 0, F --> and (not Cond), F
7850 if (isZeroOrZeroSplat(Src: True, /* AllowUndefs */ true)) {
7851 MatchInfo = [=](MachineIRBuilder &B) {
7852 B.setInstrAndDebugLoc(*Select);
7853 // First the not.
7854 Register Inner = MRI.createGenericVirtualRegister(Ty: CondTy);
7855 B.buildNot(Dst: Inner, Src0: Cond);
7856 // Then an ext to match the destination register.
7857 Register Ext = MRI.createGenericVirtualRegister(Ty: TrueTy);
7858 B.buildZExtOrTrunc(Res: Ext, Op: Inner);
7859 auto FreezeFalse = B.buildFreeze(Dst: TrueTy, Src: False);
7860 B.buildAnd(Dst: DstReg, Src0: Ext, Src1: FreezeFalse);
7861 };
7862 return true;
7863 }
7864
7865 return false;
7866}
7867
7868bool CombinerHelper::matchSelectIMinMax(const MachineOperand &MO,
7869 BuildFnTy &MatchInfo) const {
7870 Register DstReg = MO.getReg();
7871 Register CondReg, True, False;
7872 if (!mi_match(R: DstReg, MRI,
7873 P: m_GISelect(Src0: m_Reg(R&: CondReg), Src1: m_Reg(R&: True), Src2: m_Reg(R&: False))))
7874 return false;
7875
7876 CmpInst::Predicate Pred;
7877 Register CmpLHS, CmpRHS;
7878 if (!mi_match(R: CondReg, MRI,
7879 P: m_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: CmpLHS), R: m_Reg(R&: CmpRHS))))
7880 return false;
7881
7882 LLT DstTy = MRI.getType(Reg: DstReg);
7883 if (DstTy.isPointerOrPointerVector())
7884 return false;
7885
7886 // We want to fold the icmp and replace the select.
7887 if (!MRI.hasOneNonDBGUse(RegNo: CondReg))
7888 return false;
7889
7890 // We need a larger or smaller predicate for
7891 // canonicalization.
7892 if (CmpInst::isEquality(pred: Pred))
7893 return false;
7894
7895 // We can swap CmpLHS and CmpRHS for higher hitrate.
7896 if (True == CmpRHS && False == CmpLHS) {
7897 std::swap(a&: CmpLHS, b&: CmpRHS);
7898 Pred = CmpInst::getSwappedPredicate(pred: Pred);
7899 }
7900
7901 // (icmp X, Y) ? X : Y -> integer minmax.
7902 // see matchSelectPattern in ValueTracking.
7903 // Legality between G_SELECT and integer minmax can differ.
7904 if (True != CmpLHS || False != CmpRHS)
7905 return false;
7906
7907 switch (Pred) {
7908 case ICmpInst::ICMP_UGT:
7909 case ICmpInst::ICMP_UGE: {
7910 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_UMAX, DstTy}))
7911 return false;
7912 MatchInfo = [=](MachineIRBuilder &B) { B.buildUMax(Dst: DstReg, Src0: True, Src1: False); };
7913 return true;
7914 }
7915 case ICmpInst::ICMP_SGT:
7916 case ICmpInst::ICMP_SGE: {
7917 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SMAX, DstTy}))
7918 return false;
7919 MatchInfo = [=](MachineIRBuilder &B) { B.buildSMax(Dst: DstReg, Src0: True, Src1: False); };
7920 return true;
7921 }
7922 case ICmpInst::ICMP_ULT:
7923 case ICmpInst::ICMP_ULE: {
7924 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_UMIN, DstTy}))
7925 return false;
7926 MatchInfo = [=](MachineIRBuilder &B) { B.buildUMin(Dst: DstReg, Src0: True, Src1: False); };
7927 return true;
7928 }
7929 case ICmpInst::ICMP_SLT:
7930 case ICmpInst::ICMP_SLE: {
7931 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SMIN, DstTy}))
7932 return false;
7933 MatchInfo = [=](MachineIRBuilder &B) { B.buildSMin(Dst: DstReg, Src0: True, Src1: False); };
7934 return true;
7935 }
7936 default:
7937 return false;
7938 }
7939}
7940
7941// (neg (min/max x, (neg x))) --> (max/min x, (neg x))
7942bool CombinerHelper::matchSimplifyNegMinMax(MachineInstr &MI,
7943 BuildFnTy &MatchInfo) const {
7944 assert(MI.getOpcode() == TargetOpcode::G_SUB);
7945 Register DestReg = MI.getOperand(i: 0).getReg();
7946 LLT DestTy = MRI.getType(Reg: DestReg);
7947
7948 Register X;
7949 Register Sub0;
7950 auto NegPattern = m_all_of(preds: m_Neg(Src: m_DeferredReg(R&: X)), preds: m_Reg(R&: Sub0));
7951 if (mi_match(R: DestReg, MRI,
7952 P: m_Neg(Src: m_OneUse(SP: m_any_of(preds: m_GSMin(L: m_Reg(R&: X), R: NegPattern),
7953 preds: m_GSMax(L: m_Reg(R&: X), R: NegPattern),
7954 preds: m_GUMin(L: m_Reg(R&: X), R: NegPattern),
7955 preds: m_GUMax(L: m_Reg(R&: X), R: NegPattern)))))) {
7956 MachineInstr *MinMaxMI;
7957 if (!mi_match(R: MI.getOperand(i: 2).getReg(), MRI, P: m_MInstr(MI&: MinMaxMI)))
7958 return false;
7959 unsigned NewOpc = getInverseGMinMaxOpcode(MinMaxOpc: MinMaxMI->getOpcode());
7960 if (isLegal(Query: {NewOpc, {DestTy}})) {
7961 MatchInfo = [=](MachineIRBuilder &B) {
7962 B.buildInstr(Opc: NewOpc, DstOps: {DestReg}, SrcOps: {X, Sub0});
7963 };
7964 return true;
7965 }
7966 }
7967
7968 return false;
7969}
7970
7971bool CombinerHelper::matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo) const {
7972 GSelect *Select = cast<GSelect>(Val: &MI);
7973
7974 if (tryFoldSelectOfConstants(Select, MatchInfo))
7975 return true;
7976
7977 if (tryFoldBoolSelectToLogic(Select, MatchInfo))
7978 return true;
7979
7980 return false;
7981}
7982
7983/// Fold (icmp Pred1 V1, C1) && (icmp Pred2 V2, C2)
7984/// or (icmp Pred1 V1, C1) || (icmp Pred2 V2, C2)
7985/// into a single comparison using range-based reasoning.
7986/// see InstCombinerImpl::foldAndOrOfICmpsUsingRanges.
7987bool CombinerHelper::tryFoldAndOrOrICmpsUsingRanges(
7988 GLogicalBinOp *Logic, BuildFnTy &MatchInfo) const {
7989 assert(Logic->getOpcode() != TargetOpcode::G_XOR && "unexpected xor");
7990 bool IsAnd = Logic->getOpcode() == TargetOpcode::G_AND;
7991 Register DstReg = Logic->getReg(Idx: 0);
7992 Register LHS = Logic->getLHSReg();
7993 Register RHS = Logic->getRHSReg();
7994 unsigned Flags = Logic->getFlags();
7995
7996 // We need an G_ICMP on the LHS register.
7997 GICmp *Cmp1 = getOpcodeDef<GICmp>(Reg: LHS, MRI);
7998 if (!Cmp1)
7999 return false;
8000
8001 // We need an G_ICMP on the RHS register.
8002 GICmp *Cmp2 = getOpcodeDef<GICmp>(Reg: RHS, MRI);
8003 if (!Cmp2)
8004 return false;
8005
8006 // We want to fold the icmps.
8007 if (!MRI.hasOneNonDBGUse(RegNo: Cmp1->getReg(Idx: 0)) ||
8008 !MRI.hasOneNonDBGUse(RegNo: Cmp2->getReg(Idx: 0)))
8009 return false;
8010
8011 APInt C1;
8012 APInt C2;
8013 std::optional<ValueAndVReg> MaybeC1 =
8014 getIConstantVRegValWithLookThrough(VReg: Cmp1->getRHSReg(), MRI);
8015 if (!MaybeC1)
8016 return false;
8017 C1 = MaybeC1->Value;
8018
8019 std::optional<ValueAndVReg> MaybeC2 =
8020 getIConstantVRegValWithLookThrough(VReg: Cmp2->getRHSReg(), MRI);
8021 if (!MaybeC2)
8022 return false;
8023 C2 = MaybeC2->Value;
8024
8025 Register R1 = Cmp1->getLHSReg();
8026 Register R2 = Cmp2->getLHSReg();
8027 CmpInst::Predicate Pred1 = Cmp1->getCond();
8028 CmpInst::Predicate Pred2 = Cmp2->getCond();
8029 LLT CmpTy = MRI.getType(Reg: Cmp1->getReg(Idx: 0));
8030 LLT CmpOperandTy = MRI.getType(Reg: R1);
8031
8032 if (CmpOperandTy.isPointer())
8033 return false;
8034
8035 // We build ands, adds, and constants of type CmpOperandTy.
8036 // They must be legal to build.
8037 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_AND, CmpOperandTy}) ||
8038 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ADD, CmpOperandTy}) ||
8039 !isConstantLegalOrBeforeLegalizer(Ty: CmpOperandTy))
8040 return false;
8041
8042 // Look through add of a constant offset on R1, R2, or both operands. This
8043 // allows us to interpret the R + C' < C'' range idiom into a proper range.
8044 std::optional<APInt> Offset1;
8045 std::optional<APInt> Offset2;
8046 if (R1 != R2) {
8047 if (GAdd *Add = getOpcodeDef<GAdd>(Reg: R1, MRI)) {
8048 std::optional<ValueAndVReg> MaybeOffset1 =
8049 getIConstantVRegValWithLookThrough(VReg: Add->getRHSReg(), MRI);
8050 if (MaybeOffset1) {
8051 R1 = Add->getLHSReg();
8052 Offset1 = MaybeOffset1->Value;
8053 }
8054 }
8055 if (GAdd *Add = getOpcodeDef<GAdd>(Reg: R2, MRI)) {
8056 std::optional<ValueAndVReg> MaybeOffset2 =
8057 getIConstantVRegValWithLookThrough(VReg: Add->getRHSReg(), MRI);
8058 if (MaybeOffset2) {
8059 R2 = Add->getLHSReg();
8060 Offset2 = MaybeOffset2->Value;
8061 }
8062 }
8063 }
8064
8065 if (R1 != R2)
8066 return false;
8067
8068 // We calculate the icmp ranges including maybe offsets.
8069 ConstantRange CR1 = ConstantRange::makeExactICmpRegion(
8070 Pred: IsAnd ? ICmpInst::getInversePredicate(pred: Pred1) : Pred1, Other: C1);
8071 if (Offset1)
8072 CR1 = CR1.subtract(CI: *Offset1);
8073
8074 ConstantRange CR2 = ConstantRange::makeExactICmpRegion(
8075 Pred: IsAnd ? ICmpInst::getInversePredicate(pred: Pred2) : Pred2, Other: C2);
8076 if (Offset2)
8077 CR2 = CR2.subtract(CI: *Offset2);
8078
8079 bool CreateMask = false;
8080 APInt LowerDiff;
8081 std::optional<ConstantRange> CR = CR1.exactUnionWith(CR: CR2);
8082 if (!CR) {
8083 // We need non-wrapping ranges.
8084 if (CR1.isWrappedSet() || CR2.isWrappedSet())
8085 return false;
8086
8087 // Check whether we have equal-size ranges that only differ by one bit.
8088 // In that case we can apply a mask to map one range onto the other.
8089 LowerDiff = CR1.getLower() ^ CR2.getLower();
8090 APInt UpperDiff = (CR1.getUpper() - 1) ^ (CR2.getUpper() - 1);
8091 APInt CR1Size = CR1.getUpper() - CR1.getLower();
8092 if (!LowerDiff.isPowerOf2() || LowerDiff != UpperDiff ||
8093 CR1Size != CR2.getUpper() - CR2.getLower())
8094 return false;
8095
8096 CR = CR1.getLower().ult(RHS: CR2.getLower()) ? CR1 : CR2;
8097 CreateMask = true;
8098 }
8099
8100 if (IsAnd)
8101 CR = CR->inverse();
8102
8103 CmpInst::Predicate NewPred;
8104 APInt NewC, Offset;
8105 CR->getEquivalentICmp(Pred&: NewPred, RHS&: NewC, Offset);
8106
8107 // We take the result type of one of the original icmps, CmpTy, for
8108 // the to be build icmp. The operand type, CmpOperandTy, is used for
8109 // the other instructions and constants to be build. The types of
8110 // the parameters and output are the same for add and and. CmpTy
8111 // and the type of DstReg might differ. That is why we zext or trunc
8112 // the icmp into the destination register.
8113
8114 MatchInfo = [=](MachineIRBuilder &B) {
8115 if (CreateMask && Offset != 0) {
8116 auto TildeLowerDiff = B.buildConstant(Res: CmpOperandTy, Val: ~LowerDiff);
8117 auto And = B.buildAnd(Dst: CmpOperandTy, Src0: R1, Src1: TildeLowerDiff); // the mask.
8118 auto OffsetC = B.buildConstant(Res: CmpOperandTy, Val: Offset);
8119 auto Add = B.buildAdd(Dst: CmpOperandTy, Src0: And, Src1: OffsetC, Flags);
8120 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
8121 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: Add, Op1: NewCon);
8122 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
8123 } else if (CreateMask && Offset == 0) {
8124 auto TildeLowerDiff = B.buildConstant(Res: CmpOperandTy, Val: ~LowerDiff);
8125 auto And = B.buildAnd(Dst: CmpOperandTy, Src0: R1, Src1: TildeLowerDiff); // the mask.
8126 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
8127 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: And, Op1: NewCon);
8128 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
8129 } else if (!CreateMask && Offset != 0) {
8130 auto OffsetC = B.buildConstant(Res: CmpOperandTy, Val: Offset);
8131 auto Add = B.buildAdd(Dst: CmpOperandTy, Src0: R1, Src1: OffsetC, Flags);
8132 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
8133 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: Add, Op1: NewCon);
8134 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
8135 } else if (!CreateMask && Offset == 0) {
8136 auto NewCon = B.buildConstant(Res: CmpOperandTy, Val: NewC);
8137 auto ICmp = B.buildICmp(Pred: NewPred, Res: CmpTy, Op0: R1, Op1: NewCon);
8138 B.buildZExtOrTrunc(Res: DstReg, Op: ICmp);
8139 } else {
8140 llvm_unreachable("unexpected configuration of CreateMask and Offset");
8141 }
8142 };
8143 return true;
8144}
8145
8146bool CombinerHelper::tryFoldLogicOfFCmps(GLogicalBinOp *Logic,
8147 BuildFnTy &MatchInfo) const {
8148 assert(Logic->getOpcode() != TargetOpcode::G_XOR && "unexpecte xor");
8149 Register DestReg = Logic->getReg(Idx: 0);
8150 Register LHS = Logic->getLHSReg();
8151 Register RHS = Logic->getRHSReg();
8152 bool IsAnd = Logic->getOpcode() == TargetOpcode::G_AND;
8153
8154 // We need a compare on the LHS register.
8155 GFCmp *Cmp1 = getOpcodeDef<GFCmp>(Reg: LHS, MRI);
8156 if (!Cmp1)
8157 return false;
8158
8159 // We need a compare on the RHS register.
8160 GFCmp *Cmp2 = getOpcodeDef<GFCmp>(Reg: RHS, MRI);
8161 if (!Cmp2)
8162 return false;
8163
8164 LLT CmpTy = MRI.getType(Reg: Cmp1->getReg(Idx: 0));
8165 LLT CmpOperandTy = MRI.getType(Reg: Cmp1->getLHSReg());
8166
8167 // We build one fcmp, want to fold the fcmps, replace the logic op,
8168 // and the fcmps must have the same shape.
8169 if (!isLegalOrBeforeLegalizer(
8170 Query: {TargetOpcode::G_FCMP, {CmpTy, CmpOperandTy}}) ||
8171 !MRI.hasOneNonDBGUse(RegNo: Logic->getReg(Idx: 0)) ||
8172 !MRI.hasOneNonDBGUse(RegNo: Cmp1->getReg(Idx: 0)) ||
8173 !MRI.hasOneNonDBGUse(RegNo: Cmp2->getReg(Idx: 0)) ||
8174 MRI.getType(Reg: Cmp1->getLHSReg()) != MRI.getType(Reg: Cmp2->getLHSReg()))
8175 return false;
8176
8177 CmpInst::Predicate PredL = Cmp1->getCond();
8178 CmpInst::Predicate PredR = Cmp2->getCond();
8179 Register LHS0 = Cmp1->getLHSReg();
8180 Register LHS1 = Cmp1->getRHSReg();
8181 Register RHS0 = Cmp2->getLHSReg();
8182 Register RHS1 = Cmp2->getRHSReg();
8183
8184 if (LHS0 == RHS1 && LHS1 == RHS0) {
8185 // Swap RHS operands to match LHS.
8186 PredR = CmpInst::getSwappedPredicate(pred: PredR);
8187 std::swap(a&: RHS0, b&: RHS1);
8188 }
8189
8190 if (LHS0 == RHS0 && LHS1 == RHS1) {
8191 // We determine the new predicate.
8192 unsigned CmpCodeL = getFCmpCode(CC: PredL);
8193 unsigned CmpCodeR = getFCmpCode(CC: PredR);
8194 unsigned NewPred = IsAnd ? CmpCodeL & CmpCodeR : CmpCodeL | CmpCodeR;
8195 unsigned Flags = Cmp1->getFlags() | Cmp2->getFlags();
8196 MatchInfo = [=](MachineIRBuilder &B) {
8197 // The fcmp predicates fill the lower part of the enum.
8198 FCmpInst::Predicate Pred = static_cast<FCmpInst::Predicate>(NewPred);
8199 if (Pred == FCmpInst::FCMP_FALSE &&
8200 isConstantLegalOrBeforeLegalizer(Ty: CmpTy)) {
8201 auto False = B.buildConstant(Res: CmpTy, Val: 0);
8202 B.buildZExtOrTrunc(Res: DestReg, Op: False);
8203 } else if (Pred == FCmpInst::FCMP_TRUE &&
8204 isConstantLegalOrBeforeLegalizer(Ty: CmpTy)) {
8205 auto True =
8206 B.buildConstant(Res: CmpTy, Val: getICmpTrueVal(TLI: getTargetLowering(),
8207 IsVector: CmpTy.isVector() /*isVector*/,
8208 IsFP: true /*isFP*/));
8209 B.buildZExtOrTrunc(Res: DestReg, Op: True);
8210 } else { // We take the predicate without predicate optimizations.
8211 auto Cmp = B.buildFCmp(Pred, Res: CmpTy, Op0: LHS0, Op1: LHS1, Flags);
8212 B.buildZExtOrTrunc(Res: DestReg, Op: Cmp);
8213 }
8214 };
8215 return true;
8216 }
8217
8218 return false;
8219}
8220
8221bool CombinerHelper::matchAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const {
8222 GAnd *And = cast<GAnd>(Val: &MI);
8223
8224 if (tryFoldAndOrOrICmpsUsingRanges(Logic: And, MatchInfo))
8225 return true;
8226
8227 if (tryFoldLogicOfFCmps(Logic: And, MatchInfo))
8228 return true;
8229
8230 return false;
8231}
8232
8233bool CombinerHelper::matchOr(MachineInstr &MI, BuildFnTy &MatchInfo) const {
8234 GOr *Or = cast<GOr>(Val: &MI);
8235
8236 if (tryFoldAndOrOrICmpsUsingRanges(Logic: Or, MatchInfo))
8237 return true;
8238
8239 if (tryFoldLogicOfFCmps(Logic: Or, MatchInfo))
8240 return true;
8241
8242 return false;
8243}
8244
8245bool CombinerHelper::matchAddOverflow(MachineInstr &MI,
8246 BuildFnTy &MatchInfo) const {
8247 GAddCarryOut *Add = cast<GAddCarryOut>(Val: &MI);
8248
8249 // Addo has no flags
8250 Register Dst = Add->getReg(Idx: 0);
8251 Register Carry = Add->getReg(Idx: 1);
8252 Register LHS = Add->getLHSReg();
8253 Register RHS = Add->getRHSReg();
8254 bool IsSigned = Add->isSigned();
8255 LLT DstTy = MRI.getType(Reg: Dst);
8256 LLT CarryTy = MRI.getType(Reg: Carry);
8257
8258 // Fold addo, if the carry is dead -> add, undef.
8259 if (MRI.use_nodbg_empty(RegNo: Carry) &&
8260 isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ADD, {DstTy}})) {
8261 MatchInfo = [=](MachineIRBuilder &B) {
8262 B.buildAdd(Dst, Src0: LHS, Src1: RHS);
8263 B.buildUndef(Res: Carry);
8264 };
8265 return true;
8266 }
8267
8268 // Canonicalize constant to RHS.
8269 if (isConstantOrConstantVectorI(Src: LHS) && !isConstantOrConstantVectorI(Src: RHS)) {
8270 if (IsSigned) {
8271 MatchInfo = [=](MachineIRBuilder &B) {
8272 B.buildSAddo(Res: Dst, CarryOut: Carry, Op0: RHS, Op1: LHS);
8273 };
8274 return true;
8275 }
8276 // !IsSigned
8277 MatchInfo = [=](MachineIRBuilder &B) {
8278 B.buildUAddo(Res: Dst, CarryOut: Carry, Op0: RHS, Op1: LHS);
8279 };
8280 return true;
8281 }
8282
8283 std::optional<APInt> MaybeLHS = getConstantOrConstantSplatVector(Src: LHS);
8284 std::optional<APInt> MaybeRHS = getConstantOrConstantSplatVector(Src: RHS);
8285
8286 // Fold addo(c1, c2) -> c3, carry.
8287 if (MaybeLHS && MaybeRHS && isConstantLegalOrBeforeLegalizer(Ty: DstTy) &&
8288 isConstantLegalOrBeforeLegalizer(Ty: CarryTy)) {
8289 bool Overflow;
8290 APInt Result = IsSigned ? MaybeLHS->sadd_ov(RHS: *MaybeRHS, Overflow)
8291 : MaybeLHS->uadd_ov(RHS: *MaybeRHS, Overflow);
8292 MatchInfo = [=](MachineIRBuilder &B) {
8293 B.buildConstant(Res: Dst, Val: Result);
8294 B.buildConstant(Res: Carry, Val: Overflow);
8295 };
8296 return true;
8297 }
8298
8299 // Fold (addo x, 0) -> x, no carry
8300 if (MaybeRHS && *MaybeRHS == 0 && isConstantLegalOrBeforeLegalizer(Ty: CarryTy)) {
8301 MatchInfo = [=](MachineIRBuilder &B) {
8302 B.buildCopy(Res: Dst, Op: LHS);
8303 B.buildConstant(Res: Carry, Val: 0);
8304 };
8305 return true;
8306 }
8307
8308 // Given 2 constant operands whose sum does not overflow:
8309 // uaddo (X +nuw C0), C1 -> uaddo X, C0 + C1
8310 // saddo (X +nsw C0), C1 -> saddo X, C0 + C1
8311 GAdd *AddLHS = getOpcodeDef<GAdd>(Reg: LHS, MRI);
8312 if (MaybeRHS && AddLHS && MRI.hasOneNonDBGUse(RegNo: Add->getReg(Idx: 0)) &&
8313 ((IsSigned && AddLHS->getFlag(Flag: MachineInstr::MIFlag::NoSWrap)) ||
8314 (!IsSigned && AddLHS->getFlag(Flag: MachineInstr::MIFlag::NoUWrap)))) {
8315 std::optional<APInt> MaybeAddRHS =
8316 getConstantOrConstantSplatVector(Src: AddLHS->getRHSReg());
8317 if (MaybeAddRHS) {
8318 bool Overflow;
8319 APInt NewC = IsSigned ? MaybeAddRHS->sadd_ov(RHS: *MaybeRHS, Overflow)
8320 : MaybeAddRHS->uadd_ov(RHS: *MaybeRHS, Overflow);
8321 if (!Overflow && isConstantLegalOrBeforeLegalizer(Ty: DstTy)) {
8322 if (IsSigned) {
8323 MatchInfo = [=](MachineIRBuilder &B) {
8324 auto ConstRHS = B.buildConstant(Res: DstTy, Val: NewC);
8325 B.buildSAddo(Res: Dst, CarryOut: Carry, Op0: AddLHS->getLHSReg(), Op1: ConstRHS);
8326 };
8327 return true;
8328 }
8329 // !IsSigned
8330 MatchInfo = [=](MachineIRBuilder &B) {
8331 auto ConstRHS = B.buildConstant(Res: DstTy, Val: NewC);
8332 B.buildUAddo(Res: Dst, CarryOut: Carry, Op0: AddLHS->getLHSReg(), Op1: ConstRHS);
8333 };
8334 return true;
8335 }
8336 }
8337 };
8338
8339 // We try to combine addo to non-overflowing add.
8340 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_ADD, {DstTy}}) ||
8341 !isConstantLegalOrBeforeLegalizer(Ty: CarryTy))
8342 return false;
8343
8344 // We try to combine uaddo to non-overflowing add.
8345 if (!IsSigned) {
8346 ConstantRange CRLHS =
8347 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: LHS), /*IsSigned=*/false);
8348 ConstantRange CRRHS =
8349 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: RHS), /*IsSigned=*/false);
8350
8351 switch (CRLHS.unsignedAddMayOverflow(Other: CRRHS)) {
8352 case ConstantRange::OverflowResult::MayOverflow:
8353 return false;
8354 case ConstantRange::OverflowResult::NeverOverflows: {
8355 MatchInfo = [=](MachineIRBuilder &B) {
8356 B.buildAdd(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoUWrap);
8357 B.buildConstant(Res: Carry, Val: 0);
8358 };
8359 return true;
8360 }
8361 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
8362 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
8363 MatchInfo = [=](MachineIRBuilder &B) {
8364 B.buildAdd(Dst, Src0: LHS, Src1: RHS);
8365 B.buildConstant(Res: Carry, Val: 1);
8366 };
8367 return true;
8368 }
8369 }
8370 return false;
8371 }
8372
8373 // We try to combine saddo to non-overflowing add.
8374
8375 // If LHS and RHS each have at least two sign bits, then there is no signed
8376 // overflow.
8377 if (VT->computeNumSignBits(R: RHS) > 1 && VT->computeNumSignBits(R: LHS) > 1) {
8378 MatchInfo = [=](MachineIRBuilder &B) {
8379 B.buildAdd(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoSWrap);
8380 B.buildConstant(Res: Carry, Val: 0);
8381 };
8382 return true;
8383 }
8384
8385 ConstantRange CRLHS =
8386 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: LHS), /*IsSigned=*/true);
8387 ConstantRange CRRHS =
8388 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: RHS), /*IsSigned=*/true);
8389
8390 switch (CRLHS.signedAddMayOverflow(Other: CRRHS)) {
8391 case ConstantRange::OverflowResult::MayOverflow:
8392 return false;
8393 case ConstantRange::OverflowResult::NeverOverflows: {
8394 MatchInfo = [=](MachineIRBuilder &B) {
8395 B.buildAdd(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoSWrap);
8396 B.buildConstant(Res: Carry, Val: 0);
8397 };
8398 return true;
8399 }
8400 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
8401 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
8402 MatchInfo = [=](MachineIRBuilder &B) {
8403 B.buildAdd(Dst, Src0: LHS, Src1: RHS);
8404 B.buildConstant(Res: Carry, Val: 1);
8405 };
8406 return true;
8407 }
8408 }
8409
8410 return false;
8411}
8412
8413void CombinerHelper::applyBuildFnMO(const MachineOperand &MO,
8414 BuildFnTy &MatchInfo) const {
8415 MachineInstr *Root = getDefIgnoringCopies(Reg: MO.getReg(), MRI);
8416 MatchInfo(Builder);
8417 Root->eraseFromParent();
8418}
8419
8420bool CombinerHelper::matchFPowIExpansion(MachineInstr &MI,
8421 int64_t Exponent) const {
8422 bool OptForSize = MI.getMF()->getFunction().hasOptSize();
8423 return getTargetLowering().isBeneficialToExpandPowI(Exponent, OptForSize);
8424}
8425
8426void CombinerHelper::applyExpandFPowI(MachineInstr &MI,
8427 int64_t Exponent) const {
8428 auto [Dst, Base] = MI.getFirst2Regs();
8429 LLT Ty = MRI.getType(Reg: Dst);
8430 int64_t ExpVal = Exponent;
8431
8432 if (ExpVal == 0) {
8433 Builder.buildFConstant(Res: Dst, Val: 1.0);
8434 MI.removeFromParent();
8435 return;
8436 }
8437
8438 if (ExpVal < 0)
8439 ExpVal = -ExpVal;
8440
8441 // We use the simple binary decomposition method from SelectionDAG ExpandPowI
8442 // to generate the multiply sequence. There are more optimal ways to do this
8443 // (for example, powi(x,15) generates one more multiply than it should), but
8444 // this has the benefit of being both really simple and much better than a
8445 // libcall.
8446 std::optional<SrcOp> Res;
8447 SrcOp CurSquare = Base;
8448 while (ExpVal > 0) {
8449 if (ExpVal & 1) {
8450 if (!Res)
8451 Res = CurSquare;
8452 else
8453 Res = Builder.buildFMul(Dst: Ty, Src0: *Res, Src1: CurSquare);
8454 }
8455
8456 CurSquare = Builder.buildFMul(Dst: Ty, Src0: CurSquare, Src1: CurSquare);
8457 ExpVal >>= 1;
8458 }
8459
8460 // If the original exponent was negative, invert the result, producing
8461 // 1/(x*x*x).
8462 if (Exponent < 0)
8463 Res = Builder.buildFDiv(Dst: Ty, Src0: Builder.buildFConstant(Res: Ty, Val: 1.0), Src1: *Res,
8464 Flags: MI.getFlags());
8465
8466 Builder.buildCopy(Res: Dst, Op: *Res);
8467 MI.eraseFromParent();
8468}
8469
8470bool CombinerHelper::matchFoldAPlusC1MinusC2(const MachineInstr &MI,
8471 BuildFnTy &MatchInfo) const {
8472 // fold (A+C1)-C2 -> A+(C1-C2)
8473 const GSub *Sub = cast<GSub>(Val: &MI);
8474 Register A, C1Reg;
8475 if (!mi_match(R: Sub->getLHSReg(), MRI, P: m_GAdd(L: m_Reg(R&: A), R: m_Reg(R&: C1Reg))))
8476 return false;
8477
8478 if (!MRI.hasOneNonDBGUse(RegNo: Sub->getLHSReg()))
8479 return false;
8480
8481 APInt C2 = getIConstantFromReg(VReg: Sub->getRHSReg(), MRI);
8482 APInt C1 = getIConstantFromReg(VReg: C1Reg, MRI);
8483
8484 Register Dst = Sub->getReg(Idx: 0);
8485 LLT DstTy = MRI.getType(Reg: Dst);
8486
8487 MatchInfo = [=](MachineIRBuilder &B) {
8488 auto Const = B.buildConstant(Res: DstTy, Val: C1 - C2);
8489 B.buildAdd(Dst, Src0: A, Src1: Const);
8490 };
8491
8492 return true;
8493}
8494
8495bool CombinerHelper::matchFoldC2MinusAPlusC1(const MachineInstr &MI,
8496 BuildFnTy &MatchInfo) const {
8497 // fold C2-(A+C1) -> (C2-C1)-A
8498 const GSub *Sub = cast<GSub>(Val: &MI);
8499 Register A, C1Reg;
8500 if (!mi_match(R: Sub->getRHSReg(), MRI, P: m_GAdd(L: m_Reg(R&: A), R: m_Reg(R&: C1Reg))))
8501 return false;
8502
8503 if (!MRI.hasOneNonDBGUse(RegNo: Sub->getRHSReg()))
8504 return false;
8505
8506 APInt C2 = getIConstantFromReg(VReg: Sub->getLHSReg(), MRI);
8507 APInt C1 = getIConstantFromReg(VReg: C1Reg, MRI);
8508
8509 Register Dst = Sub->getReg(Idx: 0);
8510 LLT DstTy = MRI.getType(Reg: Dst);
8511
8512 MatchInfo = [=](MachineIRBuilder &B) {
8513 auto Const = B.buildConstant(Res: DstTy, Val: C2 - C1);
8514 B.buildSub(Dst, Src0: Const, Src1: A);
8515 };
8516
8517 return true;
8518}
8519
8520bool CombinerHelper::matchFoldAMinusC1MinusC2(const MachineInstr &MI,
8521 BuildFnTy &MatchInfo) const {
8522 // fold (A-C1)-C2 -> A-(C1+C2)
8523 const GSub *Sub1 = cast<GSub>(Val: &MI);
8524 Register A, C1Reg;
8525 if (!mi_match(R: Sub1->getLHSReg(), MRI, P: m_GSub(L: m_Reg(R&: A), R: m_Reg(R&: C1Reg))))
8526 return false;
8527
8528 if (!MRI.hasOneNonDBGUse(RegNo: Sub1->getLHSReg()))
8529 return false;
8530
8531 APInt C2 = getIConstantFromReg(VReg: Sub1->getRHSReg(), MRI);
8532 APInt C1 = getIConstantFromReg(VReg: C1Reg, MRI);
8533
8534 Register Dst = Sub1->getReg(Idx: 0);
8535 LLT DstTy = MRI.getType(Reg: Dst);
8536
8537 MatchInfo = [=](MachineIRBuilder &B) {
8538 auto Const = B.buildConstant(Res: DstTy, Val: C1 + C2);
8539 B.buildSub(Dst, Src0: A, Src1: Const);
8540 };
8541
8542 return true;
8543}
8544
8545bool CombinerHelper::matchFoldC1Minus2MinusC2(const MachineInstr &MI,
8546 BuildFnTy &MatchInfo) const {
8547 // fold (C1-A)-C2 -> (C1-C2)-A
8548 const GSub *Sub1 = cast<GSub>(Val: &MI);
8549 Register C1Reg, A;
8550 if (!mi_match(R: Sub1->getLHSReg(), MRI, P: m_GSub(L: m_Reg(R&: C1Reg), R: m_Reg(R&: A))))
8551 return false;
8552
8553 if (!MRI.hasOneNonDBGUse(RegNo: Sub1->getLHSReg()))
8554 return false;
8555
8556 APInt C2 = getIConstantFromReg(VReg: Sub1->getRHSReg(), MRI);
8557 APInt C1 = getIConstantFromReg(VReg: C1Reg, MRI);
8558
8559 Register Dst = Sub1->getReg(Idx: 0);
8560 LLT DstTy = MRI.getType(Reg: Dst);
8561
8562 MatchInfo = [=](MachineIRBuilder &B) {
8563 auto Const = B.buildConstant(Res: DstTy, Val: C1 - C2);
8564 B.buildSub(Dst, Src0: Const, Src1: A);
8565 };
8566
8567 return true;
8568}
8569
8570bool CombinerHelper::matchFoldAMinusC1PlusC2(const MachineInstr &MI,
8571 BuildFnTy &MatchInfo) const {
8572 // fold ((A-C1)+C2) -> (A+(C2-C1))
8573 const GAdd *Add = cast<GAdd>(Val: &MI);
8574 Register A, C1Reg;
8575 if (!mi_match(R: Add->getLHSReg(), MRI, P: m_GSub(L: m_Reg(R&: A), R: m_Reg(R&: C1Reg))))
8576 return false;
8577
8578 if (!MRI.hasOneNonDBGUse(RegNo: Add->getLHSReg()))
8579 return false;
8580
8581 APInt C2 = getIConstantFromReg(VReg: Add->getRHSReg(), MRI);
8582 APInt C1 = getIConstantFromReg(VReg: C1Reg, MRI);
8583
8584 Register Dst = Add->getReg(Idx: 0);
8585 LLT DstTy = MRI.getType(Reg: Dst);
8586
8587 MatchInfo = [=](MachineIRBuilder &B) {
8588 auto Const = B.buildConstant(Res: DstTy, Val: C2 - C1);
8589 B.buildAdd(Dst, Src0: A, Src1: Const);
8590 };
8591
8592 return true;
8593}
8594
8595bool CombinerHelper::matchUnmergeValuesAnyExtBuildVector(
8596 const MachineInstr &MI, BuildFnTy &MatchInfo) const {
8597 const GUnmerge *Unmerge = cast<GUnmerge>(Val: &MI);
8598
8599 if (!MRI.hasOneNonDBGUse(RegNo: Unmerge->getSourceReg()))
8600 return false;
8601
8602 LLT DstTy = MRI.getType(Reg: Unmerge->getReg(Idx: 0));
8603
8604 // $bv:_(<8 x s8>) = G_BUILD_VECTOR ....
8605 // $any:_(<8 x s16>) = G_ANYEXT $bv
8606 // $uv:_(<4 x s16>), $uv1:_(<4 x s16>) = G_UNMERGE_VALUES $any
8607 //
8608 // ->
8609 //
8610 // $any:_(s16) = G_ANYEXT $bv[0]
8611 // $any1:_(s16) = G_ANYEXT $bv[1]
8612 // $any2:_(s16) = G_ANYEXT $bv[2]
8613 // $any3:_(s16) = G_ANYEXT $bv[3]
8614 // $any4:_(s16) = G_ANYEXT $bv[4]
8615 // $any5:_(s16) = G_ANYEXT $bv[5]
8616 // $any6:_(s16) = G_ANYEXT $bv[6]
8617 // $any7:_(s16) = G_ANYEXT $bv[7]
8618 // $uv:_(<4 x s16>) = G_BUILD_VECTOR $any, $any1, $any2, $any3
8619 // $uv1:_(<4 x s16>) = G_BUILD_VECTOR $any4, $any5, $any6, $any7
8620
8621 // We want to unmerge into vectors.
8622 if (!DstTy.isFixedVector())
8623 return false;
8624
8625 Register AnySrcReg;
8626 if (!mi_match(R: Unmerge->getSourceReg(), MRI, P: m_GAnyExt(Src: m_Reg(R&: AnySrcReg))))
8627 return false;
8628
8629 GBuildVector *BV;
8630 if (mi_match(R: AnySrcReg, MRI, P: m_GBuildVector(Inst&: BV))) {
8631 // G_UNMERGE_VALUES G_ANYEXT G_BUILD_VECTOR
8632
8633 if (!MRI.hasOneNonDBGUse(RegNo: BV->getReg(Idx: 0)))
8634 return false;
8635
8636 // FIXME: check element types?
8637 if (BV->getNumSources() % Unmerge->getNumDefs() != 0)
8638 return false;
8639
8640 LLT BigBvTy = MRI.getType(Reg: BV->getReg(Idx: 0));
8641 LLT SmallBvTy = DstTy;
8642 LLT SmallBvElemenTy = SmallBvTy.getElementType();
8643
8644 if (!isLegalOrBeforeLegalizer(
8645 Query: {TargetOpcode::G_BUILD_VECTOR, {SmallBvTy, SmallBvElemenTy}}))
8646 return false;
8647
8648 // We check the legality of scalar anyext.
8649 if (!isLegalOrBeforeLegalizer(
8650 Query: {TargetOpcode::G_ANYEXT,
8651 {SmallBvElemenTy, BigBvTy.getElementType()}}))
8652 return false;
8653
8654 MatchInfo = [=](MachineIRBuilder &B) {
8655 // Build into each G_UNMERGE_VALUES def
8656 // a small build vector with anyext from the source build vector.
8657 for (unsigned I = 0; I < Unmerge->getNumDefs(); ++I) {
8658 SmallVector<Register> Ops;
8659 for (unsigned J = 0; J < SmallBvTy.getNumElements(); ++J) {
8660 Register SourceArray =
8661 BV->getSourceReg(I: I * SmallBvTy.getNumElements() + J);
8662 auto AnyExt = B.buildAnyExt(Res: SmallBvElemenTy, Op: SourceArray);
8663 Ops.push_back(Elt: AnyExt.getReg(Idx: 0));
8664 }
8665 B.buildBuildVector(Res: Unmerge->getOperand(i: I).getReg(), Ops);
8666 };
8667 };
8668 return true;
8669 };
8670
8671 return false;
8672}
8673
8674bool CombinerHelper::matchShuffleUndefRHS(MachineInstr &MI,
8675 BuildFnTy &MatchInfo) const {
8676
8677 bool Changed = false;
8678 auto &Shuffle = cast<GShuffleVector>(Val&: MI);
8679 ArrayRef<int> OrigMask = Shuffle.getMask();
8680 SmallVector<int, 16> NewMask;
8681 const LLT SrcTy = MRI.getType(Reg: Shuffle.getSrc1Reg());
8682 const unsigned NumSrcElems = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
8683 const unsigned NumDstElts = OrigMask.size();
8684 for (unsigned i = 0; i != NumDstElts; ++i) {
8685 int Idx = OrigMask[i];
8686 if (Idx >= (int)NumSrcElems) {
8687 Idx = -1;
8688 Changed = true;
8689 }
8690 NewMask.push_back(Elt: Idx);
8691 }
8692
8693 if (!Changed)
8694 return false;
8695
8696 MatchInfo = [&, NewMask = std::move(NewMask)](MachineIRBuilder &B) {
8697 B.buildShuffleVector(Res: MI.getOperand(i: 0), Src1: MI.getOperand(i: 1), Src2: MI.getOperand(i: 2),
8698 Mask: std::move(NewMask));
8699 };
8700
8701 return true;
8702}
8703
8704static void commuteMask(MutableArrayRef<int> Mask, const unsigned NumElems) {
8705 const unsigned MaskSize = Mask.size();
8706 for (unsigned I = 0; I < MaskSize; ++I) {
8707 int Idx = Mask[I];
8708 if (Idx < 0)
8709 continue;
8710
8711 if (Idx < (int)NumElems)
8712 Mask[I] = Idx + NumElems;
8713 else
8714 Mask[I] = Idx - NumElems;
8715 }
8716}
8717
8718bool CombinerHelper::matchShuffleDisjointMask(MachineInstr &MI,
8719 BuildFnTy &MatchInfo) const {
8720
8721 auto &Shuffle = cast<GShuffleVector>(Val&: MI);
8722 // If any of the two inputs is already undef, don't check the mask again to
8723 // prevent infinite loop
8724 if (getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: Shuffle.getSrc1Reg(), MRI))
8725 return false;
8726
8727 if (getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: Shuffle.getSrc2Reg(), MRI))
8728 return false;
8729
8730 const LLT DstTy = MRI.getType(Reg: Shuffle.getReg(Idx: 0));
8731 const LLT Src1Ty = MRI.getType(Reg: Shuffle.getSrc1Reg());
8732 if (!isLegalOrBeforeLegalizer(
8733 Query: {TargetOpcode::G_SHUFFLE_VECTOR, {DstTy, Src1Ty}}))
8734 return false;
8735
8736 ArrayRef<int> Mask = Shuffle.getMask();
8737 const unsigned NumSrcElems = Src1Ty.getNumElements();
8738
8739 bool TouchesSrc1 = false;
8740 bool TouchesSrc2 = false;
8741 const unsigned NumElems = Mask.size();
8742 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
8743 if (Mask[Idx] < 0)
8744 continue;
8745
8746 if (Mask[Idx] < (int)NumSrcElems)
8747 TouchesSrc1 = true;
8748 else
8749 TouchesSrc2 = true;
8750 }
8751
8752 if (TouchesSrc1 == TouchesSrc2)
8753 return false;
8754
8755 Register NewSrc1 = Shuffle.getSrc1Reg();
8756 SmallVector<int, 16> NewMask(Mask);
8757 if (TouchesSrc2) {
8758 NewSrc1 = Shuffle.getSrc2Reg();
8759 commuteMask(Mask: NewMask, NumElems: NumSrcElems);
8760 }
8761
8762 MatchInfo = [=, &Shuffle](MachineIRBuilder &B) {
8763 auto Undef = B.buildUndef(Res: Src1Ty);
8764 B.buildShuffleVector(Res: Shuffle.getReg(Idx: 0), Src1: NewSrc1, Src2: Undef, Mask: NewMask);
8765 };
8766
8767 return true;
8768}
8769
8770bool CombinerHelper::matchSuboCarryOut(const MachineInstr &MI,
8771 BuildFnTy &MatchInfo) const {
8772 const GSubCarryOut *Subo = cast<GSubCarryOut>(Val: &MI);
8773
8774 Register Dst = Subo->getReg(Idx: 0);
8775 Register LHS = Subo->getLHSReg();
8776 Register RHS = Subo->getRHSReg();
8777 Register Carry = Subo->getCarryOutReg();
8778 LLT DstTy = MRI.getType(Reg: Dst);
8779 LLT CarryTy = MRI.getType(Reg: Carry);
8780
8781 // Check legality before known bits.
8782 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_SUB, {DstTy}}) ||
8783 !isConstantLegalOrBeforeLegalizer(Ty: CarryTy))
8784 return false;
8785
8786 ConstantRange KBLHS =
8787 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: LHS),
8788 /* IsSigned=*/Subo->isSigned());
8789 ConstantRange KBRHS =
8790 ConstantRange::fromKnownBits(Known: VT->getKnownBits(R: RHS),
8791 /* IsSigned=*/Subo->isSigned());
8792
8793 if (Subo->isSigned()) {
8794 // G_SSUBO
8795 switch (KBLHS.signedSubMayOverflow(Other: KBRHS)) {
8796 case ConstantRange::OverflowResult::MayOverflow:
8797 return false;
8798 case ConstantRange::OverflowResult::NeverOverflows: {
8799 MatchInfo = [=](MachineIRBuilder &B) {
8800 B.buildSub(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoSWrap);
8801 B.buildConstant(Res: Carry, Val: 0);
8802 };
8803 return true;
8804 }
8805 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
8806 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
8807 MatchInfo = [=](MachineIRBuilder &B) {
8808 B.buildSub(Dst, Src0: LHS, Src1: RHS);
8809 B.buildConstant(Res: Carry, Val: getICmpTrueVal(TLI: getTargetLowering(),
8810 /*isVector=*/IsVector: CarryTy.isVector(),
8811 /*isFP=*/IsFP: false));
8812 };
8813 return true;
8814 }
8815 }
8816 return false;
8817 }
8818
8819 // G_USUBO
8820 switch (KBLHS.unsignedSubMayOverflow(Other: KBRHS)) {
8821 case ConstantRange::OverflowResult::MayOverflow:
8822 return false;
8823 case ConstantRange::OverflowResult::NeverOverflows: {
8824 MatchInfo = [=](MachineIRBuilder &B) {
8825 B.buildSub(Dst, Src0: LHS, Src1: RHS, Flags: MachineInstr::MIFlag::NoUWrap);
8826 B.buildConstant(Res: Carry, Val: 0);
8827 };
8828 return true;
8829 }
8830 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
8831 case ConstantRange::OverflowResult::AlwaysOverflowsHigh: {
8832 MatchInfo = [=](MachineIRBuilder &B) {
8833 B.buildSub(Dst, Src0: LHS, Src1: RHS);
8834 B.buildConstant(Res: Carry, Val: getICmpTrueVal(TLI: getTargetLowering(),
8835 /*isVector=*/IsVector: CarryTy.isVector(),
8836 /*isFP=*/IsFP: false));
8837 };
8838 return true;
8839 }
8840 }
8841
8842 return false;
8843}
8844
8845// Fold (ctlz (xor x, (sra x, bitwidth-1))) -> (add (ctls x), 1).
8846// Fold (ctlz (or (shl (xor x, (sra x, bitwidth-1)), 1), 1) -> (ctls x)
8847bool CombinerHelper::matchCtls(MachineInstr &CtlzMI,
8848 BuildFnTy &MatchInfo) const {
8849 assert((CtlzMI.getOpcode() == TargetOpcode::G_CTLZ ||
8850 CtlzMI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_POISON) &&
8851 "Expected G_CTLZ variant");
8852
8853 const Register Dst = CtlzMI.getOperand(i: 0).getReg();
8854 Register Src = CtlzMI.getOperand(i: 1).getReg();
8855
8856 LLT Ty = MRI.getType(Reg: Dst);
8857 LLT SrcTy = MRI.getType(Reg: Src);
8858
8859 if (!(Ty.isValid() && Ty.isScalar()))
8860 return false;
8861
8862 if (!LI)
8863 return false;
8864
8865 SmallVector<LLT, 2> QueryTypes = {Ty, SrcTy};
8866 LegalityQuery Query(TargetOpcode::G_CTLS, QueryTypes);
8867
8868 switch (LI->getAction(Query).Action) {
8869 default:
8870 return false;
8871 case LegalizeActions::Legal:
8872 case LegalizeActions::Custom:
8873 case LegalizeActions::WidenScalar:
8874 break;
8875 }
8876
8877 // Src = or(shl(V, 1), 1) -> Src=V; NeedAdd = False
8878 Register V;
8879 bool NeedAdd = true;
8880 if (mi_match(R: Src, MRI,
8881 P: m_OneUse(SP: m_GOr(L: m_OneUse(SP: m_GShl(L: m_Reg(R&: V), R: m_SpecificICst(RequestedValue: 1))),
8882 R: m_SpecificICst(RequestedValue: 1))))) {
8883 NeedAdd = false;
8884 Src = V;
8885 }
8886
8887 unsigned BitWidth = Ty.getScalarSizeInBits();
8888
8889 Register X;
8890 if (!mi_match(R: Src, MRI,
8891 P: m_OneUse(SP: m_GXor(L: m_Reg(R&: X), R: m_OneUse(SP: m_GAShr(
8892 L: m_DeferredReg(R&: X),
8893 R: m_SpecificICst(RequestedValue: BitWidth - 1)))))))
8894 return false;
8895
8896 MatchInfo = [=](MachineIRBuilder &B) {
8897 if (!NeedAdd) {
8898 B.buildCTLS(Dst, Src0: X);
8899 return;
8900 }
8901
8902 auto Ctls = B.buildCTLS(Dst: Ty, Src0: X);
8903 auto One = B.buildConstant(Res: Ty, Val: 1);
8904
8905 B.buildAdd(Dst, Src0: Ctls, Src1: One);
8906 };
8907
8908 return true;
8909}
8910
8911// Fold shr ( add ( ext X, ext Y ), 1 ) -> avgfloor ( x, y )
8912// Fold shr ( add ( ext X, ext Y, 1 ), 1 ) -> avgceil ( x, y )
8913bool CombinerHelper::matchAVG(MachineInstr &MI, MachineRegisterInfo &MRI,
8914 Register X, Register Y,
8915 unsigned TargetOpc) const {
8916 assert((MI.getOpcode() == TargetOpcode::G_LSHR ||
8917 MI.getOpcode() == TargetOpcode::G_ASHR) &&
8918 "Expected G_LSHR/G_ASHR");
8919
8920 LLT XTy = MRI.getType(Reg: X);
8921 return XTy == MRI.getType(Reg: Y) && isLegal(Query: {TargetOpc, {XTy}});
8922}
8923
8924static unsigned getCountZeroPoisonOpcode(const MachineInstr &MI) {
8925 assert((MI.getOpcode() == TargetOpcode::G_CTLZ ||
8926 MI.getOpcode() == TargetOpcode::G_CTTZ) &&
8927 "Expected count-zero opcode");
8928 switch (MI.getOpcode()) {
8929 case TargetOpcode::G_CTLZ:
8930 return TargetOpcode::G_CTLZ_ZERO_POISON;
8931 case TargetOpcode::G_CTTZ:
8932 return TargetOpcode::G_CTTZ_ZERO_POISON;
8933 default:
8934 llvm_unreachable("Unexpected count-zero opcode");
8935 }
8936}
8937
8938bool CombinerHelper::matchCountZeroToZeroPoison(MachineInstr &MI) const {
8939 if (!VT)
8940 return false;
8941
8942 unsigned ZPOpc = getCountZeroPoisonOpcode(MI);
8943 Register Src = MI.getOperand(i: 1).getReg();
8944 if (!VT->isKnownNeverZero(R: Src))
8945 return false;
8946
8947 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
8948 LLT SrcTy = MRI.getType(Reg: Src);
8949 return isLegalOrBeforeLegalizer(Query: {ZPOpc, {DstTy, SrcTy}});
8950}
8951
8952void CombinerHelper::applyCountZeroToZeroPoison(MachineInstr &MI) const {
8953 replaceOpcodeWith(FromMI&: MI, ToOpcode: getCountZeroPoisonOpcode(MI));
8954}
8955